"""Split Ren'Py source into logical lines organised as an indentation tree.""" import re from dataclasses import dataclass, field class ParseError(Exception): def __init__(self, msg: str, filename: str = "", lineno: int = 0): super().__init__(f"{filename}:{lineno}: {msg}" if filename else msg) self.msg = msg self.filename = filename self.lineno = lineno @dataclass class Line: text: str # logical line, comments removed, continuation lines joined lineno: int indent: int physical: list[str] # original physical lines (for python blocks) block: list["Line"] = field(default_factory=list) def block_source(self) -> str: """Raw source of the nested block, for python: blocks.""" out: list[str] = [] for child in self.block: out.extend(child.physical) out.append(child.block_source()) return "\n".join(s for s in out if s) _QUOTES = ('"""', "'''", '"', "'", "`") def _scan(line: str, state: dict) -> tuple[str, bool]: """Scan one physical line. Returns text without comment and whether it continues.""" out: list[str] = [] i, n = 0, len(line) while i < n: q = state["quote"] c = line[i] if q: if c == "\\" and i + 1 < n: out.append(line[i:i + 2]) i += 2 continue if line.startswith(q, i): state["quote"] = None out.append(q) i += len(q) continue out.append(c) i += 1 continue if c == "#": break for quote in _QUOTES: if line.startswith(quote, i): state["quote"] = quote out.append(quote) i += len(quote) break else: if c in "([{": state["depth"] += 1 elif c in ")]}": state["depth"] = max(0, state["depth"] - 1) out.append(c) i += 1 text = "".join(out) cont = state["depth"] > 0 or state["quote"] is not None if not cont and text.rstrip().endswith("\\") and not text.rstrip().endswith("\\\\"): text = text.rstrip()[:-1] cont = True return text, cont def logical_lines(source: str, filename: str = "") -> list[Line]: source = source.lstrip("").replace("\r\n", "\n").replace("\r", "\n") physical = source.split("\n") result: list[Line] = [] i = 0 while i < len(physical): first = physical[i].replace("\t", " ") state = {"depth": 0, "quote": None} parts: list[str] = [] raw: list[str] = [] start = i while True: raw.append(physical[i]) text, cont = _scan(physical[i].replace("\t", " ") if i != start else first, state) parts.append(text) i += 1 if not cont or i >= len(physical): break joined = "\n".join(parts).strip() if not joined: continue indent = len(first) - len(first.lstrip(" ")) result.append(Line(joined, start + 1, indent, raw)) return result def build_tree(lines: list[Line], filename: str = "") -> list[Line]: root: list[Line] = [] stack: list[tuple[int, list[Line]]] = [(-1, root)] prev: Line | None = None for line in lines: if prev is not None and line.indent > prev.indent and prev.text.endswith(":"): stack.append((line.indent, prev.block)) else: while stack[-1][0] > line.indent: stack.pop() if stack[-1][0] != line.indent and len(stack) > 1: # inconsistent dedent; attach to nearest enclosing block stack.append((line.indent, stack[-1][1])) stack[-1][1].append(line) prev = line return root def parse_tree(source: str, filename: str = "") -> list[Line]: return build_tree(logical_lines(source, filename), filename) # --- in-line tokenizer ------------------------------------------------------- _WORD = re.compile(r"[A-Za-z_\u0080-￿][\w\u0080-￿]*") _DOTTED = re.compile(r"[A-Za-z_\u0080-￿][\w\u0080-￿]*(\.[A-Za-z_\u0080-￿][\w\u0080-￿]*)*") _LABEL = re.compile(r"\.?[A-Za-z_\u0080-￿][\w\u0080-￿]*(\.[A-Za-z_\u0080-￿][\w\u0080-￿]*)*") _IMAGE_PART = re.compile(r"[-\w\u0080-￿]+") _NUMBER = re.compile(r"[-+]?(\d+\.?\d*|\.\d+)([eE][-+]?\d+)?") _STRING_START = re.compile(r"[rRuUbB]{0,2}(\"\"\"|'''|\"|'|`)") def renpy_string(body: str, raw: bool = False) -> str: """Apply Ren'Py string semantics: collapse whitespace, process escapes.""" if raw: return body body = re.sub(r"\\\n", "", body) body = re.sub(r"[ \n]+", " ", body) def dequote(m: re.Match) -> str: c = m.group(1) if c == "n": return "\n" if c.startswith("u"): return chr(int(c[1:], 16)) return c return re.sub(r"\\(u[0-9a-fA-F]{1,4}|.)", dequote, body) class Cursor: def __init__(self, text: str, filename: str = "", lineno: int = 0): self.text = text self.pos = 0 self.filename = filename self.lineno = lineno def error(self, msg: str) -> ParseError: return ParseError(f"{msg}: {self.text[:80]!r}", self.filename, self.lineno) def ws(self) -> None: while self.pos < len(self.text) and self.text[self.pos] in " \n\t": self.pos += 1 def eol(self) -> bool: self.ws() return self.pos >= len(self.text) def peek(self, s: str) -> bool: self.ws() return self.text.startswith(s, self.pos) def char(self, s: str) -> bool: if self.peek(s): self.pos += len(s) return True return False def _regex(self, rx: re.Pattern) -> str | None: self.ws() m = rx.match(self.text, self.pos) if not m: return None self.pos = m.end() return m.group(0) def keyword(self, kw: str) -> bool: self.ws() save = self.pos word = self._regex(_WORD) if word == kw: return True self.pos = save return False def peek_word(self) -> str | None: save = self.pos word = self._regex(_WORD) self.pos = save return word def word(self) -> str | None: return self._regex(_WORD) def dotted(self) -> str | None: return self._regex(_DOTTED) def label_name(self) -> str | None: return self._regex(_LABEL) def image_part(self) -> str | None: return self._regex(_IMAGE_PART) def number(self) -> str | None: return self._regex(_NUMBER) def string_raw(self) -> str | None: """Match a string literal and return its source text (with quotes).""" self.ws() m = _STRING_START.match(self.text, self.pos) if not m: return None quote = m.group(1) i = m.end() while i < len(self.text): if self.text[i] == "\\": i += 2 continue if self.text.startswith(quote, i): src = self.text[self.pos:i + len(quote)] self.pos = i + len(quote) return src i += 1 raise self.error("незакрытая строка") def string(self) -> str | None: src = self.string_raw() if src is None: return None m = _STRING_START.match(src) prefix, quote = src[: m.start(1)], m.group(1) return renpy_string(src[m.end(): -len(quote)], raw="r" in prefix.lower()) def rest(self) -> str: self.ws() s = self.text[self.pos:] self.pos = len(self.text) return s.strip() def until(self, stop_words: set[str], stop_chars: str = "") -> str: """Python-ish text up to a top-level stop word, stop char or end.""" self.ws() start = i = self.pos depth = 0 n = len(self.text) while i < n: c = self.text[i] m = _STRING_START.match(self.text, i) if c in "\"'`rRuUbB" else None if m and (c in "\"'`" or m.end() > i + 1): save = self.pos self.pos = i self.string_raw() i = self.pos self.pos = save continue if c in "([{": depth += 1 elif c in ")]}": depth -= 1 elif depth == 0: if c in stop_chars: break if (c.isalpha() or c == "_") and (i == 0 or not (self.text[i - 1].isalnum() or self.text[i - 1] in "_.")): w = _WORD.match(self.text, i) if w and w.group(0) in stop_words and self.text[i - 1:i].isspace(): break i = w.end() if w else i + 1 continue i += 1 self.pos = i return self.text[start:i].strip()