291 lines
9.1 KiB
Python
291 lines
9.1 KiB
Python
"""Split Ren'Py source into logical lines organised as an indentation tree."""
|
||
|
||
import re
|
||
from dataclasses import dataclass, field
|
||
|
||
|
||
class ParseError(Exception):
|
||
def __init__(self, msg: str, filename: str = "", lineno: int = 0):
|
||
super().__init__(f"{filename}:{lineno}: {msg}" if filename else msg)
|
||
self.msg = msg
|
||
self.filename = filename
|
||
self.lineno = lineno
|
||
|
||
|
||
@dataclass
|
||
class Line:
|
||
text: str # logical line, comments removed, continuation lines joined
|
||
lineno: int
|
||
indent: int
|
||
physical: list[str] # original physical lines (for python blocks)
|
||
block: list["Line"] = field(default_factory=list)
|
||
|
||
def block_source(self) -> str:
|
||
"""Raw source of the nested block, for python: blocks."""
|
||
out: list[str] = []
|
||
for child in self.block:
|
||
out.extend(child.physical)
|
||
out.append(child.block_source())
|
||
return "\n".join(s for s in out if s)
|
||
|
||
|
||
_QUOTES = ('"""', "'''", '"', "'", "`")
|
||
|
||
|
||
def _scan(line: str, state: dict) -> tuple[str, bool]:
|
||
"""Scan one physical line. Returns text without comment and whether it continues."""
|
||
out: list[str] = []
|
||
i, n = 0, len(line)
|
||
while i < n:
|
||
q = state["quote"]
|
||
c = line[i]
|
||
if q:
|
||
if c == "\\" and i + 1 < n:
|
||
out.append(line[i:i + 2])
|
||
i += 2
|
||
continue
|
||
if line.startswith(q, i):
|
||
state["quote"] = None
|
||
out.append(q)
|
||
i += len(q)
|
||
continue
|
||
out.append(c)
|
||
i += 1
|
||
continue
|
||
if c == "#":
|
||
break
|
||
for quote in _QUOTES:
|
||
if line.startswith(quote, i):
|
||
state["quote"] = quote
|
||
out.append(quote)
|
||
i += len(quote)
|
||
break
|
||
else:
|
||
if c in "([{":
|
||
state["depth"] += 1
|
||
elif c in ")]}":
|
||
state["depth"] = max(0, state["depth"] - 1)
|
||
out.append(c)
|
||
i += 1
|
||
text = "".join(out)
|
||
cont = state["depth"] > 0 or state["quote"] is not None
|
||
if not cont and text.rstrip().endswith("\\") and not text.rstrip().endswith("\\\\"):
|
||
text = text.rstrip()[:-1]
|
||
cont = True
|
||
return text, cont
|
||
|
||
|
||
def logical_lines(source: str, filename: str = "") -> list[Line]:
|
||
source = source.lstrip("").replace("\r\n", "\n").replace("\r", "\n")
|
||
physical = source.split("\n")
|
||
result: list[Line] = []
|
||
i = 0
|
||
while i < len(physical):
|
||
first = physical[i].replace("\t", " ")
|
||
state = {"depth": 0, "quote": None}
|
||
parts: list[str] = []
|
||
raw: list[str] = []
|
||
start = i
|
||
while True:
|
||
raw.append(physical[i])
|
||
text, cont = _scan(physical[i].replace("\t", " ") if i != start else first, state)
|
||
parts.append(text)
|
||
i += 1
|
||
if not cont or i >= len(physical):
|
||
break
|
||
joined = "\n".join(parts).strip()
|
||
if not joined:
|
||
continue
|
||
indent = len(first) - len(first.lstrip(" "))
|
||
result.append(Line(joined, start + 1, indent, raw))
|
||
return result
|
||
|
||
|
||
# decompiled `layeredimage name: tag`: the block still follows
|
||
LAYERED_HEADER = re.compile(r"^layeredimage\s+[^:]+:\s+\w+$")
|
||
|
||
|
||
def build_tree(lines: list[Line], filename: str = "") -> list[Line]:
|
||
root: list[Line] = []
|
||
stack: list[tuple[int, list[Line]]] = [(-1, root)]
|
||
prev: Line | None = None
|
||
for line in lines:
|
||
if prev is not None and line.indent > prev.indent and (
|
||
prev.text.endswith(":") or LAYERED_HEADER.match(prev.text)):
|
||
stack.append((line.indent, prev.block))
|
||
else:
|
||
while stack[-1][0] > line.indent:
|
||
stack.pop()
|
||
if stack[-1][0] != line.indent and len(stack) > 1:
|
||
# inconsistent dedent; attach to nearest enclosing block
|
||
stack.append((line.indent, stack[-1][1]))
|
||
stack[-1][1].append(line)
|
||
prev = line
|
||
return root
|
||
|
||
|
||
def parse_tree(source: str, filename: str = "") -> list[Line]:
|
||
return build_tree(logical_lines(source, filename), filename)
|
||
|
||
|
||
# --- in-line tokenizer -------------------------------------------------------
|
||
|
||
_WORD = re.compile(r"[A-Za-z_\u0080-][\w\u0080-]*")
|
||
_DOTTED = re.compile(r"[A-Za-z_\u0080-][\w\u0080-]*(\.[A-Za-z_\u0080-][\w\u0080-]*)*")
|
||
_LABEL = re.compile(r"\.?[A-Za-z_\u0080-][\w\u0080-]*(\.[A-Za-z_\u0080-][\w\u0080-]*)*")
|
||
_IMAGE_PART = re.compile(r"[-\w\u0080-]+")
|
||
_NUMBER = re.compile(r"[-+]?(\d+\.?\d*|\.\d+)([eE][-+]?\d+)?")
|
||
_STRING_START = re.compile(r"[rRuUbB]{0,2}(\"\"\"|'''|\"|'|`)")
|
||
|
||
|
||
def renpy_string(body: str, raw: bool = False) -> str:
|
||
"""Apply Ren'Py string semantics: collapse whitespace, process escapes."""
|
||
if raw:
|
||
return body
|
||
body = re.sub(r"\\\n", "", body)
|
||
body = re.sub(r"[ \n]+", " ", body)
|
||
|
||
def dequote(m: re.Match) -> str:
|
||
c = m.group(1)
|
||
if c == "n":
|
||
return "\n"
|
||
if c.startswith("u"):
|
||
return chr(int(c[1:], 16))
|
||
return c
|
||
|
||
return re.sub(r"\\(u[0-9a-fA-F]{1,4}|.)", dequote, body)
|
||
|
||
|
||
class Cursor:
|
||
def __init__(self, text: str, filename: str = "", lineno: int = 0):
|
||
self.text = text
|
||
self.pos = 0
|
||
self.filename = filename
|
||
self.lineno = lineno
|
||
|
||
def error(self, msg: str) -> ParseError:
|
||
return ParseError(f"{msg}: {self.text[:80]!r}", self.filename, self.lineno)
|
||
|
||
def ws(self) -> None:
|
||
while self.pos < len(self.text) and self.text[self.pos] in " \n\t":
|
||
self.pos += 1
|
||
|
||
def eol(self) -> bool:
|
||
self.ws()
|
||
return self.pos >= len(self.text)
|
||
|
||
def peek(self, s: str) -> bool:
|
||
self.ws()
|
||
return self.text.startswith(s, self.pos)
|
||
|
||
def char(self, s: str) -> bool:
|
||
if self.peek(s):
|
||
self.pos += len(s)
|
||
return True
|
||
return False
|
||
|
||
def _regex(self, rx: re.Pattern) -> str | None:
|
||
self.ws()
|
||
m = rx.match(self.text, self.pos)
|
||
if not m:
|
||
return None
|
||
self.pos = m.end()
|
||
return m.group(0)
|
||
|
||
def keyword(self, kw: str) -> bool:
|
||
self.ws()
|
||
save = self.pos
|
||
word = self._regex(_WORD)
|
||
if word == kw:
|
||
return True
|
||
self.pos = save
|
||
return False
|
||
|
||
def peek_word(self) -> str | None:
|
||
save = self.pos
|
||
word = self._regex(_WORD)
|
||
self.pos = save
|
||
return word
|
||
|
||
def word(self) -> str | None:
|
||
return self._regex(_WORD)
|
||
|
||
def dotted(self) -> str | None:
|
||
return self._regex(_DOTTED)
|
||
|
||
def label_name(self) -> str | None:
|
||
return self._regex(_LABEL)
|
||
|
||
def image_part(self) -> str | None:
|
||
return self._regex(_IMAGE_PART)
|
||
|
||
def number(self) -> str | None:
|
||
return self._regex(_NUMBER)
|
||
|
||
def string_raw(self) -> str | None:
|
||
"""Match a string literal and return its source text (with quotes)."""
|
||
self.ws()
|
||
m = _STRING_START.match(self.text, self.pos)
|
||
if not m:
|
||
return None
|
||
quote = m.group(1)
|
||
i = m.end()
|
||
while i < len(self.text):
|
||
if self.text[i] == "\\":
|
||
i += 2
|
||
continue
|
||
if self.text.startswith(quote, i):
|
||
src = self.text[self.pos:i + len(quote)]
|
||
self.pos = i + len(quote)
|
||
return src
|
||
i += 1
|
||
raise self.error("незакрытая строка")
|
||
|
||
def string(self) -> str | None:
|
||
src = self.string_raw()
|
||
if src is None:
|
||
return None
|
||
m = _STRING_START.match(src)
|
||
prefix, quote = src[: m.start(1)], m.group(1)
|
||
return renpy_string(src[m.end(): -len(quote)], raw="r" in prefix.lower())
|
||
|
||
def rest(self) -> str:
|
||
self.ws()
|
||
s = self.text[self.pos:]
|
||
self.pos = len(self.text)
|
||
return s.strip()
|
||
|
||
def until(self, stop_words: set[str], stop_chars: str = "") -> str:
|
||
"""Python-ish text up to a top-level stop word, stop char or end."""
|
||
self.ws()
|
||
start = i = self.pos
|
||
depth = 0
|
||
n = len(self.text)
|
||
while i < n:
|
||
c = self.text[i]
|
||
m = _STRING_START.match(self.text, i) if c in "\"'`rRuUbB" else None
|
||
if m and (c in "\"'`" or m.end() > i + 1):
|
||
save = self.pos
|
||
self.pos = i
|
||
self.string_raw()
|
||
i = self.pos
|
||
self.pos = save
|
||
continue
|
||
if c in "([{":
|
||
depth += 1
|
||
elif c in ")]}":
|
||
depth -= 1
|
||
elif depth == 0:
|
||
if c in stop_chars:
|
||
break
|
||
if (c.isalpha() or c == "_") and (i == 0 or not (self.text[i - 1].isalnum()
|
||
or self.text[i - 1] in "_.")):
|
||
w = _WORD.match(self.text, i)
|
||
if w and w.group(0) in stop_words and self.text[i - 1:i].isspace():
|
||
break
|
||
i = w.end() if w else i + 1
|
||
continue
|
||
i += 1
|
||
self.pos = i
|
||
return self.text[start:i].strip()
|