Initial local translation API with persistent projects and batch jobs

local_translator/protection.py

@@ -0,0 +1,63 @@
+from collections import Counter
+from dataclasses import dataclass
+import re
+import uuid
+
+
+BUILTIN_RULES = {
+    "newlines": r"\r\n|\r|\n",
+    "escapes": r"\\(?:[nrt\\\"']|u[0-9a-fA-F]{4}|x[0-9a-fA-F]{2})",
+    "dollar": r"\$\{[^{}\r\n]+\}|\$[A-Za-z_][A-Za-z0-9_]*",
+    "braces": r"\{\{[^{}\r\n]*\}\}|\{[^{}\r\n]+\}",
+    "printf": r"%%|%[A-Za-z_][A-Za-z0-9_]*%|%(?:\d+\$)?[-+#0 ]*(?:\d+|\*)?(?:\.(?:\d+|\*))?(?:hh|ll|[hlLzjt])?[diuoxXfFeEgGaAcspn]",
+    "tags": r"</?[A-Za-z][^<>\r\n]*>",
+    "square": r"\[[^\[\]\r\n]+\]",
+    "entities": r"&(?:[A-Za-z][A-Za-z0-9]*|#\d+|#x[0-9a-fA-F]+);",
+    "controls": r"[\x00-\x09\x0b\x0c\x0e-\x1f]",
+}
+
+
+@dataclass
+class ProtectedText:
+    text: str
+    tokens: dict[str, str]
+    rules: list[str]
+    original: str
+
+    def restore(self, translated: str) -> str:
+        # Unique sentinels must occur exactly once and in their original order.
+        found = re.findall(r"__LT_[a-f0-9]+_\d+__", translated)
+        if found != list(self.tokens):
+            raise ValueError("Protected tokens were changed, removed, duplicated or reordered")
+        for token, original in self.tokens.items():
+            translated = translated.replace(token, original)
+        # Also reject newly invented markup/placeholders matching active rules.
+        for pattern in self.rules:
+            before = Counter(match.group() for match in re.finditer(pattern, self.original))
+            after = Counter(match.group() for match in re.finditer(pattern, translated))
+            if before != after:
+                raise ValueError("Protected syntax changed or was introduced")
+        return translated
+
+
+def protect(text: str, project) -> ProtectedText:
+    # Custom rules take precedence over built-ins at the same offset.
+    rules = [r"__LT_[a-f0-9]+_\d+__"] + [r.pattern for r in project.protection_rules]
+    rules.extend(pattern for name, pattern in BUILTIN_RULES.items() if name not in project.disabled_rules)
+    if not rules:
+        return ProtectedText(text, {}, [], text)
+    matches = []
+    for priority, pattern in enumerate(rules):
+        matches.extend((m.start(), priority, m.end(), m.group()) for m in re.finditer(pattern, text) if m.end() > m.start())
+    matches.sort(key=lambda item: (item[0], item[1], -item[2]))
+    tokens, pieces, position = {}, [], 0
+    nonce = uuid.uuid4().hex
+    for start, _, end, value in matches:
+        if start < position:
+            continue
+        token = f"__LT_{nonce}_{len(tokens)}__"
+        pieces.extend((text[position:start], token))
+        tokens[token] = value
+        position = end
+    pieces.append(text[position:])
+    return ProtectedText("".join(pieces), tokens, rules, text)