Introduce diffusion-inpaint restoration engine in HVideoTool: added support for a new restoration method that regenerates masked regions via an external SwarmUI server, requiring YOLO detections for mask creation. Updated configuration management to include diffusion parameters, enhanced the UI for engine selection, and improved documentation in README and CLAUDE.md to guide users on the new functionality.

2026-06-08 06:21:44 +03:00
parent 8a366ed43d
commit 15f89b395d
14 changed files with 903 additions and 70 deletions
@@ -47,6 +47,17 @@ _SETTING_KEYS = (
    "dm_model",
    "dm_gpu",
    "dm_feed_restored",
+    "diff_backend",
+    "diff_url",
+    "diff_model",
+    "diff_prompt",
+    "diff_negative",
+    "diff_steps",
+    "diff_cfg",
+    "diff_denoise",
+    "diff_seed",
+    "diff_mask_dilate",
+    "diff_mask_blur",
 )


@@ -37,6 +37,12 @@ class Restorer(ABC):
    #: leave this False; the temporal DeepMosaics (BVDNet) sets it True.
    temporal: bool = False

+    #: Whether this engine needs the frame's detections (it builds an inpaint mask from
+    #: them). DeepMosaics locates the mosaic itself, so it leaves this False and the
+    #: caller passes ``[]``; the diffusion engine sets it True and must be fed the real
+    #: detections (a frame with none comes back unchanged).
+    needs_detections: bool = False
+
    @property
    def name(self) -> str:
        return type(self).__name__
@@ -0,0 +1,103 @@
+"""Diffusion-inpaint restoration — redraw censored regions with a diffusion backend.
+
+Unlike DeepMosaics (which *reconstructs* mosaic from its residual low-frequency data and
+locates it itself), this engine *regenerates* the masked region with a diffusion inpaint
+model: it builds a mask from the YOLO detections and hands ``(image, mask, params)`` to a
+pluggable :class:`DiffusionBackend` (SwarmUI is the first, see ``swarmui.py``).
+
+Consequences of that design:
+  - It **needs detections** (``needs_detections = True``) — a frame with none comes back
+    unchanged (no mask → nothing to regenerate). The caller feeds it the real detections.
+  - It's **per-frame** (``temporal = False``): each frame is generated independently, so a
+    video sequence will flicker. Best for stills / single frames, not coherent clips.
+  - The backend runs in a **separate process/server** (e.g. SwarmUI over HTTP), so this
+    path adds **no torch dependency** to the app and keeps the heavy model out-of-process.
+
+The backend is abstract so other diffusion servers (ComfyUI/A1111) can be added later as
+another :class:`DiffusionBackend`, without touching the restorer or the UI.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+
+import numpy as np
+
+from ..detection.types import Detection
+from .base import CancelCheck, Cancelled, Restorer
+from .mask import detections_to_mask, mask_is_empty
+
+
+@dataclass(frozen=True)
+class InpaintParams:
+    """Generation knobs handed to a :class:`DiffusionBackend`."""
+
+    prompt: str = ""
+    negative: str = ""
+    model: str | None = None       # checkpoint name as the backend knows it (None = current)
+    steps: int = 30
+    cfg: float = 7.0
+    denoise: float = 1.0           # 0..1 — how much to regenerate under the mask (1 = full)
+    seed: int = -1                 # -1 = random each call
+    mask_blur: int = 8             # px feather applied by the backend at its mask edge
+
+
+class DiffusionBackend(ABC):
+    """A diffusion inpaint engine reachable from our process (typically over HTTP)."""
+
+    @property
+    def name(self) -> str:
+        return type(self).__name__
+
+    @abstractmethod
+    def inpaint(
+        self,
+        image_bgr: np.ndarray,
+        mask: np.ndarray,
+        params: InpaintParams,
+        should_cancel: CancelCheck | None = None,
+    ) -> np.ndarray:
+        """Regenerate the white area of ``mask`` in ``image_bgr``; return a new BGR image."""
+        raise NotImplementedError
+
+
+class DiffusionRestorer(Restorer):
+    """Restorer that masks the detected regions and inpaints them via a backend."""
+
+    temporal = False
+    needs_detections = True
+
+    def __init__(
+        self,
+        backend: DiffusionBackend,
+        params: InpaintParams,
+        *,
+        mask_dilate: int = 4,
+        mask_blur: int = 8,
+    ) -> None:
+        self._backend = backend
+        self._params = params
+        self._dilate = mask_dilate
+        self._blur = mask_blur
+
+    @property
+    def name(self) -> str:
+        return f"Diffusion({self._backend.name})"
+
+    def restore(
+        self,
+        image: np.ndarray,
+        detections: list[Detection],
+        should_cancel: CancelCheck | None = None,
+    ) -> np.ndarray:
+        if should_cancel is not None and should_cancel():
+            raise Cancelled("Восстановление отменено")
+        if not detections:
+            return image.copy()  # no detections → no mask → nothing to regenerate
+        mask = detections_to_mask(
+            detections, image.shape, dilate=self._dilate, blur=self._blur
+        )
+        if mask_is_empty(mask):
+            return image.copy()
+        return self._backend.inpaint(image, mask, self._params, should_cancel)
@@ -1,13 +1,16 @@
 """Restorer factory: build a Restorer from the app config.

-Only DeepMosaics is supported (the cv2 inpaint baseline was removed — it filled but
-did not reconstruct). The DeepMosaics network code is vendored (``_deepmosaics/``,
-GPL-3.0) and run in-process; the user supplies only the weights (+ ``mosaic_position.pth``
-alongside). A CUDA GPU is recommended.
+Engines:
+- ``deepmosaics``: per-frame generative mosaic removal (image model). Vendored network
+  code (``_deepmosaics/``, GPL-3.0) run in-process; user supplies only the weights
+  (+ ``mosaic_position.pth`` alongside). Locates the mosaic itself — needs no detections.
+- ``deepmosaics_video``: temporal variant (BVDNet) using neighbouring frames for coherence
+  — needs the ``clean_*_video.pth`` weights and a contiguous frame sequence.
+- ``diffusion``: diffusion-inpaint that *regenerates* masked regions via an external
+  diffusion server (SwarmUI). Builds the mask from the YOLO detections, so it **needs
+  detections** (see ``restorer_needs_detections``); adds no torch dep (runs out-of-process).

- ``deepmosaics``: per-frame generative mosaic removal (image model).
- ``deepmosaics_video``: temporal variant (BVDNet) that uses neighbouring frames for
-  coherence — needs the ``clean_*_video.pth`` weights and a contiguous frame sequence.
+A CUDA GPU is recommended (for DeepMosaics in-process; for diffusion it's the server's GPU).
 """

 from __future__ import annotations
@@ -36,8 +39,42 @@ def build_restorer(name: str = "deepmosaics", config: AppConfig | None = None) -
            config.dm_dir, config.dm_model, config.dm_gpu,
            feed_restored=getattr(config, "dm_feed_restored", True),
        )
+    if name == "diffusion":
+        backend_name = (getattr(config, "diff_backend", "swarmui") or "swarmui")
+        if backend_name != "swarmui":
+            raise ValueError(
+                f"Diffusion-бэкенд не поддержан: {backend_name!r} (доступен только swarmui)."
+            )
+        from .diffusion import DiffusionRestorer, InpaintParams
+        from .swarmui import SwarmUIBackend
+
+        backend = SwarmUIBackend(config.diff_url)
+        params = InpaintParams(
+            prompt=config.diff_prompt,
+            negative=config.diff_negative,
+            model=config.diff_model,
+            steps=config.diff_steps,
+            cfg=config.diff_cfg,
+            denoise=config.diff_denoise,
+            seed=config.diff_seed,
+            mask_blur=config.diff_mask_blur,
+        )
+        return DiffusionRestorer(
+            backend, params,
+            mask_dilate=config.diff_mask_dilate, mask_blur=config.diff_mask_blur,
+        )
    if name == "lada":
        raise ValueError(
            "Движок LADA пока не подключён. Используйте DeepMosaics. См. README."
        )
    raise ValueError(f"Неизвестный режим восстановления: {name!r}")
+
+
+def restorer_needs_detections(name: str) -> bool:
+    """Whether engine ``name`` needs the frame's detections (to build an inpaint mask).
+
+    Lets the UI decide — *without* building the engine — whether to feed real detections
+    and whether to require that detection has been computed. Mirrors
+    ``Restorer.needs_detections`` for the engines that build lazily on a worker thread.
+    """
+    return name == "diffusion"
@@ -0,0 +1,57 @@
+"""Build an inpaint mask (255 = regenerate) from detections.
+
+Used by the diffusion restorer. Unlike DeepMosaics — which locates the mosaic itself —
+a diffusion-inpaint backend needs an explicit mask of the region to redraw. We rasterise
+each detection's polygon (or its bbox when there's no polygon) onto a single-channel
+uint8 mask, optionally growing (dilate) and feathering (blur) the edges so the inpaint
+blends into the surrounding pixels.
+
+Pure NumPy/OpenCV — no torch, no Qt.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+import cv2
+import numpy as np
+
+from ..detection.types import Detection
+
+
+def detections_to_mask(
+    detections: Sequence[Detection],
+    shape: tuple[int, ...],
+    *,
+    dilate: int = 0,
+    blur: int = 0,
+) -> np.ndarray:
+    """Rasterise ``detections`` onto a single-channel uint8 mask (255 = regenerate).
+
+    ``shape`` is the image shape (``(h, w)`` or ``(h, w, c)``). ``dilate`` grows the mask
+    by that many pixels (ellipse kernel) so the inpaint covers the censored edge; ``blur``
+    feathers the edge with a Gaussian so the boundary blends. Both are no-ops at 0.
+    """
+    h, w = int(shape[0]), int(shape[1])
+    mask = np.zeros((h, w), dtype=np.uint8)
+    for d in detections:
+        if len(d.polygon) >= 3:
+            poly = np.array(
+                [[int(round(x)), int(round(y))] for x, y in d.polygon], dtype=np.int32
+            )
+            cv2.fillPoly(mask, [poly], 255)
+        else:
+            x, y, bw, bh = (int(round(v)) for v in d.bbox)
+            cv2.rectangle(mask, (x, y), (x + bw, y + bh), 255, thickness=-1)
+    if dilate > 0:
+        k = 2 * int(dilate) + 1
+        mask = cv2.dilate(mask, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (k, k)))
+    if blur > 0:
+        k = 2 * int(blur) + 1
+        mask = cv2.GaussianBlur(mask, (k, k), 0)
+    return mask
+
+
+def mask_is_empty(mask: np.ndarray) -> bool:
+    """True if nothing is masked (so there's nothing to inpaint)."""
+    return not bool(np.any(mask))
@@ -0,0 +1,133 @@
+"""SwarmUI diffusion backend — talk to a running SwarmUI server over HTTP.
+
+SwarmUI (a REST wrapper over ComfyUI) exposes ``/API/GetNewSession`` to obtain a session
+id, then ``/API/GenerateText2Image`` to run a generation. For inpaint we send the frame
+and the mask as base64 PNG plus the prompt/params, and read the produced image back.
+
+Implementation notes:
+  - Uses only stdlib ``urllib`` — **no new dependency**; the diffusion model runs in
+    SwarmUI's own process (so our app never imports torch on this path).
+  - Exact API field names drift between SwarmUI versions, so the request body is built in
+    one place (:meth:`_build_payload`) for easy tuning; errors surface the URL + a hint.
+  - The response may carry image data inline (``data:`` URI) or as a server-relative path
+    — :meth:`_fetch_image_bytes` handles both.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import urllib.error
+import urllib.request
+
+import cv2
+import numpy as np
+
+from .base import Cancelled
+from .diffusion import DiffusionBackend, InpaintParams
+
+
+class SwarmUIBackend(DiffusionBackend):
+    def __init__(self, url: str | None, timeout: float = 600.0) -> None:
+        self._url = (url or "http://localhost:7801").rstrip("/")
+        self._timeout = timeout
+        self._session: str | None = None
+
+    @property
+    def name(self) -> str:
+        return "SwarmUI"
+
+    # ------------------------------------------------------------------ HTTP
+    def _post(self, route: str, payload: dict) -> dict:
+        data = json.dumps(payload).encode("utf-8")
+        req = urllib.request.Request(
+            self._url + route, data=data, headers={"Content-Type": "application/json"}
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=self._timeout) as resp:
+                return json.loads(resp.read().decode("utf-8"))
+        except urllib.error.URLError as e:
+            raise RuntimeError(
+                f"Не удалось связаться со SwarmUI ({self._url}{route}): {e}.\n"
+                "Проверьте, что сервер SwarmUI запущен и адрес верный "
+                "(Файл → Движок восстановления…)."
+            ) from e
+
+    def ping(self) -> str:
+        """Open a fresh session to verify the server is reachable; return the session id.
+
+        Used by the settings dialog's "Проверить соединение" — forces a new
+        ``GetNewSession`` (ignores any cached id) so repeated checks really re-test, and
+        raises a clear RuntimeError (URL + hint) if the server is down/unreachable.
+        """
+        self._session = None
+        return self._session_id()
+
+    def _session_id(self) -> str:
+        if self._session is None:
+            r = self._post("/API/GetNewSession", {})
+            self._session = r.get("session_id") or r.get("sessionId")
+            if not self._session:
+                raise RuntimeError(f"SwarmUI не вернул session_id: {r}")
+        return self._session
+
+    @staticmethod
+    def _b64_png(img: np.ndarray) -> str:
+        ok, buf = cv2.imencode(".png", img)
+        if not ok:
+            raise RuntimeError("Не удалось закодировать изображение в PNG для SwarmUI")
+        return base64.b64encode(buf.tobytes()).decode("ascii")
+
+    def _build_payload(
+        self, session: str, image_b64: str, mask_b64: str, params: InpaintParams, h: int, w: int
+    ) -> dict:
+        """Map our params onto SwarmUI's GenerateText2Image body (centralised for tuning)."""
+        payload = {
+            "session_id": session,
+            "images": 1,
+            "prompt": params.prompt,
+            "negativeprompt": params.negative,
+            "width": w,
+            "height": h,
+            "steps": int(params.steps),
+            "cfgscale": float(params.cfg),
+            "seed": int(params.seed),
+            "initimage": image_b64,
+            "maskimage": mask_b64,                       # white = regenerate
+            "initimagecreativity": float(params.denoise),  # 0..1 inpaint denoise
+            "maskblur": int(params.mask_blur),
+        }
+        if params.model:
+            payload["model"] = params.model
+        return payload
+
+    # --------------------------------------------------------------- backend
+    def inpaint(self, image_bgr, mask, params, should_cancel=None):
+        if should_cancel is not None and should_cancel():
+            raise Cancelled("Восстановление отменено")
+        session = self._session_id()
+        h, w = image_bgr.shape[:2]
+        payload = self._build_payload(
+            session, self._b64_png(image_bgr), self._b64_png(mask), params, h, w
+        )
+        if should_cancel is not None and should_cancel():
+            raise Cancelled("Восстановление отменено")
+        resp = self._post("/API/GenerateText2Image", payload)
+        return self._decode_result(resp)
+
+    def _decode_result(self, resp: dict) -> np.ndarray:
+        images = resp.get("images") or []
+        if not images:
+            raise RuntimeError(f"SwarmUI не вернул изображений (ответ: {resp})")
+        raw = self._fetch_image_bytes(images[0])
+        arr = cv2.imdecode(np.frombuffer(raw, np.uint8), cv2.IMREAD_COLOR)
+        if arr is None:
+            raise RuntimeError("Не удалось декодировать результат SwarmUI")
+        return arr
+
+    def _fetch_image_bytes(self, ref: str) -> bytes:
+        if ref.startswith("data:"):  # inline base64 data URI
+            return base64.b64decode(ref.split(",", 1)[1])
+        url = ref if ref.startswith("http") else f"{self._url}/{ref.lstrip('/')}"
+        with urllib.request.urlopen(url, timeout=self._timeout) as r:
+            return r.read()