From 623bab775f5ce60beec2f89d9ddd3a429bf4f5d5 Mon Sep 17 00:00:00 2001
From: Leonid Pershin <lpershin@syntellect.ru>
Date: Wed, 9 Sep 2026 02:59:12 +0300
Subject: [PATCH] Anchor the picture to the first frame, not to zero
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Gating frames on absolute stream position overshot in the other
direction: audio starts flowing before the first frame is decoded, so
pinning frame 0 to audio second 0 puts the whole clip permanently in
debt, and frames arrive already past due.

The first frame now records the audio position it arrived at, and the
rest keep their spacing from there — the drift correction stays, the
startup difference is no longer treated as lateness. That offset is
logged, so the residual can be trimmed with VIDEO_SYNC_OFFSET_MS, which
the publisher now applies itself instead of ffmpeg (one mechanism, and it
works in both directions).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 src/core/video.ts             | 15 ++++++++++++++-
 src/sources/video-pipeline.ts |  6 ------
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/src/core/video.ts b/src/core/video.ts
index 08c0c13..c492137 100644
--- a/src/core/video.ts
+++ b/src/core/video.ts
@@ -62,6 +62,8 @@ export class VideoPublisher {
   private produced = 0;
   private published = 0;
   private dropped = 0;
+  /** Audio position when the first frame showed up; the picture is hung off it. */
+  private anchor: { audioSeconds: number; frameIndex: number } | null = null;
   private pump: NodeJS.Timeout | null = null;
   private firstFrameTimer: NodeJS.Timeout | null = null;
 
@@ -152,10 +154,20 @@ export class VideoPublisher {
     const { fps } = this.options;
     const now = this.options.audioClock();
     const tolerance = 1 / (2 * fps);
+    const offset = config.VIDEO_SYNC_OFFSET_MS / 1000;
 
     while (this.queue.length > 0) {
       const frame = this.queue[0] as QueuedFrame;
-      const dueAt = frame.index / fps;
+      // The first frame sets the relationship between the two streams instead of
+      // assuming both start at zero: audio begins flowing before the picture is
+      // decoded, and pinning frames to an absolute zero puts the whole clip in
+      // debt. From there the spacing is held, which is what stops the drift.
+      if (!this.anchor) {
+        this.anchor = { audioSeconds: now, frameIndex: frame.index };
+        log.info({ audioSeconds: Number(now.toFixed(2)) }, "video anchored to the audio clock");
+      }
+      const dueAt =
+        this.anchor.audioSeconds + (frame.index - this.anchor.frameIndex) / fps + offset;
       if (now + tolerance < dueAt) break;
 
       this.queue.shift();
@@ -214,5 +226,6 @@ export class VideoPublisher {
     this.produced = 0;
     this.published = 0;
     this.dropped = 0;
+    this.anchor = null;
   }
 }
diff --git a/src/sources/video-pipeline.ts b/src/sources/video-pipeline.ts
index 526f807..063cc33 100644
--- a/src/sources/video-pipeline.ts
+++ b/src/sources/video-pipeline.ts
@@ -64,11 +64,6 @@ export function openVideoPipeline(
   const filters = [`scale=${width}:${height}`, `fps=${fps}`, "format=yuv420p"].join(",");
 
   const seek = seekSeconds > 0 ? ["-ss", seekSeconds.toFixed(2)] : [];
-  // Hand-tuning room for lip sync; the two tracks are published separately.
-  const offset =
-    config.VIDEO_SYNC_OFFSET_MS !== 0
-      ? ["-itsoffset", (config.VIDEO_SYNC_OFFSET_MS / 1000).toFixed(3)]
-      : [];
   // Split: fd 0 video in, fd 3 audio in, fd 4 frames out.
   // Progressive: fd 0 everything in, fd 3 frames out.
   const videoOutFd = split ? 4 : 3;
@@ -79,7 +74,6 @@ export function openVideoPipeline(
       "-hide_banner",
       "-loglevel",
       "error",
-      ...offset,
       ...seek,
       "-re",
       "-i",