Decode only what is shown, and re-peg the picture when it falls behind

The drift kept growing because the video branch could not keep real time:
VIDEO_HEIGHT is a bounding box, but it was also used to pick the source
format, so a 640x320 picture was decoded from a 720p stream — four times
the work for the same result. The download now asks for the smallest
standard rung that covers the output height.

Dropping late frames only helps when lateness is momentary; when the
source runs slower than the sound, every frame is late and the gap grows
without bound. The timeline is now re-pegged to the current audio
position instead: the picture jumps once and holds, so desync stays
bounded whatever the machine can manage.

A "video sync" line every ten seconds reports where each side is, how
many frames are queued, dropped and re-pegged, which is what separates a
slow decoder from a wrong frame rate.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

src/core/video.ts

@@ -14,8 +14,10 @@ const rtc = require("@livekit/rtc-node") as typeof import("@livekit/rtc-node");
 const log = logger.child({ mod: "video" });
 /** Give up on the video track if the source never produced a picture. */
 const FIRST_FRAME_TIMEOUT_MS = 15_000;
-/** How far behind the sound a frame may be before it is dropped instead of shown. */
+/** How far the picture may fall behind the sound before the timeline is re-pegged. */
 const MAX_LATE_SECONDS = 0.25;
+/** How often the sync state is reported while a clip plays. */
+const REPORT_EVERY_MS = 10_000;
 /**
  * Frames wait in memory until the sound catches up, so the queue is bounded by
  * bytes rather than count — 720p frames are four times the size of 360p ones.
@@ -62,8 +64,10 @@ export class VideoPublisher {
   private produced = 0;
   private published = 0;
   private dropped = 0;
+  private resyncs = 0;
   /** Audio position when the first frame showed up; the picture is hung off it. */
   private anchor: { audioSeconds: number; frameIndex: number } | null = null;
+  private reporter: NodeJS.Timeout | null = null;
   private pump: NodeJS.Timeout | null = null;
   private firstFrameTimer: NodeJS.Timeout | null = null;
 
@@ -108,6 +112,9 @@ export class VideoPublisher {
     this.pump = setInterval(() => this.release(), 5);
     this.pump.unref?.();
 
+    this.reporter = setInterval(() => this.report(), REPORT_EVERY_MS);
+    this.reporter.unref?.();
+
     this.firstFrameTimer = setTimeout(() => {
       if (this.published > 0) return;
       log.warn("no video frames arrived, dropping the video track");
@@ -172,8 +179,12 @@ export class VideoPublisher {
 
       this.queue.shift();
       if (now > dueAt + MAX_LATE_SECONDS) {
-        this.dropped += 1;
+        // Dropping late frames only helps when lateness is momentary. If the
-        continue;
+        // source runs slower than the sound, every frame is late and dropping
+        // them lets the gap grow without bound, so the timeline is re-pegged to
+        // where the sound is now: the picture jumps once, then holds.
+        this.anchor = { audioSeconds: now, frameIndex: frame.index };
+        this.resyncs += 1;
       }
       try {
         this.source.captureFrame(
@@ -186,7 +197,29 @@ export class VideoPublisher {
     }
   }
 
+  /** Periodic sync state: the numbers needed to tell late frames from bad rates. */
+  private report(): void {
+    if (this.published === 0 || !this.anchor) return;
+    const audio = this.options.audioClock();
+    const videoContent =
+      this.anchor.audioSeconds + (this.produced - this.anchor.frameIndex) / this.options.fps;
+    log.info(
+      {
+        audio: Number(audio.toFixed(2)),
+        video: Number(videoContent.toFixed(2)),
+        aheadBy: Number((videoContent - audio).toFixed(2)),
+        queued: this.queue.length,
+        published: this.published,
+        dropped: this.dropped,
+        resyncs: this.resyncs,
+      },
+      "video sync",
+    );
+  }
+
   async stop(): Promise<void> {
+    if (this.reporter) clearInterval(this.reporter);
+    this.reporter = null;
     for (const timer of [this.firstFrameTimer, this.pump]) {
       if (timer) clearTimeout(timer as NodeJS.Timeout);
     }
@@ -201,7 +234,10 @@ export class VideoPublisher {
     this.queue = [];
 
     if (this.published > 0) {
-      log.info({ published: this.published, dropped: this.dropped }, "video track finished");
+      log.info(
+        { published: this.published, dropped: this.dropped, resyncs: this.resyncs },
+        "video track finished",
+      );
     }
 
     const { room, trackSid } = this;
@@ -226,6 +262,7 @@ export class VideoPublisher {
     this.produced = 0;
     this.published = 0;
     this.dropped = 0;
+    this.resyncs = 0;
     this.anchor = null;
   }
 }

src/sources/index.ts

@@ -246,7 +246,15 @@ export async function openPlayback(
     : null;
   if (selection) {
     const size = fitWithin(selection.width, selection.height);
-    log.info({ title: track.title, mode: selection.mode, ...size }, "playing with video");
+    log.info(
+      {
+        title: track.title,
+        mode: selection.mode,
+        source: `${selection.width}x${selection.height}`,
+        output: `${size.width}x${size.height}`,
+      },
+      "playing with video",
+    );
     const pipeline = openVideoPipeline(track.url, seekSeconds, selection.mode, size);
     return {
       input: pipeline.audio,

src/sources/video-pipeline.ts

@@ -3,7 +3,14 @@ import { createRequire } from "node:module";
 import type { Duplex, Readable, Writable } from "node:stream";
 import { config } from "../config.js";
 import { logger } from "../logger.js";
-import { AUDIO_FORMAT, downloadArgs, progressiveFormat, videoFormat, type VideoMode } from "./ytdlp.js";
+import {
+  AUDIO_FORMAT,
+  downloadArgs,
+  progressiveFormat,
+  sourceHeightFor,
+  videoFormat,
+  type VideoMode,
+} from "./ytdlp.js";
 
 const require = createRequire(import.meta.url);
 /** ffmpeg-static ships the binary revoice.js already relies on. */
@@ -56,7 +63,12 @@ export function openVideoPipeline(
   const fps = config.VIDEO_FPS;
   const split = mode === "split";
 
-  const videoSource = spawnDownload(pageUrl, split ? videoFormat(height) : progressiveFormat(height));
+  // Download no more than we render: the frame is scaled down anyway.
+  const sourceHeight = sourceHeightFor(height, config.VIDEO_HEIGHT);
+  const videoSource = spawnDownload(
+    pageUrl,
+    split ? videoFormat(sourceHeight) : progressiveFormat(sourceHeight),
+  );
   const audioSource = split ? spawnDownload(pageUrl, AUDIO_FORMAT) : null;
 
   // No padding: the size already matches the clip's proportions, and black bars

src/sources/ytdlp.ts

@@ -417,6 +417,17 @@ export function videoFormat(maxHeight: number): string {
   ].join("/");
 }
 
+/**
+ * The height to actually download for a given output height. Fetching more than
+ * we render costs real CPU — decoding 720p to show 320 lines is four times the
+ * work — so we ask for the smallest standard rung that still covers the output.
+ */
+export function sourceHeightFor(outputHeight: number, ceiling: number): number {
+  const rungs = [144, 240, 360, 480, 720, 1080, 1440, 2160];
+  const rung = rungs.find((value) => value >= outputHeight) ?? outputHeight;
+  return Math.min(rung, ceiling);
+}
+
 /** Audio stream to pair with the picture. */
 export const AUDIO_FORMAT = "ba/b";