From 9f024f0415b83a026ef5a3088e40423ad77bbd8d Mon Sep 17 00:00:00 2001 From: Leonid Pershin Date: Wed, 9 Sep 2026 03:04:06 +0300 Subject: [PATCH] Decode only what is shown, and re-peg the picture when it falls behind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The drift kept growing because the video branch could not keep real time: VIDEO_HEIGHT is a bounding box, but it was also used to pick the source format, so a 640x320 picture was decoded from a 720p stream — four times the work for the same result. The download now asks for the smallest standard rung that covers the output height. Dropping late frames only helps when lateness is momentary; when the source runs slower than the sound, every frame is late and the gap grows without bound. The timeline is now re-pegged to the current audio position instead: the picture jumps once and holds, so desync stays bounded whatever the machine can manage. A "video sync" line every ten seconds reports where each side is, how many frames are queued, dropped and re-pegged, which is what separates a slow decoder from a wrong frame rate. Co-Authored-By: Claude Opus 5 --- src/core/video.ts | 45 +++++++++++++++++++++++++++++++---- src/sources/index.ts | 10 +++++++- src/sources/video-pipeline.ts | 16 +++++++++++-- src/sources/ytdlp.ts | 11 +++++++++ 4 files changed, 75 insertions(+), 7 deletions(-) diff --git a/src/core/video.ts b/src/core/video.ts index c492137..dcc2940 100644 --- a/src/core/video.ts +++ b/src/core/video.ts @@ -14,8 +14,10 @@ const rtc = require("@livekit/rtc-node") as typeof import("@livekit/rtc-node"); const log = logger.child({ mod: "video" }); /** Give up on the video track if the source never produced a picture. */ const FIRST_FRAME_TIMEOUT_MS = 15_000; -/** How far behind the sound a frame may be before it is dropped instead of shown. */ +/** How far the picture may fall behind the sound before the timeline is re-pegged. */ const MAX_LATE_SECONDS = 0.25; +/** How often the sync state is reported while a clip plays. */ +const REPORT_EVERY_MS = 10_000; /** * Frames wait in memory until the sound catches up, so the queue is bounded by * bytes rather than count — 720p frames are four times the size of 360p ones. @@ -62,8 +64,10 @@ export class VideoPublisher { private produced = 0; private published = 0; private dropped = 0; + private resyncs = 0; /** Audio position when the first frame showed up; the picture is hung off it. */ private anchor: { audioSeconds: number; frameIndex: number } | null = null; + private reporter: NodeJS.Timeout | null = null; private pump: NodeJS.Timeout | null = null; private firstFrameTimer: NodeJS.Timeout | null = null; @@ -108,6 +112,9 @@ export class VideoPublisher { this.pump = setInterval(() => this.release(), 5); this.pump.unref?.(); + this.reporter = setInterval(() => this.report(), REPORT_EVERY_MS); + this.reporter.unref?.(); + this.firstFrameTimer = setTimeout(() => { if (this.published > 0) return; log.warn("no video frames arrived, dropping the video track"); @@ -172,8 +179,12 @@ export class VideoPublisher { this.queue.shift(); if (now > dueAt + MAX_LATE_SECONDS) { - this.dropped += 1; - continue; + // Dropping late frames only helps when lateness is momentary. If the + // source runs slower than the sound, every frame is late and dropping + // them lets the gap grow without bound, so the timeline is re-pegged to + // where the sound is now: the picture jumps once, then holds. + this.anchor = { audioSeconds: now, frameIndex: frame.index }; + this.resyncs += 1; } try { this.source.captureFrame( @@ -186,7 +197,29 @@ export class VideoPublisher { } } + /** Periodic sync state: the numbers needed to tell late frames from bad rates. */ + private report(): void { + if (this.published === 0 || !this.anchor) return; + const audio = this.options.audioClock(); + const videoContent = + this.anchor.audioSeconds + (this.produced - this.anchor.frameIndex) / this.options.fps; + log.info( + { + audio: Number(audio.toFixed(2)), + video: Number(videoContent.toFixed(2)), + aheadBy: Number((videoContent - audio).toFixed(2)), + queued: this.queue.length, + published: this.published, + dropped: this.dropped, + resyncs: this.resyncs, + }, + "video sync", + ); + } + async stop(): Promise { + if (this.reporter) clearInterval(this.reporter); + this.reporter = null; for (const timer of [this.firstFrameTimer, this.pump]) { if (timer) clearTimeout(timer as NodeJS.Timeout); } @@ -201,7 +234,10 @@ export class VideoPublisher { this.queue = []; if (this.published > 0) { - log.info({ published: this.published, dropped: this.dropped }, "video track finished"); + log.info( + { published: this.published, dropped: this.dropped, resyncs: this.resyncs }, + "video track finished", + ); } const { room, trackSid } = this; @@ -226,6 +262,7 @@ export class VideoPublisher { this.produced = 0; this.published = 0; this.dropped = 0; + this.resyncs = 0; this.anchor = null; } } diff --git a/src/sources/index.ts b/src/sources/index.ts index 4e79cec..083e521 100644 --- a/src/sources/index.ts +++ b/src/sources/index.ts @@ -246,7 +246,15 @@ export async function openPlayback( : null; if (selection) { const size = fitWithin(selection.width, selection.height); - log.info({ title: track.title, mode: selection.mode, ...size }, "playing with video"); + log.info( + { + title: track.title, + mode: selection.mode, + source: `${selection.width}x${selection.height}`, + output: `${size.width}x${size.height}`, + }, + "playing with video", + ); const pipeline = openVideoPipeline(track.url, seekSeconds, selection.mode, size); return { input: pipeline.audio, diff --git a/src/sources/video-pipeline.ts b/src/sources/video-pipeline.ts index 063cc33..e77a3ad 100644 --- a/src/sources/video-pipeline.ts +++ b/src/sources/video-pipeline.ts @@ -3,7 +3,14 @@ import { createRequire } from "node:module"; import type { Duplex, Readable, Writable } from "node:stream"; import { config } from "../config.js"; import { logger } from "../logger.js"; -import { AUDIO_FORMAT, downloadArgs, progressiveFormat, videoFormat, type VideoMode } from "./ytdlp.js"; +import { + AUDIO_FORMAT, + downloadArgs, + progressiveFormat, + sourceHeightFor, + videoFormat, + type VideoMode, +} from "./ytdlp.js"; const require = createRequire(import.meta.url); /** ffmpeg-static ships the binary revoice.js already relies on. */ @@ -56,7 +63,12 @@ export function openVideoPipeline( const fps = config.VIDEO_FPS; const split = mode === "split"; - const videoSource = spawnDownload(pageUrl, split ? videoFormat(height) : progressiveFormat(height)); + // Download no more than we render: the frame is scaled down anyway. + const sourceHeight = sourceHeightFor(height, config.VIDEO_HEIGHT); + const videoSource = spawnDownload( + pageUrl, + split ? videoFormat(sourceHeight) : progressiveFormat(sourceHeight), + ); const audioSource = split ? spawnDownload(pageUrl, AUDIO_FORMAT) : null; // No padding: the size already matches the clip's proportions, and black bars diff --git a/src/sources/ytdlp.ts b/src/sources/ytdlp.ts index 049a61d..98161e1 100644 --- a/src/sources/ytdlp.ts +++ b/src/sources/ytdlp.ts @@ -417,6 +417,17 @@ export function videoFormat(maxHeight: number): string { ].join("/"); } +/** + * The height to actually download for a given output height. Fetching more than + * we render costs real CPU — decoding 720p to show 320 lines is four times the + * work — so we ask for the smallest standard rung that still covers the output. + */ +export function sourceHeightFor(outputHeight: number, ceiling: number): number { + const rungs = [144, 240, 360, 480, 720, 1080, 1440, 2160]; + const rung = rungs.find((value) => value >= outputHeight) ?? outputHeight; + return Math.min(rung, ceiling); +} + /** Audio stream to pair with the picture. */ export const AUDIO_FORMAT = "ba/b";