diff --git a/src/core/video.ts b/src/core/video.ts index c492137..dcc2940 100644 --- a/src/core/video.ts +++ b/src/core/video.ts @@ -14,8 +14,10 @@ const rtc = require("@livekit/rtc-node") as typeof import("@livekit/rtc-node"); const log = logger.child({ mod: "video" }); /** Give up on the video track if the source never produced a picture. */ const FIRST_FRAME_TIMEOUT_MS = 15_000; -/** How far behind the sound a frame may be before it is dropped instead of shown. */ +/** How far the picture may fall behind the sound before the timeline is re-pegged. */ const MAX_LATE_SECONDS = 0.25; +/** How often the sync state is reported while a clip plays. */ +const REPORT_EVERY_MS = 10_000; /** * Frames wait in memory until the sound catches up, so the queue is bounded by * bytes rather than count — 720p frames are four times the size of 360p ones. @@ -62,8 +64,10 @@ export class VideoPublisher { private produced = 0; private published = 0; private dropped = 0; + private resyncs = 0; /** Audio position when the first frame showed up; the picture is hung off it. */ private anchor: { audioSeconds: number; frameIndex: number } | null = null; + private reporter: NodeJS.Timeout | null = null; private pump: NodeJS.Timeout | null = null; private firstFrameTimer: NodeJS.Timeout | null = null; @@ -108,6 +112,9 @@ export class VideoPublisher { this.pump = setInterval(() => this.release(), 5); this.pump.unref?.(); + this.reporter = setInterval(() => this.report(), REPORT_EVERY_MS); + this.reporter.unref?.(); + this.firstFrameTimer = setTimeout(() => { if (this.published > 0) return; log.warn("no video frames arrived, dropping the video track"); @@ -172,8 +179,12 @@ export class VideoPublisher { this.queue.shift(); if (now > dueAt + MAX_LATE_SECONDS) { - this.dropped += 1; - continue; + // Dropping late frames only helps when lateness is momentary. If the + // source runs slower than the sound, every frame is late and dropping + // them lets the gap grow without bound, so the timeline is re-pegged to + // where the sound is now: the picture jumps once, then holds. + this.anchor = { audioSeconds: now, frameIndex: frame.index }; + this.resyncs += 1; } try { this.source.captureFrame( @@ -186,7 +197,29 @@ export class VideoPublisher { } } + /** Periodic sync state: the numbers needed to tell late frames from bad rates. */ + private report(): void { + if (this.published === 0 || !this.anchor) return; + const audio = this.options.audioClock(); + const videoContent = + this.anchor.audioSeconds + (this.produced - this.anchor.frameIndex) / this.options.fps; + log.info( + { + audio: Number(audio.toFixed(2)), + video: Number(videoContent.toFixed(2)), + aheadBy: Number((videoContent - audio).toFixed(2)), + queued: this.queue.length, + published: this.published, + dropped: this.dropped, + resyncs: this.resyncs, + }, + "video sync", + ); + } + async stop(): Promise { + if (this.reporter) clearInterval(this.reporter); + this.reporter = null; for (const timer of [this.firstFrameTimer, this.pump]) { if (timer) clearTimeout(timer as NodeJS.Timeout); } @@ -201,7 +234,10 @@ export class VideoPublisher { this.queue = []; if (this.published > 0) { - log.info({ published: this.published, dropped: this.dropped }, "video track finished"); + log.info( + { published: this.published, dropped: this.dropped, resyncs: this.resyncs }, + "video track finished", + ); } const { room, trackSid } = this; @@ -226,6 +262,7 @@ export class VideoPublisher { this.produced = 0; this.published = 0; this.dropped = 0; + this.resyncs = 0; this.anchor = null; } } diff --git a/src/sources/index.ts b/src/sources/index.ts index 4e79cec..083e521 100644 --- a/src/sources/index.ts +++ b/src/sources/index.ts @@ -246,7 +246,15 @@ export async function openPlayback( : null; if (selection) { const size = fitWithin(selection.width, selection.height); - log.info({ title: track.title, mode: selection.mode, ...size }, "playing with video"); + log.info( + { + title: track.title, + mode: selection.mode, + source: `${selection.width}x${selection.height}`, + output: `${size.width}x${size.height}`, + }, + "playing with video", + ); const pipeline = openVideoPipeline(track.url, seekSeconds, selection.mode, size); return { input: pipeline.audio, diff --git a/src/sources/video-pipeline.ts b/src/sources/video-pipeline.ts index 063cc33..e77a3ad 100644 --- a/src/sources/video-pipeline.ts +++ b/src/sources/video-pipeline.ts @@ -3,7 +3,14 @@ import { createRequire } from "node:module"; import type { Duplex, Readable, Writable } from "node:stream"; import { config } from "../config.js"; import { logger } from "../logger.js"; -import { AUDIO_FORMAT, downloadArgs, progressiveFormat, videoFormat, type VideoMode } from "./ytdlp.js"; +import { + AUDIO_FORMAT, + downloadArgs, + progressiveFormat, + sourceHeightFor, + videoFormat, + type VideoMode, +} from "./ytdlp.js"; const require = createRequire(import.meta.url); /** ffmpeg-static ships the binary revoice.js already relies on. */ @@ -56,7 +63,12 @@ export function openVideoPipeline( const fps = config.VIDEO_FPS; const split = mode === "split"; - const videoSource = spawnDownload(pageUrl, split ? videoFormat(height) : progressiveFormat(height)); + // Download no more than we render: the frame is scaled down anyway. + const sourceHeight = sourceHeightFor(height, config.VIDEO_HEIGHT); + const videoSource = spawnDownload( + pageUrl, + split ? videoFormat(sourceHeight) : progressiveFormat(sourceHeight), + ); const audioSource = split ? spawnDownload(pageUrl, AUDIO_FORMAT) : null; // No padding: the size already matches the clip's proportions, and black bars diff --git a/src/sources/ytdlp.ts b/src/sources/ytdlp.ts index 049a61d..98161e1 100644 --- a/src/sources/ytdlp.ts +++ b/src/sources/ytdlp.ts @@ -417,6 +417,17 @@ export function videoFormat(maxHeight: number): string { ].join("/"); } +/** + * The height to actually download for a given output height. Fetching more than + * we render costs real CPU — decoding 720p to show 320 lines is four times the + * work — so we ask for the smallest standard rung that still covers the output. + */ +export function sourceHeightFor(outputHeight: number, ceiling: number): number { + const rungs = [144, 240, 360, 480, 720, 1080, 1440, 2160]; + const rung = rungs.find((value) => value >= outputHeight) ?? outputHeight; + return Math.min(rung, ceiling); +} + /** Audio stream to pair with the picture. */ export const AUDIO_FORMAT = "ba/b";