diff --git a/src/core/video.ts b/src/core/video.ts index 08c0c13..c492137 100644 --- a/src/core/video.ts +++ b/src/core/video.ts @@ -62,6 +62,8 @@ export class VideoPublisher { private produced = 0; private published = 0; private dropped = 0; + /** Audio position when the first frame showed up; the picture is hung off it. */ + private anchor: { audioSeconds: number; frameIndex: number } | null = null; private pump: NodeJS.Timeout | null = null; private firstFrameTimer: NodeJS.Timeout | null = null; @@ -152,10 +154,20 @@ export class VideoPublisher { const { fps } = this.options; const now = this.options.audioClock(); const tolerance = 1 / (2 * fps); + const offset = config.VIDEO_SYNC_OFFSET_MS / 1000; while (this.queue.length > 0) { const frame = this.queue[0] as QueuedFrame; - const dueAt = frame.index / fps; + // The first frame sets the relationship between the two streams instead of + // assuming both start at zero: audio begins flowing before the picture is + // decoded, and pinning frames to an absolute zero puts the whole clip in + // debt. From there the spacing is held, which is what stops the drift. + if (!this.anchor) { + this.anchor = { audioSeconds: now, frameIndex: frame.index }; + log.info({ audioSeconds: Number(now.toFixed(2)) }, "video anchored to the audio clock"); + } + const dueAt = + this.anchor.audioSeconds + (frame.index - this.anchor.frameIndex) / fps + offset; if (now + tolerance < dueAt) break; this.queue.shift(); @@ -214,5 +226,6 @@ export class VideoPublisher { this.produced = 0; this.published = 0; this.dropped = 0; + this.anchor = null; } } diff --git a/src/sources/video-pipeline.ts b/src/sources/video-pipeline.ts index 526f807..063cc33 100644 --- a/src/sources/video-pipeline.ts +++ b/src/sources/video-pipeline.ts @@ -64,11 +64,6 @@ export function openVideoPipeline( const filters = [`scale=${width}:${height}`, `fps=${fps}`, "format=yuv420p"].join(","); const seek = seekSeconds > 0 ? ["-ss", seekSeconds.toFixed(2)] : []; - // Hand-tuning room for lip sync; the two tracks are published separately. - const offset = - config.VIDEO_SYNC_OFFSET_MS !== 0 - ? ["-itsoffset", (config.VIDEO_SYNC_OFFSET_MS / 1000).toFixed(3)] - : []; // Split: fd 0 video in, fd 3 audio in, fd 4 frames out. // Progressive: fd 0 everything in, fd 3 frames out. const videoOutFd = split ? 4 : 3; @@ -79,7 +74,6 @@ export function openVideoPipeline( "-hide_banner", "-loglevel", "error", - ...offset, ...seek, "-re", "-i",