From 0b17b880d686d6c2521cf42e50881d0d912c31b9 Mon Sep 17 00:00:00 2001 From: Leonid Pershin Date: Wed, 9 Sep 2026 02:29:29 +0300 Subject: [PATCH] Fix video codec, aspect and placement Three problems visible at once on a running instance: the picture stuttered and drifted behind the sound, it was letterboxed oddly, and it appeared as a separate tile instead of coming from the bot. - YouTube handed us AV1 (format 398). Software-decoding AV1 at 720p does not sustain real time on a small server, which explains both the stutter and the drift; H.264 is now requested first, VP9 second. - The frame was padded into a fixed box, so a clip whose proportions differed got black bars baked in and then more from the client. Size is now a bounding box and the frame keeps the clip's own proportions. - Video was published as a screen share, which every client renders as its own tile. VIDEO_SOURCE=camera (the new default) puts it inside the bot's tile; "screen" keeps the old behaviour. VIDEO_SYNC_OFFSET_MS is there for the residual drift, since audio and video travel as two separately published tracks. Measured loudnorm first to rule it out as the cause of the desync: it adds 0 ms. Co-Authored-By: Claude Opus 5 --- .env.example | 12 +++++++++++ README.md | 17 +++++++++++++-- src/config.ts | 5 +++++ src/core/video.ts | 13 +++++++++--- src/sources/index.ts | 27 ++++++++++++++++++++---- src/sources/video-pipeline.ts | 25 +++++++++++++++------- src/sources/ytdlp.ts | 39 ++++++++++++++++++++++++++++------- 7 files changed, 113 insertions(+), 25 deletions(-) diff --git a/.env.example b/.env.example index a44cd7f..2bde08c 100644 --- a/.env.example +++ b/.env.example @@ -83,10 +83,22 @@ YTDLP_JS_RUNTIME=node # Это лишь разрешение: сам показ включается тумблером в панели или командой # !video, и по умолчанию выключен. При false тумблер в панели не показывается. VIDEO_ENABLED=false + +# Рамка, в которую вписывается кадр: пропорции клипа сохраняются, чёрные поля +# не добавляются. 640x360 выбрано из осторожности — видео грузит CPU. VIDEO_WIDTH=640 VIDEO_HEIGHT=360 VIDEO_FPS=24 +# camera — картинка внутри плитки бота; screen — отдельной плиткой, +# как демонстрация экрана. +VIDEO_SOURCE=camera + +# Подстройка синхронизации, мс. Плюс задерживает видео, минус — торопит. +# Звук и картинка публикуются двумя дорожками, поэтому идеального совпадения +# «из коробки» не гарантируется. +VIDEO_SYNC_OFFSET_MS=0 + DEFAULT_VOLUME=60 MAX_QUEUE_SIZE=500 diff --git a/README.md b/README.md index b547e9b..0bd3e89 100644 --- a/README.md +++ b/README.md @@ -240,6 +240,8 @@ VIDEO_ENABLED=true VIDEO_WIDTH=640 VIDEO_HEIGHT=360 VIDEO_FPS=24 +VIDEO_SOURCE=camera +VIDEO_SYNC_OFFSET_MS=0 ``` Что нужно на стороне Stoat: у роли бота — право **Video** в канале, а на инстансе включённое @@ -259,9 +261,20 @@ VIDEO_FPS=24 - **Один процесс ffmpeg, два выхода:** PCM для голосовой дорожки и сырые I420-кадры для видео. Темп задаёт `-re`, иначе кадры улетали бы вперёд звука и съедали память — распакованный кадр 720p весит 1.4 МБ. +- **Кодек важнее разрешения.** YouTube по умолчанию отдаёт AV1, а его программное + декодирование не вытягивает реальное время на слабом сервере: картинка дёргается и уползает + от звука. Поэтому запрашивается сначала H.264, затем VP9, и только потом что придётся. +- **Размер — это рамка, а не жёсткий кадр.** `VIDEO_WIDTH`/`VIDEO_HEIGHT` задают ограничение, в + которое кадр вписывается с сохранением пропорций; чёрные поля не добавляются, их при + необходимости рисует сам клиент. +- **Где показывается.** `VIDEO_SOURCE=camera` (по умолчанию) — картинка внутри плитки бота; + `screen` — отдельной плиткой, как демонстрация экрана. +- **Синхронизация.** Звук и картинка публикуются двумя дорожками, поэтому совпадение зависит от + того, успевает ли сервер декодировать в реальном времени. Если картинка стабильно + опережает или отстаёт, подстройте `VIDEO_SYNC_OFFSET_MS` (плюс задерживает видео). - **Цена.** Кодирование видео ложится на CPU сервера и держится всё время трека, в отличие от - почти бесплатного звука. 360p по умолчанию выбран из осторожности; поднимайте до 720p, если - машина тянет, и снижайте `VIDEO_FPS`, если нагрузка мешает. + почти бесплатного звука. 360p по умолчанию выбран из осторожности; поднимайте, если машина + тянет, и снижайте `VIDEO_FPS`, если нагрузка мешает. - Живые трансляции и не-YouTube источники играют звуком, как раньше. ## Прокси, когда YouTube недоступен diff --git a/src/config.ts b/src/config.ts index 4e81b5f..e78f67c 100644 --- a/src/config.ts +++ b/src/config.ts @@ -69,9 +69,14 @@ const schema = z.object({ .transform((value) => value === "true"), // YouTube only serves 360p as a single progressive file, and only such files // can be streamed, so that is the realistic default. + // Treated as a bounding box: the frame keeps the clip's own proportions. VIDEO_WIDTH: z.coerce.number().int().positive().default(640), VIDEO_HEIGHT: z.coerce.number().int().positive().default(360), VIDEO_FPS: z.coerce.number().int().min(1).max(60).default(24), + /** "camera" shows up inside the bot's tile, "screen" as a separate one. */ + VIDEO_SOURCE: z.enum(["camera", "screen"]).default("camera"), + /** Nudge the picture against the sound, in milliseconds; positive delays video. */ + VIDEO_SYNC_OFFSET_MS: z.coerce.number().int().default(0), DEFAULT_VOLUME: z.coerce.number().min(0).max(200).default(60), MAX_QUEUE_SIZE: z.coerce.number().int().positive().default(500), diff --git a/src/core/video.ts b/src/core/video.ts index 713e142..03923fd 100644 --- a/src/core/video.ts +++ b/src/core/video.ts @@ -1,6 +1,7 @@ import { createRequire } from "node:module"; import type { Readable } from "node:stream"; import type { LocalVideoTrack, Room, VideoSource } from "@livekit/rtc-node"; +import { config } from "../config.js"; import { logger } from "../logger.js"; // revoice.js pulls in the CommonJS build of @livekit/rtc-node, and the native @@ -49,10 +50,13 @@ export class VideoPublisher { async start(room: Room, stream: Readable, onGiveUp?: (reason: string) => void): Promise { const { width, height } = this.options; const source = new rtc.VideoSource(width, height); - const track = rtc.LocalVideoTrack.createVideoTrack("screen", source); + // As a camera the picture appears inside the bot's own tile; a screen share + // is rendered as a separate tile by every client. + const asScreen = config.VIDEO_SOURCE === "screen"; + const track = rtc.LocalVideoTrack.createVideoTrack(asScreen ? "screen" : "video", source); const options = new rtc.TrackPublishOptions(); - options.source = rtc.TrackSource.SOURCE_SCREENSHARE; + options.source = asScreen ? rtc.TrackSource.SOURCE_SCREENSHARE : rtc.TrackSource.SOURCE_CAMERA; const participant = room.localParticipant; if (!participant) throw new Error("room has no local participant yet"); @@ -75,7 +79,10 @@ export class VideoPublisher { }, FIRST_FRAME_TIMEOUT_MS); this.firstFrameTimer.unref?.(); - log.info({ width, height, sid: publication.sid }, "screen share published"); + log.info( + { width, height, source: config.VIDEO_SOURCE, sid: publication.sid }, + "video track published", + ); } private consume(chunk: Buffer): void { diff --git a/src/sources/index.ts b/src/sources/index.ts index d0d0d42..bccbb28 100644 --- a/src/sources/index.ts +++ b/src/sources/index.ts @@ -183,6 +183,24 @@ function canShowVideo(track: Track, wanted: boolean): boolean { return wanted && config.VIDEO_ENABLED && track.source === "youtube" && !track.isLive; } +/** + * Fits the clip inside the configured box without distorting or padding it: the + * published frame keeps the source proportions, and the client letterboxes it + * however it likes. Encoders want even dimensions. + */ +export function fitWithin( + sourceWidth: number | null, + sourceHeight: number | null, +): { width: number; height: number } { + const maxWidth = config.VIDEO_WIDTH; + const maxHeight = config.VIDEO_HEIGHT; + const even = (value: number) => Math.max(2, Math.round(value / 2) * 2); + if (!sourceWidth || !sourceHeight) return { width: even(maxWidth), height: even(maxHeight) }; + + const scale = Math.min(maxWidth / sourceWidth, maxHeight / sourceHeight, 1); + return { width: even(sourceWidth * scale), height: even(sourceHeight * scale) }; +} + const HTTP_RESILIENCE = [ "-reconnect", "1", "-reconnect_streamed", "1", @@ -223,12 +241,13 @@ export async function openPlayback( // The format is checked before committing to the video pipeline: audio comes // out of that same pipeline, so falling back afterwards would kill the sound. - const videoMode = canShowVideo(track, options.video ?? false) + const selection = canShowVideo(track, options.video ?? false) ? await ytdlp.selectVideoMode(track.url, config.VIDEO_HEIGHT) : null; - if (videoMode) { - log.info({ title: track.title, mode: videoMode }, "playing with video"); - const pipeline = openVideoPipeline(track.url, seekSeconds, videoMode); + if (selection) { + const size = fitWithin(selection.width, selection.height); + log.info({ title: track.title, mode: selection.mode, ...size }, "playing with video"); + const pipeline = openVideoPipeline(track.url, seekSeconds, selection.mode, size); return { input: pipeline.audio, inputOptions: pipeline.audioInputOptions, diff --git a/src/sources/video-pipeline.ts b/src/sources/video-pipeline.ts index efd0e07..526f807 100644 --- a/src/sources/video-pipeline.ts +++ b/src/sources/video-pipeline.ts @@ -46,21 +46,29 @@ function spawnDownload(pageUrl: string, format: string) { * stops the video from racing ahead: a decoded 720p frame is 1.4 MB, so an * unpaced stream would eat memory in seconds. */ -export function openVideoPipeline(pageUrl: string, seekSeconds = 0, mode: VideoMode = "split"): VideoPipeline { - const { VIDEO_WIDTH: width, VIDEO_HEIGHT: height, VIDEO_FPS: fps } = config; +export function openVideoPipeline( + pageUrl: string, + seekSeconds = 0, + mode: VideoMode = "split", + size: { width: number; height: number } = { width: config.VIDEO_WIDTH, height: config.VIDEO_HEIGHT }, +): VideoPipeline { + const { width, height } = size; + const fps = config.VIDEO_FPS; const split = mode === "split"; const videoSource = spawnDownload(pageUrl, split ? videoFormat(height) : progressiveFormat(height)); const audioSource = split ? spawnDownload(pageUrl, AUDIO_FORMAT) : null; - const filters = [ - `scale=${width}:${height}:force_original_aspect_ratio=decrease`, - `pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2`, - `fps=${fps}`, - "format=yuv420p", - ].join(","); + // No padding: the size already matches the clip's proportions, and black bars + // baked into the frame would sit inside whatever bars the client adds. + const filters = [`scale=${width}:${height}`, `fps=${fps}`, "format=yuv420p"].join(","); const seek = seekSeconds > 0 ? ["-ss", seekSeconds.toFixed(2)] : []; + // Hand-tuning room for lip sync; the two tracks are published separately. + const offset = + config.VIDEO_SYNC_OFFSET_MS !== 0 + ? ["-itsoffset", (config.VIDEO_SYNC_OFFSET_MS / 1000).toFixed(3)] + : []; // Split: fd 0 video in, fd 3 audio in, fd 4 frames out. // Progressive: fd 0 everything in, fd 3 frames out. const videoOutFd = split ? 4 : 3; @@ -71,6 +79,7 @@ export function openVideoPipeline(pageUrl: string, seekSeconds = 0, mode: VideoM "-hide_banner", "-loglevel", "error", + ...offset, ...seek, "-re", "-i", diff --git a/src/sources/ytdlp.ts b/src/sources/ytdlp.ts index b92081e..049a61d 100644 --- a/src/sources/ytdlp.ts +++ b/src/sources/ytdlp.ts @@ -348,6 +348,13 @@ export function downloadArgs(): string[] { export type VideoMode = "progressive" | "split"; +export interface VideoSelection { + mode: VideoMode; + /** Source dimensions, so the frame can keep its own proportions. */ + width: number | null; + height: number | null; +} + /** * Decides how a clip can be played with picture, in one lookup: * @@ -357,25 +364,29 @@ export type VideoMode = "progressive" | "split"; * parallel and paired by ffmpeg. YouTube increasingly offers only this. * - null — no usable picture; the track plays as audio. */ -export async function selectVideoMode(pageUrl: string, maxHeight: number): Promise { +export async function selectVideoMode( + pageUrl: string, + maxHeight: number, +): Promise { try { const { stdout } = await runYtDlp([ ...baseArgs({ download: true }), "-f", - `b[height<=${maxHeight}]/bv*[height<=${maxHeight}]/b`, + `b[height<=${maxHeight}][vcodec^=avc1]/${videoFormat(maxHeight)}`, "--no-playlist", "--print", - "%(format_id)s|%(acodec)s|%(vcodec)s", + "%(format_id)s|%(acodec)s|%(vcodec)s|%(width)s|%(height)s", pageUrl, ]); - const [formatId, acodec, vcodec] = stdout.trim().split("|"); + const [formatId, acodec, vcodec, width, height] = stdout.trim().split("|"); if (!formatId || !vcodec || vcodec === "none") { log.warn({ pageUrl }, "no video format offered, playing audio only"); return null; } const mode: VideoMode = acodec && acodec !== "none" ? "progressive" : "split"; - log.info({ pageUrl, formatId, mode }, "video format selected"); - return mode; + const size = { width: Number(width) || null, height: Number(height) || null }; + log.info({ pageUrl, formatId, mode, ...size }, "video format selected"); + return { mode, ...size }; } catch (err) { // Worth seeing: this is the difference between "clip plays" and "sound only". const reason = err instanceof Error ? err.message : String(err); @@ -389,9 +400,21 @@ export function progressiveFormat(maxHeight: number): string { return `b[height<=${maxHeight}]`; } -/** Video-only stream, paired with AUDIO_FORMAT by ffmpeg. */ +/** + * Video-only stream, paired with AUDIO_FORMAT by ffmpeg. H.264 is asked for + * first and VP9 second: YouTube's default pick is often AV1, which software + * decoding cannot sustain in real time on a modest server — the picture then + * stutters and drifts behind the sound. + */ export function videoFormat(maxHeight: number): string { - return `bv*[height<=${maxHeight}]/b[height<=${maxHeight}]/b`; + const within = `[height<=${maxHeight}]`; + return [ + `bv*${within}[vcodec^=avc1]`, + `bv*${within}[vcodec^=vp9]`, + `bv*${within}`, + `b${within}`, + "b", + ].join("/"); } /** Audio stream to pair with the picture. */