mrleo1nid/stoat-mbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix video codec, aspect and placement

Browse Source 
Three problems visible at once on a running instance: the picture
stuttered and drifted behind the sound, it was letterboxed oddly, and it
appeared as a separate tile instead of coming from the bot.

- YouTube handed us AV1 (format 398). Software-decoding AV1 at 720p does
  not sustain real time on a small server, which explains both the
  stutter and the drift; H.264 is now requested first, VP9 second.
- The frame was padded into a fixed box, so a clip whose proportions
  differed got black bars baked in and then more from the client. Size is
  now a bounding box and the frame keeps the clip's own proportions.
- Video was published as a screen share, which every client renders as
  its own tile. VIDEO_SOURCE=camera (the new default) puts it inside the
  bot's tile; "screen" keeps the old behaviour.

VIDEO_SYNC_OFFSET_MS is there for the residual drift, since audio and
video travel as two separately published tracks. Measured loudnorm first
to rule it out as the cause of the desync: it adds 0 ms.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Leonid Pershin
2026-09-09 02:29:29 +03:00
co-authored by Claude Opus 5
parent 9285fd7fbd
commit 0b17b880d6
7 changed files with 113 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/sources/index.ts
+23 -4
View File
@@ -183,6 +183,24 @@ function canShowVideo(track: Track, wanted: boolean): boolean {
return wanted && config.VIDEO_ENABLED && track.source === "youtube" && !track.isLive;
}
/**
* Fits the clip inside the configured box without distorting or padding it: the
* published frame keeps the source proportions, and the client letterboxes it
* however it likes. Encoders want even dimensions.
*/
export function fitWithin(
sourceWidth: number | null,
sourceHeight: number | null,
): { width: number; height: number } {
const maxWidth = config.VIDEO_WIDTH;
const maxHeight = config.VIDEO_HEIGHT;
const even = (value: number) => Math.max(2, Math.round(value / 2) * 2);
if (!sourceWidth || !sourceHeight) return { width: even(maxWidth), height: even(maxHeight) };
const scale = Math.min(maxWidth / sourceWidth, maxHeight / sourceHeight, 1);
return { width: even(sourceWidth * scale), height: even(sourceHeight * scale) };
}
const HTTP_RESILIENCE = [
"-reconnect", "1",
"-reconnect_streamed", "1",
@@ -223,12 +241,13 @@ export async function openPlayback(
// The format is checked before committing to the video pipeline: audio comes
// out of that same pipeline, so falling back afterwards would kill the sound.
const videoMode = canShowVideo(track, options.video ?? false)
const selection = canShowVideo(track, options.video ?? false)
? await ytdlp.selectVideoMode(track.url, config.VIDEO_HEIGHT)
: null;
if (videoMode) {
log.info({ title: track.title, mode: videoMode }, "playing with video");
const pipeline = openVideoPipeline(track.url, seekSeconds, videoMode);
if (selection) {
const size = fitWithin(selection.width, selection.height);
log.info({ title: track.title, mode: selection.mode, ...size }, "playing with video");
const pipeline = openVideoPipeline(track.url, seekSeconds, selection.mode, size);
return {
input: pipeline.audio,
inputOptions: pipeline.audioInputOptions,
src/sources/video-pipeline.ts
+17 -8
View File
@@ -46,21 +46,29 @@ function spawnDownload(pageUrl: string, format: string) {
* stops the video from racing ahead: a decoded 720p frame is 1.4 MB, so an
* unpaced stream would eat memory in seconds.
*/
export function openVideoPipeline(pageUrl: string, seekSeconds = 0, mode: VideoMode = "split"): VideoPipeline {
const { VIDEO_WIDTH: width, VIDEO_HEIGHT: height, VIDEO_FPS: fps } = config;
export function openVideoPipeline(
pageUrl: string,
seekSeconds = 0,
mode: VideoMode = "split",
size: { width: number; height: number } = { width: config.VIDEO_WIDTH, height: config.VIDEO_HEIGHT },
): VideoPipeline {
const { width, height } = size;
const fps = config.VIDEO_FPS;
const split = mode === "split";
const videoSource = spawnDownload(pageUrl, split ? videoFormat(height) : progressiveFormat(height));
const audioSource = split ? spawnDownload(pageUrl, AUDIO_FORMAT) : null;
const filters = [
`scale=${width}:${height}:force_original_aspect_ratio=decrease`,
`pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2`,
`fps=${fps}`,
"format=yuv420p",
].join(",");
// No padding: the size already matches the clip's proportions, and black bars
// baked into the frame would sit inside whatever bars the client adds.
const filters = [`scale=${width}:${height}`, `fps=${fps}`, "format=yuv420p"].join(",");
const seek = seekSeconds > 0 ? ["-ss", seekSeconds.toFixed(2)] : [];
// Hand-tuning room for lip sync; the two tracks are published separately.
const offset =
config.VIDEO_SYNC_OFFSET_MS !== 0
? ["-itsoffset", (config.VIDEO_SYNC_OFFSET_MS / 1000).toFixed(3)]
: [];
// Split: fd 0 video in, fd 3 audio in, fd 4 frames out.
// Progressive: fd 0 everything in, fd 3 frames out.
const videoOutFd = split ? 4 : 3;
@@ -71,6 +79,7 @@ export function openVideoPipeline(pageUrl: string, seekSeconds = 0, mode: VideoM
"-hide_banner",
"-loglevel",
"error",
...offset,
...seek,
"-re",
"-i",
src/sources/ytdlp.ts
+31 -8
View File
@@ -348,6 +348,13 @@ export function downloadArgs(): string[] {
export type VideoMode = "progressive" | "split";
export interface VideoSelection {
mode: VideoMode;
/** Source dimensions, so the frame can keep its own proportions. */
width: number | null;
height: number | null;
}
/**
* Decides how a clip can be played with picture, in one lookup:
*
@@ -357,25 +364,29 @@ export type VideoMode = "progressive" | "split";
* parallel and paired by ffmpeg. YouTube increasingly offers only this.
* - null — no usable picture; the track plays as audio.
*/
export async function selectVideoMode(pageUrl: string, maxHeight: number): Promise<VideoMode | null> {
export async function selectVideoMode(
pageUrl: string,
maxHeight: number,
): Promise<VideoSelection | null> {
try {
const { stdout } = await runYtDlp([
...baseArgs({ download: true }),
"-f",
`b[height<=${maxHeight}]/bv*[height<=${maxHeight}]/b`,
`b[height<=${maxHeight}][vcodec^=avc1]/${videoFormat(maxHeight)}`,
"--no-playlist",
"--print",
"%(format_id)s|%(acodec)s|%(vcodec)s",
"%(format_id)s|%(acodec)s|%(vcodec)s|%(width)s|%(height)s",
pageUrl,
]);
const [formatId, acodec, vcodec] = stdout.trim().split("|");
const [formatId, acodec, vcodec, width, height] = stdout.trim().split("|");
if (!formatId || !vcodec || vcodec === "none") {
log.warn({ pageUrl }, "no video format offered, playing audio only");
return null;
}
const mode: VideoMode = acodec && acodec !== "none" ? "progressive" : "split";
log.info({ pageUrl, formatId, mode }, "video format selected");
return mode;
const size = { width: Number(width) || null, height: Number(height) || null };
log.info({ pageUrl, formatId, mode, ...size }, "video format selected");
return { mode, ...size };
} catch (err) {
// Worth seeing: this is the difference between "clip plays" and "sound only".
const reason = err instanceof Error ? err.message : String(err);
@@ -389,9 +400,21 @@ export function progressiveFormat(maxHeight: number): string {
return `b[height<=${maxHeight}]`;
}
/** Video-only stream, paired with AUDIO_FORMAT by ffmpeg. */
/**
* Video-only stream, paired with AUDIO_FORMAT by ffmpeg. H.264 is asked for
* first and VP9 second: YouTube's default pick is often AV1, which software
* decoding cannot sustain in real time on a modest server — the picture then
* stutters and drifts behind the sound.
*/
export function videoFormat(maxHeight: number): string {
return `bv*[height<=${maxHeight}]/b[height<=${maxHeight}]/b`;
const within = `[height<=${maxHeight}]`;
return [
`bv*${within}[vcodec^=avc1]`,
`bv*${within}[vcodec^=vp9]`,
`bv*${within}`,
`b${within}`,
"b",
].join("/");
}
/** Audio stream to pair with the picture. */