Fix video codec, aspect and placement

Three problems visible at once on a running instance: the picture
stuttered and drifted behind the sound, it was letterboxed oddly, and it
appeared as a separate tile instead of coming from the bot.

- YouTube handed us AV1 (format 398). Software-decoding AV1 at 720p does
  not sustain real time on a small server, which explains both the
  stutter and the drift; H.264 is now requested first, VP9 second.
- The frame was padded into a fixed box, so a clip whose proportions
  differed got black bars baked in and then more from the client. Size is
  now a bounding box and the frame keeps the clip's own proportions.
- Video was published as a screen share, which every client renders as
  its own tile. VIDEO_SOURCE=camera (the new default) puts it inside the
  bot's tile; "screen" keeps the old behaviour.

VIDEO_SYNC_OFFSET_MS is there for the residual drift, since audio and
video travel as two separately published tracks. Measured loudnorm first
to rule it out as the cause of the desync: it adds 0 ms.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

.env.example

@@ -83,10 +83,22 @@ YTDLP_JS_RUNTIME=node
 # Это лишь разрешение: сам показ включается тумблером в панели или командой
 # !video, и по умолчанию выключен. При false тумблер в панели не показывается.
 VIDEO_ENABLED=false
+
+# Рамка, в которую вписывается кадр: пропорции клипа сохраняются, чёрные поля
+# не добавляются. 640x360 выбрано из осторожности — видео грузит CPU.
 VIDEO_WIDTH=640
 VIDEO_HEIGHT=360
 VIDEO_FPS=24
 
+# camera — картинка внутри плитки бота; screen — отдельной плиткой,
+# как демонстрация экрана.
+VIDEO_SOURCE=camera
+
+# Подстройка синхронизации, мс. Плюс задерживает видео, минус — торопит.
+# Звук и картинка публикуются двумя дорожками, поэтому идеального совпадения
+# «из коробки» не гарантируется.
+VIDEO_SYNC_OFFSET_MS=0
+
 DEFAULT_VOLUME=60
 MAX_QUEUE_SIZE=500
 

README.md

@@ -240,6 +240,8 @@ VIDEO_ENABLED=true
 VIDEO_WIDTH=640
 VIDEO_HEIGHT=360
 VIDEO_FPS=24
+VIDEO_SOURCE=camera
+VIDEO_SYNC_OFFSET_MS=0
 ```
 
 Что нужно на стороне Stoat: у роли бота — право **Video** в канале, а на инстансе включённое
@@ -259,9 +261,20 @@ VIDEO_FPS=24
 - **Один процесс ffmpeg, два выхода:** PCM для голосовой дорожки и сырые I420-кадры для видео.
   Темп задаёт `-re`, иначе кадры улетали бы вперёд звука и съедали память — распакованный кадр
   720p весит 1.4 МБ.
+- **Кодек важнее разрешения.** YouTube по умолчанию отдаёт AV1, а его программное
+  декодирование не вытягивает реальное время на слабом сервере: картинка дёргается и уползает
+  от звука. Поэтому запрашивается сначала H.264, затем VP9, и только потом что придётся.
+- **Размер — это рамка, а не жёсткий кадр.** `VIDEO_WIDTH`/`VIDEO_HEIGHT` задают ограничение, в
+  которое кадр вписывается с сохранением пропорций; чёрные поля не добавляются, их при
+  необходимости рисует сам клиент.
+- **Где показывается.** `VIDEO_SOURCE=camera` (по умолчанию) — картинка внутри плитки бота;
+  `screen` — отдельной плиткой, как демонстрация экрана.
+- **Синхронизация.** Звук и картинка публикуются двумя дорожками, поэтому совпадение зависит от
+  того, успевает ли сервер декодировать в реальном времени. Если картинка стабильно
+  опережает или отстаёт, подстройте `VIDEO_SYNC_OFFSET_MS` (плюс задерживает видео).
 - **Цена.** Кодирование видео ложится на CPU сервера и держится всё время трека, в отличие от
-  почти бесплатного звука. 360p по умолчанию выбран из осторожности; поднимайте до 720p, если
+  почти бесплатного звука. 360p по умолчанию выбран из осторожности; поднимайте, если машина
-  машина тянет, и снижайте `VIDEO_FPS`, если нагрузка мешает.
+  тянет, и снижайте `VIDEO_FPS`, если нагрузка мешает.
 - Живые трансляции и не-YouTube источники играют звуком, как раньше.
 
 ## Прокси, когда YouTube недоступен

src/config.ts

@@ -69,9 +69,14 @@ const schema = z.object({
     .transform((value) => value === "true"),
   // YouTube only serves 360p as a single progressive file, and only such files
   // can be streamed, so that is the realistic default.
+  // Treated as a bounding box: the frame keeps the clip's own proportions.
   VIDEO_WIDTH: z.coerce.number().int().positive().default(640),
   VIDEO_HEIGHT: z.coerce.number().int().positive().default(360),
   VIDEO_FPS: z.coerce.number().int().min(1).max(60).default(24),
+  /** "camera" shows up inside the bot's tile, "screen" as a separate one. */
+  VIDEO_SOURCE: z.enum(["camera", "screen"]).default("camera"),
+  /** Nudge the picture against the sound, in milliseconds; positive delays video. */
+  VIDEO_SYNC_OFFSET_MS: z.coerce.number().int().default(0),
 
   DEFAULT_VOLUME: z.coerce.number().min(0).max(200).default(60),
   MAX_QUEUE_SIZE: z.coerce.number().int().positive().default(500),

src/core/video.ts

@@ -1,6 +1,7 @@
 import { createRequire } from "node:module";
 import type { Readable } from "node:stream";
 import type { LocalVideoTrack, Room, VideoSource } from "@livekit/rtc-node";
+import { config } from "../config.js";
 import { logger } from "../logger.js";
 
 // revoice.js pulls in the CommonJS build of @livekit/rtc-node, and the native
@@ -49,10 +50,13 @@ export class VideoPublisher {
   async start(room: Room, stream: Readable, onGiveUp?: (reason: string) => void): Promise<void> {
     const { width, height } = this.options;
     const source = new rtc.VideoSource(width, height);
-    const track = rtc.LocalVideoTrack.createVideoTrack("screen", source);
+    // As a camera the picture appears inside the bot's own tile; a screen share
+    // is rendered as a separate tile by every client.
+    const asScreen = config.VIDEO_SOURCE === "screen";
+    const track = rtc.LocalVideoTrack.createVideoTrack(asScreen ? "screen" : "video", source);
 
     const options = new rtc.TrackPublishOptions();
-    options.source = rtc.TrackSource.SOURCE_SCREENSHARE;
+    options.source = asScreen ? rtc.TrackSource.SOURCE_SCREENSHARE : rtc.TrackSource.SOURCE_CAMERA;
 
     const participant = room.localParticipant;
     if (!participant) throw new Error("room has no local participant yet");
@@ -75,7 +79,10 @@ export class VideoPublisher {
     }, FIRST_FRAME_TIMEOUT_MS);
     this.firstFrameTimer.unref?.();
 
-    log.info({ width, height, sid: publication.sid }, "screen share published");
+    log.info(
+      { width, height, source: config.VIDEO_SOURCE, sid: publication.sid },
+      "video track published",
+    );
   }
 
   private consume(chunk: Buffer): void {

src/sources/index.ts

@@ -183,6 +183,24 @@ function canShowVideo(track: Track, wanted: boolean): boolean {
   return wanted && config.VIDEO_ENABLED && track.source === "youtube" && !track.isLive;
 }
 
+/**
+ * Fits the clip inside the configured box without distorting or padding it: the
+ * published frame keeps the source proportions, and the client letterboxes it
+ * however it likes. Encoders want even dimensions.
+ */
+export function fitWithin(
+  sourceWidth: number | null,
+  sourceHeight: number | null,
+): { width: number; height: number } {
+  const maxWidth = config.VIDEO_WIDTH;
+  const maxHeight = config.VIDEO_HEIGHT;
+  const even = (value: number) => Math.max(2, Math.round(value / 2) * 2);
+  if (!sourceWidth || !sourceHeight) return { width: even(maxWidth), height: even(maxHeight) };
+
+  const scale = Math.min(maxWidth / sourceWidth, maxHeight / sourceHeight, 1);
+  return { width: even(sourceWidth * scale), height: even(sourceHeight * scale) };
+}
+
 const HTTP_RESILIENCE = [
   "-reconnect", "1",
   "-reconnect_streamed", "1",
@@ -223,12 +241,13 @@ export async function openPlayback(
 
   // The format is checked before committing to the video pipeline: audio comes
   // out of that same pipeline, so falling back afterwards would kill the sound.
-  const videoMode = canShowVideo(track, options.video ?? false)
+  const selection = canShowVideo(track, options.video ?? false)
     ? await ytdlp.selectVideoMode(track.url, config.VIDEO_HEIGHT)
     : null;
-  if (videoMode) {
+  if (selection) {
-    log.info({ title: track.title, mode: videoMode }, "playing with video");
+    const size = fitWithin(selection.width, selection.height);
-    const pipeline = openVideoPipeline(track.url, seekSeconds, videoMode);
+    log.info({ title: track.title, mode: selection.mode, ...size }, "playing with video");
+    const pipeline = openVideoPipeline(track.url, seekSeconds, selection.mode, size);
     return {
       input: pipeline.audio,
       inputOptions: pipeline.audioInputOptions,

src/sources/video-pipeline.ts

@@ -46,21 +46,29 @@ function spawnDownload(pageUrl: string, format: string) {
  * stops the video from racing ahead: a decoded 720p frame is 1.4 MB, so an
  * unpaced stream would eat memory in seconds.
  */
-export function openVideoPipeline(pageUrl: string, seekSeconds = 0, mode: VideoMode = "split"): VideoPipeline {
+export function openVideoPipeline(
-  const { VIDEO_WIDTH: width, VIDEO_HEIGHT: height, VIDEO_FPS: fps } = config;
+  pageUrl: string,
+  seekSeconds = 0,
+  mode: VideoMode = "split",
+  size: { width: number; height: number } = { width: config.VIDEO_WIDTH, height: config.VIDEO_HEIGHT },
+): VideoPipeline {
+  const { width, height } = size;
+  const fps = config.VIDEO_FPS;
   const split = mode === "split";
 
   const videoSource = spawnDownload(pageUrl, split ? videoFormat(height) : progressiveFormat(height));
   const audioSource = split ? spawnDownload(pageUrl, AUDIO_FORMAT) : null;
 
-  const filters = [
+  // No padding: the size already matches the clip's proportions, and black bars
-    `scale=${width}:${height}:force_original_aspect_ratio=decrease`,
+  // baked into the frame would sit inside whatever bars the client adds.
-    `pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2`,
+  const filters = [`scale=${width}:${height}`, `fps=${fps}`, "format=yuv420p"].join(",");
-    `fps=${fps}`,
-    "format=yuv420p",
-  ].join(",");
 
   const seek = seekSeconds > 0 ? ["-ss", seekSeconds.toFixed(2)] : [];
+  // Hand-tuning room for lip sync; the two tracks are published separately.
+  const offset =
+    config.VIDEO_SYNC_OFFSET_MS !== 0
+      ? ["-itsoffset", (config.VIDEO_SYNC_OFFSET_MS / 1000).toFixed(3)]
+      : [];
   // Split: fd 0 video in, fd 3 audio in, fd 4 frames out.
   // Progressive: fd 0 everything in, fd 3 frames out.
   const videoOutFd = split ? 4 : 3;
@@ -71,6 +79,7 @@ export function openVideoPipeline(pageUrl: string, seekSeconds = 0, mode: VideoM
       "-hide_banner",
       "-loglevel",
       "error",
+      ...offset,
       ...seek,
       "-re",
       "-i",

src/sources/ytdlp.ts

@@ -348,6 +348,13 @@ export function downloadArgs(): string[] {
 
 export type VideoMode = "progressive" | "split";
 
+export interface VideoSelection {
+  mode: VideoMode;
+  /** Source dimensions, so the frame can keep its own proportions. */
+  width: number | null;
+  height: number | null;
+}
+
 /**
  * Decides how a clip can be played with picture, in one lookup:
  *
@@ -357,25 +364,29 @@ export type VideoMode = "progressive" | "split";
  *   parallel and paired by ffmpeg. YouTube increasingly offers only this.
  * - null — no usable picture; the track plays as audio.
  */
-export async function selectVideoMode(pageUrl: string, maxHeight: number): Promise<VideoMode | null> {
+export async function selectVideoMode(
+  pageUrl: string,
+  maxHeight: number,
+): Promise<VideoSelection | null> {
   try {
     const { stdout } = await runYtDlp([
       ...baseArgs({ download: true }),
       "-f",
-      `b[height<=${maxHeight}]/bv*[height<=${maxHeight}]/b`,
+      `b[height<=${maxHeight}][vcodec^=avc1]/${videoFormat(maxHeight)}`,
       "--no-playlist",
       "--print",
-      "%(format_id)s|%(acodec)s|%(vcodec)s",
+      "%(format_id)s|%(acodec)s|%(vcodec)s|%(width)s|%(height)s",
       pageUrl,
     ]);
-    const [formatId, acodec, vcodec] = stdout.trim().split("|");
+    const [formatId, acodec, vcodec, width, height] = stdout.trim().split("|");
     if (!formatId || !vcodec || vcodec === "none") {
       log.warn({ pageUrl }, "no video format offered, playing audio only");
       return null;
     }
     const mode: VideoMode = acodec && acodec !== "none" ? "progressive" : "split";
-    log.info({ pageUrl, formatId, mode }, "video format selected");
+    const size = { width: Number(width) || null, height: Number(height) || null };
-    return mode;
+    log.info({ pageUrl, formatId, mode, ...size }, "video format selected");
+    return { mode, ...size };
   } catch (err) {
     // Worth seeing: this is the difference between "clip plays" and "sound only".
     const reason = err instanceof Error ? err.message : String(err);
@@ -389,9 +400,21 @@ export function progressiveFormat(maxHeight: number): string {
   return `b[height<=${maxHeight}]`;
 }
 
-/** Video-only stream, paired with AUDIO_FORMAT by ffmpeg. */
+/**
+ * Video-only stream, paired with AUDIO_FORMAT by ffmpeg. H.264 is asked for
+ * first and VP9 second: YouTube's default pick is often AV1, which software
+ * decoding cannot sustain in real time on a modest server — the picture then
+ * stutters and drifts behind the sound.
+ */
 export function videoFormat(maxHeight: number): string {
-  return `bv*[height<=${maxHeight}]/b[height<=${maxHeight}]/b`;
+  const within = `[height<=${maxHeight}]`;
+  return [
+    `bv*${within}[vcodec^=avc1]`,
+    `bv*${within}[vcodec^=vp9]`,
+    `bv*${within}`,
+    `b${within}`,
+    "b",
+  ].join("/");
 }
 
 /** Audio stream to pair with the picture. */