Fall back to separate video and audio downloads

Playing with video only worked for clips YouTube still serves as a single
progressive file, and it mostly does not: the log showed "Requested
format is not available" and every track fell back to sound only.

A progressive format is still preferred when offered — one download, one
decode — but when there is none, video and audio are fetched in parallel
and handed to one ffmpeg over separate pipes, which pairs them by
timestamp. This also lifts the 360p ceiling: the height is now a CPU
choice rather than a limit.

The extra-descriptor plumbing (two inputs, two outputs on one ffmpeg) was
verified against synthetic media: exactly 5.0s of PCM and 120 whole
frames at 24 fps, no partial tail.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

.env.example

@@ -77,8 +77,9 @@ YTDLP_JS_RUNTIME=node
 
 # Показывать клип в голосовом канале как демонстрацию экрана (только YouTube).
 # Требует прав Video у бота и включённого видео в конфигурации инстанса.
-# YouTube отдаёт одним файлом только 360p, а стримить можно лишь такой формат —
-# отсюда размер по умолчанию. Кодирование видео заметно грузит CPU сервера.
+# 360p по умолчанию из осторожности: кодирование видео заметно грузит CPU
+# сервера. Можно поднять до 720 (VIDEO_WIDTH=1280, VIDEO_HEIGHT=720), если
+# машина позволяет.
 # Это лишь разрешение: сам показ включается тумблером в панели или командой
 # !video, и по умолчанию выключен. При false тумблер в панели не показывается.
 VIDEO_ENABLED=false

README.md

@@ -249,15 +249,19 @@ VIDEO_FPS=24
 
 Как это устроено и почему так:
 
-- **Только YouTube и только 360p.** Стримить можно лишь прогрессивный формат — один файл со
-  звуком и картинкой. Раздельные дорожки (720p и выше) yt-dlp обязан сначала скачать целиком
-  и лишь потом склеить, то есть воспроизведение началось бы после полной загрузки. Отдавать
-  ffmpeg прямые ссылки на CDN тоже нельзя: YouTube их для сторонних клиентов подвешивает.
+- **Только YouTube.** Для остальных источников картинки нет, трек играет звуком.
+- **Два способа получить картинку.** Если YouTube отдаёт прогрессивный формат (один файл со
+  звуком и видео) — используется он: одна загрузка, один декодер. Такие форматы встречаются всё
+  реже, поэтому есть запасной путь: видео и звук качаются двумя процессами параллельно и
+  сводятся одним ffmpeg по таймкодам. Склейка средствами yt-dlp не годится — он скачивает оба
+  потока целиком, прежде чем выдать первый байт; прямые ссылки на CDN тоже: YouTube подвешивает
+  их для сторонних клиентов.
 - **Один процесс ffmpeg, два выхода:** PCM для голосовой дорожки и сырые I420-кадры для видео.
   Темп задаёт `-re`, иначе кадры улетали бы вперёд звука и съедали память — распакованный кадр
   720p весит 1.4 МБ.
 - **Цена.** Кодирование видео ложится на CPU сервера и держится всё время трека, в отличие от
-  почти бесплатного звука. Ставьте `VIDEO_FPS` пониже, если нагрузка мешает.
+  почти бесплатного звука. 360p по умолчанию выбран из осторожности; поднимайте до 720p, если
+  машина тянет, и снижайте `VIDEO_FPS`, если нагрузка мешает.
 - Живые трансляции и не-YouTube источники играют звуком, как раньше.
 
 ## Прокси, когда YouTube недоступен

src/sources/index.ts

@@ -223,12 +223,12 @@ export async function openPlayback(
 
   // The format is checked before committing to the video pipeline: audio comes
   // out of that same pipeline, so falling back afterwards would kill the sound.
-  if (
-    canShowVideo(track, options.video ?? false) &&
-    (await ytdlp.hasProgressiveVideo(track.url, config.VIDEO_HEIGHT))
-  ) {
-    log.info({ title: track.title }, "playing with video");
-    const pipeline = openVideoPipeline(track.url, seekSeconds);
+  const videoMode = canShowVideo(track, options.video ?? false)
+    ? await ytdlp.selectVideoMode(track.url, config.VIDEO_HEIGHT)
+    : null;
+  if (videoMode) {
+    log.info({ title: track.title, mode: videoMode }, "playing with video");
+    const pipeline = openVideoPipeline(track.url, seekSeconds, videoMode);
     return {
       input: pipeline.audio,
       inputOptions: pipeline.audioInputOptions,

src/sources/video-pipeline.ts

@@ -1,9 +1,9 @@
-import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
+import { spawn, type ChildProcess, type ChildProcessWithoutNullStreams } from "node:child_process";
 import { createRequire } from "node:module";
-import type { Readable } from "node:stream";
+import type { Duplex, Readable, Writable } from "node:stream";
 import { config } from "../config.js";
 import { logger } from "../logger.js";
-import { downloadArgs, progressiveFormat } from "./ytdlp.js";
+import { AUDIO_FORMAT, downloadArgs, progressiveFormat, videoFormat, type VideoMode } from "./ytdlp.js";
 
 const require = createRequire(import.meta.url);
 /** ffmpeg-static ships the binary revoice.js already relies on. */
@@ -23,33 +23,35 @@ export interface VideoPipeline {
   kill(): void;
 }
 
-/**
- * One download, one decode, two outputs: PCM for the voice track and I420 frames
- * for the screen share.
- *
- * yt-dlp does the fetching (so cookies and the proxy apply as everywhere else)
- * and hands ffmpeg a progressive stream. `-re` paces the decode at playback
- * speed: it keeps picture and sound together and stops the video from racing
- * ahead — a decoded 720p frame is 1.4 MB, so an unpaced stream would eat memory
- * in seconds.
- */
-export function openVideoPipeline(pageUrl: string, seekSeconds = 0): VideoPipeline {
-  const { VIDEO_WIDTH: width, VIDEO_HEIGHT: height, VIDEO_FPS: fps } = config;
-
-  const ytdlp = spawn(
+function spawnDownload(pageUrl: string, format: string) {
+  return spawn(
     config.YTDLP_PATH,
-    [
-      ...downloadArgs(),
-      "-f",
-      progressiveFormat(height),
-      "--no-playlist",
-      "--quiet",
-      "-o",
-      "-",
-      pageUrl,
-    ],
+    [...downloadArgs(), "-f", format, "--no-playlist", "--quiet", "-o", "-", pageUrl],
     { stdio: ["ignore", "pipe", "pipe"], windowsHide: true },
   );
+}
+
+/**
+ * Decodes a clip into the two things a call needs: PCM for the voice track and
+ * raw I420 frames for the screen share, both out of a single ffmpeg.
+ *
+ * A progressive format (one file with both streams) is the cheap path — one
+ * download, one decode. When YouTube offers no such format, which is now the
+ * common case, picture and sound are downloaded in parallel and fed to ffmpeg
+ * over separate pipes, and it pairs them by timestamp.
+ *
+ * Everything is fetched by yt-dlp, so cookies and the proxy apply as everywhere
+ * else — handing CDN URLs to ffmpeg instead does not work, YouTube stalls them.
+ * `-re` paces the decode at playback speed, which keeps the two in step and
+ * stops the video from racing ahead: a decoded 720p frame is 1.4 MB, so an
+ * unpaced stream would eat memory in seconds.
+ */
+export function openVideoPipeline(pageUrl: string, seekSeconds = 0, mode: VideoMode = "split"): VideoPipeline {
+  const { VIDEO_WIDTH: width, VIDEO_HEIGHT: height, VIDEO_FPS: fps } = config;
+  const split = mode === "split";
+
+  const videoSource = spawnDownload(pageUrl, split ? videoFormat(height) : progressiveFormat(height));
+  const audioSource = split ? spawnDownload(pageUrl, AUDIO_FORMAT) : null;
 
   const filters = [
     `scale=${width}:${height}:force_original_aspect_ratio=decrease`,
@@ -58,19 +60,25 @@ export function openVideoPipeline(pageUrl: string, seekSeconds = 0): VideoPipeli
     "format=yuv420p",
   ].join(",");
 
+  const seek = seekSeconds > 0 ? ["-ss", seekSeconds.toFixed(2)] : [];
+  // Split: fd 0 video in, fd 3 audio in, fd 4 frames out.
+  // Progressive: fd 0 everything in, fd 3 frames out.
+  const videoOutFd = split ? 4 : 3;
+
   const ffmpeg = spawn(
     FFMPEG_PATH,
     [
       "-hide_banner",
       "-loglevel",
       "error",
-      ...(seekSeconds > 0 ? ["-ss", seekSeconds.toFixed(2)] : []),
+      ...seek,
       "-re",
       "-i",
       "pipe:0",
+      ...(split ? [...seek, "-re", "-i", "pipe:3"] : []),
       // Audio branch → stdout.
       "-map",
-      "0:a:0?",
+      split ? "1:a:0" : "0:a:0",
       "-f",
       "s16le",
       "-ar",
@@ -78,27 +86,56 @@ export function openVideoPipeline(pageUrl: string, seekSeconds = 0): VideoPipeli
       "-ac",
       "2",
       "pipe:1",
-      // Video branch → the extra pipe on fd 3.
+      // Video branch → its own pipe.
       "-map",
       "0:v:0",
       "-vf",
       filters,
       "-f",
       "rawvideo",
-      "pipe:3",
+      `pipe:${videoOutFd}`,
     ],
-    { stdio: ["pipe", "pipe", "pipe", "pipe"], windowsHide: true },
-  ) as ChildProcessWithoutNullStreams & { stdio: Readable[] };
+    {
+      stdio: split
+        ? ["pipe", "pipe", "pipe", "pipe", "pipe"]
+        : ["pipe", "pipe", "pipe", "pipe"],
+      windowsHide: true,
+    },
+  ) as ChildProcessWithoutNullStreams & { stdio: Array<Duplex & Readable> };
 
-  ytdlp.stdout.pipe(ffmpeg.stdin);
+  const videoOut = ffmpeg.stdio[videoOutFd] as Readable;
+  videoSource.stdout.pipe(ffmpeg.stdin);
 
-  ytdlp.stderr.setEncoding("utf8");
-  ytdlp.stderr.on("data", (chunk: string) => log.warn({ ytdlp: chunk.trim() }, "video download"));
+  const pipes: Array<Readable | Writable | Duplex> = [
+    videoSource.stdout,
+    ffmpeg.stdin,
+    ffmpeg.stdout,
+    videoOut,
+  ];
+  if (audioSource) {
+    const audioIn = ffmpeg.stdio[3] as Duplex;
+    audioSource.stdout.pipe(audioIn);
+    pipes.push(audioSource.stdout, audioIn);
+  }
+
+  const children: ChildProcess[] = [videoSource, ffmpeg];
+  if (audioSource) children.push(audioSource);
+
+  for (const [name, child] of [
+    ["video", videoSource],
+    ["audio", audioSource],
+  ] as const) {
+    if (!child) continue;
+    child.stderr.setEncoding("utf8");
+    child.stderr.on("data", (chunk: string) =>
+      log.warn({ stream: name, ytdlp: chunk.trim() }, "video pipeline download"),
+    );
+  }
   ffmpeg.stderr.setEncoding("utf8");
   ffmpeg.stderr.on("data", (chunk: string) => log.warn({ ffmpeg: chunk.trim() }, "ffmpeg"));
 
   // A closed pipe when the other side goes away is expected, not an error.
-  for (const stream of [ytdlp.stdout, ffmpeg.stdin, ffmpeg.stdout]) {
+  for (const stream of pipes) {
     stream.on("error", (err: NodeJS.ErrnoException) => {
       if (err.code !== "EPIPE") log.debug({ err }, "pipeline stream error");
     });
@@ -108,14 +145,15 @@ export function openVideoPipeline(pageUrl: string, seekSeconds = 0): VideoPipeli
   return {
     audio: ffmpeg.stdout,
     audioInputOptions: ["-f", "s16le", "-ar", "48000", "-ac", "2"],
-    video: ffmpeg.stdio[3] as Readable,
+    video: videoOut,
     width,
     height,
     kill: () => {
       if (killed) return;
       killed = true;
-      if (ytdlp.exitCode === null) ytdlp.kill("SIGKILL");
-      if (ffmpeg.exitCode === null) ffmpeg.kill("SIGKILL");
+      for (const child of children) {
+        if (child.exitCode === null) child.kill("SIGKILL");
+      }
     },
   };
 }

src/sources/ytdlp.ts

@@ -346,43 +346,57 @@ export function downloadArgs(): string[] {
   return baseArgs({ download: true });
 }
 
+export type VideoMode = "progressive" | "split";
+
 /**
- * Checks whether YouTube still offers a progressive format (video and audio in
- * one file) at this size. Only those can be streamed: anything that needs
- * merging makes yt-dlp download both streams in full before writing a byte, and
- * direct CDN URLs are no longer fetchable by ffmpeg — YouTube stalls them.
+ * Decides how a clip can be played with picture, in one lookup:
+ *
+ * - "progressive" — one file carries both streams. Cheapest: a single download
+ *   and a single decode.
+ * - "split" — video and audio come separately, so they are downloaded in
+ *   parallel and paired by ffmpeg. YouTube increasingly offers only this.
+ * - null — no usable picture; the track plays as audio.
  */
-export async function hasProgressiveVideo(pageUrl: string, maxHeight: number): Promise<boolean> {
+export async function selectVideoMode(pageUrl: string, maxHeight: number): Promise<VideoMode | null> {
   try {
     const { stdout } = await runYtDlp([
       ...baseArgs({ download: true }),
       "-f",
-      progressiveFormat(maxHeight),
+      `b[height<=${maxHeight}]/bv*[height<=${maxHeight}]/b`,
       "--no-playlist",
       "--print",
-      "%(format_id)s",
+      "%(format_id)s|%(acodec)s|%(vcodec)s",
       pageUrl,
     ]);
-    const formatId = stdout.trim();
-    if (!formatId) {
-      log.warn({ pageUrl }, "no progressive format offered, playing audio only");
-      return false;
+    const [formatId, acodec, vcodec] = stdout.trim().split("|");
+    if (!formatId || !vcodec || vcodec === "none") {
+      log.warn({ pageUrl }, "no video format offered, playing audio only");
+      return null;
     }
-    log.info({ pageUrl, formatId }, "progressive format for video playback");
-    return true;
+    const mode: VideoMode = acodec && acodec !== "none" ? "progressive" : "split";
+    log.info({ pageUrl, formatId, mode }, "video format selected");
+    return mode;
   } catch (err) {
     // Worth seeing: this is the difference between "clip plays" and "sound only".
     const reason = err instanceof Error ? err.message : String(err);
     log.warn({ pageUrl, reason }, "format lookup failed, playing audio only");
-    return false;
+    return null;
   }
 }
 
-/** Best single-file format within the height budget. */
+/** Single file with both streams, when YouTube still offers one. */
 export function progressiveFormat(maxHeight: number): string {
-  return `b[height<=${maxHeight}]/b`;
+  return `b[height<=${maxHeight}]`;
 }
 
+/** Video-only stream, paired with AUDIO_FORMAT by ffmpeg. */
+export function videoFormat(maxHeight: number): string {
+  return `bv*[height<=${maxHeight}]/b[height<=${maxHeight}]/b`;
+}
+
+/** Audio stream to pair with the picture. */
+export const AUDIO_FORMAT = "ba/b";
+
 /** Private, disposable copy of the cookie jar for concurrent reads. */
 async function copyCookies(): Promise<string | null> {
   if (!config.YTDLP_COOKIES) return null;