Publish the clip as a screen share alongside the audio

VIDEO_ENABLED makes the bot publish a second LiveKit track with the picture. Stoat only grants screen_share in the call token when the bot has the Video permission and the instance has video enabled, so a refusal is reported in chat and playback continues with sound alone. Two constraints shaped the pipeline, both found by testing rather than assumption: - Only progressive formats can be streamed. Separate video+audio streams make yt-dlp download both in full before muxing a single byte, and direct CDN URLs handed to ffmpeg simply hang — YouTube no longer serves them to other clients. That caps video at the 360p single file YouTube offers, and the format is checked before committing to the video path, since audio would otherwise come from the same broken pipeline. - One ffmpeg with two outputs, paced by -re: an unpaced decode races ahead of the sound and eats memory at 1.4 MB per frame. The same CDN-URL finding removes the audio seek shortcut, which resolved such a URL and would have hung the same way; seeking now decodes up to the offset like it already did behind a proxy. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-09 01:10:11 +03:00
parent 24dfcf9232
commit 49b87dc172
10 changed files with 387 additions and 23 deletions
@@ -3,6 +3,7 @@ import { config } from "../config.js";
 import { UserFacingError, type Requester, type SearchResult, type Track } from "../types.js";
 import * as direct from "./direct.js";
 import * as local from "./local.js";
+import { openVideoPipeline } from "./video-pipeline.js";
 import * as ytdlp from "./ytdlp.js";

 export { checkAvailable as checkYtDlp, checkCookies, checkProxy, describeProxy } from "./ytdlp.js";
@@ -165,6 +166,13 @@ export interface PlaybackInput {
  cleanup(): void;
  /** Resolves with a reason if the downloader died on its own, for reporting. */
  failure?: Promise<string | null>;
+  /** Raw I420 frames to publish as a screen share, when video is on. */
+  video?: { stream: Readable; width: number; height: number };
+}
+
+/** Only YouTube reliably carries a picture worth showing next to the audio. */
+function canShowVideo(track: Track): boolean {
+  return config.VIDEO_ENABLED && track.source === "youtube" && !track.isLive;
 }

 const HTTP_RESILIENCE = [
@@ -201,16 +209,9 @@ export async function openPlayback(track: Track, seekSeconds = 0): Promise<Playb
    };
  }

-  if (seekSeconds > 0 && !config.YTDLP_PROXY) {
-    // Seeking over a pipe would mean decoding everything up to the offset, so we
-    // resolve the CDN URL instead and let ffmpeg do an HTTP range request.
-    const streamUrl = await ytdlp.resolveStreamUrl(track.url);
-    return { input: streamUrl, inputOptions: [...HTTP_RESILIENCE, ...seekOptions], cleanup: () => {} };
-  }
-
-  // With a proxy configured we always pipe through yt-dlp, which honours it;
-  // handing a CDN URL to ffmpeg would leak the request past the proxy (and would
-  // simply fail for SOCKS). Seeking then costs a decode up to the offset.
+  // Everything goes through yt-dlp: YouTube stalls direct CDN URLs fetched by
+  // anything else, and ffmpeg would bypass a SOCKS proxy anyway. Seeking
+  // therefore costs a decode up to the offset instead of an HTTP range request.
  const proc = ytdlp.openAudioStream(track.url);
  return {
    input: proc.stream,
@@ -0,0 +1,121 @@
+import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
+import { createRequire } from "node:module";
+import type { Readable } from "node:stream";
+import { config } from "../config.js";
+import { logger } from "../logger.js";
+import { downloadArgs, progressiveFormat } from "./ytdlp.js";
+
+const require = createRequire(import.meta.url);
+/** ffmpeg-static ships the binary revoice.js already relies on. */
+const FFMPEG_PATH = require("ffmpeg-static") as string;
+
+const log = logger.child({ mod: "video-pipeline" });
+
+export interface VideoPipeline {
+  /** Raw PCM for the audio track. */
+  audio: Readable;
+  /** ffmpeg input options describing that raw PCM. */
+  audioInputOptions: string[];
+  /** Raw I420 frames for the screen share. */
+  video: Readable;
+  width: number;
+  height: number;
+  kill(): void;
+}
+
+/**
+ * One download, one decode, two outputs: PCM for the voice track and I420 frames
+ * for the screen share.
+ *
+ * yt-dlp does the fetching (so cookies and the proxy apply as everywhere else)
+ * and hands ffmpeg a progressive stream. `-re` paces the decode at playback
+ * speed: it keeps picture and sound together and stops the video from racing
+ * ahead — a decoded 720p frame is 1.4 MB, so an unpaced stream would eat memory
+ * in seconds.
+ */
+export function openVideoPipeline(pageUrl: string, seekSeconds = 0): VideoPipeline {
+  const { VIDEO_WIDTH: width, VIDEO_HEIGHT: height, VIDEO_FPS: fps } = config;
+
+  const ytdlp = spawn(
+    config.YTDLP_PATH,
+    [
+      ...downloadArgs(),
+      "-f",
+      progressiveFormat(height),
+      "--no-playlist",
+      "--quiet",
+      "-o",
+      "-",
+      pageUrl,
+    ],
+    { stdio: ["ignore", "pipe", "pipe"], windowsHide: true },
+  );
+
+  const filters = [
+    `scale=${width}:${height}:force_original_aspect_ratio=decrease`,
+    `pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2`,
+    `fps=${fps}`,
+    "format=yuv420p",
+  ].join(",");
+
+  const ffmpeg = spawn(
+    FFMPEG_PATH,
+    [
+      "-hide_banner",
+      "-loglevel",
+      "error",
+      ...(seekSeconds > 0 ? ["-ss", seekSeconds.toFixed(2)] : []),
+      "-re",
+      "-i",
+      "pipe:0",
+      // Audio branch → stdout.
+      "-map",
+      "0:a:0?",
+      "-f",
+      "s16le",
+      "-ar",
+      "48000",
+      "-ac",
+      "2",
+      "pipe:1",
+      // Video branch → the extra pipe on fd 3.
+      "-map",
+      "0:v:0",
+      "-vf",
+      filters,
+      "-f",
+      "rawvideo",
+      "pipe:3",
+    ],
+    { stdio: ["pipe", "pipe", "pipe", "pipe"], windowsHide: true },
+  ) as ChildProcessWithoutNullStreams & { stdio: Readable[] };
+
+  ytdlp.stdout.pipe(ffmpeg.stdin);
+
+  ytdlp.stderr.setEncoding("utf8");
+  ytdlp.stderr.on("data", (chunk: string) => log.warn({ ytdlp: chunk.trim() }, "video download"));
+  ffmpeg.stderr.setEncoding("utf8");
+  ffmpeg.stderr.on("data", (chunk: string) => log.warn({ ffmpeg: chunk.trim() }, "ffmpeg"));
+
+  // A closed pipe when the other side goes away is expected, not an error.
+  for (const stream of [ytdlp.stdout, ffmpeg.stdin, ffmpeg.stdout]) {
+    stream.on("error", (err: NodeJS.ErrnoException) => {
+      if (err.code !== "EPIPE") log.debug({ err }, "pipeline stream error");
+    });
+  }
+
+  let killed = false;
+  return {
+    audio: ffmpeg.stdout,
+    audioInputOptions: ["-f", "s16le", "-ar", "48000", "-ac", "2"],
+    video: ffmpeg.stdio[3] as Readable,
+    width,
+    height,
+    kill: () => {
+      if (killed) return;
+      killed = true;
+      if (ytdlp.exitCode === null) ytdlp.kill("SIGKILL");
+      if (ffmpeg.exitCode === null) ffmpeg.kill("SIGKILL");
+    },
+  };
+}
@@ -341,19 +341,38 @@ export async function resolveUrl(
  return { tracks: [toTrack(root, requestedBy)], playlist: null };
 }

-/** Resolves a direct, time-limited media URL for a page URL (used when seeking). */
-export async function resolveStreamUrl(pageUrl: string): Promise<string> {
-  const { stdout } = await runYtDlp([
-    ...baseArgs({ download: true }),
-    "-f",
-    "bestaudio/best",
-    "--no-playlist",
-    "-g",
-    pageUrl,
-  ]);
-  const url = stdout.split(/\r?\n/).map((l) => l.trim()).find(Boolean);
-  if (!url) throw new UserFacingError("Не удалось получить ссылку на аудиопоток");
-  return url;
+/** yt-dlp arguments for a download, exposed for the video pipeline. */
+export function downloadArgs(): string[] {
+  return baseArgs({ download: true });
+}
+
+/**
+ * Checks whether YouTube still offers a progressive format (video and audio in
+ * one file) at this size. Only those can be streamed: anything that needs
+ * merging makes yt-dlp download both streams in full before writing a byte, and
+ * direct CDN URLs are no longer fetchable by ffmpeg — YouTube stalls them.
+ */
+export async function hasProgressiveVideo(pageUrl: string, maxHeight: number): Promise<boolean> {
+  try {
+    const { stdout } = await runYtDlp([
+      ...baseArgs({ download: true }),
+      "-f",
+      progressiveFormat(maxHeight),
+      "--no-playlist",
+      "--print",
+      "%(format_id)s",
+      pageUrl,
+    ]);
+    return stdout.trim().length > 0;
+  } catch (err) {
+    log.debug({ err, pageUrl }, "no progressive format for video playback");
+    return false;
+  }
+}
+
+/** Best single-file format within the height budget. */
+export function progressiveFormat(maxHeight: number): string {
+  return `b[height<=${maxHeight}]/b`;
 }

 /** Private, disposable copy of the cookie jar for concurrent reads. */