Fix video codec, aspect and placement
Three problems visible at once on a running instance: the picture stuttered and drifted behind the sound, it was letterboxed oddly, and it appeared as a separate tile instead of coming from the bot. - YouTube handed us AV1 (format 398). Software-decoding AV1 at 720p does not sustain real time on a small server, which explains both the stutter and the drift; H.264 is now requested first, VP9 second. - The frame was padded into a fixed box, so a clip whose proportions differed got black bars baked in and then more from the client. Size is now a bounding box and the frame keeps the clip's own proportions. - Video was published as a screen share, which every client renders as its own tile. VIDEO_SOURCE=camera (the new default) puts it inside the bot's tile; "screen" keeps the old behaviour. VIDEO_SYNC_OFFSET_MS is there for the residual drift, since audio and video travel as two separately published tracks. Measured loudnorm first to rule it out as the cause of the desync: it adds 0 ms. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
9285fd7fbd
commit
0b17b880d6
+10
-3
@@ -1,6 +1,7 @@
|
||||
import { createRequire } from "node:module";
|
||||
import type { Readable } from "node:stream";
|
||||
import type { LocalVideoTrack, Room, VideoSource } from "@livekit/rtc-node";
|
||||
import { config } from "../config.js";
|
||||
import { logger } from "../logger.js";
|
||||
|
||||
// revoice.js pulls in the CommonJS build of @livekit/rtc-node, and the native
|
||||
@@ -49,10 +50,13 @@ export class VideoPublisher {
|
||||
async start(room: Room, stream: Readable, onGiveUp?: (reason: string) => void): Promise<void> {
|
||||
const { width, height } = this.options;
|
||||
const source = new rtc.VideoSource(width, height);
|
||||
const track = rtc.LocalVideoTrack.createVideoTrack("screen", source);
|
||||
// As a camera the picture appears inside the bot's own tile; a screen share
|
||||
// is rendered as a separate tile by every client.
|
||||
const asScreen = config.VIDEO_SOURCE === "screen";
|
||||
const track = rtc.LocalVideoTrack.createVideoTrack(asScreen ? "screen" : "video", source);
|
||||
|
||||
const options = new rtc.TrackPublishOptions();
|
||||
options.source = rtc.TrackSource.SOURCE_SCREENSHARE;
|
||||
options.source = asScreen ? rtc.TrackSource.SOURCE_SCREENSHARE : rtc.TrackSource.SOURCE_CAMERA;
|
||||
|
||||
const participant = room.localParticipant;
|
||||
if (!participant) throw new Error("room has no local participant yet");
|
||||
@@ -75,7 +79,10 @@ export class VideoPublisher {
|
||||
}, FIRST_FRAME_TIMEOUT_MS);
|
||||
this.firstFrameTimer.unref?.();
|
||||
|
||||
log.info({ width, height, sid: publication.sid }, "screen share published");
|
||||
log.info(
|
||||
{ width, height, source: config.VIDEO_SOURCE, sid: publication.sid },
|
||||
"video track published",
|
||||
);
|
||||
}
|
||||
|
||||
private consume(chunk: Buffer): void {
|
||||
|
||||
Reference in New Issue
Block a user