Fix duplicate movie audio during scene transitions

frontend/src/player/movie.ts

@@ -4,12 +4,55 @@ export function applyMovieMask(picture: Uint8ClampedArray, mask: Uint8ClampedArr
   for (let i = 3; i < picture.length; i += 4) picture[i] = Math.round(picture[i] * mask[i - 3] / 255);
 }
 
+const movieDisposers = new WeakMap<HTMLElement, () => void>();
+
+/** Removing a media element from the DOM does not stop its playback. */
+export function disposeMovies(root: Element) {
+  for (const element of [root, ...root.querySelectorAll('video, canvas')]) {
+    const dispose = movieDisposers.get(element as HTMLElement);
+    if (dispose) { dispose(); movieDisposers.delete(element as HTMLElement); }
+    else if (element.tagName.toLowerCase() === 'video') stopVideo(element as HTMLVideoElement);
+  }
+}
+
+function stopVideo(video: HTMLVideoElement) {
+  video.autoplay = false;
+  video.pause?.();
+  video.removeAttribute('src');
+  video.load?.();
+}
+
+/** Transition pictures are frozen frames, never additional autoplay players. */
+export function snapshotDisplayables(root: HTMLElement): HTMLElement {
+  const copy = root.cloneNode(true) as HTMLElement;
+  const originals = root.querySelectorAll('video, canvas');
+  const copies = copy.querySelectorAll('video, canvas');
+  originals.forEach((element, index) => {
+    const source = element as HTMLVideoElement | HTMLCanvasElement;
+    const target = copies[index];
+    const video = source.tagName.toLowerCase() === 'video';
+    const frame = video ? document.createElement('canvas') : target as HTMLCanvasElement;
+    frame.width = video ? (source as HTMLVideoElement).videoWidth || source.offsetWidth || 300 : (source as HTMLCanvasElement).width;
+    frame.height = video ? (source as HTMLVideoElement).videoHeight || source.offsetHeight || 150 : (source as HTMLCanvasElement).height;
+    if (video) {
+      frame.className = target.className;
+      frame.style.cssText = (target as HTMLElement).style.cssText;
+      frame.style.width = `${source.offsetWidth || frame.width}px`;
+      frame.style.height = `${source.offsetHeight || frame.height}px`;
+    }
+    if (!video || (source as HTMLVideoElement).readyState >= 2) frame.getContext?.('2d')?.drawImage(source, 0, 0);
+    if (video) { disposeMovies(target); target.replaceWith(frame); }
+  });
+  return copy;
+}
+
 export function movieDisplayable(source: string, mask: string | null, loop = true): HTMLElement {
   const video = document.createElement("video");
   video.src = source;
   video.autoplay = true;
   video.loop = loop;
   video.muted = true;
+  video.defaultMuted = true;
   video.playsInline = true;
   if (!mask) return video;
   const alpha = document.createElement("video");
@@ -25,6 +68,7 @@ export function movieDisplayable(source: string, mask: string | null, loop = tru
   if (!ctx || !maskCtx) throw new Error("Не удалось создать поверхность для видео с маской");
   let last = -1;
   let failed = false;
+  movieDisposers.set(canvas, () => { failed = true; stopVideo(video); stopVideo(alpha); });
   const draw = () => {
     if (failed) return;
     if (!canvas.isConnected) { video.pause(); alpha.pause(); return; }

frontend/src/player/renderer.ts

@@ -4,7 +4,7 @@
 import { AtlRunner, bindArguments, blockProperties, type AtlBlock } from "./atl";
 import { blendMode, layoutTransform } from "./rendertransform";
 import { ColorMatrix, matrixFilter } from "./colormatrix";
-import { movieDisplayable } from "./movie";
+import { disposeMovies, movieDisplayable, snapshotDisplayables } from "./movie";
 import type { AtSpec, ImageDef, ShownImage, Transition } from "./types";
 
 const POSITIONS: Record<string, Record<string, number>> = {
@@ -169,6 +169,7 @@ export class Renderer {
     for (const e of layer.shown.values()) {
       e.resize?.disconnect();
       e.runners.forEach((r) => r.runner.stop());
+      disposeMovies(e.el);
       e.el.remove();
     }
     layer.shown.clear();
@@ -196,6 +197,7 @@ export class Renderer {
     prev?.runners.forEach((r) => r.runner.stop());
     prev?.resize?.disconnect();
     if (prev) {
+      disposeMovies(prev.el);
       prev.el.replaceWith(el);
     } else {
       const behind = opts.behind?.map((t) => layer.shown.get(t)?.el).find(Boolean);
@@ -268,12 +270,13 @@ export class Renderer {
     const exit = e.runners.find((r) => r.runner.hasHideHandler(r.block));
     if (!exit || this.fast) {
       e.runners.forEach((r) => r.runner.stop());
+      disposeMovies(e.el);
       e.el.remove();
       return;
     }
     // `on hide` animation: the picture leaves the scene state at once but stays visible until it ends
     e.runners.forEach((r) => r.runner !== exit.runner && r.runner.stop());
-    void exit.runner.hide(exit.block).finally(() => e.el.remove());
+    void exit.runner.hide(exit.block).finally(() => { disposeMovies(e.el); e.el.remove(); });
   }
 
   isShown(tag: string, layer = "master") {
@@ -425,6 +428,7 @@ export class Renderer {
     child.style.gridArea = "1 / 1";
     const old = [...box.children];
     if (!transition || this.fast) {
+      old.forEach(disposeMovies);
       box.replaceChildren(child);
       return;
     }
@@ -433,7 +437,7 @@ export class Renderer {
     child.animate([{ opacity: 0 }, { opacity: 1 }], { duration: (transition.d ?? 0.5) * 1000, fill: "forwards" })
       .onfinish = () => {
         child.style.opacity = "";
-        old.forEach((e) => e.remove());
+        old.forEach((e) => { disposeMovies(e); e.remove(); });
       };
   }
 
@@ -489,6 +493,7 @@ export class Renderer {
         // was removed before any timer observed it attached.
         if (!box.isConnected) return;
         i = last ? 0 : i + 1;
+        disposeMovies(box);
         box.replaceChildren(this.build(frames[i].img, depth + 1));
         tick();
       }, d * 1000);
@@ -629,7 +634,7 @@ export class Renderer {
     box.className = "pg-layer";
     for (const layer of this.layers.values()) {
       if (this.depth(layer.name) > 0) continue;
-      const copy = layer.el.cloneNode(true) as HTMLElement;
+      const copy = snapshotDisplayables(layer.el);
       copy.querySelector(":scope > .pg-usr")?.remove();
       box.append(copy);
     }

frontend/src/player/screens.ts

@@ -8,7 +8,7 @@ import type { AudioSystem } from "./audio";
 import { isNativeAction } from "./musicroom";
 import { galleryClass } from "./gallery";
 import { parseExpression } from "./pyparse";
-import { movieDisplayable } from "./movie";
+import { disposeMovies, movieDisplayable } from "./movie";
 import { RenderHost, isDisplayableInstance } from "./render";
 import { Matrix4, blendMode } from "./rendertransform";
 import { strftime } from "./pymodules";
@@ -358,7 +358,10 @@ export class ScreenManager {
     inst.runners.forEach((r) => r.stop());
     inst.runners = [];
     inst.el.remove();
-    for (const id of [...this.movies.keys()]) if (id.startsWith(`${inst.name}|`)) this.movies.delete(id);
+    for (const [id, element] of [...this.movies]) if (id.startsWith(`${inst.name}|`)) {
+      disposeMovies(element);
+      this.movies.delete(id);
+    }
     if (forget) this.shown.delete(inst.name);
   }
 
@@ -462,13 +465,13 @@ export class ScreenManager {
       this.nodes(inst.def.body, ctx, children);
       inst.el.replaceChildren(...children);
     });
-    // a video taken out of the page pauses by itself; carry on, and forget the ones no screen shows now
+    // Preserve cached players on redraw; explicitly stop those no screen shows.
     for (const [id, element] of [...this.movies]) {
       if (this.moviesSeen.has(id)) {
         const video = element instanceof HTMLVideoElement ? element : element.querySelector("video");
         if (video && video.paused && !video.ended) void video.play().catch(() => {});
       } else if (id.startsWith(`${inst.name}|`)) {
-        if (element instanceof HTMLVideoElement) element.pause();
+        disposeMovies(element);
         this.movies.delete(id);
       }
     }

frontend/tests/mist-gallery-playback-browser.test.mjs

@@ -6,7 +6,7 @@ import { fileURLToPath, pathToFileURL } from "node:url";
 import { test } from "node:test";
 import { build } from "esbuild";
 
-test("MIST gallery scene accepts Skip and menu clicks, keeps video centered and bounds overlapping music", {
+test("MIST gallery plays normal transitions without extra audio and supports Skip, video layout and exit", {
   skip: !process.env.PGAMES_PLAYWRIGHT_PATH || !process.env.PGAMES_MIST_RUNTIME || !process.env.PGAMES_MIST_MANIFEST || !process.env.PGAMES_MIST_BLOBS,
 }, async () => {
   const { chromium } = await import(pathToFileURL(process.env.PGAMES_PLAYWRIGHT_PATH + "/index.mjs").href);
@@ -22,7 +22,7 @@ test("MIST gallery scene accepts Skip and menu clicks, keeps video centered and
       {list: async () => [], get: async () => null, put: async () => {}, remove: async () => {}},
       'mist-playback-browser', () => {}, tr, async () => tr);
     e.settings.cps = 0;
-    e.renderer.transitionsOff = true;
+    e.renderer.transitionsOff = false;
     e.audio.setVolumes({music: 0, sound: 0, voice: 0});
     e.audio.setMixer('ambiant', 0);
     const play = e.audio.play.bind(e.audio);
@@ -68,8 +68,25 @@ test("MIST gallery scene accepts Skip and menu clicks, keeps video centered and
     await page.waitForFunction(() => [...document.querySelectorAll('.pg-simgbtn img')].every(el => el.complete && el.naturalWidth > 0));
     await page.locator('[data-test="robgallery"] .pg-simgbtn').nth(1).click();
     await page.waitForFunction(() => !!window.engine.onAdvance && !window.engine.screens.shown.size);
+    // Ordinary viewing must not start extra media players when a movie is
+    // captured as the outgoing picture of the next transition.
+    for (let line = 0; line < 12; line++) {
+      if (await page.locator('video.pg-image').count()) break;
+      const before = await page.evaluate(() => JSON.stringify(window.engine.cur));
+      await page.mouse.click(480, 270);
+      await page.waitForFunction(before => JSON.stringify(window.engine.cur) !== before && !!window.engine.onAdvance, before);
+    }
+    await page.waitForFunction(() => document.querySelector('video.pg-image')?.readyState >= 2);
+    const snapshot = await page.evaluate(() => {
+      window.firstSceneVideo = document.querySelector('video.pg-image');
+      window.engine.renderer.commit();
+      return window.engine.renderer.committed.querySelectorAll('video').length;
+    });
+    assert.equal(snapshot, 0, 'transition snapshots must not contain additional movie players');
     await page.locator('[data-act="skip"]').click();
     await page.getByRole('button', { name: 'End Scene.', exact: true }).waitFor();
+    assert.equal(await page.evaluate(() => window.firstSceneVideo.paused && !window.firstSceneVideo.hasAttribute('src')), true,
+      'the previous scene movie is stopped and unloaded even when a transition snapshot exists');
     await page.getByRole('button', { name: 'Keep Looking.', exact: true }).click();
     await page.waitForFunction(() => document.querySelector('.pg-image video, video.pg-image')?.videoWidth > 0);
     const state = await page.evaluate(() => {

frontend/tests/movie.test.mjs

@@ -6,7 +6,46 @@ import { parseHTML } from "linkedom";
 
 const source = readFileSync(new URL("../src/player/movie.ts", import.meta.url), "utf8");
 const { outputText } = ts.transpileModule(source, { compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext } });
-const { applyMovieMask, movieDisplayable } = await import(`data:text/javascript;base64,${Buffer.from(outputText).toString("base64")}`);
+const { applyMovieMask, movieDisplayable, disposeMovies, snapshotDisplayables } = await import(`data:text/javascript;base64,${Buffer.from(outputText).toString("base64")}`);
+
+test("transition snapshots copy video and canvas pixels without cloning active players", () => {
+  const { document } = parseHTML('<html><body><div><video class="pg-image" style="left: 10px"></video><canvas width="4" height="2"></canvas></div></body></html>');
+  globalThis.document = document;
+  const create = document.createElement.bind(document);
+  const drawn = [];
+  const root = document.querySelector('div');
+  const video = root.querySelector('video');
+  const canvas = root.querySelector('canvas');
+  Object.assign(video, { readyState: 2, videoWidth: 4, videoHeight: 2, muted: true, autoplay: true });
+  const canvasProto = Object.getPrototypeOf(canvas);
+  const previous = canvasProto.getContext;
+  canvasProto.getContext = function () { return { drawImage: source => drawn.push(source) }; };
+  try {
+    const copy = snapshotDisplayables(root);
+    assert.equal(copy.querySelectorAll('video').length, 0);
+    assert.equal(copy.querySelectorAll('canvas').length, 2);
+    assert.deepEqual(drawn, [video, canvas]);
+    assert.equal(copy.querySelector('.pg-image').style.left, '10px');
+    assert.equal(root.querySelector('video'), video, 'snapshot leaves the live movie in place');
+  } finally { canvasProto.getContext = previous; document.createElement = create; }
+});
+
+test("disposing a removed movie stops and unloads its player immediately", () => {
+  const { document } = parseHTML('<html><body></body></html>');
+  globalThis.document = document;
+  const video = movieDisplayable('/movie.webm', null);
+  let paused = false, unloaded = false;
+  video.pause = () => { paused = true; };
+  video.load = () => { unloaded = true; };
+  assert.equal(video.defaultMuted, true);
+  document.body.append(video);
+  video.remove();
+  disposeMovies(video);
+  assert.equal(paused, true);
+  assert.equal(unloaded, true);
+  assert.equal(video.autoplay, false);
+  assert.equal(video.hasAttribute('src'), false);
+});
 
 test("video opacity uses the mask's red channel and retains existing transparency", () => {
   const pixels = new Uint8ClampedArray([200, 10, 20, 255, 30, 40, 50, 128]);