Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions packages/core/src/canaryRegistry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,16 @@ export const CANARIES: readonly CanaryDefinition[] = [
owner: "vance",
sunsetAfter: "2026-11-15",
},
{
name: "audio-track-mute",
percentage: 0,
description:
"Label the visibility control as Mute on audio tracks, and make preview " +
"silence data-hidden audio the way the render already does. Fixes a " +
"shipped preview/export mismatch, so it is gated separately.",
owner: "vance",
sunsetAfter: "2026-12-15",
},
] as const;

export function findCanary(name: string): CanaryDefinition | undefined {
Expand Down
95 changes: 95 additions & 0 deletions packages/core/src/runtime/init.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { initSandboxRuntimeModular } from "./init";
import { TYPEGPU_PRESENT_HEARTBEAT_MS } from "./adapters/typegpu";
import { WebAudioTransport } from "./webAudioTransport";
import type { RuntimeTimelineLike } from "./types";

function createMockTimeline(duration: number): RuntimeTimelineLike {
Expand Down Expand Up @@ -1260,6 +1261,100 @@ describe("initSandboxRuntimeModular", () => {
expect(hiddenClip.style.display).toBe("");
});

it("excludes a data-hidden audio clip from Web Audio scheduling", () => {
const root = document.createElement("div");
root.setAttribute("data-composition-id", "main");
root.setAttribute("data-root", "true");
root.setAttribute("data-start", "0");
root.setAttribute("data-duration", "10");
root.setAttribute("data-width", "1920");
root.setAttribute("data-height", "1080");
document.body.appendChild(root);

const hiddenAudio = document.createElement("audio");
hiddenAudio.setAttribute("data-start", "0");
hiddenAudio.setAttribute("data-duration", "10");
hiddenAudio.setAttribute("data-hidden", "");
hiddenAudio.load = () => {};
hiddenAudio.play = vi.fn(() => Promise.resolve());
root.appendChild(hiddenAudio);

const audibleAudio = document.createElement("audio");
audibleAudio.setAttribute("data-start", "0");
audibleAudio.setAttribute("data-duration", "10");
audibleAudio.load = () => {};
audibleAudio.play = vi.fn(() => Promise.resolve());
root.appendChild(audibleAudio);

window.__timelines = { main: createMockTimeline(10) };
initSandboxRuntimeModular();

const decodeSpy = vi
.spyOn(WebAudioTransport.prototype, "decodeAudioElement")
.mockResolvedValue(null);

const player = window.__player;
player?.play();
player?.seek(0);

expect(decodeSpy).toHaveBeenCalledTimes(1);
expect(decodeSpy.mock.calls[0]?.[0]).toBe(audibleAudio);
});

it("batches a mid-playback data-hidden toggle into exactly one Web Audio reschedule", () => {
const root = document.createElement("div");
root.setAttribute("data-composition-id", "main");
root.setAttribute("data-root", "true");
root.setAttribute("data-start", "0");
root.setAttribute("data-duration", "10");
root.setAttribute("data-width", "1920");
root.setAttribute("data-height", "1080");
document.body.appendChild(root);

// Two separately-toggled audio clips (not a wrapper div — the visibility
// sweep only walks [data-start] nodes, so the attribute must sit on each
// timed element itself, matching how the eye button hides per-element).
const audioA = document.createElement("audio");
audioA.setAttribute("data-start", "0");
audioA.setAttribute("data-duration", "10");
audioA.setAttribute("data-hidden", "");
audioA.load = () => {};
audioA.play = vi.fn(() => Promise.resolve());
root.appendChild(audioA);

const audioB = document.createElement("audio");
audioB.setAttribute("data-start", "0");
audioB.setAttribute("data-duration", "10");
audioB.setAttribute("data-hidden", "");
audioB.load = () => {};
audioB.play = vi.fn(() => Promise.resolve());
root.appendChild(audioB);

window.__timelines = { main: createMockTimeline(10) };
initSandboxRuntimeModular();

const player = window.__player;
// play() alone (no seek) already runs one visibility pass while the clock
// is playing, registering both clips as hidden — the baseline this test
// toggles away from.
player?.play();

const decodeSpy = vi
.spyOn(WebAudioTransport.prototype, "decodeAudioElement")
.mockResolvedValue(null);
const generationSpy = vi.spyOn(WebAudioTransport.prototype, "startGeneration");

// Both become visible in the SAME sync pass — must still be one reschedule.
// keepPlaying: a plain seek() pauses the clock before re-syncing visibility,
// which would make the hiddenAudioDirty branch's isPlaying() gate a no-op.
audioA.removeAttribute("data-hidden");
audioB.removeAttribute("data-hidden");
player?.seek(1, { keepPlaying: true });

expect(generationSpy).toHaveBeenCalledTimes(1);
expect(decodeSpy).toHaveBeenCalledTimes(2);
});

it("does not stamp Studio timing on GSAP targets inside authored timed clips", () => {
withStudioIframe(() => {
const root = document.createElement("div");
Expand Down
15 changes: 15 additions & 0 deletions packages/core/src/runtime/init.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1917,6 +1917,13 @@ export function initSandboxRuntimeModular(): void {
};
const dataHiddenDisplayRestores = new WeakMap<HTMLElement, string>();
const dataHiddenDisplayNodes = new WeakSet<HTMLElement>();
// A data-hidden toggle on (or affecting) an audio element must re-schedule
// WebAudio playback so the hidden clip's source is dropped/restored mid-
// playback. Batched to one call per syncTimedElementVisibility pass, not
// one per toggled node (schedulePlayback replaces the whole active set).
let hiddenAudioDirty = false;
const nodeAffectsAudio = (node: HTMLElement): boolean =>
node.matches("audio[data-start]") || node.querySelector("audio[data-start]") !== null;

const syncTimedElementVisibility = (
currentTime: number,
Expand All @@ -1930,6 +1937,7 @@ export function initSandboxRuntimeModular(): void {
if (!dataHiddenDisplayNodes.has(rawNode)) {
dataHiddenDisplayRestores.set(rawNode, rawNode.style.getPropertyValue("display"));
dataHiddenDisplayNodes.add(rawNode);
if (nodeAffectsAudio(rawNode)) hiddenAudioDirty = true;
}
rawNode.style.display = "none";
if (rawNode instanceof HTMLVideoElement || rawNode instanceof HTMLImageElement) {
Expand All @@ -1947,6 +1955,7 @@ export function initSandboxRuntimeModular(): void {
}
dataHiddenDisplayRestores.delete(rawNode);
dataHiddenDisplayNodes.delete(rawNode);
if (nodeAffectsAudio(rawNode)) hiddenAudioDirty = true;
}

let isVisibleNow = isTimedElementVisibleAt(rawNode, currentTime);
Expand Down Expand Up @@ -1976,6 +1985,10 @@ export function initSandboxRuntimeModular(): void {
rawNode.style.display = "none";
}
}
if (hiddenAudioDirty && clock.isPlaying()) {
scheduleWebAudioForActiveClips();
}
hiddenAudioDirty = false;
};

const syncMediaForCurrentState = () => {
Expand Down Expand Up @@ -2921,6 +2934,7 @@ export function initSandboxRuntimeModular(): void {
let foundActive = false;
for (const rawEl of audioEls) {
if (!(rawEl instanceof HTMLMediaElement) || !rawEl.isConnected) continue;
if (rawEl.closest("[data-hidden]")) continue;
const start = Number.parseFloat(rawEl.dataset.start ?? "");
const durAttr = Number.parseFloat(rawEl.dataset.duration ?? "");
const end = Number.isFinite(durAttr) && durAttr > 0 ? start + durAttr : Infinity;
Expand Down Expand Up @@ -3031,6 +3045,7 @@ export function initSandboxRuntimeModular(): void {
const audioEls = document.querySelectorAll("audio[data-start]");
for (const rawEl of audioEls) {
if (!(rawEl instanceof HTMLMediaElement) || !rawEl.isConnected) continue;
if (rawEl.closest("[data-hidden]")) continue;
const compStart = Number.parseFloat(rawEl.dataset.start ?? "");
if (!Number.isFinite(compStart)) continue;
const mediaStart =
Expand Down
38 changes: 38 additions & 0 deletions packages/core/src/runtime/media.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -460,6 +460,44 @@ describe("syncRuntimeMedia", () => {
expect(clip.el.play).toHaveBeenCalled();
});

describe("data-hidden silences preview volume", () => {
it("zeroes effective volume for a clip under a data-hidden ancestor", () => {
const clip = createMockClip({ start: 0, end: 10, volume: 0.8 });
Object.defineProperty(clip.el, "readyState", { value: 4, writable: true });
const hiddenAncestor = document.createElement("div");
hiddenAncestor.setAttribute("data-hidden", "");
document.body.appendChild(hiddenAncestor);
hiddenAncestor.appendChild(clip.el);

let seen = -1;
syncRuntimeMedia({
clips: [clip],
timeSeconds: 1,
playing: true,
playbackRate: 1,
onElementVolume: (_el, v) => {
seen = v;
},
});

expect(seen).toBe(0);
});

it("does not touch el.muted when silencing a hidden clip (RULES trap: transport owns el.muted)", () => {
const clip = createMockClip({ start: 0, end: 10, volume: 0.8 });
Object.defineProperty(clip.el, "readyState", { value: 4, writable: true });
const hiddenAncestor = document.createElement("div");
hiddenAncestor.setAttribute("data-hidden", "");
document.body.appendChild(hiddenAncestor);
hiddenAncestor.appendChild(clip.el);
clip.el.muted = false;

syncRuntimeMedia({ clips: [clip], timeSeconds: 1, playing: true, playbackRate: 1 });

expect(clip.el.muted).toBe(false);
});
});

describe("play() storm guard (unplayable elements)", () => {
it("does not play() an element with a media error", () => {
const clip = createMockClip({ start: 0, end: 10 });
Expand Down
5 changes: 4 additions & 1 deletion packages/core/src/runtime/media.ts
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,10 @@ export function syncRuntimeMedia(params: {
authorVolume = fallbackAuthorVolume;
}

const effectiveVolume = clampVolume(authorVolume * userVol);
// A data-hidden ancestor is silent in the export (audioMixer.ts drops
// it); preview must match. Folded into the per-tick volume, not
// el.muted (RULES trap: el.muted is the transport's ownership flag).
const effectiveVolume = el.closest("[data-hidden]") ? 0 : clampVolume(authorVolume * userVol);
el.volume = effectiveVolume;
lastRuntimeAppliedVolume.set(el, effectiveVolume);
params.onElementVolume?.(el, effectiveVolume);
Expand Down
81 changes: 81 additions & 0 deletions packages/studio/src/hooks/timelineTrackVisibility.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,87 @@ describe("toggleTimelineTrackHidden", () => {

expect(recordEdit.mock.calls[0]?.[0]?.label).toBe("Show track 2");
});

it("labels an audio-only track Mute/Unmute instead of Hide/Show", async () => {
const files = new Map([
["index.html", `<div id="voiceover" data-start="0" data-duration="2"></div>`],
]);
stubProjectFiles(files);

const recordEdit = vi.fn();

await toggleTimelineTrackHidden({
projectId: "project-1",
activeCompPath: "index.html",
timelineElements: [element({ id: "voiceover", domId: "voiceover", track: 0, tag: "audio" })],
track: 0,
hidden: true,
previewIframe: null,
writeProjectFile: async () => {},
recordEdit,
domEditSaveTimestampRef: { current: 0 },
pendingTimelineEditPathRef: { current: new Set() },
});

expect(recordEdit.mock.calls[0]?.[0]?.label).toBe("Mute track 1");
});

it("labels unmuting an audio-only track back on", async () => {
const files = new Map([
["index.html", `<div id="voiceover" data-start="0" data-duration="2" data-hidden=""></div>`],
]);
stubProjectFiles(files);

const recordEdit = vi.fn();

await toggleTimelineTrackHidden({
projectId: "project-1",
activeCompPath: "index.html",
timelineElements: [
element({ id: "voiceover", domId: "voiceover", track: 0, tag: "audio", hidden: true }),
],
track: 0,
hidden: false,
previewIframe: null,
writeProjectFile: async () => {},
recordEdit,
domEditSaveTimestampRef: { current: 0 },
pendingTimelineEditPathRef: { current: new Set() },
});

expect(recordEdit.mock.calls[0]?.[0]?.label).toBe("Unmute track 1");
});

it("keeps Hide/Show wording for a mixed (audio + visual) track", async () => {
const files = new Map([
[
"index.html",
`<div id="voiceover" data-start="0" data-duration="2"></div>
<div id="caption" data-start="0" data-duration="2"></div>`,
],
]);
stubProjectFiles(files);

const recordEdit = vi.fn();

await toggleTimelineTrackHidden({
projectId: "project-1",
activeCompPath: "index.html",
timelineElements: [
element({ id: "voiceover", domId: "voiceover", track: 0, tag: "audio" }),
element({ id: "caption", domId: "caption", track: 0, tag: "div" }),
],
track: 0,
hidden: true,
previewIframe: null,
writeProjectFile: async () => {},
recordEdit,
domEditSaveTimestampRef: { current: 0 },
pendingTimelineEditPathRef: { current: new Set() },
});

expect(recordEdit.mock.calls[0]?.[0]?.label).toBe("Hide track 1");
});
});

describe("toggleTimelineElementHidden", () => {
Expand Down
14 changes: 12 additions & 2 deletions packages/studio/src/hooks/timelineTrackVisibility.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
trackDisplaySuffix,
} from "../player/components/timelineTrackDisplay";
import { saveProjectFilesWithHistory } from "../utils/studioFileHistory";
import { isAudioTimelineElement } from "../utils/timelineInspector";
import { readTagSnippetByTarget, type PatchOperation } from "../utils/sourcePatcher";
import {
applyPatchByTarget,
Expand Down Expand Up @@ -218,12 +219,21 @@ export async function toggleTimelineTrackHidden({
const suffix = trackDisplaySuffix(
trackDisplayNumber(timelineTrackOrder(timelineElements), track),
);
const trackElements = timelineElements.filter((element) => element.track === track);
const isAudioOnlyTrack = trackElements.length > 0 && trackElements.every(isAudioTimelineElement);
const label = isAudioOnlyTrack
? hidden
? `Mute track${suffix}`
: `Unmute track${suffix}`
: hidden
? `Hide track${suffix}`
: `Show track${suffix}`;
return setElementsHidden({
projectId,
activeCompPath,
elements: timelineElements.filter((element) => element.track === track),
elements: trackElements,
hidden,
label: hidden ? `Hide track${suffix}` : `Show track${suffix}`,
label,
previewIframe,
writeProjectFile,
recordEdit,
Expand Down
Loading
Loading