From 2db07f92f4f11af746e0fd96ae3192a1c5dcf1f1 Mon Sep 17 00:00:00 2001 From: Duncan Owen <283775822+duncsdownunder@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:02:24 +1000 Subject: [PATCH 1/3] Fix cut boundary drift in cut-silences/cut-mistakes retiming Both apply-cuts.mjs and cut-silences.mjs computed retimed transcript timestamps by exact float subtraction, then fed those same floats to ffmpeg's trim/atrim filters, which can only cut on real frame boundaries. Each cut silently lost a fraction of a frame versus the math, and the error compounded linearly with cut count (measured up to ~1.5s drift across 163 cuts on a real 48-minute podcast render), so any downstream tool trusting the retimed transcript against the actual rendered video would land on the wrong word once enough cuts had accumulated. Snap every cut boundary to the nearest real video frame (detected via ffprobe) before computing the retimed transcript, when --video is given. Testing on real footage shows this substantially reduces drift but does not fully eliminate it (real frame spacing isn't perfectly uniform), so both SKILL.md files now document the residual limitation and require re-transcribing the final render to diff against intended text before calling any such cut "done" -- frame strips alone can't catch a uniform time offset on a locked-off single camera. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01PbLt6aLpY5gruwSHS2AXaC --- .agents/skills/cut-mistakes/SKILL.md | 27 +++++++++++++++++ .../cut-mistakes/scripts/apply-cuts.mjs | 29 +++++++++++++++++-- .agents/skills/cut-silences/SKILL.md | 22 +++++++++++++- .../cut-silences/scripts/cut-silences.mjs | 29 ++++++++++++++++++- .claude/skills/cut-mistakes/SKILL.md | 27 +++++++++++++++++ .../cut-mistakes/scripts/apply-cuts.mjs | 29 +++++++++++++++++-- .claude/skills/cut-silences/SKILL.md | 22 +++++++++++++- .../cut-silences/scripts/cut-silences.mjs | 29 ++++++++++++++++++- 8 files changed, 206 insertions(+), 8 deletions(-) diff --git a/.agents/skills/cut-mistakes/SKILL.md b/.agents/skills/cut-mistakes/SKILL.md index 94cf1b80..cda333a0 100644 --- a/.agents/skills/cut-mistakes/SKILL.md +++ b/.agents/skills/cut-mistakes/SKILL.md @@ -65,6 +65,33 @@ node scripts/build-edl-review.mjs .mistakes-edl.json \ npx serve . -p 8080 -n ``` +## Known limitation: retimed-transcript precision (read before reusing timestamps downstream) + +`apply-cuts.mjs` computes `.mistakes-transcript.json`'s word timings by +exact float subtraction, then feeds those same floats to ffmpeg's +`trim`/`atrim`. ffmpeg can only cut on real frame boundaries, so each cut's +actual rendered position can differ from the math by a fraction of a frame, +and — because this runs after `cut-silences`, which typically makes 100-300+ +cuts of its own — the error compounds across two stages. The script now +snaps every cut boundary to the nearest real frame (via `ffprobe`'s +`r_frame_rate` when `--video` is given) before computing anything, which +substantially reduces drift, but testing found it does not fully eliminate +it on real camera footage (frame spacing isn't perfectly uniform) — expect +low-single-digit milliseconds of residual drift per cut, which can still add +up to a noticeable offset (measured: ~0.3s across 45 cuts on real footage) +on a file with many cuts. + +**Practical rule**: if you need this transcript's timestamps to point at +exact words in the *rendered* video — for another cutting pass, a beat-sync +tool, or anything requiring frame-accurate timing — do not trust +`.mistakes-transcript.json` on its own once more than a few dozen cuts +have been applied (by this tool or by `cut-silences` before it). Re-transcribe +the actual rendered video directly for the region you need, and — before +calling any such downstream cut "done" — re-transcribe the *final render* +and diff it against the intended text. Frame strips / contact sheets alone +cannot catch this: a locked-off single-camera shot looks identical a second +early. + ## Notes - A very clean delivery may yield few or zero real cuts — that's a valid outcome; don't cut natural speech to hit a quota. diff --git a/.agents/skills/cut-mistakes/scripts/apply-cuts.mjs b/.agents/skills/cut-mistakes/scripts/apply-cuts.mjs index 62973c4e..b7eb7e31 100644 --- a/.agents/skills/cut-mistakes/scripts/apply-cuts.mjs +++ b/.agents/skills/cut-mistakes/scripts/apply-cuts.mjs @@ -69,6 +69,31 @@ if (!Number.isFinite(duration)) { if (!Number.isFinite(duration) || duration <= 0) die('Invalid source duration'); if (approved.some(c => c.start < 0 || c.end > duration)) die('Cut range is outside source duration bounds'); +// ---- frame rate (for boundary snapping, video renders only) ---- +// ffmpeg's trim/atrim filters cut on real source frame boundaries, not exact +// float seconds. If the retimed transcript is computed from un-snapped +// floats, each cut's actual rendered position silently differs from the math +// by a fraction of a frame, and that error compounds linearly with cut count. +// Snapping every delete-range edge to the nearest real frame time BEFORE +// computing anything downstream keeps the retimed transcript closer to what +// ffmpeg actually renders. Only meaningful (and only applied) when --video is +// given -- with no video there is no rendered file to drift from, and +// snapping would just add pointless rounding to an otherwise-exact edit. +let fps = null; +if (opts.video && existsSync(resolve(opts.video))) { + fps = 30; + const probe = spawnSync("ffprobe", [ + "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=r_frame_rate", + "-of", "default=noprint_wrappers=1:nokey=1", resolve(opts.video), + ]); + const raw = probe.stdout?.toString().trim(); + if (raw && raw.includes("/")) { + const [n, d] = raw.split("/").map(Number); + if (Number.isFinite(n) && Number.isFinite(d) && d > 0) fps = n / d; + } else if (Number.isFinite(Number(raw))) fps = Number(raw); +} +const snap = (t) => (fps ? Math.round(t * fps) / fps : t); + function mergeRanges(ranges) { const sorted = ranges.map((r) => ({ start: Number(r.start), end: Number(r.end), reasons: r.reason ? [r.reason] : [] })) .sort((a, b) => a.start - b.start); @@ -80,7 +105,7 @@ function mergeRanges(ranges) { } return merged; } -const mergedDeletes = mergeRanges(approved); +const mergedDeletes = mergeRanges(approved).map((r) => ({ ...r, start: snap(r.start), end: snap(r.end) })).filter((r) => r.end > r.start); const keepRanges = []; let cursor = 0; @@ -149,7 +174,7 @@ writeFileSync(join(outDir, `${stem}.mistakes-decisions.md`), [ ].join("\n")); const summary = { - inputDuration: Number(duration.toFixed(3)), editedDuration, + inputDuration: Number(duration.toFixed(3)), fps, editedDuration, removed: Number(removedTotal.toFixed(3)), removedPct: Number(((removedTotal / duration) * 100).toFixed(1)), cuts: mergedDeletes.length, editedWords: editedWords.length, outDir, }; diff --git a/.agents/skills/cut-silences/SKILL.md b/.agents/skills/cut-silences/SKILL.md index 8615b65d..c5aac015 100644 --- a/.agents/skills/cut-silences/SKILL.md +++ b/.agents/skills/cut-silences/SKILL.md @@ -80,4 +80,24 @@ The JSON summary printed to stdout includes `removed`, `removedPct`, range count ## Hand-off to the next agent -Pass `.silence-transcript.json` (and the `silenced.mp4` if rendered) to the **cut-mistakes** agent. Because timestamps are already on the edited timeline, downstream beat timing and `scripts/validate-beat-sync.mjs` work without further adjustment. +Pass `.silence-transcript.json` (and the `silenced.mp4` if rendered) to the **cut-mistakes** agent. + +## Known limitation: retimed-transcript precision + +`.silence-transcript.json`'s word timings are computed by exact float +subtraction, then the same floats are fed to ffmpeg's `trim`/`atrim`, which +can only cut on real frame boundaries. Each cut's actual rendered position +can therefore differ from the math by a fraction of a frame, and a typical +silence pass makes 100-300+ cuts, so the error compounds. Boundaries are now +snapped to the nearest real frame (via `ffprobe`'s `r_frame_rate` when +`--video` is given) before computing anything, which substantially reduces +drift but — measured on real camera footage — does not fully eliminate it +(real frame spacing isn't perfectly uniform; expect low-single-digit +milliseconds of residual drift per cut, which can add up to a real offset +across hundreds of cuts). Do not assume this file's timestamps land exactly +on the right word in `silenced.mp4` once cut-mistakes (or anything else) has +compounded more cuts on top — for anything requiring frame-accurate timing +against the actual rendered video, re-transcribe that video directly for the +region you need, and re-transcribe any final render to diff against intended +text before calling a cut "done". A locked-off single camera shot looks +identical a second early, so frame strips alone won't catch this. diff --git a/.agents/skills/cut-silences/scripts/cut-silences.mjs b/.agents/skills/cut-silences/scripts/cut-silences.mjs index fd34ec6a..2781ecd9 100644 --- a/.agents/skills/cut-silences/scripts/cut-silences.mjs +++ b/.agents/skills/cut-silences/scripts/cut-silences.mjs @@ -103,6 +103,32 @@ if (!Number.isFinite(duration)) { } if (!Number.isFinite(duration)) duration = words.at(-1).end + opts.tailPad; +// ---- frame rate (for boundary snapping, video renders only) ---- +// ffmpeg's trim/atrim filters cut on real source frame boundaries, not exact +// float seconds. If the retimed transcript is computed from un-snapped floats, +// each cut's actual rendered position silently differs from the math by a +// fraction of a frame, and that error compounds linearly with cut count (a +// typical silence pass makes 100-300+ cuts). Snapping every delete-range edge +// to the nearest real frame time BEFORE computing anything downstream keeps +// the retimed transcript closer to what ffmpeg actually renders. Only +// meaningful (and only applied) when --video is given -- with no video there +// is no rendered file to drift from, and snapping would just add pointless +// rounding to an otherwise-exact edit. +let fps = null; +if (opts.video && existsSync(resolve(opts.video))) { + fps = 30; + const probe = spawnSync("ffprobe", [ + "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=r_frame_rate", + "-of", "default=noprint_wrappers=1:nokey=1", resolve(opts.video), + ]); + const raw = probe.stdout?.toString().trim(); + if (raw && raw.includes("/")) { + const [n, d] = raw.split("/").map(Number); + if (Number.isFinite(n) && Number.isFinite(d) && d > 0) fps = n / d; + } else if (Number.isFinite(Number(raw))) fps = Number(raw); +} +const snap = (t) => (fps ? Math.round(t * fps) / fps : t); + // ---- build the delete ranges (silence only) ---- function mergeRanges(ranges) { const sorted = ranges @@ -147,7 +173,7 @@ for (let i = 1; i < words.length; i++) { } } -const mergedDeletes = mergeRanges(deleteRanges); +const mergedDeletes = mergeRanges(deleteRanges).map((r) => ({ ...r, start: snap(r.start), end: snap(r.end) })).filter((r) => r.end > r.start); // ---- keep ranges = the complement of the deletes ---- const keepRanges = []; @@ -255,6 +281,7 @@ writeFileSync(decPath, decisions); const summary = { sourceDuration: Number(duration.toFixed(3)), + fps, editedDuration, removed: Number(removedTotal.toFixed(3)), removedPct: Number(((removedTotal / duration) * 100).toFixed(1)), diff --git a/.claude/skills/cut-mistakes/SKILL.md b/.claude/skills/cut-mistakes/SKILL.md index 475178d9..a76af69d 100644 --- a/.claude/skills/cut-mistakes/SKILL.md +++ b/.claude/skills/cut-mistakes/SKILL.md @@ -65,6 +65,33 @@ node scripts/build-edl-review.mjs .mistakes-edl.json \ npx serve . -p 8080 -n ``` +## Known limitation: retimed-transcript precision (read before reusing timestamps downstream) + +`apply-cuts.mjs` computes `.mistakes-transcript.json`'s word timings by +exact float subtraction, then feeds those same floats to ffmpeg's +`trim`/`atrim`. ffmpeg can only cut on real frame boundaries, so each cut's +actual rendered position can differ from the math by a fraction of a frame, +and — because this runs after `cut-silences`, which typically makes 100-300+ +cuts of its own — the error compounds across two stages. The script now +snaps every cut boundary to the nearest real frame (via `ffprobe`'s +`r_frame_rate` when `--video` is given) before computing anything, which +substantially reduces drift, but testing found it does not fully eliminate +it on real camera footage (frame spacing isn't perfectly uniform) — expect +low-single-digit milliseconds of residual drift per cut, which can still add +up to a noticeable offset (measured: ~0.3s across 45 cuts on real footage) +on a file with many cuts. + +**Practical rule**: if you need this transcript's timestamps to point at +exact words in the *rendered* video — for another cutting pass, a beat-sync +tool, or anything requiring frame-accurate timing — do not trust +`.mistakes-transcript.json` on its own once more than a few dozen cuts +have been applied (by this tool or by `cut-silences` before it). Re-transcribe +the actual rendered video directly for the region you need, and — before +calling any such downstream cut "done" — re-transcribe the *final render* +and diff it against the intended text. Frame strips / contact sheets alone +cannot catch this: a locked-off single-camera shot looks identical a second +early. + ## Notes - A very clean delivery may yield few or zero real cuts — that's a valid outcome; don't cut natural speech to hit a quota. diff --git a/.claude/skills/cut-mistakes/scripts/apply-cuts.mjs b/.claude/skills/cut-mistakes/scripts/apply-cuts.mjs index 62973c4e..b7eb7e31 100644 --- a/.claude/skills/cut-mistakes/scripts/apply-cuts.mjs +++ b/.claude/skills/cut-mistakes/scripts/apply-cuts.mjs @@ -69,6 +69,31 @@ if (!Number.isFinite(duration)) { if (!Number.isFinite(duration) || duration <= 0) die('Invalid source duration'); if (approved.some(c => c.start < 0 || c.end > duration)) die('Cut range is outside source duration bounds'); +// ---- frame rate (for boundary snapping, video renders only) ---- +// ffmpeg's trim/atrim filters cut on real source frame boundaries, not exact +// float seconds. If the retimed transcript is computed from un-snapped +// floats, each cut's actual rendered position silently differs from the math +// by a fraction of a frame, and that error compounds linearly with cut count. +// Snapping every delete-range edge to the nearest real frame time BEFORE +// computing anything downstream keeps the retimed transcript closer to what +// ffmpeg actually renders. Only meaningful (and only applied) when --video is +// given -- with no video there is no rendered file to drift from, and +// snapping would just add pointless rounding to an otherwise-exact edit. +let fps = null; +if (opts.video && existsSync(resolve(opts.video))) { + fps = 30; + const probe = spawnSync("ffprobe", [ + "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=r_frame_rate", + "-of", "default=noprint_wrappers=1:nokey=1", resolve(opts.video), + ]); + const raw = probe.stdout?.toString().trim(); + if (raw && raw.includes("/")) { + const [n, d] = raw.split("/").map(Number); + if (Number.isFinite(n) && Number.isFinite(d) && d > 0) fps = n / d; + } else if (Number.isFinite(Number(raw))) fps = Number(raw); +} +const snap = (t) => (fps ? Math.round(t * fps) / fps : t); + function mergeRanges(ranges) { const sorted = ranges.map((r) => ({ start: Number(r.start), end: Number(r.end), reasons: r.reason ? [r.reason] : [] })) .sort((a, b) => a.start - b.start); @@ -80,7 +105,7 @@ function mergeRanges(ranges) { } return merged; } -const mergedDeletes = mergeRanges(approved); +const mergedDeletes = mergeRanges(approved).map((r) => ({ ...r, start: snap(r.start), end: snap(r.end) })).filter((r) => r.end > r.start); const keepRanges = []; let cursor = 0; @@ -149,7 +174,7 @@ writeFileSync(join(outDir, `${stem}.mistakes-decisions.md`), [ ].join("\n")); const summary = { - inputDuration: Number(duration.toFixed(3)), editedDuration, + inputDuration: Number(duration.toFixed(3)), fps, editedDuration, removed: Number(removedTotal.toFixed(3)), removedPct: Number(((removedTotal / duration) * 100).toFixed(1)), cuts: mergedDeletes.length, editedWords: editedWords.length, outDir, }; diff --git a/.claude/skills/cut-silences/SKILL.md b/.claude/skills/cut-silences/SKILL.md index 55e53799..2a627b6b 100644 --- a/.claude/skills/cut-silences/SKILL.md +++ b/.claude/skills/cut-silences/SKILL.md @@ -80,4 +80,24 @@ The JSON summary printed to stdout includes `removed`, `removedPct`, range count ## Hand-off to the next agent -Pass `.silence-transcript.json` (and the `silenced.mp4` if rendered) to the **cut-mistakes** agent. Because timestamps are already on the edited timeline, downstream beat timing and `scripts/validate-beat-sync.mjs` work without further adjustment. +Pass `.silence-transcript.json` (and the `silenced.mp4` if rendered) to the **cut-mistakes** agent. + +## Known limitation: retimed-transcript precision + +`.silence-transcript.json`'s word timings are computed by exact float +subtraction, then the same floats are fed to ffmpeg's `trim`/`atrim`, which +can only cut on real frame boundaries. Each cut's actual rendered position +can therefore differ from the math by a fraction of a frame, and a typical +silence pass makes 100-300+ cuts, so the error compounds. Boundaries are now +snapped to the nearest real frame (via `ffprobe`'s `r_frame_rate` when +`--video` is given) before computing anything, which substantially reduces +drift but — measured on real camera footage — does not fully eliminate it +(real frame spacing isn't perfectly uniform; expect low-single-digit +milliseconds of residual drift per cut, which can add up to a real offset +across hundreds of cuts). Do not assume this file's timestamps land exactly +on the right word in `silenced.mp4` once cut-mistakes (or anything else) has +compounded more cuts on top — for anything requiring frame-accurate timing +against the actual rendered video, re-transcribe that video directly for the +region you need, and re-transcribe any final render to diff against intended +text before calling a cut "done". A locked-off single camera shot looks +identical a second early, so frame strips alone won't catch this. diff --git a/.claude/skills/cut-silences/scripts/cut-silences.mjs b/.claude/skills/cut-silences/scripts/cut-silences.mjs index fd34ec6a..2781ecd9 100644 --- a/.claude/skills/cut-silences/scripts/cut-silences.mjs +++ b/.claude/skills/cut-silences/scripts/cut-silences.mjs @@ -103,6 +103,32 @@ if (!Number.isFinite(duration)) { } if (!Number.isFinite(duration)) duration = words.at(-1).end + opts.tailPad; +// ---- frame rate (for boundary snapping, video renders only) ---- +// ffmpeg's trim/atrim filters cut on real source frame boundaries, not exact +// float seconds. If the retimed transcript is computed from un-snapped floats, +// each cut's actual rendered position silently differs from the math by a +// fraction of a frame, and that error compounds linearly with cut count (a +// typical silence pass makes 100-300+ cuts). Snapping every delete-range edge +// to the nearest real frame time BEFORE computing anything downstream keeps +// the retimed transcript closer to what ffmpeg actually renders. Only +// meaningful (and only applied) when --video is given -- with no video there +// is no rendered file to drift from, and snapping would just add pointless +// rounding to an otherwise-exact edit. +let fps = null; +if (opts.video && existsSync(resolve(opts.video))) { + fps = 30; + const probe = spawnSync("ffprobe", [ + "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=r_frame_rate", + "-of", "default=noprint_wrappers=1:nokey=1", resolve(opts.video), + ]); + const raw = probe.stdout?.toString().trim(); + if (raw && raw.includes("/")) { + const [n, d] = raw.split("/").map(Number); + if (Number.isFinite(n) && Number.isFinite(d) && d > 0) fps = n / d; + } else if (Number.isFinite(Number(raw))) fps = Number(raw); +} +const snap = (t) => (fps ? Math.round(t * fps) / fps : t); + // ---- build the delete ranges (silence only) ---- function mergeRanges(ranges) { const sorted = ranges @@ -147,7 +173,7 @@ for (let i = 1; i < words.length; i++) { } } -const mergedDeletes = mergeRanges(deleteRanges); +const mergedDeletes = mergeRanges(deleteRanges).map((r) => ({ ...r, start: snap(r.start), end: snap(r.end) })).filter((r) => r.end > r.start); // ---- keep ranges = the complement of the deletes ---- const keepRanges = []; @@ -255,6 +281,7 @@ writeFileSync(decPath, decisions); const summary = { sourceDuration: Number(duration.toFixed(3)), + fps, editedDuration, removed: Number(removedTotal.toFixed(3)), removedPct: Number(((removedTotal / duration) * 100).toFixed(1)), From 4e9dd06845f1a0c03e81a1018b306db01bca6196 Mon Sep 17 00:00:00 2001 From: Duncan Owen <283775822+duncsdownunder@users.noreply.github.com> Date: Fri, 11 Sep 2026 09:23:36 +1000 Subject: [PATCH 2/3] Write trim times at microsecond precision so frame snapping actually holds The previous commit snapped cut boundaries to real frame times but then wrote them into the ffmpeg filtergraph with toFixed(3). A 30fps frame boundary is 33.333...ms, so millisecond rounding pushed the value past the frame's real timestamp, ffmpeg dropped the boundary frame, video ran one frame short of its sample-exact audio, and concat shifted every later segment a few ms in the same direction. Net effect: the snap did nothing. Controlled A/B on one identical 45-cut clip, drift measured by re-transcribing each render: pre-fix 0.27s at end of file snap + ms-precision (previous) 0.31s -- no improvement snap + us-precision (this) 0.01s, rendered duration matches the transcript's claim exactly The previous commit message and both SKILL.md files overclaimed a substantial reduction that was never measured against a controlled baseline; the docs now state the measured comparison and keep the "re-transcribe the final render and diff" requirement as defense in depth. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01PbLt6aLpY5gruwSHS2AXaC --- .agents/skills/cut-mistakes/SKILL.md | 48 +++++++++---------- .../cut-mistakes/scripts/apply-cuts.mjs | 4 +- .agents/skills/cut-silences/SKILL.md | 44 +++++++++-------- .../cut-silences/scripts/cut-silences.mjs | 4 +- .claude/skills/cut-mistakes/SKILL.md | 48 +++++++++---------- .../cut-mistakes/scripts/apply-cuts.mjs | 4 +- .claude/skills/cut-silences/SKILL.md | 44 +++++++++-------- .../cut-silences/scripts/cut-silences.mjs | 4 +- 8 files changed, 102 insertions(+), 98 deletions(-) diff --git a/.agents/skills/cut-mistakes/SKILL.md b/.agents/skills/cut-mistakes/SKILL.md index cda333a0..9b004a49 100644 --- a/.agents/skills/cut-mistakes/SKILL.md +++ b/.agents/skills/cut-mistakes/SKILL.md @@ -65,32 +65,28 @@ node scripts/build-edl-review.mjs .mistakes-edl.json \ npx serve . -p 8080 -n ``` -## Known limitation: retimed-transcript precision (read before reusing timestamps downstream) - -`apply-cuts.mjs` computes `.mistakes-transcript.json`'s word timings by -exact float subtraction, then feeds those same floats to ffmpeg's -`trim`/`atrim`. ffmpeg can only cut on real frame boundaries, so each cut's -actual rendered position can differ from the math by a fraction of a frame, -and — because this runs after `cut-silences`, which typically makes 100-300+ -cuts of its own — the error compounds across two stages. The script now -snaps every cut boundary to the nearest real frame (via `ffprobe`'s -`r_frame_rate` when `--video` is given) before computing anything, which -substantially reduces drift, but testing found it does not fully eliminate -it on real camera footage (frame spacing isn't perfectly uniform) — expect -low-single-digit milliseconds of residual drift per cut, which can still add -up to a noticeable offset (measured: ~0.3s across 45 cuts on real footage) -on a file with many cuts. - -**Practical rule**: if you need this transcript's timestamps to point at -exact words in the *rendered* video — for another cutting pass, a beat-sync -tool, or anything requiring frame-accurate timing — do not trust -`.mistakes-transcript.json` on its own once more than a few dozen cuts -have been applied (by this tool or by `cut-silences` before it). Re-transcribe -the actual rendered video directly for the region you need, and — before -calling any such downstream cut "done" — re-transcribe the *final render* -and diff it against the intended text. Frame strips / contact sheets alone -cannot catch this: a locked-off single-camera shot looks identical a second -early. +## Retimed-transcript precision (read before reusing timestamps downstream) + +`.mistakes-transcript.json` only lines up with the rendered cut if +every cut lands on a real frame. `apply-cuts.mjs` now snaps each cut edge +to the nearest source frame (via `ffprobe`'s `r_frame_rate`, only when +`--video` is given) *and* writes the filtergraph times at microsecond +precision. The second half matters: at millisecond precision a 30fps +boundary (33.333…ms) rounds past the frame's real timestamp, ffmpeg drops +the boundary frame, video runs a frame short of its sample-exact audio, and +`concat` shifts every later segment a few ms in the same direction. Snapping +alone, with ms-precision output, measured **no improvement** over the +original (0.31s vs 0.27s drift at the end of an identical 45-cut clip); +snap + µs precision measured 0.01s. See the same section in the +`cut-silences` skill for the full comparison. + +Because this stage runs on `cut-silences`' output, its input transcript +carries whatever drift that stage left behind — both scripts must be on the +fixed version for the final transcript to be trustworthy. And still +re-transcribe the final render and diff it against the intended text before +calling any downstream cut done: transcription timing has ~10–100ms jitter, +variable-frame-rate sources are not handled, and a locked-off single-camera +shot looks identical a second early in a frame strip. ## Notes diff --git a/.agents/skills/cut-mistakes/scripts/apply-cuts.mjs b/.agents/skills/cut-mistakes/scripts/apply-cuts.mjs index b7eb7e31..109eb0e6 100644 --- a/.agents/skills/cut-mistakes/scripts/apply-cuts.mjs +++ b/.agents/skills/cut-mistakes/scripts/apply-cuts.mjs @@ -146,8 +146,8 @@ function buildFilterScript(ranges) { const lines = []; for (let i = 0; i < ranges.length; i++) { const r = ranges[i]; - lines.push(`[0:v]trim=start=${r.start.toFixed(3)}:end=${r.end.toFixed(3)},setpts=PTS-STARTPTS[v${i}];`); - lines.push(`[0:a]atrim=start=${r.start.toFixed(3)}:end=${r.end.toFixed(3)},asetpts=PTS-STARTPTS[a${i}];`); + lines.push(`[0:v]trim=start=${r.start.toFixed(6)}:end=${r.end.toFixed(6)},setpts=PTS-STARTPTS[v${i}];`); + lines.push(`[0:a]atrim=start=${r.start.toFixed(6)}:end=${r.end.toFixed(6)},asetpts=PTS-STARTPTS[a${i}];`); } lines.push(`${ranges.map((_, i) => `[v${i}][a${i}]`).join("")}concat=n=${ranges.length}:v=1:a=1[v][a]`); return lines.join("\n"); diff --git a/.agents/skills/cut-silences/SKILL.md b/.agents/skills/cut-silences/SKILL.md index c5aac015..237dfe70 100644 --- a/.agents/skills/cut-silences/SKILL.md +++ b/.agents/skills/cut-silences/SKILL.md @@ -82,22 +82,28 @@ The JSON summary printed to stdout includes `removed`, `removedPct`, range count Pass `.silence-transcript.json` (and the `silenced.mp4` if rendered) to the **cut-mistakes** agent. -## Known limitation: retimed-transcript precision - -`.silence-transcript.json`'s word timings are computed by exact float -subtraction, then the same floats are fed to ffmpeg's `trim`/`atrim`, which -can only cut on real frame boundaries. Each cut's actual rendered position -can therefore differ from the math by a fraction of a frame, and a typical -silence pass makes 100-300+ cuts, so the error compounds. Boundaries are now -snapped to the nearest real frame (via `ffprobe`'s `r_frame_rate` when -`--video` is given) before computing anything, which substantially reduces -drift but — measured on real camera footage — does not fully eliminate it -(real frame spacing isn't perfectly uniform; expect low-single-digit -milliseconds of residual drift per cut, which can add up to a real offset -across hundreds of cuts). Do not assume this file's timestamps land exactly -on the right word in `silenced.mp4` once cut-mistakes (or anything else) has -compounded more cuts on top — for anything requiring frame-accurate timing -against the actual rendered video, re-transcribe that video directly for the -region you need, and re-transcribe any final render to diff against intended -text before calling a cut "done". A locked-off single camera shot looks -identical a second early, so frame strips alone won't catch this. +## Retimed-transcript precision + +`.silence-transcript.json` only lines up with `silenced.mp4` if every +cut lands on a real frame. Two things have to hold, and originally neither +did: + +1. Delete-range edges are snapped to the nearest source frame (via + `ffprobe`'s `r_frame_rate`, only when `--video` is given) before any + retiming math. +2. The filtergraph writes those times at microsecond precision. At + millisecond precision a 30fps boundary (33.333…ms) rounds past the frame's + real timestamp, ffmpeg drops the boundary frame, video runs one frame + short of its sample-exact audio, and `concat` shifts every later segment + by a few ms — always in the same direction. Across a few hundred cuts that + compounded to 1–3s. + +Measured on one identical 45-cut clip: pre-fix drift 0.27s at the end of the +file; snap-only (ms precision) 0.31s — no improvement; snap + µs precision +0.01s, with the rendered duration matching the transcript's claim exactly. + +Still re-transcribe the final render and diff it against the intended text +before calling any downstream cut done. Transcription timing itself has +~10–100ms of jitter, variable-frame-rate sources are not handled, and a +locked-off single-camera shot looks identical a second early in a frame +strip — that check is the only thing that catches a uniform time offset. diff --git a/.agents/skills/cut-silences/scripts/cut-silences.mjs b/.agents/skills/cut-silences/scripts/cut-silences.mjs index 2781ecd9..7c4f59de 100644 --- a/.agents/skills/cut-silences/scripts/cut-silences.mjs +++ b/.agents/skills/cut-silences/scripts/cut-silences.mjs @@ -219,8 +219,8 @@ function buildFilterScript(ranges) { const lines = []; for (let i = 0; i < ranges.length; i++) { const r = ranges[i]; - lines.push(`[0:v]trim=start=${r.start.toFixed(3)}:end=${r.end.toFixed(3)},setpts=PTS-STARTPTS[v${i}];`); - lines.push(`[0:a]atrim=start=${r.start.toFixed(3)}:end=${r.end.toFixed(3)},asetpts=PTS-STARTPTS[a${i}];`); + lines.push(`[0:v]trim=start=${r.start.toFixed(6)}:end=${r.end.toFixed(6)},setpts=PTS-STARTPTS[v${i}];`); + lines.push(`[0:a]atrim=start=${r.start.toFixed(6)}:end=${r.end.toFixed(6)},asetpts=PTS-STARTPTS[a${i}];`); } const inputs = ranges.map((_, i) => `[v${i}][a${i}]`).join(""); lines.push(`${inputs}concat=n=${ranges.length}:v=1:a=1[v][a]`); diff --git a/.claude/skills/cut-mistakes/SKILL.md b/.claude/skills/cut-mistakes/SKILL.md index a76af69d..41e38875 100644 --- a/.claude/skills/cut-mistakes/SKILL.md +++ b/.claude/skills/cut-mistakes/SKILL.md @@ -65,32 +65,28 @@ node scripts/build-edl-review.mjs .mistakes-edl.json \ npx serve . -p 8080 -n ``` -## Known limitation: retimed-transcript precision (read before reusing timestamps downstream) - -`apply-cuts.mjs` computes `.mistakes-transcript.json`'s word timings by -exact float subtraction, then feeds those same floats to ffmpeg's -`trim`/`atrim`. ffmpeg can only cut on real frame boundaries, so each cut's -actual rendered position can differ from the math by a fraction of a frame, -and — because this runs after `cut-silences`, which typically makes 100-300+ -cuts of its own — the error compounds across two stages. The script now -snaps every cut boundary to the nearest real frame (via `ffprobe`'s -`r_frame_rate` when `--video` is given) before computing anything, which -substantially reduces drift, but testing found it does not fully eliminate -it on real camera footage (frame spacing isn't perfectly uniform) — expect -low-single-digit milliseconds of residual drift per cut, which can still add -up to a noticeable offset (measured: ~0.3s across 45 cuts on real footage) -on a file with many cuts. - -**Practical rule**: if you need this transcript's timestamps to point at -exact words in the *rendered* video — for another cutting pass, a beat-sync -tool, or anything requiring frame-accurate timing — do not trust -`.mistakes-transcript.json` on its own once more than a few dozen cuts -have been applied (by this tool or by `cut-silences` before it). Re-transcribe -the actual rendered video directly for the region you need, and — before -calling any such downstream cut "done" — re-transcribe the *final render* -and diff it against the intended text. Frame strips / contact sheets alone -cannot catch this: a locked-off single-camera shot looks identical a second -early. +## Retimed-transcript precision (read before reusing timestamps downstream) + +`.mistakes-transcript.json` only lines up with the rendered cut if +every cut lands on a real frame. `apply-cuts.mjs` now snaps each cut edge +to the nearest source frame (via `ffprobe`'s `r_frame_rate`, only when +`--video` is given) *and* writes the filtergraph times at microsecond +precision. The second half matters: at millisecond precision a 30fps +boundary (33.333…ms) rounds past the frame's real timestamp, ffmpeg drops +the boundary frame, video runs a frame short of its sample-exact audio, and +`concat` shifts every later segment a few ms in the same direction. Snapping +alone, with ms-precision output, measured **no improvement** over the +original (0.31s vs 0.27s drift at the end of an identical 45-cut clip); +snap + µs precision measured 0.01s. See the same section in the +`cut-silences` skill for the full comparison. + +Because this stage runs on `cut-silences`' output, its input transcript +carries whatever drift that stage left behind — both scripts must be on the +fixed version for the final transcript to be trustworthy. And still +re-transcribe the final render and diff it against the intended text before +calling any downstream cut done: transcription timing has ~10–100ms jitter, +variable-frame-rate sources are not handled, and a locked-off single-camera +shot looks identical a second early in a frame strip. ## Notes diff --git a/.claude/skills/cut-mistakes/scripts/apply-cuts.mjs b/.claude/skills/cut-mistakes/scripts/apply-cuts.mjs index b7eb7e31..109eb0e6 100644 --- a/.claude/skills/cut-mistakes/scripts/apply-cuts.mjs +++ b/.claude/skills/cut-mistakes/scripts/apply-cuts.mjs @@ -146,8 +146,8 @@ function buildFilterScript(ranges) { const lines = []; for (let i = 0; i < ranges.length; i++) { const r = ranges[i]; - lines.push(`[0:v]trim=start=${r.start.toFixed(3)}:end=${r.end.toFixed(3)},setpts=PTS-STARTPTS[v${i}];`); - lines.push(`[0:a]atrim=start=${r.start.toFixed(3)}:end=${r.end.toFixed(3)},asetpts=PTS-STARTPTS[a${i}];`); + lines.push(`[0:v]trim=start=${r.start.toFixed(6)}:end=${r.end.toFixed(6)},setpts=PTS-STARTPTS[v${i}];`); + lines.push(`[0:a]atrim=start=${r.start.toFixed(6)}:end=${r.end.toFixed(6)},asetpts=PTS-STARTPTS[a${i}];`); } lines.push(`${ranges.map((_, i) => `[v${i}][a${i}]`).join("")}concat=n=${ranges.length}:v=1:a=1[v][a]`); return lines.join("\n"); diff --git a/.claude/skills/cut-silences/SKILL.md b/.claude/skills/cut-silences/SKILL.md index 2a627b6b..3ef92c39 100644 --- a/.claude/skills/cut-silences/SKILL.md +++ b/.claude/skills/cut-silences/SKILL.md @@ -82,22 +82,28 @@ The JSON summary printed to stdout includes `removed`, `removedPct`, range count Pass `.silence-transcript.json` (and the `silenced.mp4` if rendered) to the **cut-mistakes** agent. -## Known limitation: retimed-transcript precision - -`.silence-transcript.json`'s word timings are computed by exact float -subtraction, then the same floats are fed to ffmpeg's `trim`/`atrim`, which -can only cut on real frame boundaries. Each cut's actual rendered position -can therefore differ from the math by a fraction of a frame, and a typical -silence pass makes 100-300+ cuts, so the error compounds. Boundaries are now -snapped to the nearest real frame (via `ffprobe`'s `r_frame_rate` when -`--video` is given) before computing anything, which substantially reduces -drift but — measured on real camera footage — does not fully eliminate it -(real frame spacing isn't perfectly uniform; expect low-single-digit -milliseconds of residual drift per cut, which can add up to a real offset -across hundreds of cuts). Do not assume this file's timestamps land exactly -on the right word in `silenced.mp4` once cut-mistakes (or anything else) has -compounded more cuts on top — for anything requiring frame-accurate timing -against the actual rendered video, re-transcribe that video directly for the -region you need, and re-transcribe any final render to diff against intended -text before calling a cut "done". A locked-off single camera shot looks -identical a second early, so frame strips alone won't catch this. +## Retimed-transcript precision + +`.silence-transcript.json` only lines up with `silenced.mp4` if every +cut lands on a real frame. Two things have to hold, and originally neither +did: + +1. Delete-range edges are snapped to the nearest source frame (via + `ffprobe`'s `r_frame_rate`, only when `--video` is given) before any + retiming math. +2. The filtergraph writes those times at microsecond precision. At + millisecond precision a 30fps boundary (33.333…ms) rounds past the frame's + real timestamp, ffmpeg drops the boundary frame, video runs one frame + short of its sample-exact audio, and `concat` shifts every later segment + by a few ms — always in the same direction. Across a few hundred cuts that + compounded to 1–3s. + +Measured on one identical 45-cut clip: pre-fix drift 0.27s at the end of the +file; snap-only (ms precision) 0.31s — no improvement; snap + µs precision +0.01s, with the rendered duration matching the transcript's claim exactly. + +Still re-transcribe the final render and diff it against the intended text +before calling any downstream cut done. Transcription timing itself has +~10–100ms of jitter, variable-frame-rate sources are not handled, and a +locked-off single-camera shot looks identical a second early in a frame +strip — that check is the only thing that catches a uniform time offset. diff --git a/.claude/skills/cut-silences/scripts/cut-silences.mjs b/.claude/skills/cut-silences/scripts/cut-silences.mjs index 2781ecd9..7c4f59de 100644 --- a/.claude/skills/cut-silences/scripts/cut-silences.mjs +++ b/.claude/skills/cut-silences/scripts/cut-silences.mjs @@ -219,8 +219,8 @@ function buildFilterScript(ranges) { const lines = []; for (let i = 0; i < ranges.length; i++) { const r = ranges[i]; - lines.push(`[0:v]trim=start=${r.start.toFixed(3)}:end=${r.end.toFixed(3)},setpts=PTS-STARTPTS[v${i}];`); - lines.push(`[0:a]atrim=start=${r.start.toFixed(3)}:end=${r.end.toFixed(3)},asetpts=PTS-STARTPTS[a${i}];`); + lines.push(`[0:v]trim=start=${r.start.toFixed(6)}:end=${r.end.toFixed(6)},setpts=PTS-STARTPTS[v${i}];`); + lines.push(`[0:a]atrim=start=${r.start.toFixed(6)}:end=${r.end.toFixed(6)},asetpts=PTS-STARTPTS[a${i}];`); } const inputs = ranges.map((_, i) => `[v${i}][a${i}]`).join(""); lines.push(`${inputs}concat=n=${ranges.length}:v=1:a=1[v][a]`); From 7c0563fd9a3c14741859ecccd7ed08b89c8627cc Mon Sep 17 00:00:00 2001 From: Duncan Owen <283775822+duncsdownunder@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:47:50 +1000 Subject: [PATCH 3/3] Snap cut edges to the audio, add cut-phrases, document both boundary bugs Second cut-boundary fix from the podcast cutdowns project. The first (frame snapping at microsecond precision) fixed timeline drift; this one fixes cuts landing on the first syllable of the next word. Cause: ASR word timestamps are 50-150ms coarse and speakers usually start the next word inside that window, so a cut at word.end (plus any pad) clips the following onset. apply-cuts.mjs now moves each delete edge to the quietest 30ms nearby (up to 150ms outward, 30ms inward, on a 16 kHz mono extract) and never past the neighbouring kept words, before frame snapping. On by default with --video; --no-snap-audio restores the old behaviour. The helper is scripts/lib/audio-edges.mjs with its own tests. scripts/cut-phrases.mjs is the phrase-driven cutter used to build the nine ads: name each segment's first and last words, resolve them against short ground-truth transcripts of the source file itself, snap, cut with dense keyframes, assemble, then re-transcribe the assembly and diff it against the intended text. Skill and workflow docs updated: audio-guided edges section in cut-mistakes, cut-phrases + keyframe + strict-render guidance in short-form-edit, apply-cuts flag in WORKFLOW.md. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01PbLt6aLpY5gruwSHS2AXaC --- .agents/skills/cut-mistakes/SKILL.md | 11 +- .../cut-mistakes/scripts/apply-cuts.mjs | 28 +++- .agents/skills/short-form-edit/SKILL.md | 12 +- .claude/skills/cut-mistakes/SKILL.md | 11 +- .../cut-mistakes/scripts/apply-cuts.mjs | 28 +++- .claude/skills/short-form-edit/SKILL.md | 12 +- docs/WORKFLOW.md | 7 + scripts/cut-phrases.mjs | 125 ++++++++++++++++++ scripts/lib/audio-edges.mjs | 60 +++++++++ tests/audio-edges.test.mjs | 35 +++++ 10 files changed, 317 insertions(+), 12 deletions(-) create mode 100644 scripts/cut-phrases.mjs create mode 100644 scripts/lib/audio-edges.mjs create mode 100644 tests/audio-edges.test.mjs diff --git a/.agents/skills/cut-mistakes/SKILL.md b/.agents/skills/cut-mistakes/SKILL.md index 9b004a49..23837ff2 100644 --- a/.agents/skills/cut-mistakes/SKILL.md +++ b/.agents/skills/cut-mistakes/SKILL.md @@ -53,6 +53,14 @@ node .agents/skills/cut-mistakes/scripts/apply-cuts.mjs \ Writes the EDL, a re-timed `.mistakes-transcript.json` (feeds the motion-graphics agent + `validate-beat-sync.mjs`), a decisions log, and — with `--apply` — the cut video (ffmpeg trim+concat, A/V in sync). Drop `--apply` for a dry run. +With `--video`, every cut edge is **audio-snapped** before frame snapping (see below); pass `--no-snap-audio` to cut at the exact approved times instead. + +## Audio-guided edges (why cuts at word times clip the next word) + +ASR word timestamps are 50–150ms coarse, and a speaker usually starts the next word inside that window. A cut placed at the approved `end` (which is normally a word's `end`) therefore lands on the first syllable of the following word, and a cut at a word's `start` can leave the tail of the previous one — heard as "the transition is halfway through a word". `apply-cuts.mjs` now moves each delete edge to the quietest 30ms nearby (searching up to 150ms outward, 30ms inward, on a 16 kHz mono extract of the source), and never past the neighbouring kept words. The EDL records both `requested_*` and snapped times. The helper lives in `scripts/lib/audio-edges.mjs` and is shared with `scripts/cut-phrases.mjs`. + +Measured on a 41-cut set: last-80ms energy before each cut went from speech level (≈-30 dB) to -31…-78 dB; the few that stay near -31 dB are true no-gap boundaries where the cut sits inside the last word's own decay rather than on the next word's onset. + ## Reviewing the result Use the shared review tool to eyeball the cuts and spot-check boundaries: @@ -91,5 +99,6 @@ shot looks identical a second early in a frame strip. ## Notes - A very clean delivery may yield few or zero real cuts — that's a valid outcome; don't cut natural speech to hit a quota. -- Cuts between two spoken words are hard joins. They're usually clean for stutters/false starts; for tighter audio a 20-30ms fade can be added later. +- Cuts between two spoken words are hard joins. With audio snapping on they land in the gap, not on a word; for tighter audio a 20-30ms fade can be added later. +- Cutting *pieces* out of a long recording (short-form ads, cutdowns)? Use `scripts/cut-phrases.mjs` instead of hand-picking timestamps: name the first and last words of each phrase, transcribe short windows of the source itself for ground truth, and let it snap, cut, assemble, and re-transcribe-and-diff. - Hand the `mistakes-transcript.json` + the clean video to **Agent 3 (motion graphics / tiered cards)**. diff --git a/.agents/skills/cut-mistakes/scripts/apply-cuts.mjs b/.agents/skills/cut-mistakes/scripts/apply-cuts.mjs index 109eb0e6..5453f1cc 100644 --- a/.agents/skills/cut-mistakes/scripts/apply-cuts.mjs +++ b/.agents/skills/cut-mistakes/scripts/apply-cuts.mjs @@ -27,10 +27,10 @@ function die(m, c = 1) { console.error(m); exit(c); } const args = argv.slice(2); if (args.length === 0 || args.includes("--help")) { - console.log("Usage: node apply-cuts.mjs --cuts [--video in.mp4] [--output out.mp4] [--out-dir dir] [--apply]"); + console.log("Usage: node apply-cuts.mjs --cuts [--video in.mp4] [--output out.mp4] [--out-dir dir] [--apply] [--no-snap-audio]"); exit(args.length === 0 ? 1 : 0); } -const opts = { transcript: null, cuts: null, video: null, output: null, outDir: null, apply: false }; +const opts = { transcript: null, cuts: null, video: null, output: null, outDir: null, apply: false, snapAudio: true }; for (let i = 0; i < args.length; i++) { const a = args[i]; if (a === "--cuts") opts.cuts = args[++i]; @@ -38,6 +38,7 @@ for (let i = 0; i < args.length; i++) { else if (a === "--output" || a === "-o") opts.output = args[++i]; else if (a === "--out-dir") opts.outDir = args[++i]; else if (a === "--apply") opts.apply = true; + else if (a === "--no-snap-audio") opts.snapAudio = false; else if (a.startsWith("--")) die(`unknown option: ${a}`); else if (!opts.transcript) opts.transcript = a; } @@ -105,7 +106,26 @@ function mergeRanges(ranges) { } return merged; } -const mergedDeletes = mergeRanges(approved).map((r) => ({ ...r, start: snap(r.start), end: snap(r.end) })).filter((r) => r.end > r.start); +// ---- audio-guided edges (video renders only, default on) ---- +// A cut placed exactly at an ASR word edge usually clips the onset of the +// next word or leaves the tail of the last one, because word timestamps are +// 50-150ms coarse. Before frame snapping, move each delete edge to the +// quietest 30ms nearby, never past the neighbouring kept words. +let audioSnapped = false; +let rangesForRender = mergeRanges(approved); +if (fps && opts.snapAudio) { + const { extractWav, loadWav, snapRange } = await import(new URL("../../../../scripts/lib/audio-edges.mjs", import.meta.url)); + const outDirEarly = opts.outDir ? resolve(opts.outDir) : dirname(transcriptPath); + mkdirSync(outDirEarly, { recursive: true }); + const wav = loadWav(extractWav(resolve(opts.video), join(outDirEarly, `${basename(resolve(opts.video), extname(opts.video))}.16k.wav`))); + rangesForRender = rangesForRender.map((r) => { + const prevKept = words.filter((w) => w.end <= r.start).at(-1); + const nextKept = words.find((w) => w.start >= r.end); + return snapRange(wav, r, { prevEnd: prevKept ? prevKept.end : 0, nextStart: nextKept ? nextKept.start : duration }); + }); + audioSnapped = true; +} +const mergedDeletes = rangesForRender.map((r) => ({ ...r, start: snap(r.start), end: snap(r.end) })).filter((r) => r.end > r.start); const keepRanges = []; let cursor = 0; @@ -174,7 +194,7 @@ writeFileSync(join(outDir, `${stem}.mistakes-decisions.md`), [ ].join("\n")); const summary = { - inputDuration: Number(duration.toFixed(3)), fps, editedDuration, + inputDuration: Number(duration.toFixed(3)), fps, audioSnapped, editedDuration, removed: Number(removedTotal.toFixed(3)), removedPct: Number(((removedTotal / duration) * 100).toFixed(1)), cuts: mergedDeletes.length, editedWords: editedWords.length, outDir, }; diff --git a/.agents/skills/short-form-edit/SKILL.md b/.agents/skills/short-form-edit/SKILL.md index a584e642..4b135b03 100644 --- a/.agents/skills/short-form-edit/SKILL.md +++ b/.agents/skills/short-form-edit/SKILL.md @@ -50,6 +50,14 @@ this test. Treat the answer as editorial judgment until audience data exists. Render the clean cut once and derive word timing from the actual frame-aligned EDL. Lock the cut after the rough animatic, before polished graphics. Every later timing change rebuilds dependent captions, graphics, and sound from the same map. + When pulling phrases out of a long recording, use `scripts/cut-phrases.mjs`: + name each phrase's first and last words, transcribe short windows of the + source file itself for ground truth (never a retimed transcript), and let it + snap each cut to the local audio minimum. Cutting at ASR word times plus a + fixed pad lands on the next word's first syllable. Cut pieces with a keyframe + every second (`-g 30`) or the renderer warns about seek failures. The script + re-transcribes the assembly and diffs it against the intended text; do the + same on the final render. User authorization persists. An instruction to make a finished video and iterate authorizes ordinary local editing and rendering. An explicit request for Kie @@ -274,7 +282,9 @@ human reactions where they support the story. sounds. Make meaningful contacts perceptible, vary texture and density, and retain quieter conviction beats. Do not substitute louder music or a whoosh on every cut. Inspect the actual animation landing before setting each cue. -6. Run lint and inspect the live preview before rendering. Browser automation is +6. Run lint and `hyperframes validate` before every render, and render with + `--strict` so a lint error (a missing font, logo, or GSAP file) blocks the + render instead of producing a silently broken video. Browser automation is headless with Pointer Lock and pointer capture disabled in every context. Local automated review is permitted when the user authorized autonomous iteration. Render a draft, inspect it, revise, then render final quality. diff --git a/.claude/skills/cut-mistakes/SKILL.md b/.claude/skills/cut-mistakes/SKILL.md index 41e38875..334b3f3e 100644 --- a/.claude/skills/cut-mistakes/SKILL.md +++ b/.claude/skills/cut-mistakes/SKILL.md @@ -53,6 +53,14 @@ node .claude/skills/cut-mistakes/scripts/apply-cuts.mjs \ Writes the EDL, a re-timed `.mistakes-transcript.json` (feeds the motion-graphics agent + `validate-beat-sync.mjs`), a decisions log, and — with `--apply` — the cut video (ffmpeg trim+concat, A/V in sync). Drop `--apply` for a dry run. +With `--video`, every cut edge is **audio-snapped** before frame snapping (see below); pass `--no-snap-audio` to cut at the exact approved times instead. + +## Audio-guided edges (why cuts at word times clip the next word) + +ASR word timestamps are 50–150ms coarse, and a speaker usually starts the next word inside that window. A cut placed at the approved `end` (which is normally a word's `end`) therefore lands on the first syllable of the following word, and a cut at a word's `start` can leave the tail of the previous one — heard as "the transition is halfway through a word". `apply-cuts.mjs` now moves each delete edge to the quietest 30ms nearby (searching up to 150ms outward, 30ms inward, on a 16 kHz mono extract of the source), and never past the neighbouring kept words. The EDL records both `requested_*` and snapped times. The helper lives in `scripts/lib/audio-edges.mjs` and is shared with `scripts/cut-phrases.mjs`. + +Measured on a 41-cut set: last-80ms energy before each cut went from speech level (≈-30 dB) to -31…-78 dB; the few that stay near -31 dB are true no-gap boundaries where the cut sits inside the last word's own decay rather than on the next word's onset. + ## Reviewing the result Use the shared review tool to eyeball the cuts and spot-check boundaries: @@ -91,5 +99,6 @@ shot looks identical a second early in a frame strip. ## Notes - A very clean delivery may yield few or zero real cuts — that's a valid outcome; don't cut natural speech to hit a quota. -- Cuts between two spoken words are hard joins. They're usually clean for stutters/false starts; for tighter audio a 20-30ms fade can be added later. +- Cuts between two spoken words are hard joins. With audio snapping on they land in the gap, not on a word; for tighter audio a 20-30ms fade can be added later. +- Cutting *pieces* out of a long recording (short-form ads, cutdowns)? Use `scripts/cut-phrases.mjs` instead of hand-picking timestamps: name the first and last words of each phrase, transcribe short windows of the source itself for ground truth, and let it snap, cut, assemble, and re-transcribe-and-diff. - Hand the `mistakes-transcript.json` + the clean video to **Agent 3 (motion graphics / tiered cards)**. diff --git a/.claude/skills/cut-mistakes/scripts/apply-cuts.mjs b/.claude/skills/cut-mistakes/scripts/apply-cuts.mjs index 109eb0e6..5453f1cc 100644 --- a/.claude/skills/cut-mistakes/scripts/apply-cuts.mjs +++ b/.claude/skills/cut-mistakes/scripts/apply-cuts.mjs @@ -27,10 +27,10 @@ function die(m, c = 1) { console.error(m); exit(c); } const args = argv.slice(2); if (args.length === 0 || args.includes("--help")) { - console.log("Usage: node apply-cuts.mjs --cuts [--video in.mp4] [--output out.mp4] [--out-dir dir] [--apply]"); + console.log("Usage: node apply-cuts.mjs --cuts [--video in.mp4] [--output out.mp4] [--out-dir dir] [--apply] [--no-snap-audio]"); exit(args.length === 0 ? 1 : 0); } -const opts = { transcript: null, cuts: null, video: null, output: null, outDir: null, apply: false }; +const opts = { transcript: null, cuts: null, video: null, output: null, outDir: null, apply: false, snapAudio: true }; for (let i = 0; i < args.length; i++) { const a = args[i]; if (a === "--cuts") opts.cuts = args[++i]; @@ -38,6 +38,7 @@ for (let i = 0; i < args.length; i++) { else if (a === "--output" || a === "-o") opts.output = args[++i]; else if (a === "--out-dir") opts.outDir = args[++i]; else if (a === "--apply") opts.apply = true; + else if (a === "--no-snap-audio") opts.snapAudio = false; else if (a.startsWith("--")) die(`unknown option: ${a}`); else if (!opts.transcript) opts.transcript = a; } @@ -105,7 +106,26 @@ function mergeRanges(ranges) { } return merged; } -const mergedDeletes = mergeRanges(approved).map((r) => ({ ...r, start: snap(r.start), end: snap(r.end) })).filter((r) => r.end > r.start); +// ---- audio-guided edges (video renders only, default on) ---- +// A cut placed exactly at an ASR word edge usually clips the onset of the +// next word or leaves the tail of the last one, because word timestamps are +// 50-150ms coarse. Before frame snapping, move each delete edge to the +// quietest 30ms nearby, never past the neighbouring kept words. +let audioSnapped = false; +let rangesForRender = mergeRanges(approved); +if (fps && opts.snapAudio) { + const { extractWav, loadWav, snapRange } = await import(new URL("../../../../scripts/lib/audio-edges.mjs", import.meta.url)); + const outDirEarly = opts.outDir ? resolve(opts.outDir) : dirname(transcriptPath); + mkdirSync(outDirEarly, { recursive: true }); + const wav = loadWav(extractWav(resolve(opts.video), join(outDirEarly, `${basename(resolve(opts.video), extname(opts.video))}.16k.wav`))); + rangesForRender = rangesForRender.map((r) => { + const prevKept = words.filter((w) => w.end <= r.start).at(-1); + const nextKept = words.find((w) => w.start >= r.end); + return snapRange(wav, r, { prevEnd: prevKept ? prevKept.end : 0, nextStart: nextKept ? nextKept.start : duration }); + }); + audioSnapped = true; +} +const mergedDeletes = rangesForRender.map((r) => ({ ...r, start: snap(r.start), end: snap(r.end) })).filter((r) => r.end > r.start); const keepRanges = []; let cursor = 0; @@ -174,7 +194,7 @@ writeFileSync(join(outDir, `${stem}.mistakes-decisions.md`), [ ].join("\n")); const summary = { - inputDuration: Number(duration.toFixed(3)), fps, editedDuration, + inputDuration: Number(duration.toFixed(3)), fps, audioSnapped, editedDuration, removed: Number(removedTotal.toFixed(3)), removedPct: Number(((removedTotal / duration) * 100).toFixed(1)), cuts: mergedDeletes.length, editedWords: editedWords.length, outDir, }; diff --git a/.claude/skills/short-form-edit/SKILL.md b/.claude/skills/short-form-edit/SKILL.md index acda767b..e6222c64 100644 --- a/.claude/skills/short-form-edit/SKILL.md +++ b/.claude/skills/short-form-edit/SKILL.md @@ -50,6 +50,14 @@ this test. Treat the answer as editorial judgment until audience data exists. Render the clean cut once and derive word timing from the actual frame-aligned EDL. Lock the cut after the rough animatic, before polished graphics. Every later timing change rebuilds dependent captions, graphics, and sound from the same map. + When pulling phrases out of a long recording, use `scripts/cut-phrases.mjs`: + name each phrase's first and last words, transcribe short windows of the + source file itself for ground truth (never a retimed transcript), and let it + snap each cut to the local audio minimum. Cutting at ASR word times plus a + fixed pad lands on the next word's first syllable. Cut pieces with a keyframe + every second (`-g 30`) or the renderer warns about seek failures. The script + re-transcribes the assembly and diffs it against the intended text; do the + same on the final render. User authorization persists. An instruction to make a finished video and iterate authorizes ordinary local editing and rendering. An explicit request for Kie @@ -274,7 +282,9 @@ human reactions where they support the story. sounds. Make meaningful contacts perceptible, vary texture and density, and retain quieter conviction beats. Do not substitute louder music or a whoosh on every cut. Inspect the actual animation landing before setting each cue. -6. Run lint and inspect the live preview before rendering. Browser automation is +6. Run lint and `hyperframes validate` before every render, and render with + `--strict` so a lint error (a missing font, logo, or GSAP file) blocks the + render instead of producing a silently broken video. Browser automation is headless with Pointer Lock and pointer capture disabled in every context. Local automated review is permitted when the user authorized autonomous iteration. Render a draft, inspect it, revise, then render final quality. diff --git a/docs/WORKFLOW.md b/docs/WORKFLOW.md index 6721efd3..4cc8f3a4 100644 --- a/docs/WORKFLOW.md +++ b/docs/WORKFLOW.md @@ -42,6 +42,13 @@ Use `raw.mistakes-transcript.json` alongside `clean.mp4`. Copy that transcript t the project's `assets/transcript.json` for beat validation. Never use raw timings against the cleaned video. Claude users can substitute `.claude` for `.agents`. +With `--video`, each cut edge is snapped to the quietest point near the approved +time so it never lands on the next word's first syllable (`--no-snap-audio` to +disable). For cutdowns and short ads pulled out of a long recording, use +`node scripts/cut-phrases.mjs ` with a `spec.json` of phrases and +ground-truth windows; it cuts, assembles, and re-transcribes the result to diff +it against the intended text. + ## 4. Design the visual layer Read video-storytelling and the motion philosophy. Fill a beat sheet with the diff --git a/scripts/cut-phrases.mjs b/scripts/cut-phrases.mjs new file mode 100644 index 00000000..cc4c2dfe --- /dev/null +++ b/scripts/cut-phrases.mjs @@ -0,0 +1,125 @@ +#!/usr/bin/env node +// cut-phrases: cut a short-form edit out of a long recording by naming the +// phrases you want, not by trusting timestamps. +// +// Why this exists: retimed transcripts from the cut tools are a *projected* +// timeline (they can drift from the rendered file), and ASR word timestamps +// are 50-150ms coarse (a cut at `word.end` + a pad usually lands on the first +// syllable of the next word). This script instead: +// 1. resolves each segment's first/last phrase against a ground-truth +// transcript of the SOURCE FILE ITSELF (short windows, transcribed with +// scripts/transcribe-elevenlabs.mjs), +// 2. places each cut at the quietest 30ms between the phrase's edge word and +// its neighbour (scripts/lib/audio-edges.mjs), +// 3. cuts the pieces (dense keyframes, short fades), concats them with any +// pre-rendered card images, renders a low-res rough, +// 4. re-transcribes the assembly and diffs it against the intended text. +// +// Usage: +// node scripts/cut-phrases.mjs [--no-verify] +// +// /spec.json: +// { +// "source": "../assets/edited-clean.mp4", +// "windows": { "W1": { "transcript": "ground-truth/W1.json", "offset": 940 } }, +// "segments": [ { "id": "seg0", "window": "W1", "from": "first words of", "to": "last words of the phrase" } ], +// "cards": [ { "id": "card1", "image": "card1.png", "duration": 2.5 } ], +// "order": [ "seg0", "card1", "seg1" ] +// } +// A window transcript is the output of transcribe-elevenlabs.mjs run on an +// audio excerpt of the source starting at `offset` seconds, e.g. +// ffmpeg -ss 940 -t 60 -i source.mp4 -vn -ac 1 -ar 16000 W1.mp3 +// node scripts/transcribe-elevenlabs.mjs W1.mp3 --output ground-truth/W1.json --no-diarize +// Outputs (in ): pieces/seg*.mp4, pieces/full-assembly.mp4, +// animatic-rough.mp4, animatic-edl.json (resolved times + verification). + +import { readFileSync, writeFileSync, mkdirSync } from "node:fs"; +import { resolve, join, dirname, basename, extname } from "node:path"; +import { spawnSync } from "node:child_process"; +import { extractWav, loadWav, quietest } from "./lib/audio-edges.mjs"; + +const args = process.argv.slice(2); +const dir = resolve(args.find((a) => !a.startsWith("--")) ?? "."); +const verify = !args.includes("--no-verify"); +const kit = resolve(dirname(new URL(import.meta.url).pathname), ".."); +const spec = JSON.parse(readFileSync(join(dir, "spec.json"), "utf8")); +const src = resolve(dir, spec.source); +const pieces = join(dir, "pieces"); +mkdirSync(pieces, { recursive: true }); + +const norm = (s) => s.toLowerCase().replace(/[^a-z0-9$%' ]+/g, " ").replace(/\s+/g, " ").trim(); +const run = (cmd, a) => { + const r = spawnSync(cmd, a, { stdio: ["ignore", "pipe", "pipe"] }); + if (r.status !== 0) throw new Error(`${cmd} ${a.join(" ")}\n${r.stderr}`); + return r.stdout.toString(); +}; + +function loadWindow(name) { + const w = spec.windows[name]; + if (!w) throw new Error(`unknown window "${name}"`); + const t = JSON.parse(readFileSync(resolve(dir, w.transcript), "utf8")); + return t.words.filter((x) => !x.type || x.type === "word").map((x) => ({ text: norm(x.text), start: x.start + w.offset, end: x.end + w.offset })); +} +function findSeq(words, phrase, from = 0) { + const p = norm(phrase).split(" "); + outer: for (let i = from; i <= words.length - p.length; i++) { + for (let j = 0; j < p.length; j++) if (words[i + j].text !== p[j]) continue outer; + return i; + } + throw new Error(`phrase not found: "${phrase}"`); +} + +// 1) resolve segments with audio-guided edges +const FADE = 0.06; +const wav = loadWav(extractWav(src, src.replace(/\.[^.]+$/, "") + ".16k.wav")); +const segments = spec.segments.map((s) => { + const words = loadWindow(s.window); + const i0 = findSeq(words, s.from); + const i1 = findSeq(words, s.to, i0); + const n = norm(s.to).split(" ").length; + const first = words[i0], last = words[i1 + n - 1], prev = words[i0 - 1], next = words[i1 + n]; + const start = quietest(wav, Math.max(prev ? prev.end : 0, first.start - 0.25), first.start + 0.02); + const endHi = Math.min(next ? next.start : Infinity, last.end + 0.25); + const end = endHi - (last.end - 0.06) < 0.04 ? last.end - 0.01 : quietest(wav, last.end - 0.06, endHi); + return { ...s, start: +start.toFixed(3), end: +end.toFixed(3), text: words.slice(i0, i1 + n).map((w) => w.text).join(" ") }; +}); + +// 2) cut pieces (30fps, keyframe every frame-second, short fades) +for (const s of segments) { + const d = s.end - s.start; + run("ffmpeg", ["-y", "-v", "error", "-ss", s.start.toFixed(3), "-t", d.toFixed(3), "-i", src, + "-af", `afade=t=in:d=${FADE},afade=t=out:st=${(d - FADE).toFixed(3)}:d=${FADE}`, + "-c:v", "libx264", "-preset", "fast", "-crf", "18", "-r", "30", "-g", "30", "-keyint_min", "30", "-pix_fmt", "yuv420p", + "-c:a", "aac", "-b:a", "192k", "-ar", "48000", "-ac", "2", join(pieces, `${s.id}.mp4`)]); +} + +// 3) cards from pre-rendered images (silent, same stream params as pieces) +for (const c of spec.cards ?? []) { + run("ffmpeg", ["-y", "-v", "error", "-loop", "1", "-framerate", "30", "-i", resolve(dir, c.image), + "-f", "lavfi", "-i", "anullsrc=r=48000:cl=stereo", "-t", String(c.duration ?? 2.5), + "-vf", "scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2", + "-c:v", "libx264", "-preset", "fast", "-crf", "18", "-r", "30", "-g", "30", "-pix_fmt", "yuv420p", + "-c:a", "aac", "-b:a", "192k", "-shortest", join(pieces, `${c.id}.mp4`)]); +} + +// 4) concat + rough preview +writeFileSync(join(pieces, "concat-list.txt"), spec.order.map((id) => `file '${id}.mp4'`).join("\n") + "\n"); +run("ffmpeg", ["-y", "-v", "error", "-f", "concat", "-safe", "0", "-i", join(pieces, "concat-list.txt"), "-c", "copy", join(pieces, "full-assembly.mp4")]); +run("ffmpeg", ["-y", "-v", "error", "-i", join(pieces, "full-assembly.mp4"), "-vf", "scale=480:-2", "-c:v", "libx264", "-preset", "fast", "-crf", "24", "-c:a", "aac", "-b:a", "128k", join(dir, "animatic-rough.mp4")]); +const runtime = Number(run("ffprobe", ["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", join(pieces, "full-assembly.mp4")])); + +// 5) verify: re-transcribe the assembly and diff against the intended text +let verified = "skipped (--no-verify)"; +if (verify) { + run("node", [join(kit, "scripts/transcribe-elevenlabs.mjs"), join(pieces, "full-assembly.mp4"), "--output", join(pieces, "full-assembly.json"), "--no-diarize"]); + const got = JSON.parse(readFileSync(join(pieces, "full-assembly.json"), "utf8")).words.filter((w) => !w.type || w.type === "word").map((w) => norm(w.text)).join(" "); + const want = spec.order.filter((id) => segments.some((s) => s.id === id)).map((id) => segments.find((s) => s.id === id).text).join(" "); + const missing = want.split(" ").filter((w) => !got.includes(w)); + verified = got === want ? "exact" : missing.length === 0 ? "all intended words present (ASR wording differs slightly)" : `MISSING words: ${missing.join(", ")}`; +} + +writeFileSync(join(dir, "animatic-edl.json"), JSON.stringify({ + source: spec.source, note: "timestamps resolved from ground-truth window transcripts of the source file itself; edges snapped to the local audio minimum", + order: spec.order, segments, cards: spec.cards ?? [], runtime: +runtime.toFixed(2), verified: { date: new Date().toISOString().slice(0, 10), result: verified }, +}, null, 2)); +console.log(JSON.stringify({ runtime: +runtime.toFixed(2), verified, segments: segments.map((s) => `${s.id} ${s.start}-${s.end}`) }, null, 2)); diff --git a/scripts/lib/audio-edges.mjs b/scripts/lib/audio-edges.mjs new file mode 100644 index 00000000..896f8a3a --- /dev/null +++ b/scripts/lib/audio-edges.mjs @@ -0,0 +1,60 @@ +// Audio-guided cut edges. +// +// ASR word timestamps are 50-150ms coarse and speakers usually start the next +// word inside that window, so a cut placed at `word.end` (+ any fixed pad) +// tends to land on the first syllable of the following word. These helpers +// move a proposed edge to the quietest short window nearby, bounded so it can +// never enter a neighbouring kept word. +// +// Works on a 16 kHz mono 16-bit PCM WAV (see `extractWav`). + +import { readFileSync, existsSync } from "node:fs"; +import { spawnSync } from "node:child_process"; + +export const SR = 16000; +const HEADER = 44; + +export function extractWav(videoPath, wavPath) { + if (existsSync(wavPath)) return wavPath; + const r = spawnSync("ffmpeg", ["-y", "-v", "error", "-i", videoPath, "-vn", "-ac", "1", "-ar", String(SR), "-c:a", "pcm_s16le", wavPath]); + if (r.status !== 0) throw new Error(`ffmpeg wav extract failed: ${r.stderr}`); + return wavPath; +} + +export function loadWav(wavPath) { + return readFileSync(wavPath); +} + +// Mean-square energy of [a, b) seconds. Buffers may be a raw Int16 array-like +// (tests) or a WAV file buffer (skips the 44-byte header). +export function energy(buf, a, b) { + const isWav = Buffer.isBuffer(buf); + const off = isWav ? HEADER : 0; + const n = isWav ? (buf.length - HEADER) / 2 : buf.length; + const i0 = Math.max(0, Math.floor(a * SR)), i1 = Math.min(n, Math.floor(b * SR)); + let s = 0, c = 0; + for (let t = i0; t < i1; t++) { const v = (isWav ? buf.readInt16LE(off + t * 2) : buf[t]) / 32768; s += v * v; c++; } + return c ? s / c : 0; +} + +// Centre of the quietest `win`-second window inside [a, b]. Falls back to the +// midpoint when the range is narrower than one window. +export function quietest(buf, a, b, win = 0.03, step = 0.005) { + if (b - a <= win) return (a + b) / 2; + let best = a, bv = Infinity; + for (let t = a; t <= b - win + 1e-9; t += step) { const e = energy(buf, t, t + win); if (e < bv) { bv = e; best = t; } } + return best + win / 2; +} + +// Snap one delete range to audio. `prevEnd` / `nextStart` are the hard bounds +// (end of the last kept word before the range, start of the first kept word +// after it); `reach` is how far past the proposed edge the search may look. +export function snapRange(buf, range, { prevEnd = 0, nextStart = Infinity, reach = 0.15, inward = 0.03 } = {}) { + const startLo = Math.max(prevEnd, range.start - reach); + const startHi = Math.min(range.start + inward, range.end); + const endLo = Math.max(range.end - inward, range.start); + const endHi = Math.min(nextStart, range.end + reach); + const start = startHi > startLo ? quietest(buf, startLo, startHi) : range.start; + const end = endHi > endLo ? quietest(buf, endLo, endHi) : range.end; + return { ...range, start: +start.toFixed(3), end: +end.toFixed(3), requested_start: range.start, requested_end: range.end }; +} diff --git a/tests/audio-edges.test.mjs b/tests/audio-edges.test.mjs new file mode 100644 index 00000000..36f9124d --- /dev/null +++ b/tests/audio-edges.test.mjs @@ -0,0 +1,35 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { SR, energy, quietest, snapRange } from '../scripts/lib/audio-edges.mjs'; + +// Synthetic 3s clip: a "word" (loud) 0.50-1.00s, silence, a second word 1.30-1.80s. +// Int16 array-like, no WAV header. +function clip() { + const a = new Int16Array(3 * SR); + const loud = (t0, t1) => { for (let i = Math.floor(t0 * SR); i < Math.floor(t1 * SR); i++) a[i] = 12000 * Math.sin(i * 0.3); }; + loud(0.50, 1.00); loud(1.30, 1.80); + return a; +} + +test('energy separates speech from silence', () => { + const a = clip(); + assert.ok(energy(a, 0.6, 0.9) > 0.01); + assert.ok(energy(a, 1.05, 1.25) < 1e-6); +}); + +test('quietest window lands in the gap between two words', () => { + const t = quietest(clip(), 0.95, 1.35); + assert.ok(t > 1.0 && t < 1.3, `expected a point inside the 1.00-1.30 gap, got ${t}`); +}); + +test('snapRange moves an ASR-edge cut off the next word onset and never past a kept word', () => { + // A delete range that starts 40ms late (inside the first word's tail is + // fine) and ends exactly at ASR word.end of the second word, 30ms late. + const r = snapRange(clip(), { start: 1.02, end: 1.83 }, { prevEnd: 1.00, nextStart: 2.50 }); + assert.ok(r.start >= 1.00 && r.start <= 1.05); + assert.ok(r.end >= 1.80 && r.end <= 1.98, `end ${r.end} should sit in the silence after the word`); + assert.equal(r.requested_end, 1.83); + // Bounds are hard: a kept word starting right after the range pins the end. + const pinned = snapRange(clip(), { start: 1.02, end: 1.83 }, { prevEnd: 1.00, nextStart: 1.84 }); + assert.ok(pinned.end <= 1.84); +});