From 21cd8450288b377ba9773538950c8e9f43036424 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 10:03:34 -0500 Subject: [PATCH 001/296] chore: open 0.10.0-alpha.1 on develop --- plugins/dw/.claude-plugin/plugin.json | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/dw/.claude-plugin/plugin.json b/plugins/dw/.claude-plugin/plugin.json index ece26d66..6d7257a2 100644 --- a/plugins/dw/.claude-plugin/plugin.json +++ b/plugins/dw/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "dw", "description": "Compose MiniMax H3 video, MiniMax Music 3 and LTX-2.5 workflows over a dw MCP server, and cut a multi-episode series from them: which template fits which shape, the hard rules, cost, and how to judge the output. Prompt format comes from the vendors' own guides.", - "version": "0.9.0", + "version": "0.10.0-alpha.1", "author": { "name": "Don Kackman" }, diff --git a/pyproject.toml b/pyproject.toml index 6bdee733..f5266a07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ name = "diffusers-workflow" # runtime (static in TOML because importing dw at build time would drag # torch into the build environment). Release tags must match it - see # docs/RELEASING.md -version = "0.9.0" +version = "0.10.0-alpha.1" description = "Declarative workflow engine and web UI for Hugging Face Diffusers" readme = "README.md" requires-python = ">=3.10" From f1e0d480265195543fd9d98da34bf22bd0cfb263 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 13:21:22 -0500 Subject: [PATCH 002/296] feat(tasks): #604 - separate_stems returns htdemucs vocals/drums/bass/other as audio results Shares the separator loader and MPS CPU fallback with attribute_voices. Co-Authored-By: Claude Sonnet 5.5 --- docs/TASKS.md | 40 +++++++++++++ dw/tasks/task.py | 12 ++++ dw/tasks/voice_attribution.py | 76 ++++++++++++++++++++---- tests/test_separate_stems.py | 105 ++++++++++++++++++++++++++++++++++ 4 files changed, 221 insertions(+), 12 deletions(-) create mode 100644 tests/test_separate_stems.py diff --git a/docs/TASKS.md b/docs/TASKS.md index 064cccc5..da896778 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -1278,6 +1278,46 @@ without reading every command's schema. The list is exactly the commands declared `assessment=True`; `attribute_voices` stays in `commands` only, since it analyzes a song rather than checking a cut. +## Stem separation + +Split a mix into its htdemucs stems - `vocals`, `drums`, `bass` and `other` - +each returned as its own audio result. It is the separator `attribute_voices` +runs (one shared loader and cache), exposed for its own sake: transcribing the +`vocals` stem aligns sung lyrics better than transcribing the full mix, and +`drums` + `bass` + `other` is the instrumental to put under dialogue as a +score bed. + +```json +{ + "name": "stems", + "task": { + "command": "separate_stems", + "arguments": { "audio": "asset:song.mp3" } + }, + "result": { "content_type": "audio/wav", "file_name": "stems" } +}, +{ + "name": "lyrics", + "task": { + "command": "transcribe_audio", + "arguments": { "audio": "previous_result:stems.vocals" } + }, + "result": { "content_type": "application/json" } +} +``` + +| Argument | Required | Description | +| -------- | -------- | ----------- | +| `audio` | Yes | Path or URL of an audio file (or of a video file, whose soundtrack is taken), a waveform from a previous step, or an earlier step's video generated with a soundtrack | +| `sample_rate` | With a waveform | Sample rate of a directly passed waveform (files carry their own) | + +The result saves one file per stem, `-vocals`, `-drums`, `-bass` +and `-other`, and a later step reads one as `previous_result:.vocals`. +Stems are 44.1 kHz stereo, whatever the source was, and sum back to the mix. +The model is fixed (`htdemucs`, run in fp32), its weights download on first +use, and it needs the `demucs` package. On MPS a failed separation falls back +to the CPU and warns (`separation_cpu_fallback`). + ## Voice attribution Which reference voice sings each line of a song, by timbre - staging diff --git a/dw/tasks/task.py b/dw/tasks/task.py index 7b49ad5e..0e5506d3 100644 --- a/dw/tasks/task.py +++ b/dw/tasks/task.py @@ -425,6 +425,18 @@ def _handle_attribute_voices(task, arguments, previous_pipelines): return attribute_voices(device=task.device_for(arguments), **arguments) +@register_command( + "separate_stems", + implementation="dw.tasks.voice_attribution.separate_stems", + consumes_device=True, +) +def _handle_separate_stems(task, arguments, previous_pipelines): + """Split a mix into vocals, drums, bass and other stems, one audio result each""" + from .voice_attribution import separate_stems + + return separate_stems(device=task.device_for(arguments), **arguments) + + @register_command("compose_text", implementation="dw.tasks.compose_text.compose_text") def _handle_compose_text(task, arguments, previous_pipelines): """Join parts written once into one block of text""" diff --git a/dw/tasks/voice_attribution.py b/dw/tasks/voice_attribution.py index 63692050..766d3a92 100644 --- a/dw/tasks/voice_attribution.py +++ b/dw/tasks/voice_attribution.py @@ -534,7 +534,8 @@ def load(): return cached_model((COMMAND, _EMBEDDER_MODEL, str(device), str(dtype)), load) -def _run_separator(waveform, sample_rate, device, dtype): +def _run_separator_stems(waveform, sample_rate, device, dtype): + """Every htdemucs stem of a mix: ({name: (channels, samples) array}, rate).""" from demucs.apply import apply_model model = _load_separator(device, dtype) @@ -551,30 +552,42 @@ def _run_separator(waveform, sample_rate, device, dtype): stems = apply_model( model, ((mix - mean) / std)[None], device=device, split=True )[0] - vocals = stems[model.sources.index("vocals")] * std + mean - return vocals.float().cpu().numpy(), rate + return { + name: (stems[index] * std + mean).float().cpu().numpy() + for index, name in enumerate(model.sources) + }, rate -def separate_vocals(waveform, sample_rate, device, dtype): - """The vocal stem of a (channels, samples) mix, as a mono 16 kHz array. +def _run_separator(waveform, sample_rate, device, dtype): + stems, rate = _run_separator_stems(waveform, sample_rate, device, dtype) + return stems["vocals"], rate - demucs on MPS is unverified; if it fails there, separation is retried on - the CPU with a warning rather than failing the step. - """ + +def _separate_with_fallback(run, command, waveform, sample_rate, device, dtype): + """`run` (a separator entry point) on the device, retried on the CPU on + MPS. demucs on MPS is unverified; if it fails there, separation is retried + on the CPU with a warning rather than failing the step.""" from .. import get_device_type try: - vocals, rate = _run_separator(waveform, sample_rate, device, dtype) + return run(waveform, sample_rate, device, dtype) except Exception as error: if get_device_type(device) != "mps": raise emit_warning( - f"{COMMAND}: htdemucs failed on {device} ({error}); separating on " + f"{command}: htdemucs failed on {device} ({error}); separating on " "the CPU instead, which is slower", kind="separation_cpu_fallback", - command=COMMAND, + command=command, ) - vocals, rate = _run_separator(waveform, sample_rate, "cpu", torch.float32) + return run(waveform, sample_rate, "cpu", torch.float32) + + +def separate_vocals(waveform, sample_rate, device, dtype): + """The vocal stem of a (channels, samples) mix, as a mono 16 kHz array.""" + vocals, rate = _separate_with_fallback( + _run_separator, COMMAND, waveform, sample_rate, device, dtype + ) return _mono_16k(vocals, rate) @@ -809,3 +822,42 @@ def attribute_voices( "warnings": warnings, "thresholds": dict(THRESHOLDS), } + + +STEMS_COMMAND = "separate_stems" + + +def separate_stems(audio, sample_rate=None, device="cpu"): + """Task command: split a mix into its vocals, drums, bass and other stems. + + Runs htdemucs (Hybrid Transformer Demucs), the separator `attribute_voices` + uses, and returns each stem as its own audio result: the step's result + saves them as `-vocals`, `-drums`, `-bass` and + `-other`, and a later step reads one as + `previous_result:.vocals`. Transcribing the vocal stem aligns sung + lyrics better than the full mix, and `other` + `drums` + `bass` is the + instrumental to put under dialogue as a score bed. The stems sum back to + the mix. Needs the `demucs` package; on MPS a failed separation is retried + on the CPU with a warning. + + Args: + audio: Path or URL of an audio file (or of a video file, whose + soundtrack is taken), a video generated with a soundtrack, or a + waveform (which needs sample_rate alongside it) + sample_rate: Sample rate of a directly passed waveform (files carry + their own) + device: Where htdemucs runs + + Returns: + Dict of stem name to an audio track, at htdemucs' 44.1 kHz stereo + """ + from .audio_utils import as_track + + waveform, rate = waveform_and_rate(audio, sample_rate, STEMS_COMMAND) + # fp32 on every device, as attribute_voices runs it + stems, stem_rate = _separate_with_fallback( + _run_separator_stems, STEMS_COMMAND, waveform, rate, device, torch.float32 + ) + return { + name: as_track(array, stem_rate, STEMS_COMMAND) for name, array in stems.items() + } diff --git a/tests/test_separate_stems.py b/tests/test_separate_stems.py new file mode 100644 index 00000000..0dcd2aec --- /dev/null +++ b/tests/test_separate_stems.py @@ -0,0 +1,105 @@ +"""Tests for #604: separate_stems - htdemucs' four stems, each its own audio +result. demucs itself is faked (no weights, no network); the separator +plumbing around it - resample, normalize, un-normalize, one track per stem, +the MPS fallback - is the real code.""" + +import sys +import types +import unittest +from unittest.mock import patch + +import numpy +import torch + +from dw.introspection import describe_task, list_tasks +from dw.media_types import AudioTrack +from dw.tasks import model_cache +from dw.tasks.voice_attribution import separate_stems + +SOURCES = ["drums", "bass", "other", "vocals"] +RATE = 44100 + + +class FakeModel: + sources = SOURCES + samplerate = RATE + audio_channels = 2 + + def eval(self): + return self + + def to(self, **kwargs): + return self + + +def fake_demucs(apply_model): + pretrained = types.SimpleNamespace(get_model=lambda name: FakeModel()) + apply = types.SimpleNamespace(apply_model=apply_model) + return { + "demucs": types.SimpleNamespace(pretrained=pretrained, apply=apply), + "demucs.pretrained": pretrained, + "demucs.apply": apply, + } + + +def quarter_split(model, mix, device=None, split=True): + """Each stem is a distinct share of the (normalized) mix; they sum to it.""" + shares = torch.tensor([0.1, 0.2, 0.3, 0.4]).view(1, 4, 1, 1) + return mix[:, None] * shares + + +def mix(seconds=1.0, rate=RATE): + t = numpy.arange(int(seconds * rate)) / rate + wave = (0.5 * numpy.sin(2 * numpy.pi * 220 * t)).astype(numpy.float32) + return numpy.stack([wave, wave * 0.5]) + + +class TestSeparateStems(unittest.TestCase): + def setUp(self): + model_cache.clear_model_cache() + + def test_returns_each_stem_as_its_own_audio_track(self): + track = AudioTrack(mix(), RATE) + with patch.dict(sys.modules, fake_demucs(quarter_split)): + stems = separate_stems(track, device="cpu") + self.assertEqual(sorted(stems), sorted(SOURCES)) + for stem in stems.values(): + self.assertIsInstance(stem, AudioTrack) + self.assertEqual(stem.sample_rate, RATE) + self.assertEqual(stem.audio.shape, mix().shape) + + def test_stems_sum_back_to_the_mix(self): + source = mix() + with patch.dict(sys.modules, fake_demucs(quarter_split)): + stems = separate_stems(AudioTrack(source, RATE), device="cpu") + total = sum(stem.audio for stem in stems.values()) + numpy.testing.assert_allclose(total, source, atol=1e-4) + numpy.testing.assert_allclose(stems["vocals"].audio, source * 0.4, atol=1e-4) + + def test_mps_failure_retries_on_cpu_with_a_warning(self): + devices = [] + + def flaky(model, mixture, device=None, split=True): + devices.append(device) + if len(devices) == 1: + raise RuntimeError("boom") + return quarter_split(model, mixture) + + with ( + patch.dict(sys.modules, fake_demucs(flaky)), + patch("dw.get_device_type", return_value="mps"), + patch("dw.tasks.voice_attribution.emit_warning") as warn, + ): + stems = separate_stems(AudioTrack(mix(), RATE), device="mps") + self.assertEqual(devices, ["mps", "cpu"]) + self.assertEqual(len(stems), 4) + self.assertEqual(warn.call_args.kwargs["command"], "separate_stems") + + def test_task_is_registered_and_described(self): + self.assertIn("separate_stems", list_tasks()["commands"]) + names = [p["name"] for p in describe_task("separate_stems")["parameters"]] + self.assertIn("audio", names) + + +if __name__ == "__main__": + unittest.main() From e449d02cbb42c11e77047c2ce8b410f1896552d3 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 13:28:40 -0500 Subject: [PATCH 003/296] feat(tasks): #607 - ingredients_grid lays individual images out as a reference sheet Co-Authored-By: Claude Sonnet 5.5 --- docs/TASKS.md | 59 ++++++++ dw/task_domains.py | 6 + dw/tasks/image_utils.py | 256 ++++++++++++++++++++++++++++++++- dw/tasks/task.py | 11 ++ tests/test_ingredients_grid.py | 134 +++++++++++++++++ 5 files changed, 465 insertions(+), 1 deletion(-) create mode 100644 tests/test_ingredients_grid.py diff --git a/docs/TASKS.md b/docs/TASKS.md index da896778..3b39b8e7 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -1409,6 +1409,65 @@ and are cached between calls like any other model. On MPS a `separate: true` run that fails in htdemucs falls back to the CPU and warns (`separation_cpu_fallback`) rather than failing the step. +### ingredients_grid + +Lay individual images - a character, a prop, a location - out on one canvas as a +reference sheet. It is `frame_grid`'s counterpart for separate images, and builds +the still `ltx2/reference-sheet` conditions on. Pure PIL. + +```json +{ + "name": "sheet", + "task": { + "command": "ingredients_grid", + "arguments": { + "images": ["asset:hero.png", "asset:sword.png", "asset:castle.png"], + "width": 768, + "height": 448 + } + }, + "result": { "content_type": "image/png" } +} +``` + +| Argument | Default | Description | +| -------- | ------- | ----------- | +| `images` | required | Images in reading order: references, paths or `{"location": ...}` dicts; `gather:` of a `for_each` step splices in | +| `width`, `height` | 768, 448 | Canvas size in pixels | +| `layout` | `auto` | `rows`, `panels` or `auto` | +| `fit` | `contain` | `contain` scales each image to fit whole and pads with `background`; `cover` fills its cell and crops the overflow | +| `gap` | 8 | Pixels between cells, and kept clear of the canvas edge | +| `background` | `white` | Canvas colour, a name or `#hex` | +| `max_images` | 12 | More images than this is refused, never silently dropped | + +`rows` keeps every image at its own aspect ratio and justifies each row to the +canvas width. Which images share a row is searched: every way of cutting the +list, in order, into rows is scored on wasted canvas, overflow past the height, +and spread of row heights. `panels` gives every image an equal grid cell, the +column count chosen to waste the least canvas, with a short last row centred. +`auto` uses `rows` for up to 3 images and `panels` for more. Transparency is +flattened onto white. Returns one RGB image of exactly `width` x `height`. + +To build the sheet inside `ltx2/reference-sheet`, add this step first and point +`static_sheet`'s `video` at it (`"video": "previous_result:sheet"` in place of the +`reference_sheet` asset). Keep `width` and `height` the clip's own, so the sheet +reads at the size the model was trained on: + +```json +{ + "name": "sheet", + "task": { + "command": "ingredients_grid", + "arguments": { + "images": ["asset:hero.png", "asset:sword.png", "asset:castle.png"], + "width": "variable:width", + "height": "variable:height" + } + }, + "result": { "content_type": "image/png", "save": false } +} +``` + ## Data Gathering ### gather_images diff --git a/dw/task_domains.py b/dw/task_domains.py index bddd0506..5c1a3373 100644 --- a/dw/task_domains.py +++ b/dw/task_domains.py @@ -152,6 +152,12 @@ "loop_frames": {"num_frames": POSITIVE}, "upscale_h3_latents": {"width": POSITIVE, "height": POSITIVE}, "frame_grid": {"count": POSITIVE, "columns": POSITIVE, "tile_width": POSITIVE}, + "ingredients_grid": { + "width": POSITIVE, + "height": POSITIVE, + "gap": NON_NEGATIVE, + "max_images": POSITIVE, + }, "dissolve_videos": { "dissolve_frames": NON_NEGATIVE, "fade_in_frames": NON_NEGATIVE, diff --git a/dw/tasks/image_utils.py b/dw/tasks/image_utils.py index e066bbd7..f13f292e 100644 --- a/dw/tasks/image_utils.py +++ b/dw/tasks/image_utils.py @@ -1,5 +1,8 @@ -from PIL import Image +from itertools import combinations + +from PIL import Image, ImageColor, ImageOps import numpy as np +from ..argument_media import fetch_image from .borders import add_border_and_mask, add_border_and_mask_with_size from .model_cache import cached_model import torch @@ -762,3 +765,254 @@ def process_image(image, processor, device, kwargs): [92, 0, 255], ] ) + + +# --- ingredients_grid (#607) --- +# Lay individual images out as one reference sheet. +# +# `frame_grid` tiles the frames of a video; this tiles separate images - a +# character, a prop, a location - onto one canvas the size a model reads, which +# is the sheet the LTX-2.5 Ingredients template conditions on. Pure PIL. +# +# Two layouts, and `auto` picks between them from the image count: +# +# - `rows` keeps every image at its own aspect ratio and justifies each row to the +# canvas width. Which images share a row is searched: every way of cutting the +# list, in order, into rows is scored on the canvas it wastes, how far it +# overflows the height, and how unevenly tall its rows come out. +# - `panels` gives every image an equal cell of a grid whose column count is +# searched the same way, so a sheet of several elements reads as even panels. + +LAYOUTS = ("auto", "rows", "panels") +FITS = ("contain", "cover") + +# A handful of references is a row to read across; past this, even panels +AUTO_ROWS_UP_TO = 3 + +# Weights of the row-partition score. Waste dominates; overflow and uneven row +# heights are tie-breakers that keep a sheet from favouring one tiny row +_OVERFLOW_WEIGHT = 0.5 +_SPREAD_WEIGHT = 0.25 + + +def ingredients_grid( + images, + width=768, + height=448, + layout="auto", + fit="contain", + gap=8, + background="white", + max_images=12, +): + """Task command: lay out several images on one canvas as a reference sheet. + + Args: + images: The images to lay out, in reading order - a list of + previous_result / asset: / output: references, paths, or + {"location": ...} dicts. `gather:` of a for_each step splices in + width: Canvas width in pixels + height: Canvas height in pixels + layout: "rows" (aspect-preserving justified rows), "panels" (an even + grid of equal cells) or "auto" (rows for up to 3 images, panels + for more) + fit: How an image fills its cell: "contain" scales it to fit whole + and pads the rest with `background`; "cover" scales it to fill + the cell and crops the overflow + gap: Pixels between cells, and kept clear of the canvas edge + background: Canvas colour, any name or #hex PIL accepts + max_images: Refuse more images than this rather than drop some + + Returns: + One RGB PIL image of exactly width x height + """ + width = _grid_whole_number(width, "width", 1) + height = _grid_whole_number(height, "height", 1) + gap = _grid_whole_number(gap, "gap", 0) + max_images = _grid_whole_number(max_images, "max_images", 1) + if layout not in LAYOUTS: + raise ValueError( + f"ingredients_grid needs 'layout' as one of {list(LAYOUTS)}, got {layout!r}" + ) + if fit not in FITS: + raise ValueError( + f"ingredients_grid needs 'fit' as one of {list(FITS)}, got {fit!r}" + ) + try: + color = ImageColor.getrgb(background) + except (ValueError, AttributeError): + raise ValueError( + f"ingredients_grid needs 'background' as a colour name or #hex, got {background!r}" + ) + + sources = _grid_load(images) + if not sources: + raise ValueError("ingredients_grid was given no images") + if len(sources) > max_images: + raise ValueError( + f"ingredients_grid was given {len(sources)} images but 'max_images' is " + f"{max_images} - raise it, or pass fewer images" + ) + + # The margin is one gap; a canvas the gaps alone would fill has no room + inner_w, inner_h = width - 2 * gap, height - 2 * gap + if inner_w < len(sources) and inner_h < len(sources): + raise ValueError( + f"ingredients_grid cannot fit {len(sources)} images on a {width}x{height} " + f"canvas with a gap of {gap}" + ) + + aspects = [image.width / image.height for image in sources] + if layout == "auto": + layout = "rows" if len(sources) <= AUTO_ROWS_UP_TO else "panels" + if layout == "rows": + cells = _grid_row_cells(aspects, inner_w, inner_h, gap) + else: + cells = _grid_panel_cells(aspects, inner_w, inner_h, gap) + + canvas = Image.new("RGB", (width, height), color) + for source, (x, y, w, h) in zip(sources, cells): + canvas.paste(_grid_fit_to_cell(source, w, h, fit, color), (gap + x, gap + y)) + return canvas + + +def _grid_whole_number(value, name, minimum): + if isinstance(value, str): + try: + value = int(value) + except ValueError: + raise ValueError( + f"ingredients_grid needs '{name}' as a whole number, got {value!r}" + ) + if not isinstance(value, int) or isinstance(value, bool): + raise ValueError( + f"ingredients_grid needs '{name}' as a whole number, got {value!r}" + ) + if value < minimum: + raise ValueError( + f"ingredients_grid needs '{name}' of at least {minimum}, got {value}" + ) + return value + + +def _grid_load(images): + """The images as flat list of RGB PIL images, from whatever references resolved to""" + if images is None: + return [] + loaded = fetch_image(images) + return [_grid_flatten_alpha(image) for image in _grid_flatten(loaded)] + + +def _grid_flatten(value): + if isinstance(value, (list, tuple)): + return [leaf for item in value for leaf in _grid_flatten(item)] + return [value] + + +def _grid_flatten_alpha(image): + """RGB of an image, transparency composited onto white""" + if image.mode in ("RGBA", "LA") or "transparency" in image.info: + rgba = image.convert("RGBA") + base = Image.new("RGBA", rgba.size, (255, 255, 255, 255)) + return Image.alpha_composite(base, rgba).convert("RGB") + return image.convert("RGB") + + +def _grid_compositions(count): + """Every way of cutting `count` ordered items into consecutive rows, as row sizes""" + for cuts in range(count): + for chosen in combinations(range(1, count), cuts): + edges = (0, *chosen, count) + yield [edges[i + 1] - edges[i] for i in range(len(edges) - 1)] + + +def _grid_row_layout(aspects, sizes, width, height, gap): + """Score one partition into rows; returns (score, rects)""" + rows, start = [], 0 + for size in sizes: + rows.append(aspects[start : start + size]) + start += size + + heights = [(width - gap * (len(row) - 1)) / sum(row) for row in rows] + total = sum(heights) + gap * (len(rows) - 1) + overflow = max(0.0, total - height) / height + # Over-tall rows shrink uniformly to fit, which narrows them too + scale = min(1.0, height / total) + heights = [h * scale for h in heights] + + rects, y = [], 0.0 + for row, row_h in zip(rows, heights): + row_w = sum(a * row_h for a in row) + gap * (len(row) - 1) + x = (width - row_w) / 2 + for aspect in row: + rects.append((x, y, aspect * row_h, row_h)) + x += aspect * row_h + gap + y += row_h + gap + used = sum(w * h for _, _, w, h in rects) + waste = 1 - used / (width * height) + spread = (max(heights) - min(heights)) / max(heights) + return waste + _OVERFLOW_WEIGHT * overflow + _SPREAD_WEIGHT * spread, rects + + +def _grid_row_cells(aspects, width, height, gap): + best_score, best_rects = None, None + for sizes in _grid_compositions(len(aspects)): + score, rects = _grid_row_layout(aspects, sizes, width, height, gap) + if best_score is None or score < best_score: + best_score, best_rects = score, rects + return _grid_centered(best_rects, width, height) + + +def _grid_panel_cells(aspects, width, height, gap): + count = len(aspects) + best_score, best = None, None + for columns in range(1, count + 1): + rows = -(-count // columns) + cell_w = (width - gap * (columns - 1)) / columns + cell_h = (height - gap * (rows - 1)) / rows + if cell_w < 1 or cell_h < 1: + continue + # The canvas each image leaves empty when contained in its cell + used = sum(min(cell_w, cell_h * a) * min(cell_h, cell_w / a) for a in aspects) + score = 1 - used / (width * height) + if best_score is None or score < best_score: + best_score, best = score, (columns, rows, cell_w, cell_h) + if best is None: + raise ValueError( + f"ingredients_grid cannot fit {count} panels on a {width}x{height} canvas " + f"with a gap of {gap}" + ) + columns, rows, cell_w, cell_h = best + rects = [] + for index in range(count): + row, column = divmod(index, columns) + in_row = min(columns, count - row * columns) + # A short last row is centred rather than left-justified + x0 = (width - (in_row * cell_w + gap * (in_row - 1))) / 2 + rects.append( + (x0 + column * (cell_w + gap), row * (cell_h + gap), cell_w, cell_h) + ) + return _grid_centered(rects, width, height) + + +def _grid_centered(rects, width, height): + """Integer cells, the block of them centred vertically on the canvas""" + bottom = max(y + h for _, y, _, h in rects) + shift = (height - bottom) / 2 + return [ + (round(x), round(y + shift), max(1, round(w)), max(1, round(h))) + for x, y, w, h in rects + ] + + +def _grid_fit_to_cell(image, width, height, fit, background): + if fit == "cover": + return ImageOps.fit(image, (width, height), Image.LANCZOS) + scale = min(width / image.width, height / image.height) + size = (max(1, round(image.width * scale)), max(1, round(image.height * scale))) + cell = Image.new("RGB", (width, height), background) + cell.paste( + image.resize(size, Image.LANCZOS), + ((width - size[0]) // 2, (height - size[1]) // 2), + ) + return cell diff --git a/dw/tasks/task.py b/dw/tasks/task.py index 0e5506d3..dcef4fd2 100644 --- a/dw/tasks/task.py +++ b/dw/tasks/task.py @@ -282,6 +282,17 @@ def _handle_frame_grid(task, arguments, previous_pipelines): return frame_grid(**arguments) +@register_command( + "ingredients_grid", implementation="dw.tasks.image_utils.ingredients_grid" +) +def _handle_ingredients_grid(task, arguments, previous_pipelines): + """Lay individual images out as one reference sheet""" + logger.debug("Building ingredients grid") + from .image_utils import ingredients_grid + + return ingredients_grid(**arguments) + + @register_command("pair_audio", implementation="dw.tasks.pair_audio.pair_audio") def _handle_pair_audio(task, arguments, previous_pipelines): """Pair a video's frames with an audio track generated beside them""" diff --git a/tests/test_ingredients_grid.py b/tests/test_ingredients_grid.py new file mode 100644 index 00000000..c64841ed --- /dev/null +++ b/tests/test_ingredients_grid.py @@ -0,0 +1,134 @@ +"""ingredients_grid: individual images laid out as one reference sheet (#607).""" + +import pytest +from PIL import Image + +from dw.tasks.image_utils import ( + _grid_compositions as _compositions, + _grid_row_cells as _row_cells, + ingredients_grid, +) + + +def solid(width, height, color): + return Image.new("RGB", (width, height), color) + + +RED, GREEN, BLUE, BLACK = (255, 0, 0), (0, 255, 0), (0, 0, 255), (0, 0, 0) + + +class TestCanvas: + def test_is_exactly_the_requested_size(self): + sheet = ingredients_grid([solid(100, 200, RED)], width=768, height=448) + assert sheet.size == (768, 448) + assert sheet.mode == "RGB" + + def test_every_image_shows_up_on_the_sheet(self): + images = [solid(300, 200, c) for c in (RED, GREEN, BLUE)] + sheet = ingredients_grid(images, background="black", gap=4) + colors = {color for _, color in sheet.getcolors(maxcolors=1 << 20)} + assert {RED, GREEN, BLUE} <= colors + + def test_background_fills_the_unused_canvas(self): + sheet = ingredients_grid([solid(100, 100, RED)], background="#102030") + assert sheet.getpixel((0, 0)) == (16, 32, 48) + + def test_an_image_stays_in_reading_order(self): + sheet = ingredients_grid( + [solid(200, 200, RED), solid(200, 200, BLUE)], + layout="rows", + background="black", + ) + reds = [x for x in range(sheet.width) if sheet.getpixel((x, 224)) == RED] + blues = [x for x in range(sheet.width) if sheet.getpixel((x, 224)) == BLUE] + assert max(reds) < min(blues) + + def test_transparency_is_flattened_onto_white(self): + clear = Image.new("RGBA", (100, 100), (255, 0, 0, 0)) + sheet = ingredients_grid([clear], background="black", gap=0) + assert sheet.getpixel((384, 224)) == (255, 255, 255) + + +class TestLayout: + def test_auto_picks_rows_for_few_and_panels_for_many(self): + few = [solid(300, 200, RED)] * 3 + many = [solid(300, 200, RED)] * 6 + assert ( + ingredients_grid(few).tobytes() + == ingredients_grid(few, layout="rows").tobytes() + ) + assert ( + ingredients_grid(many).tobytes() + == ingredients_grid(many, layout="panels").tobytes() + ) + + def test_the_partition_search_considers_every_split(self): + assert sum(1 for _ in _compositions(4)) == 8 + assert [3] in list(_compositions(3)) + assert [1, 1, 1] in list(_compositions(3)) + + def test_wide_images_stack_in_rows_and_tall_ones_sit_side_by_side(self): + wide = _row_cells([4.0, 4.0, 4.0], 768, 448, 0) + assert len({y for _, y, _, _ in wide}) == 3 + tall = _row_cells([0.4, 0.4, 0.4], 768, 448, 0) + assert len({y for _, y, _, _ in tall}) == 1 + + def test_rows_stay_inside_the_canvas(self): + cells = _row_cells([1.0] * 7, 752, 432, 8) + for x, y, w, h in cells: + assert x >= 0 and y >= 0 and x + w <= 752 and y + h <= 432 + + def test_panels_are_equal_cells(self): + sheet = ingredients_grid( + [solid(100, 100, c) for c in (RED, GREEN, BLUE, RED)], + layout="panels", + width=400, + height=400, + gap=0, + background="black", + ) + assert sheet.getpixel((100, 100)) == RED + assert sheet.getpixel((300, 100)) == GREEN + assert sheet.getpixel((100, 300)) == BLUE + + +class TestFit: + def test_contain_pads_and_cover_crops(self): + wide = solid(400, 100, RED) + kwargs = dict(layout="panels", width=200, height=200, gap=0, background="black") + contained = ingredients_grid([wide], fit="contain", **kwargs) + covered = ingredients_grid([wide], fit="cover", **kwargs) + # the single panel is the whole canvas: contain letterboxes, cover fills + assert contained.getpixel((100, 5)) == BLACK + assert covered.getpixel((100, 5)) == RED + + +class TestRefusals: + def test_no_images(self): + with pytest.raises(ValueError, match="no images"): + ingredients_grid([]) + + def test_more_than_max_images_is_refused_not_dropped(self): + with pytest.raises(ValueError, match="max_images"): + ingredients_grid([solid(10, 10, RED)] * 3, max_images=2) + + @pytest.mark.parametrize( + "kwargs", + [ + {"layout": "spiral"}, + {"fit": "stretch"}, + {"width": 0}, + {"gap": -1}, + {"background": "notacolour"}, + ], + ) + def test_bad_arguments(self, kwargs): + with pytest.raises(ValueError, match="ingredients_grid"): + ingredients_grid([solid(10, 10, RED)], **kwargs) + + +class TestRegistered: + def test_the_command_is_registered_with_its_arguments(self): + from dw.tasks.task import _COMMAND_REGISTRY + + assert "ingredients_grid" in _COMMAND_REGISTRY From a317ea4750c30e18c825a9d887fc018744c9e57c Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 13:32:41 -0500 Subject: [PATCH 004/296] docs(skills): #610 - script-to-video shot-list contract, test render, re-roll budget; series-episodes voice prompt Co-Authored-By: Claude Sonnet 5.5 --- plugins/dw/skills/script-to-video/SKILL.md | 22 ++++++++++++++++++++-- plugins/dw/skills/series-episodes/SKILL.md | 4 ++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/plugins/dw/skills/script-to-video/SKILL.md b/plugins/dw/skills/script-to-video/SKILL.md index f0534eb0..e3ca9205 100644 --- a/plugins/dw/skills/script-to-video/SKILL.md +++ b/plugins/dw/skills/script-to-video/SKILL.md @@ -26,6 +26,20 @@ several shots, and a `dw:minimax-h3` shot runs 5.17 to 14.4 seconds (the `17*n+5` frame grid, 124 to 345 frames). For each shot, name it, note which character(s) appear, what happens, and roughly how long it runs. +Fill in this shape before writing any prompt: + +- **Cast**: per character, a voice description, sex, and the reference + asset (`asset:cast/`; step 3). +- **Locations**: each place a shot happens, named once and reused. +- **Per shot**: the above, plus `opening_state` (what is on screen, who + stands where, what is held, at frame one) and `ending_state` (the same at + the last frame). A shot's `opening_state` is the previous shot's + `ending_state`, so the cut between them is written down rather than hoped for. + +Whatever the user supplied is locked; copy it, don't improve it. A field +they left blank stays blank - ask, or mark it unspecified - and you invent +no character, place or line to fill it. + ## 3. Cast recurring characters once, before any shot generates Read `dw:series-episodes` step 0 before drawing anything: every character @@ -55,7 +69,10 @@ instead of a single draw. Read the chosen family's own skill before writing a single prompt - the hard rules (frame count, canvas, reference limits) live there, not here. -## 5. Validate and quote cost before queuing +## 5. Test-render, validate and quote cost before queuing + +Render two to four representative shots first (a dialogue shot, an action +shot, the hardest cut) and judge them before the full batch. `validate_workflow` against the chosen template with the assembled `shots` argument, read `plan.estimate` and `plan.downloads_required`, and state the @@ -72,7 +89,8 @@ a `still_running` reply means call again. After the job completes, read `get_job the warnings the engine already emits (`audio_no_headroom`, `audio_clipped`, an elision diagnostic) and `rerun_job(new_seed=True)` a shot that reads wrong - not by inventing a new heuristic, by reading what is already -surfaced. Each shot's files are organized by `subfolder` - typically `final` +surfaced. Give each shot a budget of three failed attempts, then stop and +ask the user rather than spending more. Each shot's files are organized by `subfolder` - typically `final` for the deliverable video or `intermediate` for test frames; read the manifest to determine what was written where. diff --git a/plugins/dw/skills/series-episodes/SKILL.md b/plugins/dw/skills/series-episodes/SKILL.md index 071a0fee..77c04a38 100644 --- a/plugins/dw/skills/series-episodes/SKILL.md +++ b/plugins/dw/skills/series-episodes/SKILL.md @@ -64,6 +64,10 @@ and reference it as `prompt:/` from every episode. described the same way in episode 6 as in episode 1 is a reference rather than a paragraph retyped - which is the drift this skill exists to stop. +Store each speaking character's voice description the same way, beside the +portrait: `prompt:/-voice`. It is the written half of the +voice clip, and the shot list's cast entry (`script-to-video` step 2) points at both. + ## 1-5. Per episode: generate, then recut, bed, match_levels, normalize, pair Generate the episode's shots with `templates/minimax/dialogue-short` (or From d7b7e5c79d117127e8f8c927b19795191e9066d8 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 13:36:21 -0500 Subject: [PATCH 005/296] feat(catalog): #612 - Alissonerdx H3 live-wallpaper trials, aligned-guide LoRAs recorded as blocked on #611, LMS upscaler source Co-Authored-By: Claude Sonnet 5.5 --- dw/tasks/h3_latent_upscale.py | 24 +++++++++++++--- loras/minimax-h3/head-swap.json | 27 ++++++++++++++++++ loras/minimax-h3/live-wallpaper-r32.json | 23 ++++++++++++++++ loras/minimax-h3/live-wallpaper-r64.json | 23 ++++++++++++++++ loras/minimax-h3/lms.json | 27 ++++++++++++++++++ loras/minimax-h3/style-transfer.json | 27 ++++++++++++++++++ loras/minimax-h3/vfx-edit.json | 27 ++++++++++++++++++ tests/test_h3_latent_upscale.py | 35 ++++++++++++++++++++++++ 8 files changed, 209 insertions(+), 4 deletions(-) create mode 100644 loras/minimax-h3/head-swap.json create mode 100644 loras/minimax-h3/live-wallpaper-r32.json create mode 100644 loras/minimax-h3/live-wallpaper-r64.json create mode 100644 loras/minimax-h3/lms.json create mode 100644 loras/minimax-h3/style-transfer.json create mode 100644 loras/minimax-h3/vfx-edit.json diff --git a/dw/tasks/h3_latent_upscale.py b/dw/tasks/h3_latent_upscale.py index f9e379db..bfde1733 100644 --- a/dw/tasks/h3_latent_upscale.py +++ b/dw/tasks/h3_latent_upscale.py @@ -36,6 +36,19 @@ # file is without changing its path UPSCALER_REVISION = "3f941d5d182014dd5c0a5e16330420ee2d4aa0c6" +# Other upscaler checkpoints known to be the v1 architecture, by (repo, file): +# the folder each sits in and the commit it is read at. A source not named +# here is read from its repo root at main +LMS_UPSCALER_REPO = "Alissonerdx/Minimax-H3-ComfyUI" +KNOWN_UPSCALER_SOURCES = { + (DEFAULT_UPSCALER_REPO, None): (DEFAULT_UPSCALER_SUBFOLDER, UPSCALER_REVISION), + # Sharpness-tuned from the default, same keys and shapes (#612) + (LMS_UPSCALER_REPO, "h3_upscaler_lms_v0.1.safetensors"): ( + "latent_upscaler", + "849fb3d4f434f2e9b4a4d4f7253471bf5e6719f9", + ), +} + DEFAULT_H3_REPO = "MiniMaxAI/MiniMax-H3" # H3's video VAE compresses 16x in each spatial dimension @@ -202,9 +215,10 @@ def _load_upscaler(device, model_name, weight_name): # The pin names a commit of the default repo; a caller naming another # repo gets that repo's main - default_repo = model_name == DEFAULT_UPSCALER_REPO - revision = UPSCALER_REVISION if default_repo else None - subfolder = DEFAULT_UPSCALER_SUBFOLDER if default_repo else None + source = KNOWN_UPSCALER_SOURCES.get( + (model_name, weight_name) + ) or KNOWN_UPSCALER_SOURCES.get((model_name, None)) + subfolder, revision = source if source else (None, None) def load_net(): from safetensors.torch import load_file @@ -245,7 +259,9 @@ def upscale_h3_latents( LBH-123-AI/Minimax_h3_latent_Upscaler, read at a pinned revision) weight_name: The safetensors file within it, a bare file name (default: the v1 bf16 checkpoint). In the default repo it is read from the - v1 checkpoint folder, so the v1 fp16 file is named the same way + v1 checkpoint folder, so the v1 fp16 file is named the same way. + `Alissonerdx/Minimax-H3-ComfyUI` with `h3_upscaler_lms_v0.1.safetensors` + is the sharpness-tuned LMS upscaler, read at a pinned commit Returns: The upscaled latents, (B, 24, T, height / 16, width / 16), in the diff --git a/loras/minimax-h3/head-swap.json b/loras/minimax-h3/head-swap.json new file mode 100644 index 00000000..aef66d5a --- /dev/null +++ b/loras/minimax-h3/head-swap.json @@ -0,0 +1,27 @@ +{ + "model_name": "Alissonerdx/Minimax-H3-ComfyUI", + "weight_name": "loras/minimax_h3_head_swap_v1.0_r32.safetensors", + "revision": "849fb3d4f434f2e9b4a4d4f7253471bf5e6719f9", + "base_models": [ + "MiniMaxAI/MiniMax-H3" + ], + "workflow": null, + "description": "Blocked, not judged: head replacement from a reference image, trigger 'head_swap:'. It conditions on the source video's latents laid over the whole output grid from frame 0 (ComfyUI MiniMaxH3AddGuide); diffusers' H3 keyframe layout takes only single-frame first/last anchors, so no dw workflow can feed it its guide", + "use_when": "Do not use under dw until a full-length aligned guide exists (#611)", + "scale": { + "default": 1.0 + }, + "status": "rejected", + "evidence": [ + { + "issue": 611, + "note": "Blocked on #611: needs a multi-frame source-video guide aligned to the output grid at frame_idx 0, which dw's keyframe layout cannot express. Not tried and not rejected on merit; reopen the entry when #611 lands" + } + ], + "tags": [ + "blocked", + "head-swap", + "video-to-video" + ], + "license": "apache-2.0" +} diff --git a/loras/minimax-h3/live-wallpaper-r32.json b/loras/minimax-h3/live-wallpaper-r32.json new file mode 100644 index 00000000..d9947fad --- /dev/null +++ b/loras/minimax-h3/live-wallpaper-r32.json @@ -0,0 +1,23 @@ +{ + "model_name": "Alissonerdx/Minimax-H3-ComfyUI", + "weight_name": "loras/minimax_h3_live_wallpaper_v1.0_ref2va_r32.safetensors", + "revision": "849fb3d4f434f2e9b4a4d4f7253471bf5e6719f9", + "base_models": [ + "MiniMaxAI/MiniMax-H3" + ], + "workflow": "ref2va", + "description": "Animates one source image (optionally a midpoint and an ending frame as the 2nd and 3rd references) while keeping its subject and composition, through native ref2va conditioning. Faces tend to hold better in some cases; camera behavior is less predictable (trained without camera labels). Every prompt starts 'live_wallpaper: Animate with', describes the camera (locked-off if none is wanted) and ends 'while preserving the original subject and composition.'. diffusion_model. key format, r32; the card calls these research models", + "use_when": "Bringing a still image to life with gentle motion and effects while keeping it recognisably the same picture", + "trigger": "live_wallpaper:", + "scale": { + "default": 1.0 + }, + "status": "trial", + "tags": [ + "wallpaper", + "animate", + "reference", + "image-to-video" + ], + "license": "apache-2.0" +} diff --git a/loras/minimax-h3/live-wallpaper-r64.json b/loras/minimax-h3/live-wallpaper-r64.json new file mode 100644 index 00000000..a23129f1 --- /dev/null +++ b/loras/minimax-h3/live-wallpaper-r64.json @@ -0,0 +1,23 @@ +{ + "model_name": "Alissonerdx/Minimax-H3-ComfyUI", + "weight_name": "loras/minimax_h3_live_wallpaper_v1.0_ref2va_r64.safetensors", + "revision": "849fb3d4f434f2e9b4a4d4f7253471bf5e6719f9", + "base_models": [ + "MiniMaxAI/MiniMax-H3" + ], + "workflow": "ref2va", + "description": "Animates one source image (optionally a midpoint and an ending frame as the 2nd and 3rd references) while keeping its subject and composition, through native ref2va conditioning. The one to use when the camera must follow the prompt (continued with camera-aware captions); facial fidelity still varies by image, seed and strength. Every prompt starts 'live_wallpaper: Animate with', describes the camera (locked-off if none is wanted) and ends 'while preserving the original subject and composition.'. diffusion_model. key format, r64; the card calls these research models", + "use_when": "Bringing a still image to life with gentle motion and effects while keeping it recognisably the same picture", + "trigger": "live_wallpaper:", + "scale": { + "default": 1.0 + }, + "status": "trial", + "tags": [ + "wallpaper", + "animate", + "reference", + "image-to-video" + ], + "license": "apache-2.0" +} diff --git a/loras/minimax-h3/lms.json b/loras/minimax-h3/lms.json new file mode 100644 index 00000000..8f0f8b97 --- /dev/null +++ b/loras/minimax-h3/lms.json @@ -0,0 +1,27 @@ +{ + "model_name": "Alissonerdx/Minimax-H3-ComfyUI", + "weight_name": "loras/minimax_h3_lms_v1.0_r64.safetensors", + "revision": "849fb3d4f434f2e9b4a4d4f7253471bf5e6719f9", + "base_models": [ + "MiniMaxAI/MiniMax-H3" + ], + "workflow": null, + "description": "Blocked, not judged: a fixed-caption sharpening second pass over an existing clip (see the repo's docs/lms.md). It conditions on the source video's latents laid over the whole output grid from frame 0 (ComfyUI MiniMaxH3AddGuide); diffusers' H3 keyframe layout takes only single-frame first/last anchors, so no dw workflow can feed it its guide", + "use_when": "Do not use under dw until a full-length aligned guide exists (#611)", + "scale": { + "default": 1.0 + }, + "status": "rejected", + "evidence": [ + { + "issue": 611, + "note": "Blocked on #611: needs a multi-frame source-video guide aligned to the output grid at frame_idx 0, which dw's keyframe layout cannot express. Not tried and not rejected on merit; reopen the entry when #611 lands" + } + ], + "tags": [ + "blocked", + "sharpen", + "second-pass" + ], + "license": "apache-2.0" +} diff --git a/loras/minimax-h3/style-transfer.json b/loras/minimax-h3/style-transfer.json new file mode 100644 index 00000000..fa990dda --- /dev/null +++ b/loras/minimax-h3/style-transfer.json @@ -0,0 +1,27 @@ +{ + "model_name": "Alissonerdx/Minimax-H3-ComfyUI", + "weight_name": "loras/minimax_h3_style_transfer_v1.0_r64.safetensors", + "revision": "849fb3d4f434f2e9b4a4d4f7253471bf5e6719f9", + "base_models": [ + "MiniMaxAI/MiniMax-H3" + ], + "workflow": null, + "description": "Blocked, not judged: style from an image or text onto a video, trigger 'style_transfer:'. It conditions on the source video's latents laid over the whole output grid from frame 0 (ComfyUI MiniMaxH3AddGuide); diffusers' H3 keyframe layout takes only single-frame first/last anchors, so no dw workflow can feed it its guide", + "use_when": "Do not use under dw until a full-length aligned guide exists (#611)", + "scale": { + "default": 1.0 + }, + "status": "rejected", + "evidence": [ + { + "issue": 611, + "note": "Blocked on #611: needs a multi-frame source-video guide aligned to the output grid at frame_idx 0, which dw's keyframe layout cannot express. Not tried and not rejected on merit; reopen the entry when #611 lands" + } + ], + "tags": [ + "blocked", + "style", + "video-to-video" + ], + "license": "apache-2.0" +} diff --git a/loras/minimax-h3/vfx-edit.json b/loras/minimax-h3/vfx-edit.json new file mode 100644 index 00000000..f1684f77 --- /dev/null +++ b/loras/minimax-h3/vfx-edit.json @@ -0,0 +1,27 @@ +{ + "model_name": "Alissonerdx/Minimax-H3-ComfyUI", + "weight_name": "loras/minimax_h3_vfx_edit_v1.0_r128.safetensors", + "revision": "849fb3d4f434f2e9b4a4d4f7253471bf5e6719f9", + "base_models": [ + "MiniMaxAI/MiniMax-H3" + ], + "workflow": null, + "description": "Blocked, not judged: a prompt-directed VFX edit of a video, trigger 'vfx_edit:' (an _ffp variant sits beside it). It conditions on the source video's latents laid over the whole output grid from frame 0 (ComfyUI MiniMaxH3AddGuide); diffusers' H3 keyframe layout takes only single-frame first/last anchors, so no dw workflow can feed it its guide", + "use_when": "Do not use under dw until a full-length aligned guide exists (#611)", + "scale": { + "default": 1.0 + }, + "status": "rejected", + "evidence": [ + { + "issue": 611, + "note": "Blocked on #611: needs a multi-frame source-video guide aligned to the output grid at frame_idx 0, which dw's keyframe layout cannot express. Not tried and not rejected on merit; reopen the entry when #611 lands" + } + ], + "tags": [ + "blocked", + "vfx", + "video-to-video" + ], + "license": "apache-2.0" +} diff --git a/tests/test_h3_latent_upscale.py b/tests/test_h3_latent_upscale.py index 8a026a89..5e9a5b3d 100644 --- a/tests/test_h3_latent_upscale.py +++ b/tests/test_h3_latent_upscale.py @@ -469,3 +469,38 @@ def test_upscale_h3_latents_domains_are_positive(self): if __name__ == "__main__": pytest.main([__file__, "-v"]) + + def test_the_lms_upscaler_is_read_from_its_folder_at_its_pinned_commit( + self, monkeypatch, tmp_path + ): + from safetensors.torch import save_file + + from dw.tasks import h3_latent_upscaler_model + + captured = {} + weights_path = tmp_path / "weights.safetensors" + save_file(LatentResizer3D(channels=64).state_dict(), str(weights_path)) + monkeypatch.setattr( + h3_latent_upscaler_model, + "LatentResizer3D", + lambda: LatentResizer3D(channels=64), + ) + + def fake_hf_hub_download(repo_id, filename, subfolder, revision): + captured.update(subfolder=subfolder, revision=revision) + return str(weights_path) + + monkeypatch.setattr("huggingface_hub.hf_hub_download", fake_hf_hub_download) + + hlu._load_upscaler( + "cpu", hlu.LMS_UPSCALER_REPO, "h3_upscaler_lms_v0.1.safetensors" + ) + assert captured == { + "subfolder": "latent_upscaler", + "revision": "849fb3d4f434f2e9b4a4d4f7253471bf5e6719f9", + } + # another file in that repo is not the pinned one + hlu._load_upscaler( + "cpu", hlu.LMS_UPSCALER_REPO, "h3_upscaler_lms_v0.1_fp32.safetensors" + ) + assert captured == {"subfolder": None, "revision": None} From c918b79d092289dcea159ff42999e3fd06a23307 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 13:39:48 -0500 Subject: [PATCH 006/296] docs(skills): #614 - minimax-h3 says no extension mode is a diffusers limit, not the model's Co-Authored-By: Claude Sonnet 5.5 --- plugins/dw/skills/minimax-h3/SKILL.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/dw/skills/minimax-h3/SKILL.md b/plugins/dw/skills/minimax-h3/SKILL.md index eb44d81d..d1e827c7 100644 --- a/plugins/dw/skills/minimax-h3/SKILL.md +++ b/plugins/dw/skills/minimax-h3/SKILL.md @@ -102,8 +102,9 @@ read the `workflows` guide's authoring section. - Nine steps for an eight-step LoRA: the scheduler counts sigma grid points, terminal zero included, so `denoise_total_steps` reports 8. A null `lora_model_name` drops the LoRA; raise steps and shifts too. -- Nothing carries between generations except a passed reference: no latent - memory, no extension mode. Identity rides on a picture, voice on an audio +- Nothing carries between generations except a passed reference: this + pipeline has no latent memory or extension mode (a diffusers limit, not + the model's). Identity rides on a picture, voice on an audio clip, motion/camera on a video tail, score across cuts under concat. - H3 is guidance-distilled: no `guidance_scale` or negative prompt. - Keep `release_pipeline` where the template puts it (frees Z-Image before From c561ecc6b9fea57a373c85e65316072a02467064 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 13:44:17 -0500 Subject: [PATCH 007/296] fix(tasks): #615 - restore_faces refuses upsample_img with a video input Co-Authored-By: Claude Sonnet 5.5 --- dw/tasks/task.py | 8 +++++++ tests/test_restore_faces_video.py | 38 +++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 tests/test_restore_faces_video.py diff --git a/dw/tasks/task.py b/dw/tasks/task.py index dcef4fd2..05ba3ebf 100644 --- a/dw/tasks/task.py +++ b/dw/tasks/task.py @@ -555,6 +555,14 @@ def _handle_restore_faces(task, arguments, previous_pipelines): image = arguments.pop("image") model_name = arguments.pop("model_name") from .restore_faces import restore_faces + from .video_utils import is_video + + if is_video(image) and arguments.get("upsample_img") is not None: + raise ValueError( + "restore_faces: 'upsample_img' is a single background image and " + "cannot be used with a video input (it would be pasted under " + "every frame). Remove 'upsample_img' or pass a single image." + ) device = task.device_for(arguments) return _per_frame( diff --git a/tests/test_restore_faces_video.py b/tests/test_restore_faces_video.py new file mode 100644 index 00000000..8bb843be --- /dev/null +++ b/tests/test_restore_faces_video.py @@ -0,0 +1,38 @@ +"""restore_faces refuses a single upsample_img with a video input (#615).""" + +from unittest.mock import MagicMock, patch + +import pytest +from PIL import Image + +from dw.media_types import AudioVideo +from dw.tasks.task import _handle_restore_faces + + +def _task(): + task = MagicMock() + task.device_for.return_value = "cpu" + return task + + +def test_video_with_upsample_img_is_refused(): + video = AudioVideo([Image.new("RGB", (8, 8)) for _ in range(2)], None, None, fps=8) + args = { + "image": video, + "model_name": "m", + "upsample_img": Image.new("RGB", (16, 16)), + } + with pytest.raises(ValueError, match="upsample_img"): + _handle_restore_faces(_task(), args, {}) + + +def test_image_with_upsample_img_still_runs(): + background = Image.new("RGB", (16, 16)) + args = { + "image": Image.new("RGB", (8, 8)), + "model_name": "m", + "upsample_img": background, + } + with patch("dw.tasks.restore_faces.restore_faces", return_value="out") as run: + assert _handle_restore_faces(_task(), args, {}) == "out" + assert run.call_args.kwargs["upsample_img"] is background From 837fd24a941be1fb4a2fdb7dac0159f4db985194 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 14:07:38 -0500 Subject: [PATCH 008/296] fix(tasks): #607 - ingredients_grid bounds go through check_arguments; contain uses ImageOps.pad Co-Authored-By: Claude Sonnet 5.5 --- dw/tasks/image_utils.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/dw/tasks/image_utils.py b/dw/tasks/image_utils.py index f13f292e..3628d6c2 100644 --- a/dw/tasks/image_utils.py +++ b/dw/tasks/image_utils.py @@ -3,6 +3,7 @@ from PIL import Image, ImageColor, ImageOps import numpy as np from ..argument_media import fetch_image +from ..task_domains import check_arguments from .borders import add_border_and_mask, add_border_and_mask_with_size from .model_cache import cached_model import torch @@ -826,10 +827,13 @@ def ingredients_grid( Returns: One RGB PIL image of exactly width x height """ - width = _grid_whole_number(width, "width", 1) - height = _grid_whole_number(height, "height", 1) - gap = _grid_whole_number(gap, "gap", 0) - max_images = _grid_whole_number(max_images, "max_images", 1) + width = _grid_whole_number(width, "width") + height = _grid_whole_number(height, "height") + gap = _grid_whole_number(gap, "gap") + max_images = _grid_whole_number(max_images, "max_images") + check_arguments( + "ingredients_grid", width=width, height=height, gap=gap, max_images=max_images + ) if layout not in LAYOUTS: raise ValueError( f"ingredients_grid needs 'layout' as one of {list(LAYOUTS)}, got {layout!r}" @@ -876,7 +880,7 @@ def ingredients_grid( return canvas -def _grid_whole_number(value, name, minimum): +def _grid_whole_number(value, name): if isinstance(value, str): try: value = int(value) @@ -888,10 +892,6 @@ def _grid_whole_number(value, name, minimum): raise ValueError( f"ingredients_grid needs '{name}' as a whole number, got {value!r}" ) - if value < minimum: - raise ValueError( - f"ingredients_grid needs '{name}' of at least {minimum}, got {value}" - ) return value @@ -1008,11 +1008,4 @@ def _grid_centered(rects, width, height): def _grid_fit_to_cell(image, width, height, fit, background): if fit == "cover": return ImageOps.fit(image, (width, height), Image.LANCZOS) - scale = min(width / image.width, height / image.height) - size = (max(1, round(image.width * scale)), max(1, round(image.height * scale))) - cell = Image.new("RGB", (width, height), background) - cell.paste( - image.resize(size, Image.LANCZOS), - ((width - size[0]) // 2, (height - size[1]) // 2), - ) - return cell + return ImageOps.pad(image, (width, height), Image.LANCZOS, color=background) From 81dfb2fdf119368c107ab5a1b639b3c5510022b2 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 15:17:11 -0500 Subject: [PATCH 009/296] docs(skills): #610 - script-to-video: a new seed redraws an H3 voice; change the prompt to keep it Co-Authored-By: Claude Sonnet 5.5 --- plugins/dw/skills/script-to-video/SKILL.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/plugins/dw/skills/script-to-video/SKILL.md b/plugins/dw/skills/script-to-video/SKILL.md index e3ca9205..f5317822 100644 --- a/plugins/dw/skills/script-to-video/SKILL.md +++ b/plugins/dw/skills/script-to-video/SKILL.md @@ -89,8 +89,14 @@ a `still_running` reply means call again. After the job completes, read `get_job the warnings the engine already emits (`audio_no_headroom`, `audio_clipped`, an elision diagnostic) and `rerun_job(new_seed=True)` a shot that reads wrong - not by inventing a new heuristic, by reading what is already -surfaced. Give each shot a budget of three failed attempts, then stop and -ask the user rather than spending more. Each shot's files are organized by `subfolder` - typically `final` +surfaced. On an H3 shot with dialogue a new seed also redraws the voice: in +a lem probe (#610) three seeds of one line gave a different speaker each +time without an audio reference, and only a narrower spread with one +(`voice` / `voice-timbre-reference`). So when the voice is right and the +picture is wrong, change the prompt and keep the seed; reach for +`new_seed` only when the voice is pinned by a reference or does not matter. +Give each shot a budget of three failed attempts, then stop and ask the user +rather than spending more. Each shot's files are organized by `subfolder` - typically `final` for the deliverable video or `intermediate` for test frames; read the manifest to determine what was written where. From 85672302573c8f684b1eae4173af7a3b59a89e68 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 15:25:02 -0500 Subject: [PATCH 010/296] fix(tasks): #646 - validate refuses ingredients_grid literal layout/fit/background and over-max images Co-Authored-By: Claude Sonnet 5.5 --- dw/task_domains.py | 79 ++++++++++++++++++++++++++++++++++ dw/tasks/image_utils.py | 10 +++-- tests/test_ingredients_grid.py | 39 +++++++++++++++++ 3 files changed, 125 insertions(+), 3 deletions(-) diff --git a/dw/task_domains.py b/dw/task_domains.py index 5c1a3373..2dfdec15 100644 --- a/dw/task_domains.py +++ b/dw/task_domains.py @@ -188,6 +188,79 @@ } +# command -> argument -> the literal values it accepts. Owned here so the +# command's run-time refusal and the static pass read one list +INGREDIENTS_LAYOUTS = ("auto", "rows", "panels") +INGREDIENTS_FITS = ("contain", "cover") +TASK_ARGUMENT_CHOICES = { + "ingredients_grid": { + "layout": INGREDIENTS_LAYOUTS, + "fit": INGREDIENTS_FITS, + }, +} +INGREDIENTS_DEFAULT_MAX_IMAGES = 12 + + +def _is_reference(value): + """A string that names a reference (`variable:x`, `asset:y`) rather than a + literal - run time is the right place to check those.""" + return isinstance(value, str) and ":" in value and not value.startswith("#") + + +def choice_errors(command, arguments): + """[(argument, message)] for each literal argument outside its accepted + values, worded as the command's own refusal.""" + errors = [] + for name, choices in TASK_ARGUMENT_CHOICES.get(command, {}).items(): + value = arguments.get(name) + if name not in arguments or _is_reference(value): + continue + if value not in choices: + errors.append( + ( + name, + f"{command} needs '{name}' as one of {list(choices)}, got {value!r}", + ) + ) + return errors + + +def ingredients_grid_errors(arguments): + """[(argument, message)] for the ingredients_grid rules a literal + workflow can break before it runs: a bad `layout`/`fit`/`background`, and + a literal `images` list longer than a literal `max_images`.""" + errors = choice_errors("ingredients_grid", arguments) + background = arguments.get("background") + if isinstance(background, str) and not (_is_reference(background)): + from PIL import ImageColor + + try: + ImageColor.getrgb(background) + except ValueError: + errors.append( + ( + "background", + "ingredients_grid needs 'background' as a colour name or " + f"#hex, got {background!r}", + ) + ) + images = arguments.get("images") + limit = arguments.get("max_images", INGREDIENTS_DEFAULT_MAX_IMAGES) + if isinstance(images, list) and not any( + isinstance(item, str) and item.startswith("gather:") for item in images + ): + number = as_number(limit) + if number is not None and number > 0 and len(images) > number: + errors.append( + ( + "images", + f"ingredients_grid was given {len(images)} images but " + f"'max_images' is {limit} - raise it, or pass fewer images", + ) + ) + return errors + + def in_domain(value, domain): """Whether a number satisfies a domain. Anything unmeasurable is True - a value this cannot read is not this check's to refuse.""" @@ -332,6 +405,12 @@ def task_argument_errors(workflow_definition, source_indices=None): "message": f"{message}{where}.", } ) + if command == "ingredients_grid": + for key, message in ingredients_grid_errors(arguments): + path = ("steps", source, "task", "arguments", key) + errors.append( + {"path": render_path(path), "message": f"{message}{where}."} + ) return errors diff --git a/dw/tasks/image_utils.py b/dw/tasks/image_utils.py index 3628d6c2..031d22ef 100644 --- a/dw/tasks/image_utils.py +++ b/dw/tasks/image_utils.py @@ -3,7 +3,11 @@ from PIL import Image, ImageColor, ImageOps import numpy as np from ..argument_media import fetch_image -from ..task_domains import check_arguments +from ..task_domains import ( + INGREDIENTS_FITS, + INGREDIENTS_LAYOUTS, + check_arguments, +) from .borders import add_border_and_mask, add_border_and_mask_with_size from .model_cache import cached_model import torch @@ -784,8 +788,8 @@ def process_image(image, processor, device, kwargs): # - `panels` gives every image an equal cell of a grid whose column count is # searched the same way, so a sheet of several elements reads as even panels. -LAYOUTS = ("auto", "rows", "panels") -FITS = ("contain", "cover") +LAYOUTS = INGREDIENTS_LAYOUTS +FITS = INGREDIENTS_FITS # A handful of references is a row to read across; past this, even panels AUTO_ROWS_UP_TO = 3 diff --git a/tests/test_ingredients_grid.py b/tests/test_ingredients_grid.py index c64841ed..08ece1d9 100644 --- a/tests/test_ingredients_grid.py +++ b/tests/test_ingredients_grid.py @@ -132,3 +132,42 @@ def test_the_command_is_registered_with_its_arguments(self): from dw.tasks.task import _COMMAND_REGISTRY assert "ingredients_grid" in _COMMAND_REGISTRY + + +class TestStaticValidation: + """validate refuses literal layout/fit/background and over-max image lists (#646).""" + + def _errors(self, **arguments): + from dw.task_domains import task_argument_errors + + step = { + "name": "s", + "task": {"command": "ingredients_grid", "arguments": arguments}, + } + return task_argument_errors({"steps": [step]}) + + def test_bad_layout_and_fit(self): + paths = { + e["path"] + for e in self._errors(images=["asset:a"], layout="grid", fit="stretch") + } + assert paths == { + "steps[0].task.arguments.layout", + "steps[0].task.arguments.fit", + } + + def test_bad_background(self): + assert self._errors(images=["asset:a"], background="notacolour") + + def test_too_many_images(self): + errors = self._errors(images=[f"asset:{i}" for i in range(13)]) + assert [e["path"] for e in errors] == ["steps[0].task.arguments.images"] + assert not self._errors(images=[f"asset:{i}" for i in range(13)], max_images=13) + + def test_references_left_to_run_time(self): + assert not self._errors( + images=["gather:x"], + layout="variable:l", + fit="previous_result:f", + background="#fff", + ) From d63bb6d291c59ab31e06744e04bee1a91c12b545 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 15:25:46 -0500 Subject: [PATCH 011/296] fix(tasks): #646 - name the gather prefix through references.GATHER Co-Authored-By: Claude Sonnet 5.5 --- dw/task_domains.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dw/task_domains.py b/dw/task_domains.py index 2dfdec15..0166ae0a 100644 --- a/dw/task_domains.py +++ b/dw/task_domains.py @@ -32,7 +32,7 @@ import logging import numbers -from .references import MEMBER_SEPARATOR, author_index, render_path +from .references import GATHER, MEMBER_SEPARATOR, author_index, render_path logger = logging.getLogger("dw") @@ -247,7 +247,7 @@ def ingredients_grid_errors(arguments): images = arguments.get("images") limit = arguments.get("max_images", INGREDIENTS_DEFAULT_MAX_IMAGES) if isinstance(images, list) and not any( - isinstance(item, str) and item.startswith("gather:") for item in images + isinstance(item, str) and item.startswith(GATHER) for item in images ): number = as_number(limit) if number is not None and number > 0 and len(images) > number: From 16bfa0e85f929318a48b7cbd64bac328acb70860 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 15:26:15 -0500 Subject: [PATCH 012/296] fix(tasks): #646 - use is_ref for the gather check Co-Authored-By: Claude Sonnet 5.5 --- dw/task_domains.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dw/task_domains.py b/dw/task_domains.py index 0166ae0a..322da32c 100644 --- a/dw/task_domains.py +++ b/dw/task_domains.py @@ -32,7 +32,7 @@ import logging import numbers -from .references import GATHER, MEMBER_SEPARATOR, author_index, render_path +from .references import GATHER, MEMBER_SEPARATOR, author_index, is_ref, render_path logger = logging.getLogger("dw") @@ -246,9 +246,7 @@ def ingredients_grid_errors(arguments): ) images = arguments.get("images") limit = arguments.get("max_images", INGREDIENTS_DEFAULT_MAX_IMAGES) - if isinstance(images, list) and not any( - isinstance(item, str) and item.startswith(GATHER) for item in images - ): + if isinstance(images, list) and not any(is_ref(GATHER, item) for item in images): number = as_number(limit) if number is not None and number > 0 and len(images) > number: errors.append( From d3d0b18aad2d0ad744a3fe5d393a3fb812e6b1d9 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 16:30:49 -0500 Subject: [PATCH 013/296] feat(tasks): #617 - transcribe_audio ends an open-ended chunk at the clip's duration; attribute-lines template Co-Authored-By: Claude Opus 5.5 --- dw/tasks/audio_transcription.py | 38 +++++++++--- tests/test_catalog_structure.py | 6 +- tests/test_step_cache.py | 1 + tests/test_transcribe_audio.py | 67 +++++++++++++++++++++ tests/test_voice_attribution.py | 76 ++++++++++++++++++++++++ workflows/templates/attribute-lines.json | 42 +++++++++++++ 6 files changed, 220 insertions(+), 10 deletions(-) create mode 100644 workflows/templates/attribute-lines.json diff --git a/dw/tasks/audio_transcription.py b/dw/tasks/audio_transcription.py index c39755c8..e2351dd7 100644 --- a/dw/tasks/audio_transcription.py +++ b/dw/tasks/audio_transcription.py @@ -60,7 +60,9 @@ def transcribe_audio(audio, device="cpu", sample_rate=None, **kwargs): (default: openai/whisper-base). timestamps: Unset (default) returns plain text. "segment" or "word" instead returns a dict of {text, chunks}, chunks - being a list of {start, end, text} - the step's result + being a list of {start, end, text}, every time a number of + seconds (a chunk the clip ends inside ends at the clip's + duration) - the step's result content_type must then be "application/json" rather than "text/plain", since the return shape follows the argument. @@ -125,12 +127,30 @@ def load_pipe(): if timestamps is None: return text - chunks = [ - { - "start": chunk.get("timestamp", (None, None))[0], - "end": chunk.get("timestamp", (None, None))[1], - "text": chunk.get("text", "").strip(), - } - for chunk in result.get("chunks", []) - ] + duration = len(mono) / _ASR_SAMPLE_RATE + chunks = _numeric_chunks(result.get("chunks", []), duration) return {"text": text, "chunks": chunks} + + +def _numeric_chunks(raw_chunks, duration): + """The {start, end, text} chunks, with every time a number of seconds. + + Whisper leaves a chunk's end None when the audio stops inside it - a song + cut mid-line - and a None there failed every reader that does arithmetic + on the span: attribute_voices refused the whole transcript (#488). Such a + chunk ends at the clip's own duration, and a None start takes the + previous chunk's end (0 for the first). + """ + chunks = [] + previous_end = 0.0 + for chunk in raw_chunks: + start, end = chunk.get("timestamp", (None, None)) + if start is None: + start = previous_end + if end is None: + end = max(duration, start) + chunks.append( + {"start": start, "end": end, "text": chunk.get("text", "").strip()} + ) + previous_end = end + return chunks diff --git a/tests/test_catalog_structure.py b/tests/test_catalog_structure.py index 6497ecbe..9518bc92 100644 --- a/tests/test_catalog_structure.py +++ b/tests/test_catalog_structure.py @@ -112,6 +112,7 @@ def load(path): # Templates that genuinely are utilities - processing with no generation. # Anything else that derives 'utility' is a rule that missed. UTILITIES = { + "workflows/templates/attribute-lines.json", "workflows/templates/audio-trim-fade.json", "workflows/templates/image-processors.json", "workflows/templates/recenter-crop.json", @@ -415,7 +416,10 @@ def test_no_stale_entry_in_the_allowlist(): # at 9_052 before its curated `cost` (about 15 tokens more): the latent-refine # route to 2x for a clip the caller brings, kept beside upscale-clip because # the two trade differently (source latents vs. an IC-LoRA re-render). -COMPACT_BUDGET = 9_150 +# Then to 9_200 for `templates/attribute-lines` (#617, 2026-10-05), measured +# at 9_180: about 30 tokens, the transcribe -> attribute_voices chain a +# lip-sync target check starts from (#488). +COMPACT_BUDGET = 9_200 FILTERED_BUDGET = 1_500 diff --git a/tests/test_step_cache.py b/tests/test_step_cache.py index 2d50aafd..e7f005fa 100644 --- a/tests/test_step_cache.py +++ b/tests/test_step_cache.py @@ -603,6 +603,7 @@ def test_normalized_downstream_false_when_nothing_references_it(): "attention-processor.json": { "generate_with_sdpa": "8dc3e92052c445d5dbf20bf6e8b6fe91794cfa3772bb7366e2390889b4e99459" }, + "attribute-lines.json": {}, "audio-trim-fade.json": {}, "base-and-refiner.json": { "main": "384b0dccd2842b0c235b9d325e53e48bef5bc1b5150fd2e836f154d7dde32507", diff --git a/tests/test_transcribe_audio.py b/tests/test_transcribe_audio.py index 3e8b3da3..7eaa94d3 100644 --- a/tests/test_transcribe_audio.py +++ b/tests/test_transcribe_audio.py @@ -216,6 +216,73 @@ def test_timestamps_segment_requests_true_and_returns_chunks(self, mock_pipeline result["chunks"], [{"start": 0.0, "end": 1.0, "text": "hello world"}] ) + @patch("dw.tasks.audio_transcription.hf_pipeline") + def test_an_open_ended_last_chunk_ends_at_the_clips_duration(self, mock_pipeline): + # #488: Whisper leaves the last chunk's end None when the clip stops + # inside a line, and attribute_voices refused the whole transcript + # on it - the clip's own duration is where that chunk ends + self._mock_pipe( + mock_pipeline, + "first line cut off", + chunks=[ + {"text": " first line", "timestamp": (0.0, 2.0)}, + {"text": " cut off", "timestamp": (2.5, None)}, + ], + ) + waveform, rate = self._waveform(seconds=4.0) + + result = transcribe_audio( + waveform, device="cpu", sample_rate=rate, timestamps="segment" + ) + + self.assertEqual( + result["chunks"], + [ + {"start": 0.0, "end": 2.0, "text": "first line"}, + {"start": 2.5, "end": 4.0, "text": "cut off"}, + ], + ) + + @patch("dw.tasks.audio_transcription.hf_pipeline") + def test_the_duration_is_the_clips_whatever_its_sample_rate(self, mock_pipeline): + # The duration comes from the 16 kHz mono array the model heard, so a + # 44.1 kHz stereo clip of 3 s still ends its open chunk at 3 s + self._mock_pipe( + mock_pipeline, "held", chunks=[{"text": "held", "timestamp": (1.0, None)}] + ) + waveform, rate = self._waveform(channels=2, seconds=3.0, sample_rate=44100) + + result = transcribe_audio( + waveform, device="cpu", sample_rate=rate, timestamps="word" + ) + + self.assertAlmostEqual(result["chunks"][0]["end"], 3.0, places=3) + + @patch("dw.tasks.audio_transcription.hf_pipeline") + def test_a_none_start_takes_the_previous_end_or_zero(self, mock_pipeline): + self._mock_pipe( + mock_pipeline, + "a b c", + chunks=[ + {"text": "a", "timestamp": (None, 1.0)}, + {"text": "b", "timestamp": (1.0, 1.5)}, + {"text": "c", "timestamp": (None, 2.0)}, + ], + ) + waveform, rate = self._waveform(seconds=3.0) + + result = transcribe_audio( + waveform, device="cpu", sample_rate=rate, timestamps="segment" + ) + + self.assertEqual( + [(c["start"], c["end"]) for c in result["chunks"]], + [(0.0, 1.0), (1.0, 1.5), (1.5, 2.0)], + ) + for chunk in result["chunks"]: + self.assertIsInstance(chunk["start"], float) + self.assertIsInstance(chunk["end"], float) + @patch("dw.tasks.audio_transcription.hf_pipeline") def test_timestamps_requested_under_thirty_seconds_too(self, mock_pipeline): # The 30 s long-form branch is a separate reason to ask for diff --git a/tests/test_voice_attribution.py b/tests/test_voice_attribution.py index 038cd72f..b092963f 100644 --- a/tests/test_voice_attribution.py +++ b/tests/test_voice_attribution.py @@ -782,3 +782,79 @@ def test_asset_voices_and_spans_are_not_locations(self): if __name__ == "__main__": unittest.main() + + +# --------------------------------------------------------------------------- +# The attribute-lines template: transcribe_audio -> attribute_voices (#488) +# --------------------------------------------------------------------------- + + +class TestAttributeLinesTemplate(unittest.TestCase): + TEMPLATE = "workflows/templates/attribute-lines.json" + + def _workflow(self, output_dir): + import json + + with open(self.TEMPLATE) as f: + return Workflow(json.load(f), output_dir, "") + + def test_the_template_validates_and_needs_its_voices(self): + import tempfile + + with tempfile.TemporaryDirectory() as output_dir: + workflow = self._workflow(output_dir) + self.assertEqual(workflow.validation_errors(), []) + errors = workflow.validation_errors({"audio": "asset:song.wav"}) + self.assertEqual( + [error["path"] for error in errors], + ["steps[1].task.arguments.voices"], + ) + + def test_the_transcript_reaches_attribute_voices_as_its_lines(self): + """The whole {text, chunks} transcript is attribute_voices' lines, and + a last chunk Whisper left open-ended (end None) is attributed with the + clip's duration as its end rather than refusing the transcript.""" + import tempfile + + pipe = MagicMock( + return_value={ + "text": "one two", + "chunks": [ + {"timestamp": (0.0, 4.0), "text": " one"}, + {"timestamp": (4.0, None), "text": " two"}, + ], + } + ) + pipe.type = "seq2seq_whisper" + model_cache.clear_model_cache() + with ( + tempfile.TemporaryDirectory() as output_dir, + patch("dw.tasks.audio_transcription.hf_pipeline", return_value=pipe), + patch("dw.tasks.voice_attribution._load_embedder"), + patch( + "dw.tasks.voice_attribution.embed", + return_value=numpy.array([1.0, 0.0]), + ), + patch( + "dw.tasks.voice_attribution.separate_vocals", + side_effect=lambda mix, rate, device, dtype: mix.mean(axis=0), + ), + ): + result = self._workflow(output_dir).run( + { + "audio": song_audio(10.0), + "voices": { + "a": [{"start_seconds": 0.0, "duration_seconds": 3.0}], + "b": [{"start_seconds": 5.0, "duration_seconds": 3.0}], + }, + } + ) + model_cache.clear_model_cache() + + self.assertEqual(len(result), 1) + lines = result[0]["lines"] + self.assertEqual([line["text"] for line in lines], ["one", "two"]) + self.assertEqual( + [(line["start"], line["end"]) for line in lines], + [(0.0, 4.0), (4.0, 10.0)], + ) diff --git a/workflows/templates/attribute-lines.json b/workflows/templates/attribute-lines.json new file mode 100644 index 00000000..d148a081 --- /dev/null +++ b/workflows/templates/attribute-lines.json @@ -0,0 +1,42 @@ +{ + "id": "AttributeLines", + "description": "Which singer sings each line of a song, and when. It transcribes the song with timestamps ('transcribe', a Whisper-class model), then hands the whole transcript to 'attribute_voices' as its 'lines' ('attribute'), which scores every line against each named voice's reference spans by timbre. The result is one JSON document whose lines carry numeric start and end seconds into the song, the text, and the voice with its margin, uncertain flag and reason. 'voices' is required: each singer's name mapped to reference spans into 'audio' - [{\"start_seconds\": 4.0, \"duration_seconds\": 6.0}] - at least two voices and three seconds each. 'timestamps' is 'segment' (a Whisper segment per line) or 'word' (finer lines, noisier attribution - the escape for a call-and-response line that comes back uncertain). The follow-up is the lip-sync target check in the tasks guide, 'Checking the lip-sync target': one get_output_frames call per face crop at each line's start + 0.3 s and midpoint, with 'hear', to see whose mouth is open on each line - mapping song time to the cut's time first.", + "variables": { + "audio": "asset:song.wav", + "voices": null, + "timestamps": "segment", + "model_name": "openai/whisper-base" + }, + "steps": [ + { + "name": "transcribe", + "task": { + "command": "transcribe_audio", + "arguments": { + "audio": "variable:audio", + "timestamps": "variable:timestamps", + "model_name": "variable:model_name" + } + }, + "result": { + "content_type": "application/json", + "subfolder": "intermediate" + } + }, + { + "name": "attribute", + "task": { + "command": "attribute_voices", + "arguments": { + "audio": "variable:audio", + "voices": "variable:voices", + "lines": "previous_result:transcribe" + } + }, + "result": { + "content_type": "application/json", + "subfolder": "final" + } + } + ] +} From 660527d57b38d1be873b19566077b01a673677c5 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 16:32:30 -0500 Subject: [PATCH 014/296] feat(docs): #617 - Checking the lip-sync target in TASKS.md; script-to-video points to it Co-Authored-By: Claude Opus 5.5 --- docs/TASKS.md | 53 +++++++++++++++++++++- plugins/dw/skills/script-to-video/SKILL.md | 4 ++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/docs/TASKS.md b/docs/TASKS.md index 3b39b8e7..7ac19d6e 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -1409,6 +1409,57 @@ and are cached between calls like any other model. On MPS a `separate: true` run that fails in htdemucs falls back to the CPU and warns (`separation_cpu_fallback`) rather than failing the step. +**Example:** [attribute-lines.json](../workflows/templates/attribute-lines.json) — Which singer sings each line of a song, and when: `transcribe_audio` with timestamps, its transcript handed whole to `attribute_voices` as `lines`. + +### Checking the lip-sync target + +A sung multi-shot piece can put the right song on the wrong mouth: the shot +lip-syncs whoever the prompt or the reference made most prominent, not the +singer the song gives that line to. Listening does not catch it, and one +frame per shot does not either. This check lines each sung line up with the +picture and looks at whose mouth is open on it. + +1. Run `attribute-lines` on the song with each singer's reference spans as + `voices`. Every line comes back with numeric `start` and `end` seconds into + the song, its `voice`, `margin` and `uncertain` flag. +2. Map each line's song times onto the cut's timeline (the rules below). +3. For every line with a non-null `voice`, take two moments: `start + 0.3 s`, + where the mouth has opened on the line's first syllable, and the line's + midpoint. +4. Make one `get_output_frames` call per face: `at` the moments, `crop` the + face's region, `hear: 1.0`. At most 32 moments per call, so a long song + takes more than one call per face. +5. On each line, the singer `voice` names should be the face with an open + mouth. Another face singing it is the wrong lip-sync target; both mouths + open on a solo line is a shot that lip-syncs everyone. +6. Skip the lines that are `uncertain` or have a null `voice`, and name them + in the report - a skipped line is unchecked, not passed. + +**Song time to cut time.** `attribute-lines` answers in seconds into the song +it was given; the frames are read from the cut. The offset between the two +depends on how the piece was assembled: + +- `minimax/music-video`: song time is cut time unless `trim_frames > 0`. The + template slices the song at each shot's `start_frame`, joins the shots with + `concat_videos` at `trim_frames: 0`, and lays the whole song back with + `pair_audio` at `fit: video`, so no offset applies. A trim drops frames at + every seam, and the song no longer lines up past the first one. +- `join_into_song`: add `start_frame / fps − cue_seconds` to every line time, + where `start_frame` is the first sung shot's, from the step's shots output. + That shot's frame 0 is where song time `cue_seconds` lands. +- Any other assembly: transcribe the final cut's own track instead, so the + times are already the cut's. Attribution is weaker there - dialogue ducked + under the song is in the mix too. + +**Where it misleads:** + +- A duet line - two singers on one line, or call and response inside one + Whisper segment - comes back `uncertain`. `timestamps: "word"` splits it + into words, each attributed on its own, at the cost of noisier scores. +- `start + 0.3 s` is a heuristic: a singer who opens late or holds a pickup + note is still closed-mouthed there. That is why the midpoint frame is read + too; trust the two together, not either alone. + ### ingredients_grid Lay individual images - a character, a prop, a location - out on one canvas as a @@ -1974,7 +2025,7 @@ Transcribe spoken audio to text with a local Whisper-class model. The word-corre Multi-channel audio is downmixed to mono and resampled to 16 kHz before transcription, since that is what a Whisper-class model is trained on; the source audio itself is untouched. By default the result is plain text, read with MCP's `get_output_text`. -Set `timestamps` to `"segment"` or `"word"` to get chunk timings instead — a music video cut to the lyric, or a dialogue shot checked against its line, needs the times Whisper already produces past 30 s rather than the collapsed string. The result becomes `{"text": ..., "chunks": [{"start": ..., "end": ..., "text": ...}, ...]}`, so the step's `result.content_type` must be `"application/json"` rather than `"text/plain"`, and it's read with MCP's `get_output_text` (JSON results are text). A clip under 30 s asks Whisper for timestamps explicitly when `timestamps` is set — the 30 s long-form threshold is a separate, unrelated reason to ask. +Set `timestamps` to `"segment"` or `"word"` to get chunk timings instead — a music video cut to the lyric, or a dialogue shot checked against its line, needs the times Whisper already produces past 30 s rather than the collapsed string. The result becomes `{"text": ..., "chunks": [{"start": ..., "end": ..., "text": ...}, ...]}`, so the step's `result.content_type` must be `"application/json"` rather than `"text/plain"`, and it's read with MCP's `get_output_text` (JSON results are text). A clip under 30 s asks Whisper for timestamps explicitly when `timestamps` is set — the 30 s long-form threshold is a separate, unrelated reason to ask. Every `start` and `end` is a number of seconds: a chunk the clip ends inside - a song cut mid-line, which Whisper leaves open-ended - ends at the clip's duration, so the transcript drops straight into [`attribute_voices`](#voice-attribution)' `lines`. **Example:** [transcribe-audio.json](../workflows/templates/transcribe-audio.json) — Transcribe an audio file to text. diff --git a/plugins/dw/skills/script-to-video/SKILL.md b/plugins/dw/skills/script-to-video/SKILL.md index f5317822..ea0e32ee 100644 --- a/plugins/dw/skills/script-to-video/SKILL.md +++ b/plugins/dw/skills/script-to-video/SKILL.md @@ -107,6 +107,10 @@ Once every shot exists, hand off to `templates/assemble-and-score` (or the a series) for the recut/bed/match_levels/normalize/pair pass. That skill already owns this step - do not re-derive it here. +A sung multi-shot piece is reviewed by its lip-sync target: run +`get_guide("tasks", section="Checking the lip-sync target")` on the cut +before delivering it. + To take the project home, call `export_job` once per job of the project, then fetch each zip's `open_url` and unpack it into `exports/` under the working directory (per its `next`). From 828001f72c5c890e5f932314a606caaa45f81788 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 16:37:29 -0500 Subject: [PATCH 015/296] fix(tasks): #646 - one owner for reference check, max_images default and background parse; map row names the new rules Co-Authored-By: Claude Sonnet 5.5 --- docs/ARCHITECTURE.md | 2 +- dw/task_domains.py | 37 +++++++++++++++++++------------------ dw/tasks/image_utils.py | 13 +++++-------- 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 07281392..ddf7ef65 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -31,7 +31,7 @@ one to open. | LoRAs | `dw/pipeline_processors/adapters.py`: `active_loras`, `load_loras` | A `loras` entry whose `model_name` is null is switched off. | `tests/test_lora_disable.py::TestLoadLoras::test_an_all_null_entry_loads_nothing` | | Step progress | `dw/pipeline_processors/progress.py`: `reported_progress_bars` | A pipeline with no step callback reports its denoise steps through its progress bars. | `tests/test_modular_progress.py::test_every_advance_of_the_bar_is_reported` | | Adding a task | `dw/tasks/task.py`: `register_command` | A task is a function registered with `@register_command`, and the signature of its implementation is its argument schema. | `tests/test_task_discovery.py::TestDescribeTask::test_signature_becomes_the_schema` | -| Task argument domains | `dw/task_domains.py`: `check_arguments` | A numeric domain that a signature cannot express is declared in this table, and it is checked at validation and again at run time. | `tests/test_task_domains.py::TestTheStaticPass::test_a_negative_frame_count_is_refused_at_its_path`, `tests/test_task_domains.py::TestAtRunTime::test_slice_audio_refuses_a_negative_count` | +| Task argument domains | `dw/task_domains.py`: `check_arguments` | A numeric domain that a signature cannot express is declared in this table, and it is checked at validation and again at run time. A literal enum choice (`choice_errors`) and the `ingredients_grid` image count against `max_images` (`ingredients_grid_errors`) are refused at validation by the same module. | `tests/test_task_domains.py::TestTheStaticPass::test_a_negative_frame_count_is_refused_at_its_path`, `tests/test_task_domains.py::TestAtRunTime::test_slice_audio_refuses_a_negative_count`, `tests/test_ingredients_grid.py::TestStaticValidation::test_bad_layout_and_fit` | | Variables | `dw/variables.py`: `set_variables`, `argument_errors`, `resolve_variable_values` | `argument_errors` folds a caller's arguments in exactly as `set_variables` does when a run starts, so a bad name or value is refused before anything is queued. | `tests/test_variables.py::test_argument_errors_reports_a_dict_passed_for_a_string_variable` | | List variables from the CLI | `dw/run.py`, `dw/variables.py`: `get_value` | A `name=value` string given for a list variable is split on commas, so a list of objects can only be passed as JSON over the API or MCP. | `tests/test_variables.py::test_set_variables_list_default_splits_on_comma` | | `for_each` expansion | `dw/for_each.py`: `expand_for_each` | Each entry becomes an ordinary step named `@`, after substitution and before the reference check. | `tests/test_for_each.py::TestNaming::test_member_name_joins_with_at` | diff --git a/dw/task_domains.py b/dw/task_domains.py index 322da32c..ec757876 100644 --- a/dw/task_domains.py +++ b/dw/task_domains.py @@ -32,7 +32,7 @@ import logging import numbers -from .references import GATHER, MEMBER_SEPARATOR, author_index, is_ref, render_path +from .references import DEFERRED, GATHER, MEMBER_SEPARATOR, author_index, is_ref, render_path logger = logging.getLogger("dw") @@ -201,10 +201,19 @@ INGREDIENTS_DEFAULT_MAX_IMAGES = 12 -def _is_reference(value): - """A string that names a reference (`variable:x`, `asset:y`) rather than a - literal - run time is the right place to check those.""" - return isinstance(value, str) and ":" in value and not value.startswith("#") +def ingredients_background(background): + """The RGB tuple for an ingredients_grid `background`, or ValueError worded + as the command's refusal. The one parse, shared by the static check and + the run.""" + from PIL import ImageColor + + try: + return ImageColor.getrgb(background) + except (ValueError, AttributeError): + raise ValueError( + "ingredients_grid needs 'background' as a colour name or " + f"#hex, got {background!r}" + ) def choice_errors(command, arguments): @@ -213,7 +222,7 @@ def choice_errors(command, arguments): errors = [] for name, choices in TASK_ARGUMENT_CHOICES.get(command, {}).items(): value = arguments.get(name) - if name not in arguments or _is_reference(value): + if name not in arguments or is_ref(DEFERRED, value): continue if value not in choices: errors.append( @@ -231,19 +240,11 @@ def ingredients_grid_errors(arguments): a literal `images` list longer than a literal `max_images`.""" errors = choice_errors("ingredients_grid", arguments) background = arguments.get("background") - if isinstance(background, str) and not (_is_reference(background)): - from PIL import ImageColor - + if isinstance(background, str) and not is_ref(DEFERRED, background): try: - ImageColor.getrgb(background) - except ValueError: - errors.append( - ( - "background", - "ingredients_grid needs 'background' as a colour name or " - f"#hex, got {background!r}", - ) - ) + ingredients_background(background) + except ValueError as error: + errors.append(("background", str(error))) images = arguments.get("images") limit = arguments.get("max_images", INGREDIENTS_DEFAULT_MAX_IMAGES) if isinstance(images, list) and not any(is_ref(GATHER, item) for item in images): diff --git a/dw/tasks/image_utils.py b/dw/tasks/image_utils.py index 031d22ef..1deebeb0 100644 --- a/dw/tasks/image_utils.py +++ b/dw/tasks/image_utils.py @@ -1,12 +1,14 @@ from itertools import combinations -from PIL import Image, ImageColor, ImageOps +from PIL import Image, ImageOps import numpy as np from ..argument_media import fetch_image from ..task_domains import ( + INGREDIENTS_DEFAULT_MAX_IMAGES, INGREDIENTS_FITS, INGREDIENTS_LAYOUTS, check_arguments, + ingredients_background, ) from .borders import add_border_and_mask, add_border_and_mask_with_size from .model_cache import cached_model @@ -808,7 +810,7 @@ def ingredients_grid( fit="contain", gap=8, background="white", - max_images=12, + max_images=INGREDIENTS_DEFAULT_MAX_IMAGES, ): """Task command: lay out several images on one canvas as a reference sheet. @@ -846,12 +848,7 @@ def ingredients_grid( raise ValueError( f"ingredients_grid needs 'fit' as one of {list(FITS)}, got {fit!r}" ) - try: - color = ImageColor.getrgb(background) - except (ValueError, AttributeError): - raise ValueError( - f"ingredients_grid needs 'background' as a colour name or #hex, got {background!r}" - ) + color = ingredients_background(background) sources = _grid_load(images) if not sources: From e8bb0d35c7cf00fa9f4c603f20114caff8bcefde Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 16:45:56 -0500 Subject: [PATCH 016/296] feat(tasks): #617 - attribute_voices takes a zero-length transcript line, and a line its pieces split between voices is uncertain Bounce 1: word timestamps gave start == end chunks that refused the attribute-lines template, and one Whisper segment over a whole duet came back as one confident voice. A line longer than PIECE_SECONDS is now also scored in pieces; a second voice at MIXED_LINE_SHARE or more makes it uncertain, the reason naming each voice's seconds. Co-Authored-By: Claude Opus 5.5 --- docs/TASKS.md | 18 +++-- dw/tasks/voice_attribution.py | 90 ++++++++++++++++++++---- tests/test_voice_attribution.py | 118 ++++++++++++++++++++++++++++++++ 3 files changed, 208 insertions(+), 18 deletions(-) diff --git a/docs/TASKS.md b/docs/TASKS.md index 7ac19d6e..9c9fcc1a 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -1359,7 +1359,7 @@ voiced, so a weak answer is visible as weak rather than silently accepted. | -------- | -------- | ----------- | | `audio` | Yes | The song - a path, `asset:`/`output:` reference, or an earlier step's audio or video | | `voices` | Yes | Each voice's name mapped to its reference: a list of `{start_seconds, duration_seconds}` (or `{start, end}`) spans into `audio`, or a path/`asset:` of a separate clip. At least 2 voices; names follow the variable-name pattern; each voice's reference must total at least `min_reference_seconds` | -| `lines` | No | Spans to attribute, `{start, end, text?}` (a `transcribe_audio` transcript, or its `chunks` list, drops in directly) or `{start_seconds, duration_seconds, text?}`. Omitted: the song is cut into fixed windows of `window_seconds` | +| `lines` | No | Spans to attribute, `{start, end, text?}` (a `transcribe_audio` transcript, or its `chunks` list, drops in directly) or `{start_seconds, duration_seconds, text?}`. A zero-length line (`start` = `end`, as Whisper's word timestamps sometimes give) comes back with `voice: null` rather than refusing the call. Omitted: the song is cut into fixed windows of `window_seconds` | | `windows` | No | Named spans to roll lines up into, `{name, start, end}`. Omitted: mirrors the fixed windows when `lines` is also omitted, otherwise none | | `window_seconds` | No | Length of the fixed windows used without `lines` (default `2.0`) | | `min_reference_seconds` | No | Least total reference length per voice; a shorter one is refused by name (default `3.0`) | @@ -1395,6 +1395,8 @@ well above that and well below the singing. | `min_voiced_seconds` | 0.5 s | A line or window under this much voiced time has no voice - `voice: null`, `uncertain: true` | | `uncertain_margin` | 0.05 | A line whose best score beats the runner-up by less than this is `uncertain` - the argmax is still reported | | `uncertain_share_margin` | 0.2 | A window whose leading voice's share beats the runner-up's by less than this is `uncertain` - a duet line, or a window straddling a hand-over | +| `piece_seconds` | 2.0 s | A line longer than this is also scored in pieces of this length | +| `mixed_line_share` | 0.25 | A line whose confidently attributed pieces give a second voice at least this share of their voiced time is `uncertain`, and its `reason` names each voice's seconds - one embedding of two singers can name either of them with a wide margin | | `voices_too_similar` | 0.8 | Two references scoring above this against each other make every line between them a weak answer whatever its scores say | A `voices_too_similar` pair is reported in `reference_similarity`, added to @@ -1454,8 +1456,16 @@ depends on how the piece was assembled: **Where it misleads:** - A duet line - two singers on one line, or call and response inside one - Whisper segment - comes back `uncertain`. `timestamps: "word"` splits it - into words, each attributed on its own, at the cost of noisier scores. + Whisper segment - comes back `uncertain`: its 2 s pieces name more than one + voice, and the `reason` says how many seconds each holds. + `timestamps: "word"` splits it into words, each attributed on its own, at + the cost of noisier scores. A zero-length word comes back with a null + `voice`, like any line too short to embed. +- Whisper can return one segment over most of a song, its text a repeated + hallucination, when the accompaniment drowns the words. Such a line is a + duet line by the rule above whenever two singers are in it; re-run with + `timestamps: "word"`, or transcribe the `vocals` stem of + [`separate_stems`](#stem-separation) instead. - `start + 0.3 s` is a heuristic: a singer who opens late or holds a pickup note is still closed-mouthed there. That is why the midpoint frame is read too; trust the two together, not either alone. @@ -2025,7 +2035,7 @@ Transcribe spoken audio to text with a local Whisper-class model. The word-corre Multi-channel audio is downmixed to mono and resampled to 16 kHz before transcription, since that is what a Whisper-class model is trained on; the source audio itself is untouched. By default the result is plain text, read with MCP's `get_output_text`. -Set `timestamps` to `"segment"` or `"word"` to get chunk timings instead — a music video cut to the lyric, or a dialogue shot checked against its line, needs the times Whisper already produces past 30 s rather than the collapsed string. The result becomes `{"text": ..., "chunks": [{"start": ..., "end": ..., "text": ...}, ...]}`, so the step's `result.content_type` must be `"application/json"` rather than `"text/plain"`, and it's read with MCP's `get_output_text` (JSON results are text). A clip under 30 s asks Whisper for timestamps explicitly when `timestamps` is set — the 30 s long-form threshold is a separate, unrelated reason to ask. Every `start` and `end` is a number of seconds: a chunk the clip ends inside - a song cut mid-line, which Whisper leaves open-ended - ends at the clip's duration, so the transcript drops straight into [`attribute_voices`](#voice-attribution)' `lines`. +Set `timestamps` to `"segment"` or `"word"` to get chunk timings instead — a music video cut to the lyric, or a dialogue shot checked against its line, needs the times Whisper already produces past 30 s rather than the collapsed string. The result becomes `{"text": ..., "chunks": [{"start": ..., "end": ..., "text": ...}, ...]}`, so the step's `result.content_type` must be `"application/json"` rather than `"text/plain"`, and it's read with MCP's `get_output_text` (JSON results are text). A clip under 30 s asks Whisper for timestamps explicitly when `timestamps` is set — the 30 s long-form threshold is a separate, unrelated reason to ask. Every `start` and `end` is a number of seconds: a chunk the clip ends inside - a song cut mid-line, which Whisper leaves open-ended - ends at the clip's duration, so the transcript drops straight into [`attribute_voices`](#voice-attribution)' `lines`. `"word"` can give a zero-length chunk (`start` = `end`) on a short or clipped word; `attribute_voices` reads it as a line with no voice. **Example:** [transcribe-audio.json](../workflows/templates/transcribe-audio.json) — Transcribe an audio file to text. diff --git a/dw/tasks/voice_attribution.py b/dw/tasks/voice_attribution.py index 766d3a92..decbf4e8 100644 --- a/dw/tasks/voice_attribution.py +++ b/dw/tasks/voice_attribution.py @@ -85,6 +85,15 @@ END_TOLERANCE_SECONDS = 0.1 # Default length of the fixed windows the song is cut into without `lines` WINDOW_SECONDS = 2.0 +# A line longer than this is also scored in pieces of this length, because +# one embedding of two singers can name either of them confidently: a +# Whisper segment over a whole duet - 7.4 s of one singer, 6.5 s of the +# other - came back as the second at margin 0.54 (#617) +PIECE_SECONDS = 2.0 +# A line whose confidently attributed pieces give a second voice at least +# this share of their voiced time is `uncertain` - it holds more than one +# singer, and its single verdict cannot be trusted +MIXED_LINE_SHARE = 0.25 THRESHOLDS = { "voiced_level_percentile": VOICED_LEVEL_PERCENTILE, @@ -93,6 +102,8 @@ "min_voiced_seconds": MIN_VOICED_SECONDS, "uncertain_margin": UNCERTAIN_MARGIN, "uncertain_share_margin": UNCERTAIN_SHARE_MARGIN, + "piece_seconds": PIECE_SECONDS, + "mixed_line_share": MIXED_LINE_SHARE, "voices_too_similar": VOICES_TOO_SIMILAR, } @@ -112,11 +123,14 @@ def _round(value, places=4): # --- arguments ------------------------------------------------------------- -def _span(entry, where, duration): +def _span(entry, where, duration, allow_empty=False): """A {start, end} span in seconds from either accepted shape. `{start, end}` is transcribe_audio's chunk shape (#483), so its output drops in as `lines`; `{start_seconds, duration_seconds}` is slice_audio's. + allow_empty accepts end == start: Whisper's word timestamps give a + zero-length chunk now and then, and a transcript line is attributed + (no voice, too short) rather than failing the whole call (#617). """ if not isinstance(entry, dict): raise ValueError( @@ -144,9 +158,10 @@ def _span(entry, where, duration): raise ValueError(f"{COMMAND}: {where} is not a finite span") if start < 0: raise ValueError(f"{COMMAND}: {where} starts before the audio ({start} s)") - if end <= start: + empty = allow_empty and end == start + if end <= start and not empty: raise ValueError(f"{COMMAND}: {where} ends at {end} s, not after its start") - if duration is not None and start >= duration: + if duration is not None and (start > duration or (start == duration and not empty)): raise ValueError( f"{COMMAND}: {where} starts at {start:.3f} s, past the end of the " f"{duration:.3f} s audio" @@ -295,7 +310,7 @@ def parse_lines(lines, duration, window_seconds): raise ValueError(f"{COMMAND}: 'lines' must be a non-empty list of spans") parsed = [] for index, line in enumerate(lines): - start, end = _span(line, f"lines[{index}]", duration) + start, end = _span(line, f"lines[{index}]", duration, allow_empty=True) parsed.append({"start": start, "end": end, "text": line.get("text")}) return parsed @@ -651,22 +666,66 @@ def _embed_references(references, clips, song, encoder, separate, device, dtype) return embeddings +def piece_voices(start, end, score_piece): + """Voiced seconds by voice over a line's PIECE_SECONDS pieces, counting + only the pieces score_piece(start, end) answers with a confident voice.""" + tally = {} + count = max(1, math.ceil((end - start) / PIECE_SECONDS - 1e-9)) + for index in range(count): + piece_start = start + index * PIECE_SECONDS + piece = score_piece(piece_start, min(piece_start + PIECE_SECONDS, end)) + if piece["voice"] is not None and not piece["uncertain"]: + tally[piece["voice"]] = ( + tally.get(piece["voice"], 0.0) + piece["voiced_seconds"] + ) + return tally + + +def mixed_reason(tally): + """Why a line is mixed, when a second voice holds MIXED_LINE_SHARE or + more of its pieces' voiced time; None when one voice has it.""" + total = sum(tally.values()) + if len(tally) < 2 or total <= 0: + return None + ranked = sorted(tally, key=tally.get, reverse=True) + share = tally[ranked[1]] / total + if share < MIXED_LINE_SHARE: + return None + split = ", ".join(f"'{name}' {tally[name]:.2f} s" for name in ranked) + return ( + f"its {PIECE_SECONDS:g} s pieces name more than one voice ({split}); " + f"'{ranked[1]}' holds {share:.2f} of it, at or over {MIXED_LINE_SHARE} " + '- a duet, or a segment over a hand-over; timestamps: "word" splits it' + ) + + def _attribute_lines(parsed_lines, song, encoder, embeddings, too_similar, separate): - """Score each parsed line against every voice.""" - attributed = [] - for line in parsed_lines: - voiced = song.seconds(line["start"], line["end"]) + """Score each parsed line against every voice - and a line longer than + PIECE_SECONDS in pieces too, `uncertain` when they disagree.""" + + def score(start, end): + voiced = song.seconds(start, end) embedding = None if voiced >= MIN_VOICED_SECONDS: - embedding = embed(encoder, song.samples([(line["start"], line["end"])])) + embedding = embed(encoder, song.samples([(start, end)])) + return score_line( + embedding, embeddings, voiced, too_similar, separated=separate + ) + + attributed = [] + for line in parsed_lines: + result = score(line["start"], line["end"]) + if result["voice"] is not None and line["end"] - line["start"] > PIECE_SECONDS: + reason = mixed_reason(piece_voices(line["start"], line["end"], score)) + if reason is not None: + result["uncertain"] = True + result["reason"] = "; ".join(filter(None, [result["reason"], reason])) attributed.append( { "start": _round(line["start"], 3), "end": _round(line["end"], 3), "text": line["text"], - **score_line( - embedding, embeddings, voiced, too_similar, separated=separate - ), + **result, } ) return attributed @@ -731,8 +790,11 @@ def attribute_voices( (letters, digits, '_', '-'). lines: The spans to attribute, in seconds - a list of {start, end, text?} (transcribe_audio's chunk shape) or {start_seconds, - duration_seconds, text?}. Omitted: the song is cut into fixed - windows of window_seconds. + duration_seconds, text?}. A zero-length line (a word chunk with + start == end) has no voice rather than being refused. A line + longer than 2 s is also scored in 2 s pieces, and is uncertain + when they name different voices. Omitted: the song is cut into + fixed windows of window_seconds. windows: Named spans to roll the lines up into, e.g. shots - a list of {name, start, end}. Each reports every voice's share of its voiced time. Omitted: the fixed windows when 'lines' is omitted diff --git a/tests/test_voice_attribution.py b/tests/test_voice_attribution.py index b092963f..e3fa56c9 100644 --- a/tests/test_voice_attribution.py +++ b/tests/test_voice_attribution.py @@ -19,11 +19,13 @@ from dw.tasks.task import _COMMAND_INFO from dw.tasks.voice_attribution import ( MIN_VOICED_SECONDS, + MIXED_LINE_SHARE, UNCERTAIN_MARGIN, UNCERTAIN_SHARE_MARGIN, VOICES_TOO_SIMILAR, attribute_voices, cosine, + mixed_reason, reference_similarity, roll_up, score_line, @@ -327,6 +329,122 @@ def test_a_too_similar_top_two_pair_is_uncertain(self): self.assertIn("too similar", result["reason"]) +# --------------------------------------------------------------------------- +# 4b. Transcript lines that are zero-length or hold two voices (#617) +# --------------------------------------------------------------------------- + + +def duet_audio(): + """4 s of a quiet tone ('a') then 4 s of a loud one ('b'). Both sit well + above the relative voiced floor, so all 8 s is voiced.""" + quiet = tone(4.0, amplitude=0.1) + loud = tone(4.0, amplitude=0.6) + return FakeAudio(numpy.concatenate([quiet, loud], axis=1)) + + +def loudness_embed(encoder, samples): + """'b' for any stretch holding the loud tone, else 'a' - so a whole-line + embedding over both singers names 'b' confidently, as ECAPA did on the + tester's duet.""" + if len(samples) and float(numpy.max(numpy.abs(samples))) > 0.3: + return numpy.array([0.0, 1.0]) + return numpy.array([1.0, 0.0]) + + +class TestTranscriptLines(unittest.TestCase): + def _run(self, audio, lines): + with ( + patch("dw.tasks.voice_attribution._load_embedder") as mock_load_embedder, + patch("dw.tasks.voice_attribution.embed") as mock_embed, + ): + mock_load_embedder.return_value = MagicMock() + mock_embed.side_effect = loudness_embed + return attribute_voices( + audio, + voices={ + "a": [{"start_seconds": 0, "duration_seconds": 4}], + "b": [{"start_seconds": 4, "duration_seconds": 4}], + }, + lines=lines, + separate=False, + ) + + def test_a_zero_length_word_chunk_has_no_voice_rather_than_refusing(self): + # Whisper word timestamps on the tester's duet: chunks [8] and [9] + # both {start 3.96, end 3.96} refused the whole template run + transcript = { + "text": "la la la", + "chunks": [ + {"start": 1.0, "end": 2.0, "text": "la"}, + {"start": 3.96, "end": 3.96, "text": "la"}, + {"start": 5.0, "end": 6.5, "text": "la"}, + ], + } + result = self._run(duet_audio(), transcript) + lines = result["lines"] + self.assertEqual(len(lines), 3) + self.assertEqual(lines[0]["voice"], "a") + self.assertIsNone(lines[1]["voice"]) + self.assertTrue(lines[1]["uncertain"]) + self.assertIn("voiced", lines[1]["reason"]) + self.assertEqual((lines[1]["start"], lines[1]["end"]), (3.96, 3.96)) + self.assertEqual(lines[2]["voice"], "b") + + def test_a_zero_length_chunk_at_the_very_end_is_accepted(self): + result = self._run(duet_audio(), [{"start": 8.0, "end": 8.0}]) + self.assertIsNone(result["lines"][0]["voice"]) + + def test_a_line_ending_before_it_starts_is_still_refused(self): + with self.assertRaisesRegex(ValueError, "lines\\[0\\] ends at 2.0 s"): + self._run(duet_audio(), [{"start": 3.0, "end": 2.0}]) + + def test_a_zero_length_window_is_still_refused(self): + with ( + patch("dw.tasks.voice_attribution._load_embedder"), + self.assertRaisesRegex(ValueError, "windows\\[0\\] .*not after"), + ): + attribute_voices( + duet_audio(), + voices={ + "a": [{"start_seconds": 0, "duration_seconds": 4}], + "b": [{"start_seconds": 4, "duration_seconds": 4}], + }, + lines=[{"start": 0, "end": 2}], + windows=[{"name": "w", "start": 2, "end": 2}], + separate=False, + ) + + def test_one_segment_over_both_singers_is_uncertain_and_names_the_split(self): + # The tester's segment-mode run: one chunk over the whole duet came + # back as one confident voice. Its pieces disagree, so it is uncertain + result = self._run(duet_audio(), [{"start": 0.0, "end": 8.0}]) + line = result["lines"][0] + self.assertEqual(line["voice"], "b") # the whole-line argmax, still reported + self.assertTrue(line["uncertain"]) + self.assertIn("more than one voice", line["reason"]) + self.assertIn("'a' 4.00 s", line["reason"]) + self.assertIn("'b' 4.00 s", line["reason"]) + self.assertEqual(result["thresholds"]["mixed_line_share"], MIXED_LINE_SHARE) + + def test_a_long_line_by_one_singer_stays_confident(self): + result = self._run(duet_audio(), [{"start": 0.0, "end": 4.0}]) + line = result["lines"][0] + self.assertEqual(line["voice"], "a") + self.assertFalse(line["uncertain"]) + self.assertIsNone(line["reason"]) + + def test_a_short_line_is_not_split(self): + # Inside 2 s there is one piece: nothing to disagree with + result = self._run(duet_audio(), [{"start": 3.0, "end": 5.0}]) + self.assertFalse(result["lines"][0]["uncertain"]) + + def test_mixed_reason_needs_the_second_voice_at_the_share(self): + self.assertIsNone(mixed_reason({"a": 9.0, "b": 1.0})) + self.assertIsNone(mixed_reason({"a": 4.0})) + self.assertIsNone(mixed_reason({})) + self.assertIsNotNone(mixed_reason({"a": 6.0, "b": 2.0})) + + # --------------------------------------------------------------------------- # 5. voices_too_similar warnings # --------------------------------------------------------------------------- From 4fa205ad0cbc9f5dfb2fbe11faafb61c3ac7971d Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 17:04:58 -0500 Subject: [PATCH 017/296] feat(pipeline): #618 - H3 audio-hold blocks: hold_audio generates video to a held soundtrack Co-Authored-By: Claude Opus 5.5 --- dw/hold_audio.py | 123 +++++++++ dw/output_extraction.py | 15 + dw/pipeline_processors/components.py | 5 + dw/pipeline_processors/h3_blocks.py | 392 +++++++++++++++++++++++++++ dw/pipeline_processors/pipeline.py | 34 +++ dw/validation.py | 4 + 6 files changed, 573 insertions(+) create mode 100644 dw/hold_audio.py create mode 100644 dw/pipeline_processors/h3_blocks.py diff --git a/dw/hold_audio.py b/dw/hold_audio.py new file mode 100644 index 00000000..100d7042 --- /dev/null +++ b/dw/hold_audio.py @@ -0,0 +1,123 @@ +"""A `hold_audio` argument, checked before the run when it can be. + +`hold_audio` is the MiniMax-H3 soundtrack a step generates its video to +(dw/pipeline_processors/h3_blocks.py). Only an H3 core-denoise pipeline has the +blocks that take it, and only audio can be held - so both are refused here, +where the answer is knowable without loading anything, rather than after a +checkpoint load: + +- on a step whose pipeline is not a modular pipeline, or whose + `from_pretrained` `workflow` is not one of H3's three (`t2va`, `fl2va`, + `ref2va`). A modular pipeline loaded with no `workflow` is left to the + run-time check, which asks the loaded block graph itself; +- a literal path, or an `asset:`/`output:` reference whose name carries an + extension that is not audio, and any `{"media_type": ...}` reference - those + load an image or a video. + +A `previous_result:` names no file yet and is left to the run-time check, the +same as `dw/video_extensions.py` leaves it. +""" + +from . import references +from .adapter_compatibility import FROM_PRETRAINED_KEY, H3_WORKFLOWS, WORKFLOW_KEY +from .argument_media import is_media_reference +from .for_each import MEMBER_SEPARATOR, render_path +from .pipeline_processors.h3_blocks import HOLD_AUDIO_INPUT +from .security import ALLOWED_AUDIO_EXTENSIONS + +MODULAR_PIPELINE = "ModularPipeline" + + +def _not_audio(value): + """Why this `hold_audio` value is not audio, or None - including None for + anything not yet knowable.""" + if is_media_reference(value): + # fetch_media loads it as an image or a video before the call + return ( + f"hold_audio holds a soundtrack, and this is a " + f"'{value.get('media_type')}' reference - name the audio file " + f"directly, or with 'asset:', 'output:' or 'previous_result:'" + ) + if not isinstance(value, str): + return None + if value.startswith(("http://", "https://")): + return None + if references.is_ref(references.UNRESOLVED, value): + return None + if references.is_ref((references.CONSTANT, references.PROMPT), value): + return None + ext = value.rsplit(".", 1) + if len(ext) != 2 or not ext[1] or "/" in ext[1]: + return None + ext = f".{ext[1].lower()}" + if ext in ALLOWED_AUDIO_EXTENSIONS: + return None + return ( + f"hold_audio holds a soundtrack, and '{value}' is not an audio file " + f"({', '.join(sorted(ALLOWED_AUDIO_EXTENSIONS))})" + ) + + +def _not_h3(pipeline): + """Why this step's pipeline cannot hold audio, or None when it can or + cannot be told before the load.""" + configuration = pipeline.get("configuration") + component_type = ( + configuration.get("component_type") if isinstance(configuration, dict) else None + ) + from_pretrained = pipeline.get(FROM_PRETRAINED_KEY) + workflow = ( + from_pretrained.get(WORKFLOW_KEY) if isinstance(from_pretrained, dict) else None + ) + holds = ( + f"hold_audio is a MiniMax-H3 argument, taken by its " + f"{', '.join(sorted(H3_WORKFLOWS))} workflows" + ) + if ( + isinstance(component_type, str) + and not references.is_ref(references.UNRESOLVED, component_type) + and component_type.rsplit(".", 1)[-1] != MODULAR_PIPELINE + ): + return f"{holds}, and this step loads {component_type}" + if isinstance(workflow, str) and workflow not in H3_WORKFLOWS: + return f"{holds}, and this step loads the '{workflow}' workflow" + return None + + +def hold_audio_errors(workflow_definition, source_indices=None): + """Every `hold_audio` argument refused before the run, as [{path, message}]. + + Walks the substituted, expanded definition, the convention + `video_extension_errors` follows: `source_indices` maps an expanded step + back to the one the author wrote, and a path inside a `for_each` member + names the member. + """ + steps = workflow_definition.get("steps") + if not isinstance(steps, list): + return [] + + errors = [] + for index, step in enumerate(steps): + if not isinstance(step, dict): + continue + pipeline = step.get("pipeline") + if not isinstance(pipeline, dict): + continue + arguments = pipeline.get("arguments") + if not isinstance(arguments, dict) or arguments.get(HOLD_AUDIO_INPUT) is None: + continue + source = references.author_index(source_indices, index) + name = step.get("name") + where = ( + f" in member '{name}'" + if isinstance(name, str) and MEMBER_SEPARATOR in name + else "" + ) + path = render_path(("steps", source, "pipeline", "arguments", HOLD_AUDIO_INPUT)) + for problem in (_not_h3(pipeline), _not_audio(arguments[HOLD_AUDIO_INPUT])): + if problem is not None: + errors.append({"path": path, "message": f"{problem}{where}"}) + return errors + + +__all__ = ["hold_audio_errors"] diff --git a/dw/output_extraction.py b/dw/output_extraction.py index c061efa3..8f9ea43a 100644 --- a/dw/output_extraction.py +++ b/dw/output_extraction.py @@ -9,6 +9,7 @@ import torch from .media_types import AudioTrack, AudioVideo +from .pipeline_processors.h3_blocks import HELD_AUDIO_OUTPUT, HELD_AUDIO_RATE_OUTPUT logger = logging.getLogger("dw") @@ -31,6 +32,12 @@ MODULAR_SAMPLE_RATE_KEYS = ("sampling_rate", "audio_sample_rate") +# A MiniMax-H3 step that held a soundtrack: the caller's own track, fitted to the +# video, which plays in place of the decoded audio (the VAE round trip of it) +HELD_AUDIO_KEYS = (HELD_AUDIO_OUTPUT,) +HELD_AUDIO_RATE_KEYS = (HELD_AUDIO_RATE_OUTPUT,) + + def _frames_from_attributes(result): """The frames extractor for a pipeline output that carries `.frames` directly. @@ -156,6 +163,14 @@ def modular_artifacts(result): rate_key, sample_rate = first_item(result, MODULAR_SAMPLE_RATE_KEYS) consumed_keys.add(rate_key) + held_key, held = first_item(result, HELD_AUDIO_KEYS) + if held is not None: + rate_key, held_rate = first_item(result, HELD_AUDIO_RATE_KEYS) + consumed_keys.update({held_key, rate_key}) + # One track, held under every video the call generated + audio, sample_rate = [held[0]] * len(videos), held_rate + consumed_keys.update(HELD_AUDIO_KEYS + HELD_AUDIO_RATE_KEYS) + artifacts = frames_with_audio(videos, audio, sample_rate) # Keys the video/audio pairing above did not consume still need to be saved, not diff --git a/dw/pipeline_processors/components.py b/dw/pipeline_processors/components.py index dd8f8a1d..51c6dcec 100644 --- a/dw/pipeline_processors/components.py +++ b/dw/pipeline_processors/components.py @@ -14,6 +14,7 @@ from ..events import WorkflowCancelled, emit_phase from .. import download_watch from huggingface_hub.errors import HfHubHTTPError +from .h3_blocks import insert_audio_hold from .placement import ( apply_on_demand_placement, create_components_manager, @@ -630,6 +631,10 @@ def load_component( logger.info(f"Loading components for {component_name}") component.load_components(**load_components_arguments) + # MiniMax-H3 takes `hold_audio` on every core-denoise workflow; a + # no-op on any other pipeline (dw/pipeline_processors/h3_blocks.py) + insert_audio_hold(component) + if defer_placement: # The caller places this itself, once it has finished loading the # things that alter the weights - see place_component diff --git a/dw/pipeline_processors/h3_blocks.py b/dw/pipeline_processors/h3_blocks.py new file mode 100644 index 00000000..bec4d091 --- /dev/null +++ b/dw/pipeline_processors/h3_blocks.py @@ -0,0 +1,392 @@ +"""MiniMax-H3 audio hold: generating a video to a soundtrack the caller already has. + +The one place dw modifies a diffusers modular pipeline's block graph. H3 denoises +video and audio rows in one packed sequence, and the rows it treats as conditioning +are the leading `num_condition_audio_rows` of the audio stream: pinned clean at +`t = 1.0`, never stepped by the scheduler, attended to by every generated row. That +is how a Ref2VA soundtrack reference rides along. Holding a track is the same +mechanism pointed at the target rows instead - the caller's waveform is encoded +through diffusers' own Ref2VA reference encode, written over the target audio rows, +and the count is widened to cover them, so the denoiser generates video that fits a +soundtrack it is not allowed to change. + +Two blocks go into each of H3's three core-denoise sequences (`t2va`, `fl2va`, +`ref2va`), always, as no-ops when `hold_audio` is not passed: + +- `DwH3HoldAudioStep`, before `set_timesteps` - after every `prepare_latents*` + step, so Ref2VA's reference rows are already in front and its reference-row + count check has already run against the count it expects; +- `DwH3ReleaseAudioStep`, before `after_denoise` - which slices + `audio_latents[num_condition_audio_rows:]` as the generated track, so the count + has to be back to what it was for the held rows to be decoded as the target. + +The decoded `audio` is still the VAE round trip of the held rows. The step's result +uses `dw_held_audio` instead - the caller's own waveform fitted to the video's +duration (dw/output_extraction.py), which `HELD_AUDIO_OUTPUT` names for the call +(dw/pipeline_processors/pipeline.py). +""" + +import logging + +import torch + +logger = logging.getLogger("dw") + +# The call argument, and the names the hold block leaves in the pipeline state +HOLD_AUDIO_INPUT = "hold_audio" +HELD_ROWS = "dw_held_audio_rows" +HELD_AUDIO_OUTPUT = "dw_held_audio" +HELD_AUDIO_RATE_OUTPUT = "dw_held_audio_sampling_rate" + +# The block names dw inserts, and the diffusers block each goes in front of +HOLD_BLOCK = "dw_hold_audio" +RELEASE_BLOCK = "dw_release_audio" +HOLD_BEFORE = "set_timesteps" +RELEASE_BEFORE = "after_denoise" + +# MiniMaxH3Blocks: the top-level step holding the auto denoise step, and the three +# core-denoise sequences that step chooses between +DENOISE_STEP = "denoise" +CORE_DENOISE_SEQUENCES = ("t2va", "fl2va", "ref2va") + +# Encoded silence pads a track shorter than the video: this many latents of zeros +# are encoded and the middle one taken, clear of any edge effect of the VAE's +SILENCE_LATENTS = 4 + + +def _diffusers(): + """The diffusers pieces the blocks are built from - imported on first use, so + importing this module does not import a modular pipeline.""" + from diffusers.modular_pipelines.minimax_h3 import before_encoder, encoders + from diffusers.modular_pipelines.minimax_h3.modular_pipeline import ( + MINIMAX_H3_AUDIO_LATENTS_PER_SECOND, + ) + from diffusers.modular_pipelines.minimax_h3.references import ( + MiniMaxH3AudioReference, + ) + + return ( + before_encoder.MiniMaxH3Ref2VASetupStep, + encoders.MiniMaxH3Ref2VAReferenceEncoderStep, + MiniMaxH3AudioReference, + MINIMAX_H3_AUDIO_LATENTS_PER_SECOND, + ) + + +def as_channels_samples(waveform): + """A waveform as a float32 `(channels, samples)` tensor - a bare mono + `(samples,)` one gains its channel axis.""" + waveform = torch.as_tensor(waveform).float() + if waveform.ndim == 1: + waveform = waveform[None] + return waveform + + +def fit_samples(waveform, num_samples): + """Crop a `(channels, samples)` waveform to `num_samples`, or pad it with + silence to that length.""" + missing = num_samples - waveform.shape[-1] + if missing <= 0: + return waveform[..., :num_samples] + return torch.nn.functional.pad(waveform, (0, missing)) + + +def fit_latents(latents, num_latents, silence): + """Crop `(channels, n, C)` audio latents to `num_latents` per channel, or pad + them with `silence` - one `(channels, 1, C)` latent of encoded silence, + repeated.""" + missing = num_latents - latents.shape[1] + if missing <= 0: + return latents[:, :num_latents] + padding = silence.to(latents.dtype).expand(-1, missing, -1) + return torch.cat([latents, padding], dim=1) + + +def _make_blocks(): + """The two block classes, defined against the installed diffusers.""" + from diffusers.modular_pipelines.modular_pipeline import ( + ModularPipelineBlocks, + PipelineState, + ) + from diffusers.modular_pipelines.modular_pipeline_utils import ( + InputParam, + OutputParam, + ) + + ( + setup_step, + reference_encoder, + audio_reference, + latents_per_second, + ) = _diffusers() + + class DwH3HoldAudioStep(ModularPipelineBlocks): + model_name = "minimax-h3" + + def __init__(self): + super().__init__() + # Called, not copied: the encode a Ref2VA soundtrack reference takes + self._encoder = reference_encoder() + + @property + def description(self): + return ( + "dw: holds `hold_audio` as the target soundtrack - encodes it like a Ref2VA soundtrack reference, " + "writes it over the target audio rows and widens `num_condition_audio_rows` over them, so the " + "denoiser treats it as clean conditioning. Does nothing without `hold_audio`." + ) + + @property + def inputs(self): + return [ + InputParam( + name=HOLD_AUDIO_INPUT, + type_hint=audio_reference, + default=None, + description="The soundtrack to generate the video to, as a MiniMaxH3AudioReference.", + ), + InputParam(name="audio_latents", type_hint=torch.Tensor, required=True), + InputParam(name="num_audio_latents", type_hint=int, required=True), + InputParam(name="num_condition_audio_rows", type_hint=int, default=0), + InputParam(name="num_frames", type_hint=int, required=True), + ] + + @property + def intermediate_outputs(self): + return [ + OutputParam("audio_latents", type_hint=torch.Tensor), + OutputParam("num_condition_audio_rows", type_hint=int), + OutputParam(HELD_ROWS, type_hint=int), + OutputParam(HELD_AUDIO_OUTPUT, type_hint=torch.Tensor), + OutputParam(HELD_AUDIO_RATE_OUTPUT, type_hint=int), + ] + + def _encode(self, components, waveform): + """`(channels, n, C)` normalized audio latents of a VAE-rate stereo waveform.""" + sub_state = PipelineState() + sub_state.set( + "normalized_references", + [ + audio_reference( + audio=waveform, sample_rate=components.audio_sampling_rate + ) + ], + ) + _, sub_state = self._encoder(components, sub_state) + rows = sub_state.get("audio_condition_latents")[0] + return rows.reshape( + components.audio_channels, -1, components.audio_latent_channels + ) + + @torch.no_grad() + def __call__(self, components, state): + block_state = self.get_block_state(state) + block_state.dw_held_audio_rows = 0 + block_state.dw_held_audio = None + block_state.dw_held_audio_sampling_rate = None + held = block_state.hold_audio + if held is None: + self.set_block_state(state, block_state) + return components, state + + vae_rate = components.audio_sampling_rate + duration = block_state.num_frames / components.fps + num_latents = block_state.num_audio_latents + samples_per_latent = vae_rate // latents_per_second + original = as_channels_samples(held.audio) + rate = held.sample_rate or vae_rate + + # Onto the VAE's rate and channels the way a Ref2VA reference is, then + # padded with silence to the whole target - a short track encodes + # with its silence rather than having it appended in latent space + waveform = setup_step._normalize_audio_condition( + original, rate, vae_rate, max_duration=duration + ) + waveform = fit_samples(waveform, num_latents * samples_per_latent) + latents = self._encode(components, waveform) + if latents.shape[1] < num_latents: + silence = self._encode( + components, + torch.zeros(2, SILENCE_LATENTS * samples_per_latent), + ) + middle = silence.shape[1] // 2 + latents = fit_latents( + latents, num_latents, silence[:, middle : middle + 1] + ) + else: + latents = fit_latents(latents, num_latents, None) + + # Channel-major target rows, after any Ref2VA reference rows + audio_latents = block_state.audio_latents + start = block_state.num_condition_audio_rows or 0 + rows = latents.reshape(-1, components.audio_latent_channels) + audio_latents[start : start + rows.shape[0]] = rows.to( + audio_latents.device, audio_latents.dtype + ) + block_state.audio_latents = audio_latents + block_state.num_condition_audio_rows = start + rows.shape[0] + block_state.dw_held_audio_rows = rows.shape[0] + + # What the step's result plays: the caller's own track, not the VAE + # round trip of it, fitted to the video at the rate it came at + fitted = fit_samples(original, int(round(duration * rate))) + block_state.dw_held_audio = fitted[None] + block_state.dw_held_audio_sampling_rate = int(rate) + + logger.info( + f"Holding {original.shape[-1] / rate:.2f}s of audio as the " + f"{duration:.2f}s target soundtrack ({rows.shape[0]} rows)" + ) + self.set_block_state(state, block_state) + return components, state + + class DwH3ReleaseAudioStep(ModularPipelineBlocks): + model_name = "minimax-h3" + + @property + def description(self): + return ( + "dw: restores `num_condition_audio_rows` to what it was before the hold, so the held rows decode " + "as the generated soundtrack." + ) + + @property + def inputs(self): + return [ + InputParam(name="num_condition_audio_rows", type_hint=int, default=0), + InputParam(name=HELD_ROWS, type_hint=int, default=0), + ] + + @property + def intermediate_outputs(self): + return [ + OutputParam("num_condition_audio_rows", type_hint=int), + OutputParam(HELD_ROWS, type_hint=int), + ] + + @torch.no_grad() + def __call__(self, components, state): + block_state = self.get_block_state(state) + held = block_state.dw_held_audio_rows or 0 + if held: + block_state.num_condition_audio_rows -= held + block_state.dw_held_audio_rows = 0 + self.set_block_state(state, block_state) + return components, state + + return DwH3HoldAudioStep, DwH3ReleaseAudioStep + + +_BLOCKS = None + + +def blocks(): + """(DwH3HoldAudioStep, DwH3ReleaseAudioStep), built once.""" + global _BLOCKS + if _BLOCKS is None: + _BLOCKS = _make_blocks() + return _BLOCKS + + +def core_denoise_sequences(pipeline): + """[(prefix, sequence)] for every H3 core-denoise sequence a pipeline's own + block graph holds - empty for anything that is not a MiniMax-H3 modular + pipeline. + + Two shapes. Loaded whole, `denoise` is an auto step choosing between the + three sequences, each holding `set_timesteps`. Loaded with a + `from_pretrained` `workflow` (what the catalog does), diffusers prunes the + graph to that workflow's execution blocks, flattened into the top level + under dotted names - `denoise.set_timesteps` - so the top level is the one + sequence and `denoise.` the prefix its block names carry. + + `_blocks`, not `blocks`: the property hands back a deep copy, and an edit to + that copy never reaches the graph the pipeline runs. + """ + try: + from diffusers.modular_pipelines.minimax_h3 import MiniMaxH3ModularPipeline + except ImportError: + return [] + if not isinstance(pipeline, MiniMaxH3ModularPipeline): + return [] + top_blocks = getattr(pipeline, "_blocks", None) + top = getattr(top_blocks, "sub_blocks", None) or {} + flat_prefix = f"{DENOISE_STEP}." + if f"{flat_prefix}{HOLD_BEFORE}" in top: + return [(flat_prefix, top_blocks)] + denoise = getattr(top.get(DENOISE_STEP), "sub_blocks", None) or {} + return [ + ("", denoise[name]) + for name in CORE_DENOISE_SEQUENCES + if name in denoise and hasattr(denoise[name], "sub_blocks") + ] + + +def insert_audio_hold(pipeline): + """Put the hold and release blocks into every H3 core-denoise sequence. + + Idempotent - a sequence already holding them is left alone. Returns whether + the pipeline now holds audio; a sequence missing an anchor (a diffusers that + renamed `set_timesteps` or `after_denoise`) is skipped with a warning, and a + pipeline left with none refuses `hold_audio` at the call. + """ + hold_block, release_block = blocks() + inserted = False + for prefix, sequence in core_denoise_sequences(pipeline): + sub_blocks = sequence.sub_blocks + hold_name, release_name = prefix + HOLD_BLOCK, prefix + RELEASE_BLOCK + hold_before, release_before = prefix + HOLD_BEFORE, prefix + RELEASE_BEFORE + if hold_name in sub_blocks: + inserted = True + continue + names = list(sub_blocks) + if hold_before not in names or release_before not in names: + logger.warning( + f"MiniMax-H3 has a denoise sequence with no '{hold_before}'/" + f"'{release_before}' step to anchor the audio hold to, so it " + f"cannot hold audio" + ) + continue + sub_blocks.insert(hold_name, hold_block(), names.index(hold_before)) + names = list(sub_blocks) + sub_blocks.insert(release_name, release_block(), names.index(release_before)) + inserted = True + return inserted + + +def holds_audio(pipeline): + """Whether a pipeline's block graph carries the audio hold.""" + return any( + prefix + HOLD_BLOCK in sequence.sub_blocks + for prefix, sequence in core_denoise_sequences(pipeline) + ) + + +def hold_audio_reference(value): + """A `hold_audio` argument as the MiniMaxH3AudioReference the hold block takes. + + Whatever an argument can hold audio as: a file path (an `asset:` or `output:` + reference arrives resolved to one), a step's artifact - an AudioTrack, an + AudioVideo, a bare waveform - or a reference already built. + + Raises: + ValueError: If the value is not audio + """ + from ..arguments import media_arguments + from ..security import ALLOWED_AUDIO_EXTENSIONS + + audio_reference = _diffusers()[2] + if isinstance(value, audio_reference): + return value + if isinstance(value, str): + extension = "." + value.rsplit(".", 1)[-1].lower() if "." in value else "" + if extension not in ALLOWED_AUDIO_EXTENSIONS: + raise ValueError( + f"hold_audio holds a soundtrack, and '{value}' is not an audio file " + f"({', '.join(sorted(ALLOWED_AUDIO_EXTENSIONS))})" + ) + return audio_reference.from_file(value) + try: + return audio_reference(**media_arguments(audio_reference, value)) + except ValueError as error: + raise ValueError(f"hold_audio: {error}") from error diff --git a/dw/pipeline_processors/pipeline.py b/dw/pipeline_processors/pipeline.py index ba6159d5..ed14dc9d 100644 --- a/dw/pipeline_processors/pipeline.py +++ b/dw/pipeline_processors/pipeline.py @@ -24,6 +24,13 @@ attention_slicing_requested, place_component, ) +from .h3_blocks import ( + HELD_AUDIO_OUTPUT, + HELD_AUDIO_RATE_OUTPUT, + HOLD_AUDIO_INPUT, + hold_audio_reference, + holds_audio, +) from .progress import reported_blocks, reported_progress_bars from .remote import remote_text_encoder from ..type_helpers import has_method @@ -576,6 +583,7 @@ def _execute_pipeline(self, arguments): def _call_pipeline(self, arguments, attn_backend): """Call the pipeline with optional attention backend and cache contexts.""" arguments = self._with_step_callback(arguments) + arguments = self._with_held_audio(arguments) # The load is over and the denoise loop is starting. Pipelines whose # signature has no step callback report nothing else at all, so this # is the only thing that distinguishes running from still loading @@ -600,6 +608,32 @@ def _call_pipeline(self, arguments, attn_backend): return self.pipeline(**arguments) + def _with_held_audio(self, arguments): + """`hold_audio` as the reference the H3 hold block takes, and the + held track asked for alongside `audio` (dw/pipeline_processors/h3_blocks.py). + + Raises: + ValueError: If this pipeline cannot hold audio, or the value is not audio + """ + held = arguments.get(HOLD_AUDIO_INPUT) + if held is None: + return arguments + if not holds_audio(self.pipeline): + raise ValueError( + f"Step '{self.name}': hold_audio is a MiniMax-H3 argument " + f"(t2va, fl2va or ref2va), and {type(self.pipeline).__name__} " + f"cannot hold a soundtrack" + ) + arguments = dict(arguments) + arguments[HOLD_AUDIO_INPUT] = hold_audio_reference(held) + output = arguments.get("output") + if isinstance(output, (list, tuple)) and "audio" in output: + arguments["output"] = list(output) + [ + HELD_AUDIO_OUTPUT, + HELD_AUDIO_RATE_OUTPUT, + ] + return arguments + def _takes_step_callback(self): """Whether this pipeline names `callback_on_step_end` in its own signature. Only a pipeline that names the parameter explicitly gets diff --git a/dw/validation.py b/dw/validation.py index d2f4f686..84d24c83 100644 --- a/dw/validation.py +++ b/dw/validation.py @@ -39,6 +39,7 @@ from .content_types import content_type_errors from .dissolve_frame_errors import dissolve_frame_errors from .for_each import ForEachError, entry_field_warnings +from .hold_audio import hold_audio_errors from .introspection import task_signature_errors from .kernel_availability import kernel_availability_errors from .library import SubWorkflowNotFound @@ -305,6 +306,9 @@ def _task_errors(context): "video_extensions", lambda c: video_extension_errors(c.expanded, c.source_indices), ), + # hold_audio on a pipeline with no H3 hold blocks, or holding something + # that is not audio, costs a checkpoint load otherwise (dw/hold_audio.py) + Check("hold_audio", lambda c: hold_audio_errors(c.expanded, c.source_indices)), # A result content_type no writer will accept - a bare word like "video" # validated clean and then died inside the writer (dw/content_types.py, # #168) From 132ef3fbac9a20d5433e178f669af226df64fc44 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 17:09:58 -0500 Subject: [PATCH 018/296] feat(pipeline): #618 - hold_audio tests, guide section and architecture row Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 1 + docs/WORKFLOW_GUIDE.md | 34 +++ tests/test_h3_hold_audio.py | 583 ++++++++++++++++++++++++++++++++++++ tests/test_validation.py | 1 + 4 files changed, 619 insertions(+) create mode 100644 tests/test_h3_hold_audio.py diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index ddf7ef65..549562b7 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -87,6 +87,7 @@ one to open. | H3 Ref2VA VRAM numbers | `workflows/templates/minimax/` | Every Ref2VA template declares `base_gb` 16.0, `bytes_per_voxel` 28.71 and `gb_per_reference` 1.0. | `tests/test_h3_vram_ceiling.py::test_every_ref2va_minimax_template_declares_gb_per_reference` | | Inherited VRAM ceiling | `dw/vram_inheritance.py`, `dw/server/deps.py`: `ceiling_index` | A workflow with no `vram_estimate` is matched against the catalog by pipeline identity, and is warned, never refused. | `tests/test_vram_inheritance.py::test_every_template_declaring_one_identity_declares_the_same_numbers` | | H3 adapter partition | `dw/adapter_compatibility.py` | An FL2VA LoRA on a reference step is refused, and a LoRA whose file name says neither `ref2v` nor `fl2v` is warned. | `tests/test_h3_adapters.py::TestWhatIsRefused::test_a_keyframe_adapter_on_the_reference_path`, `tests/test_h3_adapters.py::TestWhatIsRefused::test_an_unrecognised_name_is_a_warning_not_a_refusal` | +| H3 audio hold | `dw/pipeline_processors/h3_blocks.py`: `insert_audio_hold`, `core_denoise_sequences`, `dw/hold_audio.py`: `hold_audio_errors` | The one place dw modifies a diffusers modular pipeline's block graph: two blocks go in before `set_timesteps` and `after_denoise` in every H3 core-denoise sequence (the whole graph and the `workflow=`-pruned flat graph), and they are no-ops without `hold_audio`. | `tests/test_h3_hold_audio.py::TestAnchors::test_inserted_next_to_the_anchors_in_every_shape`, `tests/test_h3_hold_audio.py::TestRefusals::test_a_non_h3_step_is_refused` | | IC-LoRA reference scale | `workflows/templates/ltx2/` | The three conditioning templates run at `reference_downscale_factor: 1`, and the two upscalers at 2. | `tests/test_ltx2_ic_loras.py::TestEachTemplateMatchesItsCard::test_the_reference_is_encoded_at_the_output_resolution`, `tests/test_ltx2_ic_loras.py::TestTheGenerativeUpscaleMatchesTheSameCard::test_it_loads_the_upscaler_at_factor_two` | | IC-LoRA numbers | `workflows/templates/ltx2/` | Every number in the IC-LoRA templates comes from the vendor's model card. | `tests/test_ltx2_ic_loras.py::TestEachTemplateMatchesItsCard::test_the_strength_is_the_cards_default`, `tests/test_ltx2_ic_loras.py::TestEachTemplateMatchesItsCard::test_the_defaults_are_the_trained_bucket` | | IC-LoRA prompt genre | `prompts/ltx2/` | A stored prompt tagged `ic-lora` is checked against its trained caption form, not against the T2V paragraph rule. | `tests/test_ltx_prompt_library.py::test_an_ic_lora_prompt_is_in_its_trained_form` | diff --git a/docs/WORKFLOW_GUIDE.md b/docs/WORKFLOW_GUIDE.md index 613ddd4f..d20e445b 100644 --- a/docs/WORKFLOW_GUIDE.md +++ b/docs/WORKFLOW_GUIDE.md @@ -1587,6 +1587,40 @@ it — see [workflows/templates/minimax/last-frame-only.json](../workflows/templ See [workflows/templates/minimax/music.json](../workflows/templates/minimax/music.json) and [workflows/templates/minimax/video-with-audio.json](../workflows/templates/minimax/video-with-audio.json) for full examples. +### H3: generating to a held soundtrack + +A MiniMax-H3 step (`t2va`, `fl2va` or `ref2va`) takes a pipeline argument `hold_audio`: +an audio value, spelled `asset:`, `output:` or `previous_result:.audio` +(an `AudioTrack` or `AudioVideo`). The video is generated *to* that soundtrack. The +track goes through H3's Ref2VA reference-audio encode, is written over the target audio +rows and held clean as conditioning through denoise, so the model draws picture that +fits audio it is not allowed to change: + +```json +"pipeline": { + "arguments": { + "prompt": "a drummer in a lit rehearsal room", + "num_frames": 125, + "hold_audio": "asset:track.wav" + }, + "output": ["videos", "audio", "sampling_rate"] +} +``` + +`"audio"` must be in `output` (as it already is in `video-with-audio`) for the held +track to be what the step returns. A track longer than the video is cropped, a shorter +one padded with silence. The step's `audio` is the caller's original waveform fitted to +the video's duration (`num_frames / 24` s) at its own sample rate, not a VAE round trip. +`previous_result:base.audio` holds a track an earlier step generated, music for example. + +`run_workflow`'s `arguments` can set only a declared variable, so unless the template +has one, add `"hold_audio": "variable:hold_audio"` and a `hold_audio` variable, or edit +the argument. Validation refuses `hold_audio` on a step that is not H3, on a modular +pipeline loaded with a workflow other than `t2va`, `fl2va` or `ref2va`, on a path whose +extension is not `.wav`, `.mp3`, `.flac` or `.ogg`, and on any `{"media_type": ...}` +dict; a run refuses a pipeline with no hold blocks or a value that is not audio. Without +`hold_audio` the output is unchanged. + ### Promoting an H3 take to 768p in latent space: upscale_h3_latents and decode_h3_latents Once a 960x544 MiniMax-H3 take reads the way it should, `upscale_h3_latents` and diff --git a/tests/test_h3_hold_audio.py b/tests/test_h3_hold_audio.py new file mode 100644 index 00000000..a99d0be1 --- /dev/null +++ b/tests/test_h3_hold_audio.py @@ -0,0 +1,583 @@ +"""MiniMax-H3 audio hold (#598): the blocks dw inserts into diffusers' graph, the +refusals made before a load, and the held track a step's result plays.""" + +import copy +import json +import os +import tempfile +from types import SimpleNamespace + +import pytest +import torch + +from dw.hold_audio import hold_audio_errors +from dw.media_types import AudioTrack, AudioVideo +from dw.output_extraction import modular_artifacts +from dw.pipeline_processors import h3_blocks +from dw.pipeline_processors.h3_blocks import ( + HELD_AUDIO_OUTPUT, + HELD_AUDIO_RATE_OUTPUT, + HELD_ROWS, + HOLD_BLOCK, + RELEASE_BLOCK, + as_channels_samples, + blocks, + core_denoise_sequences, + fit_latents, + fit_samples, + hold_audio_reference, + holds_audio, + insert_audio_hold, +) +from dw.pipeline_processors.pipeline import Pipeline +from dw.workflow import workflow_from_definition + +minimax = pytest.importorskip("diffusers.modular_pipelines.minimax_h3") +from diffusers.modular_pipelines.modular_pipeline import PipelineState # noqa: E402 +from diffusers.modular_pipelines.minimax_h3.modular_pipeline import ( # noqa: E402 + MINIMAX_H3_AUDIO_LATENTS_PER_SECOND, +) +from diffusers.modular_pipelines.minimax_h3.references import ( # noqa: E402 + MiniMaxH3AudioReference, +) + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +WORKFLOWS = ("t2va", "fl2va", "ref2va") +RATE = 32000 +CHANNELS = 4 +HOP = RATE // MINIMAX_H3_AUDIO_LATENTS_PER_SECOND + + +def pipelines(): + """[(label, ModularPipeline)]: the whole graph and each pruned workflow.""" + blocks_ = minimax.MiniMaxH3Blocks() + result = [("whole", blocks_.init_pipeline())] + for name in WORKFLOWS: + result.append((name, blocks_.get_workflow(name).init_pipeline())) + return result + + +@pytest.fixture(scope="module") +def built(): + return pipelines() + + +# 1. Anchors in the installed diffusers + + +class TestAnchors: + def test_inserted_next_to_the_anchors_in_every_shape(self, built): + for label, pipeline in built: + assert insert_audio_hold(pipeline) is True, label + assert holds_audio(pipeline), label + sequences = core_denoise_sequences(pipeline) + assert sequences, label + if label == "whole": + assert [prefix for prefix, _ in sequences] == [""] * 3 + else: + assert [prefix for prefix, _ in sequences] == ["denoise."] + for prefix, sequence in sequences: + names = list(sequence.sub_blocks) + hold = names.index(prefix + HOLD_BLOCK) + release = names.index(prefix + RELEASE_BLOCK) + assert names[hold + 1] == prefix + "set_timesteps", label + assert names[release + 1] == prefix + "after_denoise", label + + def test_second_call_does_not_duplicate(self, built): + for label, pipeline in built: + insert_audio_hold(pipeline) + before = [list(s.sub_blocks) for _, s in core_denoise_sequences(pipeline)] + assert insert_audio_hold(pipeline) is True + after = [list(s.sub_blocks) for _, s in core_denoise_sequences(pipeline)] + assert before == after, label + for names in after: + assert ( + names.count(HOLD_BLOCK) + names.count("denoise." + HOLD_BLOCK) == 1 + ) + assert ( + names.count(RELEASE_BLOCK) + names.count("denoise." + RELEASE_BLOCK) + == 1 + ) + + def test_hold_audio_is_a_pipeline_input(self, built): + for label, pipeline in built: + insert_audio_hold(pipeline) + names = [param.name for param in pipeline._blocks.inputs] + assert "hold_audio" in names, label + + def test_a_pipeline_without_the_blocks_does_not_hold(self): + assert not holds_audio(minimax.MiniMaxH3Blocks().init_pipeline()) + other = SimpleNamespace(_blocks=SimpleNamespace(sub_blocks={})) + assert core_denoise_sequences(other) == [] + assert holds_audio(other) is False + assert core_denoise_sequences(object()) == [] + assert insert_audio_hold(object()) is False + + def test_an_unanchored_sequence_is_skipped(self): + pipeline = minimax.MiniMaxH3Blocks().get_workflow("t2va").init_pipeline() + pipeline._blocks.sub_blocks.pop("denoise.after_denoise") + assert insert_audio_hold(pipeline) is False + assert not holds_audio(pipeline) + + +# 2. Crop and pad + + +class TestFitting: + def test_samples_crop(self): + wave = torch.arange(10.0).repeat(2, 1) + assert torch.equal(fit_samples(wave, 4), wave[:, :4]) + + def test_samples_pad_with_silence(self): + wave = torch.ones(2, 3) + fitted = fit_samples(wave, 5) + assert fitted.shape == (2, 5) + assert torch.equal(fitted[:, :3], wave) + assert torch.count_nonzero(fitted[:, 3:]) == 0 + + def test_samples_exact_is_unchanged(self): + wave = torch.ones(2, 3) + assert torch.equal(fit_samples(wave, 3), wave) + + def test_latents_crop_per_channel(self): + latents = torch.arange(2 * 6 * 3.0).reshape(2, 6, 3) + assert torch.equal(fit_latents(latents, 4, None), latents[:, :4]) + + def test_latents_pad_with_the_repeated_silence(self): + latents = torch.ones(2, 2, 3) + silence = torch.tensor([[[1.0, 2.0, 3.0]], [[4.0, 5.0, 6.0]]]) + fitted = fit_latents(latents, 5, silence) + assert fitted.shape == (2, 5, 3) + assert torch.equal(fitted[:, :2], latents) + for i in range(2, 5): + assert torch.equal(fitted[:, i : i + 1], silence) + + def test_mono_gains_a_channel_axis(self): + assert as_channels_samples(torch.zeros(7)).shape == (1, 7) + assert as_channels_samples(torch.zeros(2, 7)).shape == (2, 7) + assert as_channels_samples(torch.zeros(2, 7, dtype=torch.float64)).dtype == ( + torch.float32 + ) + + +# 3 and 4. The real block path on CPU + + +class FakePosterior: + def __init__(self, values): + self.values = values + + def mode(self): + return self.values + + +class FakeAudioVae: + config = SimpleNamespace( + latents_mean=[0.0] * CHANNELS, latents_std=[1.0] * CHANNELS + ) + + def encode(self, x, return_dict=False): + """(batch, 1, samples) -> (batch, C, samples // HOP): each hop's mean, plus + a per-channel offset.""" + n = x.shape[-1] // HOP + hops = x[:, 0, : n * HOP].reshape(x.shape[0], n, HOP).mean(-1) + offsets = torch.arange(CHANNELS, dtype=torch.float32)[None, :, None] * 0.01 + return (FakePosterior(hops[:, None, :] + offsets),) + + +def components(): + return SimpleNamespace( + audio_sampling_rate=RATE, + fps=24, + audio_channels=2, + audio_latent_channels=CHANNELS, + _execution_device=torch.device("cpu"), + audio_vae=FakeAudioVae(), + ) + + +NUM_FRAMES = 48 # two seconds at 24 fps +NUM_LATENTS = 2 * MINIMAX_H3_AUDIO_LATENTS_PER_SECOND + + +def make_state(held=None, reference_rows=0): + rows = reference_rows + 2 * NUM_LATENTS + audio_latents = torch.randn(rows, CHANNELS, generator=torch.manual_seed(1)) + if reference_rows: + audio_latents[:reference_rows] = 7.0 + state = PipelineState() + state.set("audio_latents", audio_latents) + state.set("num_audio_latents", NUM_LATENTS) + state.set("num_condition_audio_rows", reference_rows) + state.set("num_frames", NUM_FRAMES) + if held is not None: + state.set("hold_audio", held) + return state + + +def expected_rows(waveform_32k): + """The stub encode of a stereo waveform, channel-major.""" + x = waveform_32k[:, None] + latents = FakeAudioVae().encode(x)[0].mode().transpose(1, 2) + return latents.reshape(-1, CHANNELS) + + +class TestHoldBlock: + def test_a_longer_track_is_cropped_and_written_over_the_target_rows(self): + wave = torch.randn(2, 3 * RATE, generator=torch.manual_seed(2)) + state = make_state(MiniMaxH3AudioReference(audio=wave, sample_rate=RATE)) + comps = components() + hold, _ = blocks() + hold()(comps, state) + + latents = state.get("audio_latents") + count = state.get("num_condition_audio_rows") + assert count == 2 * NUM_LATENTS + assert state.get(HELD_ROWS) == 2 * NUM_LATENTS + expected = expected_rows(wave[:, : NUM_LATENTS * HOP]) + assert torch.allclose(latents[:count], expected, atol=1e-6) + + held = state.get(HELD_AUDIO_OUTPUT) + assert held.shape == (1, 2, round(NUM_FRAMES / 24 * RATE)) + assert torch.equal(held[0], wave[:, : held.shape[-1]]) + assert state.get(HELD_AUDIO_RATE_OUTPUT) == RATE + + def test_a_shorter_track_is_padded_with_silence(self): + wave = torch.randn(2, RATE, generator=torch.manual_seed(3)) + state = make_state(MiniMaxH3AudioReference(audio=wave, sample_rate=RATE)) + comps = components() + blocks()[0]()(comps, state) + + latents = state.get("audio_latents") + rows = state.get(HELD_ROWS) + assert rows == 2 * NUM_LATENTS + per_channel = latents[:rows].reshape(2, NUM_LATENTS, CHANNELS) + half = MINIMAX_H3_AUDIO_LATENTS_PER_SECOND + real = expected_rows(wave).reshape(2, half, CHANNELS) + assert torch.allclose(per_channel[:, :half], real, atol=1e-6) + # silence is zeros through the stub, so only the channel offset remains + offsets = torch.arange(CHANNELS, dtype=torch.float32) * 0.01 + assert torch.allclose( + per_channel[:, half:], offsets.expand(2, NUM_LATENTS - half, -1), atol=1e-6 + ) + + held = state.get(HELD_AUDIO_OUTPUT) + assert held.shape == (1, 2, 2 * RATE) + assert torch.equal(held[0, :, :RATE], wave) + assert torch.count_nonzero(held[0, :, RATE:]) == 0 + + def test_the_held_track_keeps_its_own_rate_and_channels(self): + wave = torch.randn(1, 3 * 16000, generator=torch.manual_seed(4)) + state = make_state(MiniMaxH3AudioReference(audio=wave, sample_rate=16000)) + pytest.importorskip("torchaudio") + blocks()[0]()(components(), state) + + held = state.get(HELD_AUDIO_OUTPUT) + assert held.shape == (1, 1, 2 * 16000) + assert state.get(HELD_AUDIO_RATE_OUTPUT) == 16000 + assert state.get(HELD_ROWS) == 2 * NUM_LATENTS + + def test_reference_rows_in_front_are_untouched(self): + k = 6 + wave = torch.randn(2, 2 * RATE, generator=torch.manual_seed(5)) + state = make_state( + MiniMaxH3AudioReference(audio=wave, sample_rate=RATE), reference_rows=k + ) + comps = components() + hold, release = blocks() + hold()(comps, state) + + latents = state.get("audio_latents") + assert torch.equal(latents[:k], torch.full((k, CHANNELS), 7.0)) + assert state.get("num_condition_audio_rows") == k + 2 * NUM_LATENTS + assert torch.allclose( + latents[k:], expected_rows(wave[:, : NUM_LATENTS * HOP]), atol=1e-6 + ) + + release()(comps, state) + assert state.get("num_condition_audio_rows") == k + assert state.get(HELD_ROWS) == 0 + + def test_release_restores_the_count(self): + wave = torch.randn(2, 2 * RATE, generator=torch.manual_seed(6)) + state = make_state(MiniMaxH3AudioReference(audio=wave, sample_rate=RATE)) + comps = components() + hold, release = blocks() + hold()(comps, state) + assert state.get("num_condition_audio_rows") > 0 + release()(comps, state) + assert state.get("num_condition_audio_rows") == 0 + + +class TestNoHold: + def test_the_hold_block_is_a_no_op(self): + state = make_state(reference_rows=3) + before = state.get("audio_latents").clone() + blocks()[0]()(components(), state) + + assert torch.equal(state.get("audio_latents"), before) + assert state.get("num_condition_audio_rows") == 3 + assert state.get(HELD_ROWS) == 0 + assert state.get(HELD_AUDIO_OUTPUT) is None + + def test_hold_then_release_leaves_the_state_as_it_was(self): + """The real set-timesteps step needs schedulers and the packed layout, which + is impractical to stub, so this compares the state across hold and release + with no `hold_audio` instead.""" + state = make_state(reference_rows=3) + before = state.get("audio_latents").clone() + comps = components() + hold, release = blocks() + hold()(comps, state) + release()(comps, state) + + assert torch.equal(state.get("audio_latents"), before) + assert state.get("num_condition_audio_rows") == 3 + assert state.get(HELD_ROWS) == 0 + + +# 5. Refusals + + +def h3_step(workflow="t2va", **arguments): + return { + "name": "video", + "pipeline": { + "configuration": {"component_type": "ModularPipeline"}, + "from_pretrained_arguments": {"workflow": workflow}, + "arguments": arguments, + }, + } + + +def errors_for(step): + return hold_audio_errors({"steps": [step]}) + + +class TestRefusals: + def test_a_non_h3_step_is_refused(self): + step = h3_step(hold_audio="asset:track.wav") + step["pipeline"]["configuration"]["component_type"] = "StableDiffusionPipeline" + errors = errors_for(step) + assert len(errors) == 1 + assert "StableDiffusionPipeline" in errors[0]["message"] + assert errors[0]["path"].endswith("hold_audio") + + @pytest.mark.parametrize("workflow", ["i2v", "t2v", "text2image"]) + def test_a_workflow_that_is_not_an_h3_one_is_refused(self, workflow): + errors = errors_for(h3_step(workflow, hold_audio="asset:track.wav")) + assert len(errors) == 1 + assert workflow in errors[0]["message"] + + @pytest.mark.parametrize("workflow", WORKFLOWS) + def test_the_h3_workflows_are_accepted(self, workflow): + assert errors_for(h3_step(workflow, hold_audio="asset:track.wav")) == [] + + @pytest.mark.parametrize("value", ["asset:frame.png", "clip.mp4", "x.png"]) + def test_a_non_audio_path_is_refused(self, value): + errors = errors_for(h3_step(hold_audio=value)) + assert len(errors) == 1 + assert "not an audio file" in errors[0]["message"] + + def test_a_media_reference_is_refused(self): + value = {"media_type": "image", "location": "asset:frame.png"} + errors = errors_for(h3_step(hold_audio=value)) + assert len(errors) == 1 + assert "'image' reference" in errors[0]["message"] + + @pytest.mark.parametrize( + "value", ["asset:track.wav", "previous_result:base.audio", "output:a/b.mp3"] + ) + def test_audio_and_deferred_values_are_accepted(self, value): + assert errors_for(h3_step(hold_audio=value)) == [] + + def test_no_workflow_and_no_hold_audio_say_nothing(self): + step = h3_step(hold_audio="asset:t.wav") + del step["pipeline"]["from_pretrained_arguments"] + assert errors_for(step) == [] + assert errors_for(h3_step()) == [] + + +def hold_errors(definition): + workflow = workflow_from_definition(definition, tempfile.mkdtemp()) + return [ + problem + for problem in workflow.validation_errors() + if "hold_audio" in problem["path"] + ] + + +def template(*parts): + with open(os.path.join(ROOT, "workflows", "templates", *parts)) as f: + return json.load(f) + + +class TestValidationEntry: + def test_sd15_template_refuses_hold_audio(self): + definition = template("text-to-image.json") + step = definition["steps"][0] + assert ( + step["pipeline"]["configuration"]["component_type"] + == "StableDiffusionPipeline" + ) + step["pipeline"]["arguments"]["hold_audio"] = "asset:track.wav" + + problems = hold_errors(definition) + + assert len(problems) == 1 + assert "StableDiffusionPipeline" in problems[0]["message"] + + def test_h3_template_accepts_hold_audio(self): + definition = copy.deepcopy(template("minimax", "video-with-audio.json")) + definition["steps"][0]["pipeline"]["arguments"]["hold_audio"] = ( + "asset:track.wav" + ) + + assert hold_errors(definition) == [] + + def test_h3_template_refuses_a_still(self): + definition = copy.deepcopy(template("minimax", "video-with-audio.json")) + definition["steps"][0]["pipeline"]["arguments"]["hold_audio"] = ( + "asset:frame.png" + ) + + assert len(hold_errors(definition)) == 1 + + +class TestHoldAudioReference: + @pytest.mark.parametrize("value", ["x.png", "notes.txt", "noextension"]) + def test_a_non_audio_path_raises(self, value): + with pytest.raises(ValueError): + hold_audio_reference(value) + + def test_a_non_audio_value_raises(self): + with pytest.raises(ValueError): + hold_audio_reference(12) + with pytest.raises(ValueError): + hold_audio_reference(object()) + + def test_a_reference_passes_through(self): + reference = MiniMaxH3AudioReference(audio=torch.zeros(2, 10), sample_rate=8000) + assert hold_audio_reference(reference) is reference + + def test_an_audio_track_becomes_a_reference_with_its_rate(self): + wave = torch.zeros(2, 100) + reference = hold_audio_reference(AudioTrack(wave, 22050)) + assert isinstance(reference, MiniMaxH3AudioReference) + assert reference.sample_rate == 22050 + assert reference.audio.shape == (2, 100) + + +# 6. _with_held_audio + + +def ns(pipeline): + return SimpleNamespace(name="video", pipeline=pipeline) + + +class TestWithHeldAudio: + def test_no_hold_audio_leaves_the_arguments_alone(self): + arguments = {"prompt": "x", "output": ["videos", "audio"]} + assert Pipeline._with_held_audio(ns(object()), arguments) is arguments + + def test_a_non_h3_pipeline_refuses(self): + with pytest.raises(ValueError, match="MiniMax-H3"): + Pipeline._with_held_audio(ns(object()), {"hold_audio": "t.wav"}) + + def test_an_h3_pipeline_gets_the_reference_and_the_held_keys(self): + pipeline = minimax.MiniMaxH3Blocks().get_workflow("t2va").init_pipeline() + insert_audio_hold(pipeline) + reference = MiniMaxH3AudioReference(audio=torch.zeros(2, 10), sample_rate=8000) + arguments = {"hold_audio": reference, "output": ["videos", "audio"]} + + result = Pipeline._with_held_audio(ns(pipeline), arguments) + + assert result["hold_audio"] is reference + assert result["output"] == [ + "videos", + "audio", + HELD_AUDIO_OUTPUT, + HELD_AUDIO_RATE_OUTPUT, + ] + assert arguments["output"] == ["videos", "audio"] # the caller's is untouched + + def test_output_without_audio_gains_nothing(self): + pipeline = minimax.MiniMaxH3Blocks().get_workflow("t2va").init_pipeline() + insert_audio_hold(pipeline) + reference = MiniMaxH3AudioReference(audio=torch.zeros(2, 10), sample_rate=8000) + result = Pipeline._with_held_audio( + ns(pipeline), {"hold_audio": reference, "output": ["videos"]} + ) + assert result["output"] == ["videos"] + + def test_a_non_audio_value_is_refused_on_an_h3_pipeline(self): + pipeline = minimax.MiniMaxH3Blocks().get_workflow("t2va").init_pipeline() + insert_audio_hold(pipeline) + with pytest.raises(ValueError): + Pipeline._with_held_audio(ns(pipeline), {"hold_audio": "x.png"}) + + +# 7. modular_artifacts + + +class TestModularArtifacts: + def test_the_held_track_replaces_the_decoded_audio(self): + frames = [["f0", "f1"]] + decoded = torch.zeros(2, 10) + held = torch.ones(1, 2, 20) + result = { + "videos": frames, + "audio": [decoded], + "sampling_rate": 48000, + HELD_AUDIO_OUTPUT: held, + HELD_AUDIO_RATE_OUTPUT: 32000, + } + + artifacts = modular_artifacts(result) + + assert len(artifacts) == 1 + video = artifacts[0] + assert isinstance(video, AudioVideo) + assert torch.equal(video.audio, held[0]) + assert video.sample_rate == 32000 + + def test_the_held_track_stands_without_decoded_audio(self): + held = torch.ones(1, 2, 20) + artifacts = modular_artifacts( + { + "videos": [["f"]], + HELD_AUDIO_OUTPUT: held, + HELD_AUDIO_RATE_OUTPUT: 24000, + } + ) + assert len(artifacts) == 1 + assert artifacts[0].sample_rate == 24000 + + def test_the_held_keys_are_not_leftovers(self): + result = { + "videos": [["f"]], + "audio": [torch.zeros(2, 4)], + "sampling_rate": 48000, + "latents": torch.zeros(1), + HELD_AUDIO_OUTPUT: torch.ones(1, 2, 4), + HELD_AUDIO_RATE_OUTPUT: 32000, + } + artifacts = modular_artifacts(result) + + assert len(artifacts) == 2 + leftovers = artifacts[-1] + assert isinstance(leftovers, dict) + assert set(leftovers) == {"latents"} + + def test_no_held_keys_no_change(self): + audio = torch.zeros(2, 4) + artifacts = modular_artifacts( + {"videos": [["f"]], "audio": [audio], "sampling_rate": 48000} + ) + assert artifacts[0].sample_rate == 48000 + assert torch.equal(artifacts[0].audio, audio) + + +def test_module_exports_stay_in_sync(): + assert h3_blocks.HOLD_BEFORE == "set_timesteps" + assert h3_blocks.RELEASE_BEFORE == "after_denoise" diff --git a/tests/test_validation.py b/tests/test_validation.py index c8ad70eb..faa92577 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -180,6 +180,7 @@ def test_validation_does_not_import_the_workflow_module(): "fps", "reference_names", "video_extensions", + "hold_audio", "content_types", "scalar_results", "locations", From f599431bd527220ff5aec5c671d479b93223107b Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 18:02:13 -0500 Subject: [PATCH 019/296] docs(proposals): #488 - close-out record for the lip-sync target check Co-Authored-By: Claude Opus 5.5 --- .../complete/attribute-voices-complete.md | 3 +- .../lip-sync-target-check-complete.md | 116 ++++++++++++++++++ docs/proposals/todo.md | 6 + 3 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 docs/proposals/complete/lip-sync-target-check-complete.md diff --git a/docs/proposals/complete/attribute-voices-complete.md b/docs/proposals/complete/attribute-voices-complete.md index 22ae50fa..cf27d0b3 100644 --- a/docs/proposals/complete/attribute-voices-complete.md +++ b/docs/proposals/complete/attribute-voices-complete.md @@ -101,4 +101,5 @@ The plan estimated ~$7-10 total. warning. - **Ensemble lines** (both singers at once) score between the two; `share` and `margin` show it, but `voice` names one. -- A `lines: "previous_result:transcribe"` case, once #483 lands. +- A `lines: "previous_result:transcribe"` case, once #483 lands. Done + by #488 (`templates/attribute-lines`; `complete/lip-sync-target-check-complete.md`). diff --git a/docs/proposals/complete/lip-sync-target-check-complete.md b/docs/proposals/complete/lip-sync-target-check-complete.md new file mode 100644 index 00000000..fb8220ab --- /dev/null +++ b/docs/proposals/complete/lip-sync-target-check-complete.md @@ -0,0 +1,116 @@ +# Checking the lip-sync target from shipped parts (#488) + +Written by model `claude-opus-5-5` via provider `anthropic` (close-out, +2026-10-05). Plan v2 approved by Don as "build smaller" on 2026-10-05; +one stage, #617 (A), verified. + +## The report + +A field report from an Opus 5.5 session on lem (2026-09-25/26): user review +of a 24-shot H3 musical found three faults that the frame checks and +`assess_output` had passed: + +1. **Anatomy vs props:** a crowd told to clap while holding candles rendered + with three hands. +2. **Direction of travel:** "march up to the house" rendered walking toward + camera, away from the house. +3. **Lip-sync target:** with two people in frame, the wrong person's mouth + moved to the vocal (6 shots); in another the singer's mouth barely moved. + +The ask was an opt-in `assess_output` rule running a VLM (the `judge` +machinery from #119) over sampled frames. + +## Decisions + +- **Plan v1: defer.** The in-engine VLM probe, anatomy and travel-direction + checks were parked (v1 Q1/Q2). #487 (reference-leak probe, closed not + planned and folded in here) added evidence that cheap structural + heuristics can't catch this fault class: its `opening_jump_ratio` scored + the known leak at 1.06 against a clean range of 0.37-2.01. +- **Rescope (Don, 2026-10-03):** v1's third trigger fired when #483 + (timestamped transcription) and #485 (`attribute_voices`) closed. The + lip-sync target, 7 of the report's 9 faults, is reopened alone. +- **Plan v2: build smaller**, one stage (~$3-4). The proposed stage B + (`vocal_windows` in `analyze_shots`) is cut: voiced spans from the mixed + track carry no speaker identity, and `attribute_voices` already names who + sings each line and shot. The loop could already run apart from one bug + (a null `end` from Whisper made `attribute_voices` refuse the call) and + the lack of a template and a written loop. +- **Q1:** transcribe **the song** and map it onto cut time with the two + timeline rules; the final cut's track is the fallback. +- **Q2:** the null-`end` fix goes in the producer, `transcribe_audio`; + `attribute_voices._span` stays strict. +- **Q3:** one pointer line in `script-to-video`, not in `minimax-h3` or + `minimax-music3` (both near the 12,288 B skill cap). +- **Non-goals held:** no verdict or auto-regeneration, no ML model inside + `assess_output`, no change to `judge`, no VLM probe, no new MCP surface, + no automatic per-shot `singer` wiring. + +## What was built + +**Stage A, #617** (server + plugin), `develop` @ `ea458b26`: + +- **`transcribe_audio`: `end` is never null with `timestamps` set** + (`_numeric_chunks` in `dw/tasks/audio_transcription.py`). A None `end` + gets the clip's duration, measured from the 16 kHz mono array Whisper was + given; a None `start` takes the previous chunk's end, or 0. The result + shape is unchanged. +- **Template `workflows/templates/attribute-lines.json`**: variables + `audio`, `voices`, `timestamps` (default `"segment"`), `model_name`; steps + `transcribe` -> `attribute` (`lines: "previous_result:transcribe"`), both + saving `application/json`. Shape `utility`, trait `needs-input-media`. + This closes #485's deferred "`lines: previous_result:transcribe`" item. +- **`attribute_voices`, from bounce 1** (`dw/tasks/voice_attribution.py`): + - a zero-length line (`start == end`, which Whisper's word mode emits) + comes back `voice: null`, `uncertain: true` with the too-short reason, + instead of refusing the call. Voice spans and windows stay strict; + - a line longer than `piece_seconds` (2 s) is also scored in 2 s pieces; + if a second voice holds ≥ `mixed_line_share` (0.25) of the confidently + attributed pieces' voiced time, the line is `uncertain`, and the + `reason` names each voice's seconds. Both thresholds are in the result's + `thresholds` and the `docs/TASKS.md` table. This made the plan's + Risks claim ("a line spanning two singers comes back `uncertain`") true. +- **`docs/TASKS.md`**: *Checking the lip-sync target* under *Voice + attribution* (the loop, the 32-moment limit, the `music-video` and + `join_into_song` timeline rules, the fallback, where it misleads), and a + note in *Speech Transcription* that `end` is always a number. +- **Plugin**: one line in `plugins/dw/skills/script-to-video/SKILL.md` + pointing a sung multi-shot piece at that subsection. + +## Bounces per stage + +| Stage | Bounces | Cause | +|---|---|---| +| A, #617 | 1 | C-F185: word mode emitted zero-length chunks the template's own `attribute` step refused; segment mode gave one hallucinated 0-20.5 s chunk a confident single verdict. Fixed in `attribute_voices` (zero-length lines, mixed-line rule). | + +No stage comment names `usage:` figures, so per-stage cost is not recorded. +The plan estimated ~$3-4. + +## Verification notes + +- C-F185's segment arm: Whisper-base returned one chunk over the whole duet; + it came back `uncertain` by the mixed-line rule, as shipped. The case's + "at least one line names each voice" can't be met by a one-chunk + transcript, so the tester filed an amendment on dkackman/harnest and left + C-F185 `pending: #617` until it is ruled on. +- C-F188 named the section "Transcription"; the heading is "Speech + Transcription". Amendment filed on harnest. +- C-F189 passed on the chain: tiles cut to the crop with ~1 s wavs per + moment; 32 moments accepted, 33 refused before decoding. A consumer agent + can't listen to the clips, so whose voice is in each was not judged by ear. +- M-F073 (a real two-singer `minimax/music-video` round) was not run: no + such render exists to reuse. Its `pending: #617` line stays. + +## Deferred + +- **The in-engine VLM probe, anatomy and travel-direction checks.** Parked + under v1's Q1/Q2; #484's notes cover prevention for those, and #487's + evidence argues for a content judge if they come back. +- **Automatic per-shot `singer` wiring** (#485's deferral stands). +- **`plan.downloads_required` still omits the demucs and Whisper weights** + the template pulls (#485's follow-up in `dw/plan.py`). +- **Whisper hallucination over accompaniment** in segment mode. The guide + says to re-run with `timestamps: "word"` or transcribe the `vocals` stem + of `separate_stems`; no template does that for the caller. +- **M-F073**, until a two-singer `music-video` render exists (#598 stage 4's + hold-vs-reference A/B is expected to use this loop and can supply one). diff --git a/docs/proposals/todo.md b/docs/proposals/todo.md index 678427ad..a45e580a 100644 --- a/docs/proposals/todo.md +++ b/docs/proposals/todo.md @@ -94,6 +94,12 @@ remaining deferred fix is recorded in home. Record, including what was deferred (a multi-job bundle, an outputs-only zip, removing an export over MCP): `complete/job-export-bulk-download-complete.md`. +- **Checking the lip-sync target** (#488, stage #617), shipped 2026-10-05 + from shipped parts: `transcribe_audio` never emits a null `end`, an + `attribute-lines` template, a mixed-line `uncertain` rule in + `attribute_voices`, and the loop in `docs/TASKS.md`. Record, including + what was deferred (the VLM probe, anatomy and travel-direction checks, a + real two-singer round): `complete/lip-sync-target-check-complete.md`. ## Declined From c5123954cac5742a24abe578b7b11177363b14c1 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 18:16:40 -0500 Subject: [PATCH 020/296] feat(pipeline): #618 - hold_audio paths go through the location owner at run time hold_audio_reference confines a path with locations.validate_media_path against the step's workflow directory and checks the extension through security.validate_file_extension before from_file opens it. Pipeline now carries the workflow's base_dir (Workflow.base_dir, which run_base_dir reuses). Resolves the architecture review's second-owner finding. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- dw/pipeline_ownership.py | 18 +++++++- dw/pipeline_processors/h3_blocks.py | 33 ++++++++++++--- dw/pipeline_processors/pipeline.py | 6 ++- dw/workflow.py | 11 +++++ dw/workflow_run.py | 6 +-- tests/test_h3_hold_audio.py | 65 ++++++++++++++++++++++++++++- 7 files changed, 124 insertions(+), 17 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 549562b7..6175515b 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -87,7 +87,7 @@ one to open. | H3 Ref2VA VRAM numbers | `workflows/templates/minimax/` | Every Ref2VA template declares `base_gb` 16.0, `bytes_per_voxel` 28.71 and `gb_per_reference` 1.0. | `tests/test_h3_vram_ceiling.py::test_every_ref2va_minimax_template_declares_gb_per_reference` | | Inherited VRAM ceiling | `dw/vram_inheritance.py`, `dw/server/deps.py`: `ceiling_index` | A workflow with no `vram_estimate` is matched against the catalog by pipeline identity, and is warned, never refused. | `tests/test_vram_inheritance.py::test_every_template_declaring_one_identity_declares_the_same_numbers` | | H3 adapter partition | `dw/adapter_compatibility.py` | An FL2VA LoRA on a reference step is refused, and a LoRA whose file name says neither `ref2v` nor `fl2v` is warned. | `tests/test_h3_adapters.py::TestWhatIsRefused::test_a_keyframe_adapter_on_the_reference_path`, `tests/test_h3_adapters.py::TestWhatIsRefused::test_an_unrecognised_name_is_a_warning_not_a_refusal` | -| H3 audio hold | `dw/pipeline_processors/h3_blocks.py`: `insert_audio_hold`, `core_denoise_sequences`, `dw/hold_audio.py`: `hold_audio_errors` | The one place dw modifies a diffusers modular pipeline's block graph: two blocks go in before `set_timesteps` and `after_denoise` in every H3 core-denoise sequence (the whole graph and the `workflow=`-pruned flat graph), and they are no-ops without `hold_audio`. | `tests/test_h3_hold_audio.py::TestAnchors::test_inserted_next_to_the_anchors_in_every_shape`, `tests/test_h3_hold_audio.py::TestRefusals::test_a_non_h3_step_is_refused` | +| H3 audio hold | `dw/pipeline_processors/h3_blocks.py`: `insert_audio_hold`, `core_denoise_sequences`, `dw/hold_audio.py`: `hold_audio_errors` | The one place dw modifies a diffusers modular pipeline's block graph: two blocks go in before `set_timesteps` and `after_denoise` in every H3 core-denoise sequence (the whole graph and the `workflow=`-pruned flat graph), and they are no-ops without `hold_audio`. A `hold_audio` path is opened only after `locations.validate_media_path` against the step's workflow directory. | `tests/test_h3_hold_audio.py::TestAnchors::test_inserted_next_to_the_anchors_in_every_shape`, `tests/test_h3_hold_audio.py::TestHoldAudioLocation::test_a_path_outside_the_roots_is_refused_at_the_call`, `tests/test_h3_hold_audio.py::TestRefusals::test_a_non_h3_step_is_refused` | | IC-LoRA reference scale | `workflows/templates/ltx2/` | The three conditioning templates run at `reference_downscale_factor: 1`, and the two upscalers at 2. | `tests/test_ltx2_ic_loras.py::TestEachTemplateMatchesItsCard::test_the_reference_is_encoded_at_the_output_resolution`, `tests/test_ltx2_ic_loras.py::TestTheGenerativeUpscaleMatchesTheSameCard::test_it_loads_the_upscaler_at_factor_two` | | IC-LoRA numbers | `workflows/templates/ltx2/` | Every number in the IC-LoRA templates comes from the vendor's model card. | `tests/test_ltx2_ic_loras.py::TestEachTemplateMatchesItsCard::test_the_strength_is_the_cards_default`, `tests/test_ltx2_ic_loras.py::TestEachTemplateMatchesItsCard::test_the_defaults_are_the_trained_bucket` | | IC-LoRA prompt genre | `prompts/ltx2/` | A stored prompt tagged `ic-lora` is checked against its trained caption form, not against the T2V paragraph rule. | `tests/test_ltx_prompt_library.py::test_an_ic_lora_prompt_is_in_its_trained_form` | diff --git a/dw/pipeline_ownership.py b/dw/pipeline_ownership.py index a391185d..d26b480c 100644 --- a/dw/pipeline_ownership.py +++ b/dw/pipeline_ownership.py @@ -250,7 +250,13 @@ def reclaim_after_step(step_data, step_name): def wrap_resident( - cached_pipeline, step_definition, default_seed, device, output_dir, file_prefix + cached_pipeline, + step_definition, + default_seed, + device, + output_dir, + file_prefix, + base_dir=None, ): """A new Pipeline wrapper for a step around a model already resident under its key, whose shared components the caller has already @@ -266,6 +272,7 @@ def wrap_resident( cached_pipeline.pipeline, # Reuse the actual loaded model output_dir=output_dir, file_prefix=file_prefix, + base_dir=base_dir, ) # Set up generator with potentially new seed. no_generator is a # boolean - only an explicit true disables the generator - and the @@ -286,7 +293,13 @@ def wrap_resident( def load_fresh( - step_definition, shared_components, default_seed, device, output_dir, file_prefix + step_definition, + shared_components, + default_seed, + device, + output_dir, + file_prefix, + base_dir=None, ): """A step's pipeline loaded from scratch, behind the trust gate.""" pipeline = Pipeline( @@ -295,6 +308,7 @@ def load_fresh( device, output_dir=output_dir, file_prefix=file_prefix, + base_dir=base_dir, ) # Before the marker, not after it: a definition refused by the # trust gate must not have announced a load it never began, or a diff --git a/dw/pipeline_processors/h3_blocks.py b/dw/pipeline_processors/h3_blocks.py index bec4d091..acccf620 100644 --- a/dw/pipeline_processors/h3_blocks.py +++ b/dw/pipeline_processors/h3_blocks.py @@ -362,30 +362,51 @@ def holds_audio(pipeline): ) -def hold_audio_reference(value): +def hold_audio_reference(value, base_dir=None): """A `hold_audio` argument as the MiniMaxH3AudioReference the hold block takes. Whatever an argument can hold audio as: a file path (an `asset:` or `output:` reference arrives resolved to one), a step's artifact - an AudioTrack, an - AudioVideo, a bare waveform - or a reference already built. + AudioVideo, a bare waveform - or a reference already built. A path or URL goes + through the location owner (dw/locations.py) before anything opens it, the + run-time half of the check validation makes on the same argument. + + Args: + value: The argument's value + base_dir: Directory a relative path resolves against - the workflow + file's directory Raises: ValueError: If the value is not audio + SecurityError: If the path or URL is outside what the workflow may read """ from ..arguments import media_arguments - from ..security import ALLOWED_AUDIO_EXTENSIONS + from ..locations import is_http_url, validate_media_path, validate_media_url + from ..security import ( + ALLOWED_AUDIO_EXTENSIONS, + InvalidInputError, + validate_file_extension, + ) audio_reference = _diffusers()[2] if isinstance(value, audio_reference): return value if isinstance(value, str): - extension = "." + value.rsplit(".", 1)[-1].lower() if "." in value else "" - if extension not in ALLOWED_AUDIO_EXTENSIONS: + try: + validate_file_extension(value, ALLOWED_AUDIO_EXTENSIONS) + except InvalidInputError as error: raise ValueError( f"hold_audio holds a soundtrack, and '{value}' is not an audio file " f"({', '.join(sorted(ALLOWED_AUDIO_EXTENSIONS))})" + ) from error + if is_http_url(value): + location = validate_media_url(value, "hold_audio") + else: + location = validate_file_extension( + validate_media_path(value, base_dir, "hold_audio"), + ALLOWED_AUDIO_EXTENSIONS, ) - return audio_reference.from_file(value) + return audio_reference.from_file(location) try: return audio_reference(**media_arguments(audio_reference, value)) except ValueError as error: diff --git a/dw/pipeline_processors/pipeline.py b/dw/pipeline_processors/pipeline.py index ed14dc9d..0bfe444c 100644 --- a/dw/pipeline_processors/pipeline.py +++ b/dw/pipeline_processors/pipeline.py @@ -141,6 +141,7 @@ def __init__( pipeline=None, output_dir=None, file_prefix=None, + base_dir=None, ): """ Initialize pipeline with configuration and device settings. @@ -155,6 +156,8 @@ def __init__( with save_segments writes its segment files file_prefix: Naming prefix for those files, matching the step's result naming (workflow id + step name) + base_dir: The workflow file's directory - what a relative media path + in a call argument resolves against at run time """ self.pipeline_definition = _loading_copy(pipeline_definition) self.default_seed = default_seed @@ -165,6 +168,7 @@ def __init__( self.pipeline = pipeline self.output_dir = output_dir self.file_prefix = file_prefix + self.base_dir = base_dir # What a chained run calls the segment it is on, so progress can say # which one the denoise counter belongs to - it restarts per segment self.segment_label = None @@ -625,7 +629,7 @@ def _with_held_audio(self, arguments): f"cannot hold a soundtrack" ) arguments = dict(arguments) - arguments[HOLD_AUDIO_INPUT] = hold_audio_reference(held) + arguments[HOLD_AUDIO_INPUT] = hold_audio_reference(held, self.base_dir) output = arguments.get("output") if isinstance(output, (list, tuple)) and "audio" in output: arguments["output"] = list(output) + [ diff --git a/dw/workflow.py b/dw/workflow.py index 9a31441d..7e77649b 100644 --- a/dw/workflow.py +++ b/dw/workflow.py @@ -256,6 +256,14 @@ def _parent_progress_fields(self): "parent_total_steps": parent["total_steps"], } + @property + def base_dir(self): + """The directory a relative path in this workflow resolves against - + its file's own.""" + return ( + os.path.dirname(os.path.abspath(self.file_spec)) if self.file_spec else None + ) + def step_file_prefix(self, step_name): """Naming prefix for files a step writes on its own (chain segment spills), matching the workflow-id-step naming its results are saved @@ -747,6 +755,7 @@ def _pipeline_action( device, self.step_output_dir(step_definition), self.step_file_prefix(step_name), + base_dir=self.base_dir, ) # Not in cache - a redefined step frees its previous model first, @@ -763,6 +772,7 @@ def _pipeline_action( device, self.step_output_dir(step_definition), self.step_file_prefix(step_name), + base_dir=self.base_dir, ) previous_pipelines[cache_key] = pipeline return pipeline @@ -795,6 +805,7 @@ def _reference_action( previous_pipeline.pipeline, output_dir=self.step_output_dir(step_definition), file_prefix=self.step_file_prefix(step_definition["name"]), + base_dir=self.base_dir, ) def _sub_workflow_action(self, step_definition, default_seed): diff --git a/dw/workflow_run.py b/dw/workflow_run.py index ac7e779c..3f23d21b 100644 --- a/dw/workflow_run.py +++ b/dw/workflow_run.py @@ -135,11 +135,7 @@ class StepLoop: def run_base_dir(workflow): """The directory file paths in the workflow resolve against - the workflow file's own.""" - return ( - os.path.dirname(os.path.abspath(workflow.file_spec)) - if workflow.file_spec - else None - ) + return workflow.base_dir def owns_run_dir(workflow): diff --git a/tests/test_h3_hold_audio.py b/tests/test_h3_hold_audio.py index a99d0be1..127aa907 100644 --- a/tests/test_h3_hold_audio.py +++ b/tests/test_h3_hold_audio.py @@ -468,11 +468,72 @@ def test_an_audio_track_becomes_a_reference_with_its_rate(self): assert reference.audio.shape == (2, 100) +class TestHoldAudioLocation: + """A hold_audio path goes through the location owner (dw/locations.py) + at the call, not just at validation.""" + + @pytest.fixture + def opened(self, monkeypatch): + opened = [] + + def from_file(cls, location): + opened.append(location) + return cls(audio=torch.zeros(2, 10), sample_rate=8000) + + monkeypatch.setattr( + MiniMaxH3AudioReference, "from_file", classmethod(from_file) + ) + return opened + + def test_a_relative_path_resolves_against_the_workflow_directory( + self, tmp_path, opened + ): + (tmp_path / "track.wav").write_bytes(b"") + + hold_audio_reference("track.wav", str(tmp_path)) + + assert opened == [os.path.realpath(tmp_path / "track.wav")] + + def test_a_path_outside_the_roots_is_refused_at_the_call( + self, tmp_path, opened, monkeypatch + ): + from dw.security import SecurityError + + monkeypatch.delenv("DW_TRUST_WORKFLOWS") # conftest trusts by default + + workflow_dir = tmp_path / "workflow" + workflow_dir.mkdir() + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + (elsewhere / "track.wav").write_bytes(b"") + + with pytest.raises(SecurityError, match="outside every directory"): + hold_audio_reference(str(elsewhere / "track.wav"), str(workflow_dir)) + assert opened == [] + + def test_the_step_hands_its_base_dir_to_the_check(self, tmp_path, opened): + (tmp_path / "track.wav").write_bytes(b"") + pipeline = minimax.MiniMaxH3Blocks().get_workflow("t2va").init_pipeline() + insert_audio_hold(pipeline) + + Pipeline._with_held_audio( + ns(pipeline, str(tmp_path)), {"hold_audio": "track.wav"} + ) + + assert opened == [os.path.realpath(tmp_path / "track.wav")] + + def test_a_workflow_gives_its_pipelines_its_own_directory(self, tmp_path): + workflow = workflow_from_definition( + {"id": "w", "steps": []}, str(tmp_path / "out"), base_dir=str(tmp_path) + ) + assert workflow.base_dir == os.path.realpath(tmp_path) + + # 6. _with_held_audio -def ns(pipeline): - return SimpleNamespace(name="video", pipeline=pipeline) +def ns(pipeline, base_dir=None): + return SimpleNamespace(name="video", pipeline=pipeline, base_dir=base_dir) class TestWithHeldAudio: From d0258e0f1c6b1cdc5040dbec097cb572c8b1f73c Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 18:34:28 -0500 Subject: [PATCH 021/296] feat(pipeline): #618 - hold_audio in the H3 signature; bare words and numbers refused at validate get_pipeline_signature on a modular pipeline class lists its block graph's inputs, built without weights and with dw's audio hold inserted. Validation refuses a hold_audio with no audio extension or that is no string at all. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- docs/MCP.md | 2 +- dw/hold_audio.py | 25 ++++++++++------ dw/introspection.py | 60 +++++++++++++++++++++++++++++++++++++ tests/test_h3_hold_audio.py | 40 +++++++++++++++++++++++++ 5 files changed, 118 insertions(+), 11 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 6175515b..ebe591b0 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -87,7 +87,7 @@ one to open. | H3 Ref2VA VRAM numbers | `workflows/templates/minimax/` | Every Ref2VA template declares `base_gb` 16.0, `bytes_per_voxel` 28.71 and `gb_per_reference` 1.0. | `tests/test_h3_vram_ceiling.py::test_every_ref2va_minimax_template_declares_gb_per_reference` | | Inherited VRAM ceiling | `dw/vram_inheritance.py`, `dw/server/deps.py`: `ceiling_index` | A workflow with no `vram_estimate` is matched against the catalog by pipeline identity, and is warned, never refused. | `tests/test_vram_inheritance.py::test_every_template_declaring_one_identity_declares_the_same_numbers` | | H3 adapter partition | `dw/adapter_compatibility.py` | An FL2VA LoRA on a reference step is refused, and a LoRA whose file name says neither `ref2v` nor `fl2v` is warned. | `tests/test_h3_adapters.py::TestWhatIsRefused::test_a_keyframe_adapter_on_the_reference_path`, `tests/test_h3_adapters.py::TestWhatIsRefused::test_an_unrecognised_name_is_a_warning_not_a_refusal` | -| H3 audio hold | `dw/pipeline_processors/h3_blocks.py`: `insert_audio_hold`, `core_denoise_sequences`, `dw/hold_audio.py`: `hold_audio_errors` | The one place dw modifies a diffusers modular pipeline's block graph: two blocks go in before `set_timesteps` and `after_denoise` in every H3 core-denoise sequence (the whole graph and the `workflow=`-pruned flat graph), and they are no-ops without `hold_audio`. A `hold_audio` path is opened only after `locations.validate_media_path` against the step's workflow directory. | `tests/test_h3_hold_audio.py::TestAnchors::test_inserted_next_to_the_anchors_in_every_shape`, `tests/test_h3_hold_audio.py::TestHoldAudioLocation::test_a_path_outside_the_roots_is_refused_at_the_call`, `tests/test_h3_hold_audio.py::TestRefusals::test_a_non_h3_step_is_refused` | +| H3 audio hold | `dw/pipeline_processors/h3_blocks.py`: `insert_audio_hold`, `core_denoise_sequences`, `dw/hold_audio.py`: `hold_audio_errors` | The one place dw modifies a diffusers modular pipeline's block graph: two blocks go in before `set_timesteps` and `after_denoise` in every H3 core-denoise sequence (the whole graph and the `workflow=`-pruned flat graph), and they are no-ops without `hold_audio`. A `hold_audio` path is opened only after `locations.validate_media_path` against the step's workflow directory. `dw/introspection.py` builds the same graph without weights, hold included, so `get_pipeline_signature` lists a modular pipeline's block inputs. | `tests/test_h3_hold_audio.py::TestAnchors::test_inserted_next_to_the_anchors_in_every_shape`, `tests/test_h3_hold_audio.py::TestHoldAudioLocation::test_a_path_outside_the_roots_is_refused_at_the_call`, `tests/test_h3_hold_audio.py::TestRefusals::test_a_non_h3_step_is_refused`, `tests/test_h3_hold_audio.py::TestSignature::test_the_h3_signature_shows_hold_audio` | | IC-LoRA reference scale | `workflows/templates/ltx2/` | The three conditioning templates run at `reference_downscale_factor: 1`, and the two upscalers at 2. | `tests/test_ltx2_ic_loras.py::TestEachTemplateMatchesItsCard::test_the_reference_is_encoded_at_the_output_resolution`, `tests/test_ltx2_ic_loras.py::TestTheGenerativeUpscaleMatchesTheSameCard::test_it_loads_the_upscaler_at_factor_two` | | IC-LoRA numbers | `workflows/templates/ltx2/` | Every number in the IC-LoRA templates comes from the vendor's model card. | `tests/test_ltx2_ic_loras.py::TestEachTemplateMatchesItsCard::test_the_strength_is_the_cards_default`, `tests/test_ltx2_ic_loras.py::TestEachTemplateMatchesItsCard::test_the_defaults_are_the_trained_bucket` | | IC-LoRA prompt genre | `prompts/ltx2/` | A stored prompt tagged `ic-lora` is checked against its trained caption form, not against the T2V paragraph rule. | `tests/test_ltx_prompt_library.py::test_an_ic_lora_prompt_is_in_its_trained_form` | diff --git a/docs/MCP.md b/docs/MCP.md index 0437a64c..5f52592a 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -216,7 +216,7 @@ when no single workflow covers it. | `get_workflow(name, variables_only=False)` | `name` | Get one stored workflow's full JSON definition. `variables_only=true` answers with just its variables and their defaults (long strings cut to 200 characters, the cut ones named in `truncated`, including strings inside a list default, named like `shots[0].prompt`) — the cheap way to confirm what a variable defaults to | | `get_schema(section=None)` | `section` | Get the JSON schema every workflow definition must satisfy. Whole it is ~8.6k tokens, so name the part you need: `section` takes `steps`, `pipelines`, `tasks`, `result`, `variables` or `configuration` and answers `{section, sections, elsewhere, schema}` - `elsewhere` says which section holds each definition the fragment still `$ref`s. An unknown section is a 404 naming the ones there are | | `list_pipelines()` | — | List every diffusers pipeline class this installation provides | -| `get_pipeline_signature(name)` | `name` | Get a pipeline's real call arguments | +| `get_pipeline_signature(name)` | `name` | Get a pipeline's real call arguments; for a modular pipeline class (e.g. `MiniMaxH3ModularPipeline`), its block graph's inputs | | `list_classes(kind)` | `kind` | List class names of one kind: pipelines, models, schedulers, or quantization | | `get_class(name, target="init")` | `name`, `target` (`init`\|`call`\|`load`) | Get a class's argument schema from the entry point a workflow reaches it by: `init` the constructor (quantization configs, schedulers), `call` a pipeline's `__call__`, `load` `from_pretrained` plus the curated loading knobs | | `list_tasks()` | — | List every task command a workflow's task step can name | diff --git a/dw/hold_audio.py b/dw/hold_audio.py index 100d7042..421cc5db 100644 --- a/dw/hold_audio.py +++ b/dw/hold_audio.py @@ -10,9 +10,10 @@ `from_pretrained` `workflow` is not one of H3's three (`t2va`, `fl2va`, `ref2va`). A modular pipeline loaded with no `workflow` is left to the run-time check, which asks the loaded block graph itself; -- a literal path, or an `asset:`/`output:` reference whose name carries an - extension that is not audio, and any `{"media_type": ...}` reference - those - load an image or a video. +- a literal path, or an `asset:`/`output:` reference, whose name carries no + audio extension (a bare word like `hello` among them), any value that is not + a string or a mapping (a number), and any `{"media_type": ...}` reference - + those load an image or a video. A `previous_result:` names no file yet and is left to the run-time check, the same as `dw/video_extensions.py` leaves it. @@ -38,19 +39,25 @@ def _not_audio(value): f"'{value.get('media_type')}' reference - name the audio file " f"directly, or with 'asset:', 'output:' or 'previous_result:'" ) - if not isinstance(value, str): + if isinstance(value, dict): return None + if not isinstance(value, str): + # A number or a list is no file and no reference - the call refuses it + return ( + f"hold_audio holds a soundtrack, and {value!r} is not one - name an " + f"audio file directly, or with 'asset:', 'output:' or " + f"'previous_result:'" + ) if value.startswith(("http://", "https://")): return None if references.is_ref(references.UNRESOLVED, value): return None if references.is_ref((references.CONSTANT, references.PROMPT), value): return None - ext = value.rsplit(".", 1) - if len(ext) != 2 or not ext[1] or "/" in ext[1]: - return None - ext = f".{ext[1].lower()}" - if ext in ALLOWED_AUDIO_EXTENSIONS: + # A name with no extension - a bare word like 'hello' - is refused at the + # call too, which checks the extension before it opens anything + _, dot, ext = value.rpartition(".") + if dot and ext and "/" not in ext and f".{ext.lower()}" in ALLOWED_AUDIO_EXTENSIONS: return None return ( f"hold_audio holds a soundtrack, and '{value}' is not an audio file " diff --git a/dw/introspection.py b/dw/introspection.py index 7077b2c8..b237e1dc 100644 --- a/dw/introspection.py +++ b/dw/introspection.py @@ -298,6 +298,58 @@ def _callable_parameters(target_callable): return parameters, accepts_kwargs +def _modular_block_parameters(cls): + """A modular pipeline's call arguments, read from its default block graph. + + A modular pipeline's __call__ is `(state, output, **kwargs)`: what it + takes is whatever its blocks declare as inputs, so those are the honest + answer. The graph is built without weights (`init_pipeline()` with no + repository), and carries dw's own blocks - the H3 audio hold + (dw/pipeline_processors/h3_blocks.py) - as a loaded one does. Empty for + a class with no default blocks, the bare `ModularPipeline` among them. + """ + blocks_name = getattr(cls, "default_blocks_name", None) + if not isinstance(blocks_name, str): + return [] + import importlib + + from .pipeline_processors.h3_blocks import insert_audio_hold + + blocks_class = getattr(importlib.import_module(cls.__module__), blocks_name, None) + if blocks_class is None: + import diffusers + + blocks_class = getattr(diffusers, blocks_name, None) + if not isinstance(blocks_class, type): + return [] + try: + pipeline = blocks_class().init_pipeline() + insert_audio_hold(pipeline) + block_inputs = pipeline._blocks.inputs + except Exception as error: + # The signature still answers without them + logger.debug(f"Could not read {blocks_name}'s inputs: {error}") + return [] + parameters = [] + for block_input in block_inputs: + if not block_input.name: + continue + parameters.append( + { + "name": block_input.name, + "required": bool(block_input.required), + "default": _json_safe_default(block_input.default), + "annotation": ( + None + if block_input.type_hint is None + else str(block_input.type_hint) + ), + "description": block_input.description or "", + } + ) + return parameters + + def describe_class(name, target="call"): """The argument schema of a class, for form generation. @@ -321,6 +373,14 @@ def describe_class(name, target="call"): parameters, accepts_kwargs = _callable_parameters(target_callable) + if target == "call": + named = {parameter["name"] for parameter in parameters} + parameters += [ + parameter + for parameter in _modular_block_parameters(cls) + if parameter["name"] not in named + ] + if target == "load": named = {parameter["name"] for parameter in parameters} parameters = [ diff --git a/tests/test_h3_hold_audio.py b/tests/test_h3_hold_audio.py index 127aa907..729930d7 100644 --- a/tests/test_h3_hold_audio.py +++ b/tests/test_h3_hold_audio.py @@ -11,6 +11,7 @@ import torch from dw.hold_audio import hold_audio_errors +from dw.introspection import describe_pipeline from dw.media_types import AudioTrack, AudioVideo from dw.output_extraction import modular_artifacts from dw.pipeline_processors import h3_blocks @@ -379,6 +380,20 @@ def test_a_non_audio_path_is_refused(self, value): assert len(errors) == 1 assert "not an audio file" in errors[0]["message"] + @pytest.mark.parametrize("value", ["hello", "asset:track", "clip"]) + def test_a_name_with_no_audio_extension_is_refused(self, value): + errors = errors_for(h3_step(hold_audio=value)) + assert len(errors) == 1 + assert "not an audio file" in errors[0]["message"] + assert errors[0]["path"] == "steps[0].pipeline.arguments.hold_audio" + + @pytest.mark.parametrize("value", [5, 1.5, True, ["track.wav"]]) + def test_a_value_that_is_no_file_or_reference_is_refused(self, value): + errors = errors_for(h3_step(hold_audio=value)) + assert len(errors) == 1 + assert "is not one" in errors[0]["message"] + assert errors[0]["path"] == "steps[0].pipeline.arguments.hold_audio" + def test_a_media_reference_is_refused(self): value = {"media_type": "image", "location": "asset:frame.png"} errors = errors_for(h3_step(hold_audio=value)) @@ -412,6 +427,31 @@ def template(*parts): return json.load(f) +class TestSignature: + """`get_pipeline_signature` reads a modular pipeline's block inputs, so the + hold dw inserts shows up where a caller looks for H3's arguments.""" + + def test_the_h3_signature_shows_hold_audio(self): + names = { + parameter["name"]: parameter + for parameter in describe_pipeline("MiniMaxH3ModularPipeline")["parameters"] + } + assert {"hold_audio", "prompt", "num_frames"} <= set(names) + assert names["hold_audio"]["required"] is False + assert "soundtrack" in names["hold_audio"]["description"] + + def test_a_bare_modular_pipeline_keeps_its_call_signature(self): + names = [p["name"] for p in describe_pipeline("ModularPipeline")["parameters"]] + assert names == ["state", "output"] + + def test_a_non_h3_modular_pipeline_shows_no_hold(self): + names = [ + p["name"] for p in describe_pipeline("WanModularPipeline")["parameters"] + ] + assert "prompt" in names + assert "hold_audio" not in names + + class TestValidationEntry: def test_sd15_template_refuses_hold_audio(self): definition = template("text-to-image.json") From d728547337da256d8ca5653fa799d5c58a1aa800 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 18:53:19 -0500 Subject: [PATCH 022/296] feat(locations): #618 - a non-http URL (file://) is refused as a media location, at validate and at the loader Co-Authored-By: Claude Opus 5.5 --- docs/SECURITY.md | 4 ++++ dw/locations.py | 22 ++++++++++++++++++++++ tests/test_h3_hold_audio.py | 23 +++++++++++++++++++++++ tests/test_locations.py | 27 +++++++++++++++++++++++++++ 4 files changed, 76 insertions(+) diff --git a/docs/SECURITY.md b/docs/SECURITY.md index fd1ee783..74ff263f 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -144,6 +144,10 @@ One policy now answers all of it, untrusted: for a file outside is to put it in the asset library and use an `asset:` reference. Containment is checked **before** existence, so the refusal cannot be used as a file-existence oracle. +- **A URL with any other scheme** (`file://`, `s3://`, ...) is refused at + validation and at the loader, whatever the trust posture. No loader opens + one, but as a relative path it joined onto the workflow directory and + passed containment, so validation called it clean (#618). - **A glob** is contained the same way, on the fixed directory its pattern starts from, and every match is re-checked on its real path so a symlink cannot carry the expansion out. diff --git a/dw/locations.py b/dw/locations.py index cc2ad84a..d8cfc38b 100644 --- a/dw/locations.py +++ b/dw/locations.py @@ -67,6 +67,25 @@ def is_http_url(value): return isinstance(value, str) and value.startswith(("http://", "https://")) +def _refuse_other_url(location, what): + """Refuse a location written as a URL whose scheme is not http(s). + + No loader opens a `file://` (or any other scheme) URL, but as a relative + path it joined onto the workflow directory, landed inside a root and so + passed containment - a refusal at run time only because no such directory + existed, and none at validation (#618). The rule `validate_model_name` + applies to a model_name (#117), applied to every media location. + """ + text = str(location) + if "://" in text and not is_http_url(text): + scheme = text.split("://", 1)[0] + raise InvalidInputError( + f"Refusing to read {what} at '{location}': it is a '{scheme}' URL, " + f"and only http(s) URLs are fetched. Name a local file by its path " + f"or with an 'asset:' reference." + ) + + def media_roots(base_dir=None): """Every directory a workflow's own locations may point inside, resolved. @@ -138,6 +157,7 @@ def validate_media_path( PathTraversalError: If the path resolves outside every root InvalidInputError, PathTraversalError: Whatever validate_path raises """ + _refuse_other_url(location, what) # base_dir is the first root, so a relative path keeps resolving against # the workflow file exactly as it did before this check existed. # allow_create here, with the existence check moved below the containment @@ -546,6 +566,8 @@ def _check(value, base_dir, what): try: if is_http_url(value): validate_media_url(value, what) + elif "://" in value: + _refuse_other_url(value, what) elif os.path.isabs(value): validate_media_path(value, base_dir, what, require_exists=False) elif ".." in value.replace("\\", "/").split("/"): diff --git a/tests/test_h3_hold_audio.py b/tests/test_h3_hold_audio.py index 729930d7..c836e8a2 100644 --- a/tests/test_h3_hold_audio.py +++ b/tests/test_h3_hold_audio.py @@ -483,6 +483,22 @@ def test_h3_template_refuses_a_still(self): assert len(hold_errors(definition)) == 1 + @pytest.mark.parametrize( + "value", ["file:///usr/share/sounds/alsa/Front_Center.wav", "file:///x/x.wav"] + ) + def test_h3_template_refuses_a_file_url(self, value, monkeypatch): + # SE-F040: a file:// URL validated clean as a path under the workflow + monkeypatch.delenv("DW_TRUST_WORKFLOWS") # conftest trusts by default + definition = copy.deepcopy(template("minimax", "video-with-audio.json")) + definition["steps"][0]["pipeline"]["arguments"]["hold_audio"] = value + + problems = hold_errors(definition) + + assert [problem["path"] for problem in problems] == [ + "steps[0].pipeline.arguments.hold_audio" + ] + assert "'file' URL" in problems[0]["message"] + class TestHoldAudioReference: @pytest.mark.parametrize("value", ["x.png", "notes.txt", "noextension"]) @@ -551,6 +567,13 @@ def test_a_path_outside_the_roots_is_refused_at_the_call( hold_audio_reference(str(elsewhere / "track.wav"), str(workflow_dir)) assert opened == [] + def test_a_file_url_is_refused_at_the_call(self, tmp_path, opened): + from dw.security import SecurityError + + with pytest.raises(SecurityError, match="'file' URL"): + hold_audio_reference("file:///usr/share/sounds/x.wav", str(tmp_path)) + assert opened == [] + def test_the_step_hands_its_base_dir_to_the_check(self, tmp_path, opened): (tmp_path / "track.wav").write_bytes(b"") pipeline = minimax.MiniMaxH3Blocks().get_workflow("t2va").init_pipeline() diff --git a/tests/test_locations.py b/tests/test_locations.py index 3a95069b..6e8bd6e7 100644 --- a/tests/test_locations.py +++ b/tests/test_locations.py @@ -109,6 +109,17 @@ def test_trust_lifts_containment(self, trusted, workflow_dir): outside ) + @pytest.mark.parametrize( + "location", + ["file:///usr/share/sounds/x.wav", "file:///nonexistent/x.png", "s3://b/x.png"], + ) + def test_a_url_that_is_not_http_is_refused(self, location, workflow_dir): + """#618: joined onto the workflow directory as a relative path, a + file:// URL landed inside a root and passed containment. Refused + whatever the trust posture - no loader opens one as a path.""" + with pytest.raises(InvalidInputError, match="only http\\(s\\) URLs"): + validate_media_path(location, workflow_dir, "an audio argument") + def test_fetch_image_refuses_an_out_of_root_absolute_path( self, untrusted, workflow_dir ): @@ -337,6 +348,22 @@ def test_an_out_of_root_image_is_an_error_with_its_path( "steps[0].pipeline.arguments.image" ] + @pytest.mark.parametrize("key", ["image", "hold_audio"]) + def test_a_file_url_is_an_error_with_its_path(self, untrusted, workflow_dir, key): + definition = { + "steps": [ + { + "name": "edit", + "pipeline": {"arguments": {key: "file:///usr/share/sounds/x.wav"}}, + } + ] + } + errors = location_errors(definition, base_dir=workflow_dir) + assert [error["path"] for error in errors] == [ + f"steps[0].pipeline.arguments.{key}" + ] + assert "'file' URL" in errors[0]["message"] + def test_a_loopback_url_is_an_error(self, untrusted, workflow_dir): definition = { "steps": [ From 1760a48b88836fd3e35c58b0e13ae30cb0734056 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 19:03:55 -0500 Subject: [PATCH 023/296] fix(server): #647 - unrealized get_job_workflow folds recorded arguments and says why Co-Authored-By: Claude Sonnet 5.5 --- dw/server/api_models.py | 1 + dw/server/jobs.py | 9 +++++++++ dw/server/routes/jobs.py | 33 ++++++++++++++++++++++++++++++--- dw_mcp/diagnose.py | 5 ++++- dw_mcp/tools_jobs.py | 4 ++-- tests/test_server.py | 19 +++++++++++++++++++ 6 files changed, 65 insertions(+), 6 deletions(-) diff --git a/dw/server/api_models.py b/dw/server/api_models.py index eff7a8a7..e494d40f 100644 --- a/dw/server/api_models.py +++ b/dw/server/api_models.py @@ -355,6 +355,7 @@ class JobWorkflow(ApiModel): description="The variable a new-seed rerun would draw into, null when the " "workflow has none - the cue for whether to offer that at all." ) + note: str = sometimes("Present when `realized` is false: why, and what was folded.") class ExportedFile(ApiModel): diff --git a/dw/server/jobs.py b/dw/server/jobs.py index 34565611..f4ad0ad7 100644 --- a/dw/server/jobs.py +++ b/dw/server/jobs.py @@ -303,6 +303,15 @@ def realized(self, job_id): logger.debug(f"No realized workflow for job {job_id}: {e}") return None + def arguments(self, job_id): + """The arguments a job was submitted with, or {} when none are on + record.""" + job = self.jobs.get(job_id) + if job is not None: + return job.spec.get("arguments") or {} + historical = self.history.get(job_id) + return (historical or {}).get("arguments") or {} + def seed_variable(self, job_id): """The variable this job's workflow draws its seed from, or None. diff --git a/dw/server/routes/jobs.py b/dw/server/routes/jobs.py index 2ca9f43a..8544442d 100644 --- a/dw/server/routes/jobs.py +++ b/dw/server/routes/jobs.py @@ -7,6 +7,7 @@ """ import asyncio +import copy import json import logging import os @@ -311,9 +312,11 @@ def get_job_workflow(request: Request, job_id: str): and for `get_job_workflow` over MCP. `realized: true` means every mutable input is pinned - the copy the - run itself wrote. `false` means the job predates run tracking (or its - run directory is gone) and this is the definition as submitted. 404 - when neither is readable - the job itself still is.""" + run itself wrote. `false` means the run's copy is not there (the job + predates run tracking, or its run directory or workspace was deleted) + and this is the definition as submitted, with the arguments on record + laid over its variables and a `note` saying so. 404 when neither is + readable - the job itself still is.""" manager = request.app.state.job_manager if manager.get(job_id) is None: raise HTTPException(status_code=404, detail="Unknown job") @@ -323,7 +326,31 @@ def get_job_workflow(request: Request, job_id: str): raise HTTPException( status_code=404, detail="No workflow definition for this job" ) + extra = {} + if realized is None: + definition = copy.deepcopy(definition) + arguments = manager.arguments(job_id) + variables = definition.get("variables") + if isinstance(variables, dict): + folded = [name for name in arguments if name in variables] + for name in folded: + variables[name] = arguments[name] + else: + folded = [] + extra["note"] = ( + "The run's realized copy is gone (the job predates run tracking, " + "or its run directory or workspace was deleted). This is the " + "definition as submitted" + + ( + f", with the recorded arguments folded into its variables " + f"({', '.join(folded)}); seed, stored prompts and " + "output:latest references are not pinned." + if folded + else "; get_job.arguments holds what the caller passed." + ) + ) return { + **extra, "id": job_id, "definition": definition, "realized": realized is not None, diff --git a/dw_mcp/diagnose.py b/dw_mcp/diagnose.py index 220fd760..d96e53e8 100644 --- a/dw_mcp/diagnose.py +++ b/dw_mcp/diagnose.py @@ -175,13 +175,16 @@ def get_job(client, job_id): def get_job_workflow(client, job_id): """The workflow a job ran. `realized: true` means every mutable input is pinned (arguments, seed, prompts, output:latest); false means the - job predates run tracking and this is the definition as submitted. + run's copy is gone (old job, or its run directory or workspace was + deleted) and `note` says what was folded from the recorded arguments. Pass it to save_workflow to rerun it by name, or edit it and pass it to run_workflow as inline_workflow.""" body = client.get_json(api_path("api", "jobs", job_id, "workflow")) + extra = {"note": body["note"]} if body.get("note") else {} return { "job_id": job_id, "realized": bool(body.get("realized")), + **extra, "workflow": body.get("definition"), # Which variable rerun_job(new_seed=True) would draw into, null when # the workflow has none - see rerun_job on why that matters diff --git a/dw_mcp/tools_jobs.py b/dw_mcp/tools_jobs.py index 3aae9cbe..a53919d9 100644 --- a/dw_mcp/tools_jobs.py +++ b/dw_mcp/tools_jobs.py @@ -93,8 +93,8 @@ def get_job_workflow(self, job_id: str) -> dict: variables, the seed the run used, stored prompt text inlined, and any `output:.../latest/...` rewritten to the run it resolved to - so the definition reproduces that run however the library changes. When it is - false the job predates run tracking and this is the definition as - submitted. After a long inline run worth keeping, this then + false the run's copy is gone (old job, or its workspace was deleted) + and `note` says what was folded in from the recorded arguments. After a long inline run worth keeping, this then `save_workflow` is how it gets a name.""" return diagnose.get_job_workflow(self.client, job_id) diff --git a/tests/test_server.py b/tests/test_server.py index 410fc432..5ae3c51c 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -556,6 +556,25 @@ def test_job_workflow_reports_whether_it_is_realized(server): assert body["definition"]["id"] == "server_test" +def test_unrealized_job_workflow_folds_recorded_arguments_and_says_why(server): + """A job whose run copy is gone (workspace deleted) still comes back with + the caller's arguments in its variables and a note, not the declared + defaults under a "predates run tracking" reading.""" + with server(success_script) as client: + submitted = client.post( + "/api/jobs", + json={"workflow": valid_workflow(), "arguments": {"prompt": "a cat"}}, + ).json() + wait_for_status(client, submitted["id"], TERMINAL_STATES) + manager = client.app.state.job_manager + manager.jobs.pop(submitted["id"]) + body = client.get(f"/api/jobs/{submitted['id']}/workflow").json() + + assert body["realized"] is False + assert body["definition"]["variables"]["prompt"] == "a cat" + assert "prompt" in body["note"] and "deleted" in body["note"] + + def test_submit_rejects_a_real_path_outside_the_workflow_dir(server, tmp_path): """workflow_path is confined to --workflow-dir, the same as the /api/workflows CRUD routes - a real, existing file elsewhere on disk From 8d1247cfcea038007918ecbaac4a0238e8944e54 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 19:11:57 -0500 Subject: [PATCH 024/296] feat(tasks): #622 - crop_face_track: follow one face, crop a steady square, record the track Co-Authored-By: Claude Opus 5.5 --- dw/media_types.py | 12 ++ dw/result.py | 10 + dw/task_domains.py | 132 +++++++++++- dw/tasks/face_track.py | 462 +++++++++++++++++++++++++++++++++++++++++ dw/tasks/task.py | 39 ++++ tests/test_shots.py | 1 + 6 files changed, 654 insertions(+), 2 deletions(-) create mode 100644 dw/tasks/face_track.py diff --git a/dw/media_types.py b/dw/media_types.py index aacf08c0..a36c7e91 100644 --- a/dw/media_types.py +++ b/dw/media_types.py @@ -103,6 +103,18 @@ def __repr__(self): return f"Selected(value={self.value!r}, position={self.position}, score={self.score!r})" +class JsonRecord(dict): + """A JSON-safe dict a step hands on as data, and saves as a .json file. + + A dict artifact is otherwise saved one entry per file under the step's + declared content type, so a record riding beside a video (a face track + beside its crops) would be exploded key by key into files that are not + video. Marking it says what it is: the writer saves it whole, as JSON, + whatever the step's content type, and `previous_result:.` + still reads it as the dict it is. + """ + + # How far a decoded track may be off the frames' own duration and still be # treated as codec padding rather than a track of its own length. AAC codes # 1024 samples at a time, so a file's audio runs up to one such block long - diff --git a/dw/result.py b/dw/result.py index 0f8f4d15..47ebed0d 100644 --- a/dw/result.py +++ b/dw/result.py @@ -22,6 +22,7 @@ from .media_types import ( AudioTrack, AudioVideo, + JsonRecord, Selected, fit_codec_padding, sample_axis, @@ -391,6 +392,15 @@ def save_artifact( _refuse_scalar_artifact(artifact, file_base_name) + if isinstance(artifact, JsonRecord): + # Data riding beside the step's media, saved whole whatever the + # step's content type - never exploded key by key below + output_path = output_file_path(output_dir, f"{file_base_name}.json") + logger.info(f"Saving JSON record to {output_path}") + with open(output_path, "w") as file: + file.write(json.dumps(artifact, indent=4)) + return [output_path] + if isinstance(artifact, dict): return self._save_mapping_artifact( output_dir, artifact, file_base_name, content_type, extension diff --git a/dw/task_domains.py b/dw/task_domains.py index ec757876..c5717536 100644 --- a/dw/task_domains.py +++ b/dw/task_domains.py @@ -185,6 +185,13 @@ "temperature": CLOSED_UNIT, "tint": CLOSED_UNIT, }, + "crop_face_track": { + "crop_size": POSITIVE, + "padding": NON_NEGATIVE, + "gate_full": POSITIVE, + "gate_zero": POSITIVE, + "min_confidence": POSITIVE, + }, } @@ -260,6 +267,123 @@ def ingredients_grid_errors(arguments): return errors +# crop_face_track's rules past a plain domain. The crop feeds a video model +# whose latent grid is 32 pixels, the padding is a multiple of the face's own +# size added on each side, and the gate is a ramp from gate_full to gate_zero, +# so the two must be in that order. Owned here so the static pass and the +# command's run-time refusal read one rule +FACE_CROP_MULTIPLE = 32 +FACE_PADDING_MAX = 3.0 +FACE_DETECTOR_SUFFIX = ".onnx" + + +def face_track_problems( + crop_size=None, padding=None, gate_full=None, gate_zero=None, min_confidence=None +): + """[(argument, message)] for each crop_face_track rule these values break. + + A value that is None or not a number is skipped - the domain check, or + another pass, owns that complaint. + """ + problems = [] + size = as_number(crop_size) + if size is not None and size > 0: + if size != int(size) or int(size) % FACE_CROP_MULTIPLE: + problems.append( + ( + "crop_size", + f"crop_face_track needs 'crop_size' as a multiple of " + f"{FACE_CROP_MULTIPLE}, got {crop_size!r} - the crops feed a " + f"video model whose frame size moves in steps of " + f"{FACE_CROP_MULTIPLE}", + ) + ) + pad = as_number(padding) + if pad is not None and pad > FACE_PADDING_MAX: + problems.append( + ( + "padding", + f"crop_face_track needs 'padding' from 0 to {FACE_PADDING_MAX} " + f"(the face's size added on each side), got {padding!r}", + ) + ) + full, zero = as_number(gate_full), as_number(gate_zero) + for name, value, number in ( + ("gate_full", gate_full, full), + ("gate_zero", gate_zero, zero), + ): + if number is not None and number > 1: + problems.append( + ( + name, + f"crop_face_track needs '{name}' as a fraction of the frame " + f"width, at most 1.0, got {value!r}", + ) + ) + if full is not None and zero is not None and zero <= full: + problems.append( + ( + "gate_zero", + f"crop_face_track needs 'gate_zero' above 'gate_full' - " + f"strength ramps from full at gate_full down to 0 at gate_zero " + f"- got gate_full {gate_full!r} and gate_zero {gate_zero!r}", + ) + ) + confidence = as_number(min_confidence) + if confidence is not None and confidence >= 1: + problems.append( + ( + "min_confidence", + f"crop_face_track needs 'min_confidence' below 1.0, got " + f"{min_confidence!r}", + ) + ) + return problems + + +def check_face_detector_source(repo, filename): + """Refuse a detector source that is not a Hub repo id and a bare .onnx + file name in it - before anything is downloaded.""" + from .locations import validate_hub_repo_id, validate_weight_name + + validate_hub_repo_id(repo, what="detector_repo") + validate_weight_name( + filename, (FACE_DETECTOR_SUFFIX,), what="detector_file", subfolders=False + ) + + +def face_track_errors(arguments): + """[(argument, message)] for the crop_face_track rules a literal workflow + can break before it runs: the crop, padding and gate rules, and a detector + source that is not a Hub repo and an .onnx file.""" + literal = { + name: arguments.get(name) + for name in ("crop_size", "padding", "gate_full", "gate_zero", "min_confidence") + if not is_ref(DEFERRED, arguments.get(name)) + } + errors = face_track_problems(**literal) + from .locations import validate_hub_repo_id, validate_weight_name + from .security import SecurityError + + for name, check in ( + ("detector_repo", lambda v: validate_hub_repo_id(v, what=name)), + ( + "detector_file", + lambda v: validate_weight_name( + v, (FACE_DETECTOR_SUFFIX,), what=name, subfolders=False + ), + ), + ): + value = arguments.get(name) + if name not in arguments or is_ref(DEFERRED, value): + continue + try: + check(value) + except (SecurityError, ValueError) as error: + errors.append((name, str(error).rstrip("."))) + return errors + + def in_domain(value, domain): """Whether a number satisfies a domain. Anything unmeasurable is True - a value this cannot read is not this check's to refuse.""" @@ -404,8 +528,12 @@ def task_argument_errors(workflow_definition, source_indices=None): "message": f"{message}{where}.", } ) - if command == "ingredients_grid": - for key, message in ingredients_grid_errors(arguments): + extra = { + "ingredients_grid": ingredients_grid_errors, + "crop_face_track": face_track_errors, + }.get(command) + if extra is not None: + for key, message in extra(arguments): path = ("steps", source, "task", "arguments", key) errors.append( {"path": render_path(path), "message": f"{message}{where}."} diff --git a/dw/tasks/face_track.py b/dw/tasks/face_track.py new file mode 100644 index 00000000..17e0a834 --- /dev/null +++ b/dw/tasks/face_track.py @@ -0,0 +1,462 @@ +"""Follow one face through a clip and crop a steady square around it. + +A face-detail pass (a refiner run on just the face) needs the face large and +still: a small face in a wide shot has too few pixels for a model to restore, +and a crop that jitters with every detection hands the refiner a face that +shakes. This task finds the face, follows it, smooths its box, and cuts the +same padded square around it from every frame - plus a record of where each +crop came from, so a later step can paste the refined face back. + +Detection is YuNet (OpenCV's own face detector, `cv2.FaceDetectorYN`), run on +the whole frame and on four overlapping tiles enlarged 2x, so a face only a few +dozen pixels wide is still seen; the tiles' detections are merged by +non-maximum suppression. One track is kept: each frame's detection is matched +to it by overlap, confidence and distance, the box is smoothed by an +exponential moving average, and a frame the detector misses holds the last box +at a decaying strength for a few frames. The track starts again at each shot +boundary the clip records, or - for a clip that records none - where the +picture's colour histogram changes abruptly. + +Each frame also carries a strength from 0 to 1: how much a face-detail pass +should change it. A face already large in the frame has the pixels it needs, +so strength falls from 1 at `gate_full` (face width over frame width) to 0 at +`gate_zero`. +""" + +import logging + +import numpy as np +from PIL import Image + +from ..events import emit_log, emit_warning +from ..media_types import AudioVideo, JsonRecord +from ..task_domains import check_face_detector_source, face_track_problems +from .video_utils import frames_as_pil_list, load_audio_video + +logger = logging.getLogger("dw") + +# OpenCV's own upload of YuNet (MIT), read at a pinned revision so a later +# push to the repo cannot change what an existing workflow runs +DEFAULT_DETECTOR_REPO = "opencv/face_detection_yunet" +DEFAULT_DETECTOR_FILE = "face_detection_yunet_2023mar.onnx" +DETECTOR_REVISION = "3cc26e7f1014a5ee5d74a42acee58bafc9d0a310" + +# Tiling: four corner tiles, each this fraction of the frame on a side (so +# neighbours overlap), enlarged by TILE_SCALE before detection +TILE_FRACTION = 0.6 +TILE_SCALE = 2.0 +NMS_IOU = 0.4 + +# Tracking +EMA_ALPHA = 0.35 # weight of the new detection in the smoothed box +HOLD_DECAY = 0.7 # strength kept per frame the detector misses +MAX_GAP = 6 # missed frames before the track counts as lost +MATCH_DISTANCE = 1.5 # centre distance, in face widths, a match may move +HISTOGRAM_CUT = 0.5 # HSV histogram correlation below which frames are a cut + + +def _iou(a, b): + """Overlap of two [x, y, w, h] boxes, as intersection over union.""" + ax2, ay2 = a[0] + a[2], a[1] + a[3] + bx2, by2 = b[0] + b[2], b[1] + b[3] + iw = max(0.0, min(ax2, bx2) - max(a[0], b[0])) + ih = max(0.0, min(ay2, by2) - max(a[1], b[1])) + inter = iw * ih + union = a[2] * a[3] + b[2] * b[3] - inter + return inter / union if union > 0 else 0.0 + + +def nms(detections, threshold=NMS_IOU): + """Greedy non-maximum suppression over [x, y, w, h, score] rows. + + The tiles overlap and the whole frame is searched too, so one face is + usually found more than once; the most confident copy is kept. + """ + ordered = sorted( + (list(map(float, d[:5])) for d in detections), key=lambda d: -d[4] + ) + kept = [] + for detection in ordered: + if all(_iou(detection, other) < threshold for other in kept): + kept.append(detection) + return kept + + +def _tiles(width, height): + """The regions searched: the whole frame, then four overlapping corners.""" + tw, th = int(round(width * TILE_FRACTION)), int(round(height * TILE_FRACTION)) + yield 0, 0, width, height, 1.0 + for x0 in (0, width - tw): + for y0 in (0, height - th): + yield x0, y0, tw, th, TILE_SCALE + + +def detect_tiled(detect, rgb): + """Every face in a frame, searched whole and in enlarged tiles, merged. + + Args: + detect: Callable taking an RGB uint8 array and returning rows of + [x, y, w, h, score] in that array's pixels + rgb: The frame, an RGB uint8 array + """ + import cv2 + + height, width = rgb.shape[:2] + found = [] + for x0, y0, tw, th, scale in _tiles(width, height): + region = rgb[y0 : y0 + th, x0 : x0 + tw] + if scale != 1.0: + region = cv2.resize( + region, + (int(round(tw * scale)), int(round(th * scale))), + interpolation=cv2.INTER_LINEAR, + ) + for row in detect(np.ascontiguousarray(region)) or []: + x, y, w, h, score = (float(v) for v in row[:5]) + found.append([x / scale + x0, y / scale + y0, w / scale, h / scale, score]) + return nms(found) + + +def _hsv_histogram(rgb): + import cv2 + + hsv = cv2.cvtColor(rgb, cv2.COLOR_RGB2HSV) + histogram = cv2.calcHist([hsv], [0, 1], None, [32, 32], [0, 180, 0, 256]) + return cv2.normalize(histogram, histogram).flatten() + + +def histogram_cuts(frames_rgb, threshold=HISTOGRAM_CUT): + """Frame indices where the picture changes abruptly - a cut nobody recorded.""" + import cv2 + + cuts = [] + previous = None + for index, rgb in enumerate(frames_rgb): + current = _hsv_histogram(rgb) + if previous is not None: + correlation = cv2.compareHist(previous, current, cv2.HISTCMP_CORREL) + if correlation < threshold: + cuts.append((index, float(correlation))) + previous = current + return cuts + + +def shot_starts(shots): + """{frame: shot name} for each recorded shot that starts after frame 0.""" + starts = {} + for shot in shots or []: + start = shot.get("start_frame") + if isinstance(start, int) and start > 0: + starts[start] = shot.get("name") + return starts + + +def gate_strength(face_width, frame_width, gate_full, gate_zero): + """1 for a small face, 0 for a large one, linear between the two gates.""" + ratio = face_width / frame_width + if ratio <= gate_full: + return 1.0 + if ratio >= gate_zero: + return 0.0 + return (gate_zero - ratio) / (gate_zero - gate_full) + + +def _centre(box): + return box[0] + box[2] / 2, box[1] + box[3] / 2 + + +def _match(box, detections): + """The detection that continues a track, or None if none is close enough.""" + best, best_score = None, 0.0 + cx, cy = _centre(box) + for detection in detections: + dx, dy = _centre(detection) + distance = np.hypot(dx - cx, dy - cy) / max(box[2], 1.0) + if distance > MATCH_DISTANCE: + continue + score = ( + _iou(box, detection) + + 0.5 * detection[4] + + 0.5 * (1.0 - distance / MATCH_DISTANCE) + ) + if score > best_score: + best, best_score = detection, score + return best + + +def track_faces(detections_per_frame, frame_width, resets_at, gate_full, gate_zero): + """Follow one face through per-frame detections. + + Args: + detections_per_frame: For each frame, the [x, y, w, h, score] rows + already above the confidence floor + frame_width: Source width in pixels, for the gate + resets_at: {frame: (reason, detail)} - where the track must start again + gate_full, gate_zero: The strength gate (gate_strength) + Returns: + (frames, resets): a dict per frame with "box" (smoothed, or None), + "strength" and "state" ("tracked", "held", "none"), and the list of + resets that happened, each {frame, reason, detail} + """ + frames, resets = [], [] + box, hold, missed = None, 1.0, 0 + for index, detections in enumerate(detections_per_frame): + if index in resets_at: + reason, detail = resets_at[index] + resets.append({"frame": index, "reason": reason, "detail": detail}) + box = None + + match = None + if box is None: + if detections: + match = max(detections, key=lambda d: d[4]) + else: + match = _match(box, detections) + + if match is not None: + if box is None: + box = list(match[:4]) + else: + box = [ + (1 - EMA_ALPHA) * old + EMA_ALPHA * new + for old, new in zip(box, match[:4]) + ] + hold, missed, state = 1.0, 0, "tracked" + elif box is not None: + missed += 1 + if missed > MAX_GAP: + resets.append({"frame": index, "reason": "lost", "detail": None}) + box, state = None, "none" + else: + hold *= HOLD_DECAY + state = "held" + else: + state = "none" + + if box is None: + frames.append({"box": None, "strength": 0.0, "state": state}) + else: + strength = hold * gate_strength(box[2], frame_width, gate_full, gate_zero) + frames.append( + { + "box": [round(v, 2) for v in box], + "strength": round(strength, 4), + "state": state, + } + ) + return frames, resets + + +def crop_square(box, padding, width, height): + """The integer square [x, y, side, side] padded around a face box.""" + if box is None: + side = min(width, height) + return [(width - side) // 2, (height - side) // 2, side, side] + side = max(box[2], box[3]) * (1 + 2 * padding) + side = max(int(round(side)), 1) + cx, cy = _centre(box) + return [int(round(cx - side / 2)), int(round(cy - side / 2)), side, side] + + +def _cut(rgb, square, crop_size): + """Cut a square from a frame, replicating the border past its edges.""" + import cv2 + + x, y, side, _ = square + height, width = rgb.shape[:2] + left, top = max(0, -x), max(0, -y) + right, bottom = max(0, x + side - width), max(0, y + side - height) + if left or top or right or bottom: + rgb = cv2.copyMakeBorder(rgb, top, bottom, left, right, cv2.BORDER_REPLICATE) + x, y = x + left, y + top + region = rgb[y : y + side, x : x + side] + return Image.fromarray(region).resize((crop_size, crop_size), Image.LANCZOS) + + +def padding_to_8n1(count): + """(before, after): frames to add so count becomes the next 8n+1.""" + target = count if (count - 1) % 8 == 0 else count + (8 - (count - 1) % 8) + pad = target - count + return pad // 2, pad - pad // 2 + + +def pad_frames(crops, before, after): + """Warm-up and cool-down frames, reflected off each end of the crops.""" + count = len(crops) + + def reflected(offset): + # offset counts outward from the end, 1 = the frame next to it + period = 2 * (count - 1) + if period == 0: + return 0 + position = offset % period + return position if position < count else period - position + + head = [crops[reflected(before - i)] for i in range(before)] + tail = [crops[count - 1 - reflected(i + 1)] for i in range(after)] + return head + crops + tail + + +def _detector(repo, filename, min_confidence, device): + """A callable running YuNet on an RGB array, loaded once per device.""" + import cv2 + + from .. import get_device_type + from .model_cache import cached_model + + check_face_detector_source(repo, filename) + revision = DETECTOR_REVISION if repo == DEFAULT_DETECTOR_REPO else None + + use_cuda = ( + get_device_type(device) == "cuda" + and hasattr(cv2, "cuda") + and cv2.cuda.getCudaEnabledDeviceCount() > 0 + ) + + def load(): + from huggingface_hub import hf_hub_download + + path = hf_hub_download(repo_id=repo, filename=filename, revision=revision) + backend, target = ( + (cv2.dnn.DNN_BACKEND_CUDA, cv2.dnn.DNN_TARGET_CUDA) + if use_cuda + else (cv2.dnn.DNN_BACKEND_OPENCV, cv2.dnn.DNN_TARGET_CPU) + ) + return cv2.FaceDetectorYN.create( + path, "", (320, 320), min_confidence, 0.3, 5000, backend, target + ) + + model = cached_model( + ("crop_face_track", repo, filename, revision, min_confidence, use_cuda), load + ) + + def detect(rgb): + height, width = rgb.shape[:2] + model.setInputSize((width, height)) + _, faces = model.detect(cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR)) + return [] if faces is None else [row[:4].tolist() + [row[-1]] for row in faces] + + return detect + + +def crop_face_track( + clip, + crop_size=512, + padding=0.6, + gate_full=0.06, + gate_zero=0.12, + min_confidence=0.6, + detector_repo=DEFAULT_DETECTOR_REPO, + detector_file=DEFAULT_DETECTOR_FILE, + device=None, +): + """Task command: crop a steady square around the one face a clip follows. + + Args: + clip: The video - frames, an AudioVideo, or the path or URL of a video + file. Named "clip" rather than "video" so the engine hands it over + as read, with the shots it records + crop_size: Side of every crop in pixels, a multiple of 32 + padding: Space added around the face on each side, as a fraction of + its size - 0.6 gives a square 2.2 face widths across + gate_full: Face width over frame width at or below which strength is 1 + gate_zero: Face width over frame width at or above which strength is 0 + min_confidence: Detections below this score are ignored + detector_repo: Hugging Face repo holding the YuNet weights + detector_file: The .onnx file in that repo + device: Where detection runs; CUDA only when OpenCV was built with it + Returns: + {"crops": AudioVideo of crop_size squares, padded to 8n+1 frames, + "track": the record of every frame's box, crop and strength} + """ + problems = face_track_problems( + crop_size=crop_size, + padding=padding, + gate_full=gate_full, + gate_zero=gate_zero, + min_confidence=min_confidence, + ) + if problems: + raise ValueError("; ".join(message for _, message in problems)) + crop_size = int(crop_size) + + detector = _detector(detector_repo, detector_file, min_confidence, device) + + if isinstance(clip, str): + clip = load_audio_video(clip) + frames = frames_as_pil_list(clip) + if not frames: + raise ValueError("crop_face_track needs a clip with at least one frame") + fps = getattr(clip, "fps", None) + rgb = [np.asarray(frame.convert("RGB")) for frame in frames] + height, width = rgb[0].shape[:2] + + shots = getattr(clip, "shots", None) + starts = shot_starts(shots) + if starts: + resets_at = {frame: ("shot", name) for frame, name in starts.items()} + else: + resets_at = { + frame: ("cut", {"histogram_correlation": round(correlation, 4)}) + for frame, correlation in histogram_cuts(rgb) + } + + detections = [ + [d for d in detect_tiled(detector, frame) if d[4] >= min_confidence] + for frame in rgb + ] + track, resets = track_faces(detections, width, resets_at, gate_full, gate_zero) + face_found = any(entry["box"] is not None for entry in track) + + # A frame without a box borrows the nearest one, at strength 0, so the + # crop does not jump to the frame centre and back across a short miss + boxes = [entry["box"] for entry in track] + known = [i for i, b in enumerate(boxes) if b is not None] + crops = [] + for index, entry in enumerate(track): + box = entry["box"] + if box is None and known: + box = boxes[min(known, key=lambda k: abs(k - index))] + square = crop_square(box, padding, width, height) + entry["crop"] = square + crops.append(_cut(rgb[index], square, crop_size)) + + before, after = padding_to_8n1(len(crops)) + crops = pad_frames(crops, before, after) + + record = JsonRecord( + { + "source": {"width": width, "height": height, "frames": len(frames), "fps": fps}, + "crop_size": crop_size, + "padding": padding, + "gate_full": gate_full, + "gate_zero": gate_zero, + "min_confidence": min_confidence, + "detector": {"repo": detector_repo, "file": detector_file}, + "pad_before": before, + "pad_after": after, + "crop_frames": len(crops), + "face_found": face_found, + "frames": track, + "resets": resets, + } + ) + if not face_found: + record["message"] = ( + f"No face was found in {len(frames)} frames (min_confidence " + f"{min_confidence}); every strength is 0 and each crop is the " + f"centre square of the frame" + ) + emit_warning( + f"crop_face_track: {record['message']}", + kind="no_face_found", + command="crop_face_track", + frames=len(frames), + ) + tracked = sum(entry["state"] == "tracked" for entry in track) + emit_log( + f"crop_face_track: face in {tracked} of {len(frames)} frames, " + f"{len(resets)} reset(s), {len(crops)} crops of {crop_size}px " + f"({before} before, {after} after for 8n+1)" + ) + return {"crops": AudioVideo(crops, None, None, fps=fps), "track": record} diff --git a/dw/tasks/task.py b/dw/tasks/task.py index 05ba3ebf..78f64071 100644 --- a/dw/tasks/task.py +++ b/dw/tasks/task.py @@ -344,6 +344,45 @@ def _handle_stabilize_video(task, arguments, previous_pipelines): return stabilize_video(**arguments) +@register_command( + "crop_face_track", + implementation="dw.tasks.face_track.crop_face_track", + consumes_device=True, + summary=( + "Follow one face through a clip and crop a steady square around it, " + "with a per-frame record of the box and a face-detail strength." + ), + parameter_descriptions={ + "clip": ( + "Video to track - a file path, an asset:/output: reference, or an " + "earlier step's video. A clip that records its shots resets the " + "track at each boundary." + ), + "crop_size": "Side of every crop in pixels; a multiple of 32.", + "padding": ( + "Space added around the face on each side, as a fraction of its " + "size (0 to 3)." + ), + "gate_full": ( + "Face width over frame width at or below which a frame's strength is 1." + ), + "gate_zero": ( + "Face width over frame width at or above which a frame's strength " + "is 0; must exceed gate_full." + ), + "min_confidence": "Detector score below which a detection is ignored.", + "detector_repo": "Hugging Face repo holding the YuNet face detector.", + "detector_file": "The .onnx file in detector_repo.", + }, +) +def _handle_crop_face_track(task, arguments, previous_pipelines): + """Crop a steady square around the one face a clip follows""" + logger.debug("Tracking a face") + from .face_track import crop_face_track + + return crop_face_track(device=task.device_for(arguments), **arguments) + + @register_command("mix_audio", implementation="dw.tasks.audio_utils.mix_audio") def _handle_mix_audio(task, arguments, previous_pipelines): """Layer audio tracks on top of one another, rather than end to end""" diff --git a/tests/test_shots.py b/tests/test_shots.py index 52c21da5..be924596 100644 --- a/tests/test_shots.py +++ b/tests/test_shots.py @@ -82,6 +82,7 @@ def audio_video(num_frames, level, fps=4, sample_rate=100): ("dw/tasks/video_utils.py", "_decode_audio_video"): ("none", 1), ("dw/output_extraction.py", "pair_audio_with_frames"): ("none", 1), ("dw/tasks/h3_latent_upscale.py", "decode_h3_latents"): ("none", 1), + ("dw/tasks/face_track.py", "crop_face_track"): ("none", 1), } _SHOTS_HELPER_BY_DECISION = { From 18391114ee43246e105f16ad5fb7a2092bef4945 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 19:14:58 -0500 Subject: [PATCH 025/296] test(tasks): #622 - crop_face_track unit tests; TASKS.md section Co-Authored-By: Claude Opus 5.5 --- docs/TASKS.md | 70 ++++++- dw/tasks/face_track.py | 11 +- tests/test_face_track.py | 412 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 488 insertions(+), 5 deletions(-) create mode 100644 tests/test_face_track.py diff --git a/docs/TASKS.md b/docs/TASKS.md index 9c9fcc1a..bb2ed37a 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -1,6 +1,6 @@ # Task Commands -Tasks are utility operations that run outside of pipeline inference. Use them for image preprocessing, data gathering, and other non-model operations. +Tasks are utility operations that run outside of a pipeline step. Use them for preprocessing, assembly, data gathering, and analysis - most are plain code, and some run a small helper model (an upscaler, a face detector, a stem separator). ```json { @@ -480,6 +480,74 @@ assembly templates once did and what made their output visibly wider than the source. The join tasks refuse shots of different sizes, so no normalization step is needed before them. +### crop_face_track + +Follow one face through a clip and cut a steady square around it, for a +face-detail pass that needs the face large and still: + +```json +{ + "name": "face_crops", + "task": { + "command": "crop_face_track", + "arguments": { + "clip": "asset:interview.mp4", + "crop_size": 512 + } + } +} +``` + +| Argument | Required | Description | +| -------- | -------- | ----------- | +| `clip` | Yes | The video - frames, an audio+video pair, or the path or URL of a video file. Named `clip` so the engine hands it over as read, with the shots it records | +| `crop_size` | No | Side of every crop in pixels, a multiple of 32. Default `512` | +| `padding` | No | Space added around the face on each side, as a fraction of its size, `0` to `3`. Default `0.6` | +| `gate_full` | No | Face width over frame width at or below which strength is 1. Default `0.06` | +| `gate_zero` | No | Face width over frame width at or above which strength is 0; must exceed `gate_full`. Default `0.12` | +| `min_confidence` | No | Detections scoring below this are ignored; below `1`. Default `0.6` | +| `detector_repo` | No | Hub repo holding the YuNet weights. Default `opencv/face_detection_yunet`, read at a pinned revision; any other must be a Hub repo id | +| `detector_file` | No | The detector file in that repo, a bare `.onnx` name. Default `face_detection_yunet_2023mar.onnx` | +| `device` | No | Where detection runs | + +The gate defaults are provisional; tune them against real footage. + +Detection is OpenCV's YuNet, run on the full frame and on four overlapping +enlarged tiles, merged by non-maximum suppression, so a face only a few dozen +pixels wide is still seen. One track is kept and smoothed with an exponential +moving average; across a short detector miss the last box is held at a decaying +strength. The track restarts at each shot boundary the clip records, or, for a +clip that records none, at an abrupt colour-histogram change. A padded square +around the smoothed box is cut from every frame and resized to `crop_size`, then +the crops are padded to 8n+1 frames with mirrored warm-up and cool-down frames. + +Each frame carries a strength from 0 to 1: how much a face-detail pass should +change it. It is 1 when face width over frame width is at or below `gate_full`, +0 at or above `gate_zero`, linear between, and multiplied by the hold decay on +frames the detector missed. + +It returns `{crops, track}`. `crops` is a video (no audio) at the clip's frame +rate. `track` is a JSON record saved as its own `.json` file, readable by +`previous_result:.track`: + +| Field | Description | +| ----- | ----------- | +| `source` | `{width, height, frames, fps}` of the input | +| `crop_size`, `padding`, `gate_full`, `gate_zero`, `min_confidence` | The parameters used | +| `detector` | `{repo, file}` | +| `pad_before`, `pad_after` | Mirrored frames added to the start and end | +| `crop_frames` | Frames in `crops`, including the padding | +| `face_found` | Whether any frame had a face | +| `message` | Present only when no face was found | +| `frames` | One entry per source frame: `box` (the smoothed `[x, y, w, h]`, or null), `crop` (the `[x, y, side, side]` square cut), `strength`, and `state` (`tracked`, `held` or `none`) | +| `resets` | `[{frame, reason, detail}]` where the track restarted; `reason` is `shot`, `cut` or `lost` | + +A clip with no face still succeeds: every strength is 0, the crops are the +centre square, and a `no_face_found` warning is raised. `validate_workflow` +refuses a `crop_size` that is not a multiple of 32, a `gate_zero` at or below +`gate_full`, a `padding` out of range, a `detector_repo` that is not a repo id +and a `detector_file` that is not a `.onnx` name, before anything downloads. + ### video_frames The frames of a generated video, as one `(frames, height, width, channels)` diff --git a/dw/tasks/face_track.py b/dw/tasks/face_track.py index 17e0a834..05d35227 100644 --- a/dw/tasks/face_track.py +++ b/dw/tasks/face_track.py @@ -72,9 +72,7 @@ def nms(detections, threshold=NMS_IOU): The tiles overlap and the whole frame is searched too, so one face is usually found more than once; the most confident copy is kept. """ - ordered = sorted( - (list(map(float, d[:5])) for d in detections), key=lambda d: -d[4] - ) + ordered = sorted((list(map(float, d[:5])) for d in detections), key=lambda d: -d[4]) kept = [] for detection in ordered: if all(_iou(detection, other) < threshold for other in kept): @@ -426,7 +424,12 @@ def crop_face_track( record = JsonRecord( { - "source": {"width": width, "height": height, "frames": len(frames), "fps": fps}, + "source": { + "width": width, + "height": height, + "frames": len(frames), + "fps": fps, + }, "crop_size": crop_size, "padding": padding, "gate_full": gate_full, diff --git a/tests/test_face_track.py b/tests/test_face_track.py new file mode 100644 index 00000000..ce6dae75 --- /dev/null +++ b/tests/test_face_track.py @@ -0,0 +1,412 @@ +""" +Unit tests for crop_face_track (dw/tasks/face_track.py) and its rules in +dw/task_domains.py. The YuNet detector is stubbed: a coloured square drawn +into synthetic frames is the "face", and the stub finds that colour's bounding +box in whatever array it is handed (the whole frame or an enlarged tile). +""" + +import json +import os +import tempfile + +import numpy as np +import pytest +from PIL import Image + +import dw.tasks.face_track as ft +from dw.media_types import AudioVideo, JsonRecord +from dw.result import Result +from dw.security import SecurityError +from dw.task_domains import task_argument_errors + +W, H = 320, 240 +FACE = (255, 0, 0) + + +def make_frame(face=None, background=(40, 160, 60)): + """An RGB uint8 frame; face is (x, y, size) of a red square, or None.""" + frame = np.zeros((H, W, 3), dtype=np.uint8) + frame[:] = background + if face is not None: + x, y, s = face + frame[y : y + s, x : x + s] = FACE + return frame + + +def stub_detect(rgb): + mask = (rgb[..., 0] > 200) & (rgb[..., 1] < 60) & (rgb[..., 2] < 60) + ys, xs = np.nonzero(mask) + if len(xs) == 0: + return [] + x0, x1, y0, y1 = xs.min(), xs.max() + 1, ys.min(), ys.max() + 1 + return [[float(x0), float(y0), float(x1 - x0), float(y1 - y0), 0.9]] + + +@pytest.fixture +def stub_detector(monkeypatch): + monkeypatch.setattr(ft, "_detector", lambda *a, **k: stub_detect) + + +@pytest.fixture +def warnings_seen(monkeypatch): + seen = [] + monkeypatch.setattr( + ft, "emit_warning", lambda message, **data: seen.append((message, data)) + ) + return seen + + +def clip_of(arrays, shots=None, fps=24): + return AudioVideo( + [Image.fromarray(a) for a in arrays], None, None, fps=fps, shots=shots + ) + + +def det(x=100, y=100, w=16, h=16, score=0.9): + return [x, y, w, h, score] + + +class TestNms: + def test_keeps_the_most_confident_of_overlapping_boxes(self): + kept = ft.nms([[10, 10, 20, 20, 0.7], [11, 11, 20, 20, 0.95]]) + assert len(kept) == 1 + assert kept[0][4] == 0.95 + + def test_keeps_non_overlapping_boxes(self): + kept = ft.nms([[0, 0, 10, 10, 0.8], [100, 100, 10, 10, 0.9]]) + assert len(kept) == 2 + assert kept[0][4] == 0.9 + + +class TestDetectTiled: + def test_one_face_found_once_near_its_true_box(self): + frame = make_frame((10, 10, 20)) + found = ft.detect_tiled(stub_detect, frame) + assert len(found) == 1 + x, y, w, h, _ = found[0] + assert abs(x - 10) <= 2 and abs(y - 10) <= 2 + assert abs(w - 20) <= 3 and abs(h - 20) <= 3 + + def test_a_tile_only_detection_maps_back_to_frame_coords(self): + # the whole-frame search sees nothing; only an enlarged tile does + calls = [] + + def detect(rgb): + calls.append(rgb.shape) + if rgb.shape[:2] == (H, W): + return [] + return [[20.0, 30.0, 40.0, 40.0, 0.9]] + + found = ft.detect_tiled(detect, make_frame()) + assert len(calls) == 5 + # every tile reports the same tile-local box, each mapping to a different + # frame position; the top-left tile's maps to (10, 15, 20, 20) + assert any( + abs(f[0] - 10) < 1e-6 and abs(f[1] - 15) < 1e-6 and f[2] == 20 + for f in found + ) + + +class TestTracking: + def test_ema_moves_partway_to_a_jumped_face(self): + dets = [[det(100, 100)], [det(110, 100)]] + frames, _ = ft.track_faces(dets, W, {}, 0.06, 0.12) + assert frames[0]["box"][0] == 100 + expected = 100 + ft.EMA_ALPHA * 10 + assert frames[1]["box"][0] == pytest.approx(expected, abs=0.01) + assert frames[1]["state"] == "tracked" + + def test_steady_face_keeps_its_box(self): + frames, resets = ft.track_faces([[det()]] * 5, W, {}, 0.06, 0.12) + assert all(f["box"] == [100, 100, 16, 16] for f in frames) + assert resets == [] + + def test_gap_is_held_with_decaying_strength(self): + dets = [[det()]] + [[]] * 3 + frames, resets = ft.track_faces(dets, W, {}, 0.06, 0.12) + assert frames[0]["strength"] == 1.0 + for i in (1, 2, 3): + assert frames[i]["state"] == "held" + assert frames[i]["box"] == frames[0]["box"] + assert frames[i]["strength"] == pytest.approx(ft.HOLD_DECAY**i, abs=1e-3) + assert resets == [] + + def test_gap_of_max_gap_is_still_held_and_beyond_is_lost(self): + dets = [[det()]] + [[]] * (ft.MAX_GAP + 2) + frames, resets = ft.track_faces(dets, W, {}, 0.06, 0.12) + assert frames[ft.MAX_GAP]["state"] == "held" + lost_at = ft.MAX_GAP + 1 + assert frames[lost_at]["state"] == "none" + assert frames[lost_at]["box"] is None and frames[lost_at]["strength"] == 0.0 + assert resets == [{"frame": lost_at, "reason": "lost", "detail": None}] + + def test_recovery_after_a_gap_restores_full_strength(self): + dets = [[det()], [], [det()]] + frames, _ = ft.track_faces(dets, W, {}, 0.06, 0.12) + assert frames[2]["state"] == "tracked" + assert frames[2]["strength"] == 1.0 + + def test_a_reset_restarts_the_box_without_smoothing(self): + dets = [[det(100, 100)], [det(100, 100)], [det(200, 50)]] + frames, resets = ft.track_faces(dets, W, {2: ("shot", "b")}, 0.06, 0.12) + assert frames[2]["box"][:2] == [200, 50] + assert resets == [{"frame": 2, "reason": "shot", "detail": "b"}] + + +class TestShotStarts: + def test_only_starts_after_frame_zero(self): + shots = [ + {"name": "a", "start_frame": 0}, + {"name": "b", "start_frame": 5}, + {"name": "c"}, + ] + assert ft.shot_starts(shots) == {5: "b"} + assert ft.shot_starts(None) == {} + + +class TestGate: + def test_ramp(self): + assert ft.gate_strength(6, 100, 0.06, 0.12) == 1.0 + assert ft.gate_strength(3, 100, 0.06, 0.12) == 1.0 + assert ft.gate_strength(12, 100, 0.06, 0.12) == 0.0 + assert ft.gate_strength(50, 100, 0.06, 0.12) == 0.0 + assert ft.gate_strength(9, 100, 0.06, 0.12) == pytest.approx(0.5) + + def test_small_face_is_full_and_frame_filling_face_is_zero(self, stub_detector): + small = ft.crop_face_track(clip_of([make_frame((100, 100, 12))] * 3)) + assert [f["strength"] for f in small["track"]["frames"]] == [1.0] * 3 + + big = ft.crop_face_track(clip_of([make_frame((0, 0, 230))] * 3)) + assert [f["strength"] for f in big["track"]["frames"]] == [0.0] * 3 + assert big["track"]["face_found"] is True + + +class TestHistogramCuts: + def test_cut_at_a_colour_change(self): + a = make_frame((100, 100, 16), background=(40, 160, 60)) + b = make_frame((100, 100, 16), background=(200, 40, 200)) + cuts = ft.histogram_cuts([a, a, a, b, b]) + assert [index for index, _ in cuts] == [3] + + def test_no_spurious_cut_on_a_steady_clip(self): + frame = make_frame((100, 100, 16)) + assert ft.histogram_cuts([frame] * 6) == [] + + def test_end_to_end_cut_is_recorded_as_a_reset(self, stub_detector): + a = make_frame((100, 100, 16), background=(40, 160, 60)) + b = make_frame((100, 100, 16), background=(200, 40, 200)) + result = ft.crop_face_track(clip_of([a, a, a, b, b])) + resets = result["track"]["resets"] + assert [(r["frame"], r["reason"]) for r in resets] == [(3, "cut")] + + +class TestShotReset: + def test_recorded_shot_start_resets_the_track(self, stub_detector): + frames = [make_frame((100, 100, 16))] * 3 + [make_frame((200, 50, 16))] * 3 + shots = [ + {"name": "one", "start_frame": 0}, + {"name": "two", "start_frame": 3}, + ] + result = ft.crop_face_track(clip_of(frames, shots=shots)) + track = result["track"] + assert {"frame": 3, "reason": "shot", "detail": "two"} in track["resets"] + # the box starts afresh at the new face rather than gliding to it + assert track["frames"][3]["box"][0] == pytest.approx(200, abs=3) + + +class TestPadding: + @pytest.mark.parametrize( + "count,expected", + [ + (1, (0, 0)), + (8, (0, 1)), + (9, (0, 0)), + (10, (3, 4)), + (17, (0, 0)), + (30, (1, 2)), + ], + ) + def test_padding_to_8n1(self, count, expected): + before, after = ft.padding_to_8n1(count) + assert (before, after) == expected + assert (count + before + after - 1) % 8 == 0 + + def test_pad_frames_mirrors_the_ends(self): + crops = list(range(10, 15)) # 5 frames + padded = ft.pad_frames(crops, 2, 3) + assert len(padded) == 10 + assert padded[2:7] == crops + assert padded[:2] == [12, 11] + assert padded[7:] == [13, 12, 11] + + def test_pad_frames_single_frame(self): + assert ft.pad_frames(["x"], 0, 0) == ["x"] + assert ft.pad_frames(["x"], 3, 4) == ["x"] * 8 + + def test_pad_frames_without_padding_is_identity(self): + assert ft.pad_frames([1, 2, 3], 0, 0) == [1, 2, 3] + + +class TestCropFaceTrack: + def test_end_to_end_shapes_and_record(self, stub_detector): + n = 5 + result = ft.crop_face_track( + clip_of([make_frame((100, 80, 16))] * n, fps=24), crop_size=64 + ) + crops, track = result["crops"], result["track"] + assert isinstance(crops, AudioVideo) + assert crops.audio is None + assert crops.fps == 24 + assert len(crops.frames) == 9 + for frame in crops.frames: + assert isinstance(frame, Image.Image) and frame.size == (64, 64) + + assert isinstance(track, JsonRecord) + json.dumps(track) + assert len(track["frames"]) == n + for entry in track["frames"]: + assert set(entry) >= {"box", "crop", "strength", "state"} + assert entry["crop"][2] == entry["crop"][3] + assert track["crop_frames"] == 9 + assert track["pad_before"] + track["pad_after"] == 4 + assert track["source"] == {"width": W, "height": H, "frames": n, "fps": 24} + assert track["face_found"] is True + assert "message" not in track + + def test_a_single_frame_clip_works(self, stub_detector): + result = ft.crop_face_track(clip_of([make_frame((100, 80, 16))]), crop_size=32) + assert len(result["crops"].frames) == 1 + assert len(result["track"]["frames"]) == 1 + + def test_no_face_succeeds_with_zero_strength_and_a_warning( + self, monkeypatch, warnings_seen + ): + monkeypatch.setattr(ft, "_detector", lambda *a, **k: lambda rgb: []) + result = ft.crop_face_track(clip_of([make_frame()] * 4), crop_size=64) + track = result["track"] + assert track["face_found"] is False + assert all(f["strength"] == 0.0 for f in track["frames"]) + assert "no face" in track["message"].lower() + assert len(result["crops"].frames) == 9 + assert [d["kind"] for _, d in warnings_seen] == ["no_face_found"] + + def test_bad_arguments_are_refused_at_run_time(self, stub_detector): + clip = clip_of([make_frame((100, 80, 16))]) + with pytest.raises(ValueError, match="crop_size"): + ft.crop_face_track(clip, crop_size=500) + with pytest.raises(ValueError, match="gate_zero"): + ft.crop_face_track(clip, gate_full=0.2, gate_zero=0.1) + with pytest.raises(ValueError, match="gate_zero"): + ft.crop_face_track(clip, gate_full=0.1, gate_zero=0.1) + + +class TestDetectorSource: + @pytest.fixture(autouse=True) + def no_download(self, monkeypatch): + import huggingface_hub + + def fail(*a, **k): + pytest.fail("a refused detector source reached the download") + + monkeypatch.setattr(huggingface_hub, "hf_hub_download", fail) + + def test_traversing_repo_is_refused(self): + with pytest.raises(SecurityError): + ft._detector("../evil", "x.onnx", 0.6, "cpu") + + def test_non_onnx_file_is_refused(self): + with pytest.raises(SecurityError): + ft._detector("opencv/face_detection_yunet", "weights.bin", 0.6, "cpu") + + +def workflow(arguments): + return { + "id": "x", + "steps": [ + { + "name": "s", + "task": {"command": "crop_face_track", "arguments": arguments}, + } + ], + } + + +class TestStaticValidation: + @pytest.mark.parametrize( + "arguments,name", + [ + ({"clip": "v.mp4", "crop_size": 500}, "crop_size"), + ({"clip": "v.mp4", "gate_full": 0.2, "gate_zero": 0.1}, "gate_zero"), + ({"clip": "v.mp4", "gate_full": 0.1, "gate_zero": 0.1}, "gate_zero"), + ({"clip": "v.mp4", "padding": 4}, "padding"), + ({"clip": "v.mp4", "padding": -1}, "padding"), + ({"clip": "v.mp4", "detector_repo": "../x"}, "detector_repo"), + ({"clip": "v.mp4", "detector_file": "w.bin"}, "detector_file"), + ], + ) + def test_bad_literal_is_an_error_at_its_argument(self, arguments, name): + errors = task_argument_errors(workflow(arguments)) + path = f"steps[0].task.arguments.{name}" + matching = [e for e in errors if e["path"] == path] + assert matching, errors + for error in errors: + # the refusal may quote the offending value, but adds no traversal text + assert ".." not in error["message"].replace("'../x'", "") + + def test_valid_workflow_has_no_errors(self): + args = { + "clip": "v.mp4", + "crop_size": 512, + "padding": 0.6, + "gate_full": 0.06, + "gate_zero": 0.12, + "min_confidence": 0.6, + "detector_repo": "opencv/face_detection_yunet", + "detector_file": "face_detection_yunet_2023mar.onnx", + } + assert task_argument_errors(workflow(args)) == [] + + def test_variable_references_are_not_judged(self): + args = { + name: f"variable:{name}" + for name in ( + "crop_size", + "padding", + "gate_full", + "gate_zero", + "detector_repo", + "detector_file", + ) + } + args["clip"] = "v.mp4" + assert task_argument_errors(workflow(args)) == [] + + +def test_command_is_registered(): + from dw.tasks.task import _COMMAND_REGISTRY + + assert "crop_face_track" in _COMMAND_REGISTRY + + +def test_json_record_is_saved_whole_under_a_video_content_type(): + record = JsonRecord({"a": 1, "frames": [{"box": None}], "nested": {"k": "v"}}) + with tempfile.TemporaryDirectory() as temp_dir: + result = Result({"content_type": "video/mp4", "save": True}) + result.add_result(record) + result.save(temp_dir, "wf-step.0") + names = os.listdir(temp_dir) + assert len(names) == 1 and names[0].endswith(".json") + with open(os.path.join(temp_dir, names[0])) as file: + assert json.load(file) == record + + +def test_the_task_result_saves_crops_as_video_and_track_as_one_json(): + crops = AudioVideo([Image.new("RGB", (64, 64))] * 9, None, None, fps=24) + with tempfile.TemporaryDirectory() as temp_dir: + result = Result({"content_type": "video/mp4", "save": True}) + result.add_result({"crops": crops, "track": JsonRecord({"a": {"b": 1}})}) + result.save(temp_dir, "wf-step.0") + names = sorted(os.listdir(temp_dir)) + assert [os.path.splitext(n)[1] for n in names] == [".mp4", ".json"] + with open(os.path.join(temp_dir, names[1])) as file: + assert json.load(file) == {"a": {"b": 1}} From 06effee13116dff0299760d1a1122804f47f419b Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 19:16:24 -0500 Subject: [PATCH 026/296] style(tasks): #622 - ruff format task_domains Co-Authored-By: Claude Opus 5.5 --- dw/task_domains.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dw/task_domains.py b/dw/task_domains.py index c5717536..afb436b0 100644 --- a/dw/task_domains.py +++ b/dw/task_domains.py @@ -32,7 +32,14 @@ import logging import numbers -from .references import DEFERRED, GATHER, MEMBER_SEPARATOR, author_index, is_ref, render_path +from .references import ( + DEFERRED, + GATHER, + MEMBER_SEPARATOR, + author_index, + is_ref, + render_path, +) logger = logging.getLogger("dw") From de21ee3ab94360afcfe55490d0286fa60d45557e Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 19:23:43 -0500 Subject: [PATCH 027/296] fix(server): #647 - fold recorded arguments through set_variables Co-Authored-By: Claude Sonnet 5.5 --- dw/server/routes/jobs.py | 26 ++++++++++++++++++++------ tests/test_server.py | 18 ++++++++++++++++++ 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/dw/server/routes/jobs.py b/dw/server/routes/jobs.py index 8544442d..e18d1452 100644 --- a/dw/server/routes/jobs.py +++ b/dw/server/routes/jobs.py @@ -33,6 +33,7 @@ from ...plan import build_plan, gate_warnings from ...schema import format_validation_errors from ...security import SecurityError, validate_path +from ...variables import set_variables from ...library import SubWorkflowNotFound, resolve_sub_workflow_reference from ...workspace import Workspace from ..admission import ( @@ -331,12 +332,20 @@ def get_job_workflow(request: Request, job_id: str): definition = copy.deepcopy(definition) arguments = manager.arguments(job_id) variables = definition.get("variables") + folded = [] + failure = None if isinstance(variables, dict): - folded = [name for name in arguments if name in variables] - for name in folded: - variables[name] = arguments[name] - else: - folded = [] + declared = { + name: arguments[name] for name in arguments if name in variables + } + # Fold through the owner, so a value is validated and coerced to + # the default's type exactly as it was when the run started + try: + set_variables(declared, variables) + folded = list(declared) + except (SecurityError, TypeError, ValueError) as exc: + definition = copy.deepcopy(manager.definition(job_id)) + failure = str(exc) extra["note"] = ( "The run's realized copy is gone (the job predates run tracking, " "or its run directory or workspace was deleted). This is the " @@ -346,7 +355,12 @@ def get_job_workflow(request: Request, job_id: str): f"({', '.join(folded)}); seed, stored prompts and " "output:latest references are not pinned." if folded - else "; get_job.arguments holds what the caller passed." + else ( + f"; the recorded arguments could not be folded ({failure}): " + "see get_job.arguments." + if failure + else "; get_job.arguments holds what the caller passed." + ) ) ) return { diff --git a/tests/test_server.py b/tests/test_server.py index 5ae3c51c..e37f2e2d 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -575,6 +575,24 @@ def test_unrealized_job_workflow_folds_recorded_arguments_and_says_why(server): assert "prompt" in body["note"] and "deleted" in body["note"] +def test_unrealized_job_workflow_folds_arguments_coerced_like_the_run(server): + """The fold goes through set_variables, so a numeric variable passed as + a string comes back as the number the run used.""" + workflow = valid_workflow() + workflow["variables"]["steps"] = 20 + with server(success_script) as client: + submitted = client.post( + "/api/jobs", + json={"workflow": workflow, "arguments": {"steps": "30"}}, + ).json() + wait_for_status(client, submitted["id"], TERMINAL_STATES) + client.app.state.job_manager.jobs.pop(submitted["id"]) + body = client.get(f"/api/jobs/{submitted['id']}/workflow").json() + + assert body["realized"] is False + assert body["definition"]["variables"]["steps"] == 30 + + def test_submit_rejects_a_real_path_outside_the_workflow_dir(server, tmp_path): """workflow_path is confined to --workflow-dir, the same as the /api/workflows CRUD routes - a real, existing file elsewhere on disk From 448bb3e5dbda2e251e39c155a7ca103e3444050e Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 19:27:39 -0500 Subject: [PATCH 028/296] feat(tasks): #622 - NMS via cv2.dnn.NMSBoxes, 8n+1 pad via np.pad; map rows Arch-review bounce: replace the hand-written greedy NMS and reflected pad indices with the library calls, add face_track_problems / face_track_errors / check_face_detector_source to the task-domains row, and add a JSON-records-beside-media row for JsonRecord. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 3 ++- dw/tasks/face_track.py | 30 ++++++++++-------------------- tests/test_face_track.py | 9 +++++++++ 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index ebe591b0..13600c38 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -31,7 +31,7 @@ one to open. | LoRAs | `dw/pipeline_processors/adapters.py`: `active_loras`, `load_loras` | A `loras` entry whose `model_name` is null is switched off. | `tests/test_lora_disable.py::TestLoadLoras::test_an_all_null_entry_loads_nothing` | | Step progress | `dw/pipeline_processors/progress.py`: `reported_progress_bars` | A pipeline with no step callback reports its denoise steps through its progress bars. | `tests/test_modular_progress.py::test_every_advance_of_the_bar_is_reported` | | Adding a task | `dw/tasks/task.py`: `register_command` | A task is a function registered with `@register_command`, and the signature of its implementation is its argument schema. | `tests/test_task_discovery.py::TestDescribeTask::test_signature_becomes_the_schema` | -| Task argument domains | `dw/task_domains.py`: `check_arguments` | A numeric domain that a signature cannot express is declared in this table, and it is checked at validation and again at run time. A literal enum choice (`choice_errors`) and the `ingredients_grid` image count against `max_images` (`ingredients_grid_errors`) are refused at validation by the same module. | `tests/test_task_domains.py::TestTheStaticPass::test_a_negative_frame_count_is_refused_at_its_path`, `tests/test_task_domains.py::TestAtRunTime::test_slice_audio_refuses_a_negative_count`, `tests/test_ingredients_grid.py::TestStaticValidation::test_bad_layout_and_fit` | +| Task argument domains | `dw/task_domains.py`: `check_arguments` | A numeric domain that a signature cannot express is declared in this table, and it is checked at validation and again at run time. A literal enum choice (`choice_errors`) and the `ingredients_grid` image count against `max_images` (`ingredients_grid_errors`) are refused at validation by the same module. So are `crop_face_track`'s cross-argument rules (`face_track_problems`, `face_track_errors`: crop size a multiple of 32, `gate_zero` above `gate_full`, padding and confidence limits), which the task checks again at run time, and `check_face_detector_source` vets its Hub repo and `.onnx` file before any download. | `tests/test_task_domains.py::TestTheStaticPass::test_a_negative_frame_count_is_refused_at_its_path`, `tests/test_task_domains.py::TestAtRunTime::test_slice_audio_refuses_a_negative_count`, `tests/test_ingredients_grid.py::TestStaticValidation::test_bad_layout_and_fit`, `tests/test_face_track.py::TestStaticValidation::test_bad_literal_is_an_error_at_its_argument`, `tests/test_face_track.py::TestCropFaceTrack::test_bad_arguments_are_refused_at_run_time`, `tests/test_face_track.py::TestDetectorSource::test_traversing_repo_is_refused` | | Variables | `dw/variables.py`: `set_variables`, `argument_errors`, `resolve_variable_values` | `argument_errors` folds a caller's arguments in exactly as `set_variables` does when a run starts, so a bad name or value is refused before anything is queued. | `tests/test_variables.py::test_argument_errors_reports_a_dict_passed_for_a_string_variable` | | List variables from the CLI | `dw/run.py`, `dw/variables.py`: `get_value` | A `name=value` string given for a list variable is split on commas, so a list of objects can only be passed as JSON over the API or MCP. | `tests/test_variables.py::test_set_variables_list_default_splits_on_comma` | | `for_each` expansion | `dw/for_each.py`: `expand_for_each` | Each entry becomes an ordinary step named `@`, after substitution and before the reference check. | `tests/test_for_each.py::TestNaming::test_member_name_joins_with_at` | @@ -65,6 +65,7 @@ one to open. | Run-version surfaces | `dw/runs.py`: `run_versions`, `dw/server/routes/gallery.py`, `dw/server/outputs.py`, `dw_mcp/tools_catalog.py`: `list_gallery`, `ui/src/lib/pages/GalleryPage.svelte` | Only `dw/runs.py` computes the number, and every other module reads it back. | `tests/test_runs.py::TestRunVersions::test_an_output_reference_can_name_a_run_by_its_version` | | Result subfolders | `dw/subfolders.py`: `subfolder_errors`, `dw/security.py`: `SUBFOLDER_PATTERN` | A step's `result.subfolder` is checked for shape at validation. | `tests/test_subfolders.py::TestValidationErrorsIntegration::test_validation_errors_reports_a_bad_subfolder` | | Template subfolder roles | `workflows/templates/`, `dw/workflows/` | Every saving step of a template is marked `final` or `intermediate`, with at least one `final`, and packaged builtins stay unmarked. | `tests/test_template_subfolders.py::test_every_saving_step_of_a_template_names_its_role`, `tests/test_template_subfolders.py::test_the_packaged_builtins_stay_unmarked` | +| JSON records beside media | `dw/media_types.py`: `JsonRecord`, `dw/result.py`: `Result.save_artifact` | A dict marked `JsonRecord` is saved whole as one `.json`, whatever content type the step declares, and is never split into one file per key. | `tests/test_face_track.py::test_json_record_is_saved_whole_under_a_video_content_type`, `tests/test_face_track.py::test_the_task_result_saves_crops_as_video_and_track_as_one_json` | | Shots in a joined video | `dw/shots.py` | Every `AudioVideo` constructor decides what to do with `shots`, and the sample spans are measured from the joined waveform. | `tests/test_shots.py::test_every_audio_video_constructor_site_is_decided`, `tests/test_shots.py::TestConcatVideosShots::test_an_overrun_track_is_measured_not_derived` | | Audio level of a deliverable | `dw/audio_qc.py`: `warn_without_headroom`, `warn_if_written_above_full_scale` | A muxed video keeps the pre-encode headroom prediction and falls back to it only when the post-write probe cannot measure the file. | `tests/test_result.py::TestNoHeadroom::test_a_clean_video_mux_drops_the_stale_prediction`, `tests/test_result.py::TestNoHeadroom::test_an_unprobeable_video_mux_falls_back_to_the_prediction`, `tests/test_result.py::TestTheWrittenLevel::test_a_file_that_decodes_above_full_scale_warns` | | Template audio-level placement | `workflows/templates/minimax/music-video.json`, `workflows/templates/minimax/music.json` | `normalize_audio` (-3 dBFS) acts only on the track that goes into the final file, so the slices that condition the shots are untouched. | — | diff --git a/dw/tasks/face_track.py b/dw/tasks/face_track.py index 05d35227..88cc8c1d 100644 --- a/dw/tasks/face_track.py +++ b/dw/tasks/face_track.py @@ -67,17 +67,18 @@ def _iou(a, b): def nms(detections, threshold=NMS_IOU): - """Greedy non-maximum suppression over [x, y, w, h, score] rows. + """Non-maximum suppression over [x, y, w, h, score] rows (`cv2.dnn.NMSBoxes`). The tiles overlap and the whole frame is searched too, so one face is usually found more than once; the most confident copy is kept. """ - ordered = sorted((list(map(float, d[:5])) for d in detections), key=lambda d: -d[4]) - kept = [] - for detection in ordered: - if all(_iou(detection, other) < threshold for other in kept): - kept.append(detection) - return kept + import cv2 + + rows = [list(map(float, d[:5])) for d in detections] + if not rows: + return [] + keep = cv2.dnn.NMSBoxes([r[:4] for r in rows], [r[4] for r in rows], 0.0, threshold) + return [rows[i] for i in np.asarray(keep, dtype=int).reshape(-1)] def _tiles(width, height): @@ -280,19 +281,8 @@ def padding_to_8n1(count): def pad_frames(crops, before, after): """Warm-up and cool-down frames, reflected off each end of the crops.""" - count = len(crops) - - def reflected(offset): - # offset counts outward from the end, 1 = the frame next to it - period = 2 * (count - 1) - if period == 0: - return 0 - position = offset % period - return position if position < count else period - position - - head = [crops[reflected(before - i)] for i in range(before)] - tail = [crops[count - 1 - reflected(i + 1)] for i in range(after)] - return head + crops + tail + indices = np.pad(np.arange(len(crops)), (before, after), mode="reflect") + return [crops[i] for i in indices] def _detector(repo, filename, min_confidence, device): diff --git a/tests/test_face_track.py b/tests/test_face_track.py index ce6dae75..34937bc6 100644 --- a/tests/test_face_track.py +++ b/tests/test_face_track.py @@ -77,6 +77,15 @@ def test_keeps_non_overlapping_boxes(self): assert len(kept) == 2 assert kept[0][4] == 0.9 + def test_no_detections_is_empty(self): + assert ft.nms([]) == [] + + def test_float_boxes_are_suppressed_by_their_overlap(self): + # cv2.dnn.NMSBoxes reads float rows as Rect2d, so sub-pixel boxes + # from the enlarged tiles still merge with the whole-frame copy + kept = ft.nms([[10.25, 10.5, 20.0, 20.0, 0.8], [10.0, 10.0, 20.5, 20.0, 0.6]]) + assert kept == [[10.25, 10.5, 20.0, 20.0, 0.8]] + class TestDetectTiled: def test_one_face_found_once_near_its_true_box(self): From 2123a3ab7a23267d5ef51ad5eebcaffb25ad80bb Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 20:30:41 -0500 Subject: [PATCH 029/296] feat(tasks): #622 - catch a cut between two framings of one picture The HSV histogram alone missed a zoom cut on letterboxed frames: the black bars dominate both histograms (correlation 0.999), so the old box was held over the new shot until the track was lost. A frame is now also a cut when its thumbnail difference from the previous frame spikes against the frames around it. Co-Authored-By: Claude Opus 5.5 --- docs/TASKS.md | 6 ++-- dw/tasks/face_track.py | 67 ++++++++++++++++++++++++++++++++-------- tests/test_face_track.py | 59 +++++++++++++++++++++++++++++++++-- 3 files changed, 115 insertions(+), 17 deletions(-) diff --git a/docs/TASKS.md b/docs/TASKS.md index bb2ed37a..a310d7e5 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -517,7 +517,9 @@ enlarged tiles, merged by non-maximum suppression, so a face only a few dozen pixels wide is still seen. One track is kept and smoothed with an exponential moving average; across a short detector miss the last box is held at a decaying strength. The track restarts at each shot boundary the clip records, or, for a -clip that records none, at an abrupt colour-histogram change. A padded square +clip that records none, where the picture jumps: a sharp drop in its HSV +colour histogram, or a spike in its difference from the previous frame (a cut +between two framings of one picture keeps its colours). A padded square around the smoothed box is cut from every frame and resized to `crop_size`, then the crops are padded to 8n+1 frames with mirrored warm-up and cool-down frames. @@ -540,7 +542,7 @@ rate. `track` is a JSON record saved as its own `.json` file, readable by | `face_found` | Whether any frame had a face | | `message` | Present only when no face was found | | `frames` | One entry per source frame: `box` (the smoothed `[x, y, w, h]`, or null), `crop` (the `[x, y, side, side]` square cut), `strength`, and `state` (`tracked`, `held` or `none`) | -| `resets` | `[{frame, reason, detail}]` where the track restarted; `reason` is `shot`, `cut` or `lost` | +| `resets` | `[{frame, reason, detail}]` where the track restarted; `reason` is `shot`, `cut` or `lost`; a `cut`'s `detail` is `{histogram_correlation, frame_change}` | A clip with no face still succeeds: every strength is 0, the crops are the centre square, and a `no_face_found` warning is raised. `validate_workflow` diff --git a/dw/tasks/face_track.py b/dw/tasks/face_track.py index 88cc8c1d..346af823 100644 --- a/dw/tasks/face_track.py +++ b/dw/tasks/face_track.py @@ -15,7 +15,9 @@ exponential moving average, and a frame the detector misses holds the last box at a decaying strength for a few frames. The track starts again at each shot boundary the clip records, or - for a clip that records none - where the -picture's colour histogram changes abruptly. +picture changes abruptly: its colour histogram drops, or the difference from +the previous frame spikes (a cut between two framings of the same picture keeps +its colours, so the histogram alone misses it). Each frame also carries a strength from 0 to 1: how much a face-detail pass should change it. A face already large in the frame has the pixels it needs, @@ -53,6 +55,13 @@ MAX_GAP = 6 # missed frames before the track counts as lost MATCH_DISTANCE = 1.5 # centre distance, in face widths, a match may move HISTOGRAM_CUT = 0.5 # HSV histogram correlation below which frames are a cut +# A cut that keeps the colours (a zoom between two framings of one picture): +# the mean difference of small thumbnails, as a fraction of full scale, must +# reach CUT_MIN_CHANGE and CUT_SPIKE times the median of the frames around it +THUMBNAIL_SIZE = (32, 18) +CUT_MIN_CHANGE = 0.06 +CUT_SPIKE = 4.0 +CUT_WINDOW = 8 # frames each side whose differences form that median def _iou(a, b): @@ -124,19 +133,54 @@ def _hsv_histogram(rgb): return cv2.normalize(histogram, histogram).flatten() -def histogram_cuts(frames_rgb, threshold=HISTOGRAM_CUT): - """Frame indices where the picture changes abruptly - a cut nobody recorded.""" +def _thumbnail(rgb): import cv2 - cuts = [] + return cv2.resize(rgb, THUMBNAIL_SIZE, interpolation=cv2.INTER_AREA).astype( + np.float32 + ) + + +def content_cuts(frames_rgb): + """Cuts nobody recorded: [(frame index, detail)] where the picture jumps. + + A frame is a cut when its HSV histogram correlates with the previous + frame's below HISTOGRAM_CUT, or when its thumbnail difference from the + previous frame is a spike: at least CUT_MIN_CHANGE, and CUT_SPIKE times + the median difference of the CUT_WINDOW frames either side. + """ + import cv2 + + correlations, changes = [None], [0.0] previous = None - for index, rgb in enumerate(frames_rgb): - current = _hsv_histogram(rgb) + for rgb in frames_rgb: + current = (_hsv_histogram(rgb), _thumbnail(rgb)) if previous is not None: - correlation = cv2.compareHist(previous, current, cv2.HISTCMP_CORREL) - if correlation < threshold: - cuts.append((index, float(correlation))) + correlations.append( + float(cv2.compareHist(previous[0], current[0], cv2.HISTCMP_CORREL)) + ) + changes.append(float(np.mean(np.abs(current[1] - previous[1]))) / 255) previous = current + + cuts = [] + for index in range(1, len(changes)): + change = changes[index] + around = ( + changes[max(1, index - CUT_WINDOW) : index] + + changes[index + 1 : index + 1 + CUT_WINDOW] + ) + baseline = float(np.median(around)) if around else 0.0 + spike = change >= CUT_MIN_CHANGE and change >= CUT_SPIKE * baseline + if correlations[index] < HISTOGRAM_CUT or spike: + cuts.append( + ( + index, + { + "histogram_correlation": round(correlations[index], 4), + "frame_change": round(change, 4), + }, + ) + ) return cuts @@ -384,10 +428,7 @@ def crop_face_track( if starts: resets_at = {frame: ("shot", name) for frame, name in starts.items()} else: - resets_at = { - frame: ("cut", {"histogram_correlation": round(correlation, 4)}) - for frame, correlation in histogram_cuts(rgb) - } + resets_at = {frame: ("cut", detail) for frame, detail in content_cuts(rgb)} detections = [ [d for d in detect_tiled(detector, frame) if d[4] >= min_confidence] diff --git a/tests/test_face_track.py b/tests/test_face_track.py index 34937bc6..acd8089c 100644 --- a/tests/test_face_track.py +++ b/tests/test_face_track.py @@ -194,12 +194,12 @@ class TestHistogramCuts: def test_cut_at_a_colour_change(self): a = make_frame((100, 100, 16), background=(40, 160, 60)) b = make_frame((100, 100, 16), background=(200, 40, 200)) - cuts = ft.histogram_cuts([a, a, a, b, b]) + cuts = ft.content_cuts([a, a, a, b, b]) assert [index for index, _ in cuts] == [3] def test_no_spurious_cut_on_a_steady_clip(self): frame = make_frame((100, 100, 16)) - assert ft.histogram_cuts([frame] * 6) == [] + assert ft.content_cuts([frame] * 6) == [] def test_end_to_end_cut_is_recorded_as_a_reset(self, stub_detector): a = make_frame((100, 100, 16), background=(40, 160, 60)) @@ -209,6 +209,61 @@ def test_end_to_end_cut_is_recorded_as_a_reset(self, stub_detector): assert [(r["frame"], r["reason"]) for r in resets] == [(3, "cut")] +def framed_portrait(scale): + """A textured portrait with a face square, shrunk by scale and letterboxed + into a black W x H frame - two scales are two framings of one picture.""" + ph, pw = H, int(H * 0.75) + y, x = np.mgrid[0:ph, 0:pw] + portrait = np.stack( + [ + 90 + 60 * np.sin(x / 20), + 70 + 50 * np.cos(y / 16), + 60 + 40 * np.sin((x + y) / 30), + ], + axis=-1, + ).astype(np.uint8) + portrait[30:78, pw // 2 - 24 : pw // 2 + 24] = FACE + small = np.asarray( + Image.fromarray(portrait).resize( + (int(pw * scale), int(ph * scale)), Image.Resampling.NEAREST + ) + ) + frame = np.zeros((H, W, 3), dtype=np.uint8) + y0, x0 = (H - small.shape[0]) // 2, (W - small.shape[1]) // 2 + frame[y0 : y0 + small.shape[0], x0 : x0 + small.shape[1]] = small + return frame + + +class TestFramingCut: + """A cut between two framings of one picture keeps its colours: the + histogram barely moves, and the thumbnail difference spike finds it.""" + + def test_zoom_cut_is_found_though_the_histogram_holds(self): + far, near = framed_portrait(1 / 3.2), framed_portrait(1.0) + cuts = ft.content_cuts([far] * 6 + [near] * 6) + assert [index for index, _ in cuts] == [6] + assert cuts[0][1]["histogram_correlation"] > ft.HISTOGRAM_CUT + assert cuts[0][1]["frame_change"] >= ft.CUT_MIN_CHANGE + + def test_steady_pan_is_not_a_cut(self): + near = framed_portrait(1.0) + pan = [np.roll(near, 3 * i, axis=1) for i in range(20)] + assert ft.content_cuts(pan) == [] + + def test_zoom_cut_resets_the_track_with_no_held_frames(self, stub_detector): + far, near = framed_portrait(1 / 3.2), framed_portrait(1.0) + result = ft.crop_face_track(clip_of([far] * 6 + [near] * 6)) + track = result["track"] + assert [(r["frame"], r["reason"]) for r in track["resets"]] == [(6, "cut")] + states = [f["state"] for f in track["frames"]] + assert states == ["tracked"] * 12 + far_width, near_width = ( + track["frames"][5]["box"][2], + track["frames"][6]["box"][2], + ) + assert near_width > 2.5 * far_width + + class TestShotReset: def test_recorded_shot_start_resets_the_track(self, stub_detector): frames = [make_frame((100, 100, 16))] * 3 + [make_frame((200, 50, 16))] * 3 From 51f6c3e57086656d39c96981581280ffed6b62e0 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 20:43:35 -0500 Subject: [PATCH 030/296] feat(tasks): #623 - paste_face_track: feathered, strength-scaled paste-back Co-Authored-By: Claude Opus 5.5 --- dw/task_domains.py | 28 ++++++ dw/tasks/face_track.py | 192 ++++++++++++++++++++++++++++++++++++++++- dw/tasks/task.py | 36 ++++++++ tests/test_shots.py | 1 + 4 files changed, 256 insertions(+), 1 deletion(-) diff --git a/dw/task_domains.py b/dw/task_domains.py index afb436b0..b40fb21b 100644 --- a/dw/task_domains.py +++ b/dw/task_domains.py @@ -199,6 +199,9 @@ "gate_zero": POSITIVE, "min_confidence": POSITIVE, }, + "paste_face_track": { + "feather": NON_NEGATIVE, + }, } @@ -348,6 +351,30 @@ def face_track_problems( return problems +def paste_feather_problems(feather=None): + """[(argument, message)] when paste_face_track's feather is past 1 - the + fraction of the mask's radius that fades, so more than all of it is none.""" + number = as_number(feather) + if number is not None and number > 1: + return [ + ( + "feather", + f"paste_face_track needs 'feather' from 0 to 1 (the fraction " + f"of the paste's radius that fades out), got {feather!r}", + ) + ] + return [] + + +def paste_face_track_errors(arguments): + """[(argument, message)] for the paste_face_track rules a literal + workflow can break before it runs.""" + feather = arguments.get("feather") + if is_ref(DEFERRED, feather): + return [] + return paste_feather_problems(feather) + + def check_face_detector_source(repo, filename): """Refuse a detector source that is not a Hub repo id and a bare .onnx file name in it - before anything is downloaded.""" @@ -538,6 +565,7 @@ def task_argument_errors(workflow_definition, source_indices=None): extra = { "ingredients_grid": ingredients_grid_errors, "crop_face_track": face_track_errors, + "paste_face_track": paste_face_track_errors, }.get(command) if extra is not None: for key, message in extra(arguments): diff --git a/dw/tasks/face_track.py b/dw/tasks/face_track.py index 346af823..a7f9817d 100644 --- a/dw/tasks/face_track.py +++ b/dw/tasks/face_track.py @@ -23,6 +23,11 @@ should change it. A face already large in the frame has the pixels it needs, so strength falls from 1 at `gate_full` (face width over frame width) to 0 at `gate_zero`. + +`paste_face_track` is the other half: given the clip, the crops after a +face-detail pass, and the track, it puts each repaired crop back over its +square with a feathered round mask scaled by that frame's strength, so the +change fades out at the edges and is absent where the strength is 0. """ import logging @@ -32,7 +37,12 @@ from ..events import emit_log, emit_warning from ..media_types import AudioVideo, JsonRecord -from ..task_domains import check_face_detector_source, face_track_problems +from ..shots import carried_shots +from ..task_domains import ( + check_face_detector_source, + face_track_problems, + paste_feather_problems, +) from .video_utils import frames_as_pil_list, load_audio_video logger = logging.getLogger("dw") @@ -494,3 +504,183 @@ def crop_face_track( f"({before} before, {after} after for 8n+1)" ) return {"crops": AudioVideo(crops, None, None, fps=fps), "track": record} + + +def feather_mask(side, feather): + """The alpha a pasted square is blended with: radial, 1 at the centre. + + The mask is the circle inscribed in the square. Its outer `feather` + fraction of the radius falls from 1 to 0 along a smoothstep, so the + square's corners - and everything past the circle - keep the source. + """ + radius = side / 2 + coords = np.arange(side, dtype=np.float32) + 0.5 - radius + distance = np.hypot(coords[None, :], coords[:, None]) / radius + if feather <= 0: + return (distance < 1).astype(np.float32) + t = np.clip((1 - distance) / feather, 0.0, 1.0) + return t * t * (3 - 2 * t) + + +def paste_crop(source_rgb, crop, square, strength, feather, color_match): + """One frame with a repaired crop blended back over its square. + + Args: + source_rgb: The source frame, an RGB uint8 array + crop: The repaired crop, a PIL image of any size + square: [x, y, side, side] the crop was cut from; it may run past the + frame's edges, where the crop replicated the border + strength: 0 to 1, scaling the whole mask + feather: Fraction of the mask's radius that fades (feather_mask) + color_match: Shift the crop's mean colour, inside the mask, onto the + source's before blending + """ + x, y, side, _ = (int(v) for v in square) + height, width = source_rgb.shape[:2] + x0, y0 = max(0, x), max(0, y) + x1, y1 = min(width, x + side), min(height, y + side) + if x1 <= x0 or y1 <= y0: + return source_rgb + + patch = np.asarray( + crop.convert("RGB").resize((side, side), Image.LANCZOS), dtype=np.float32 + ) + mask = feather_mask(side, feather) + inside = (slice(y0 - y, y1 - y), slice(x0 - x, x1 - x)) + patch, mask = patch[inside], mask[inside] + region = source_rgb[y0:y1, x0:x1].astype(np.float32) + + if color_match: + weight = float(mask.sum()) + if weight > 0: + patch = patch + ((region - patch) * mask[..., None]).sum((0, 1)) / weight + + alpha = (mask * strength)[..., None] + out = source_rgb.copy() + out[y0:y1, x0:x1] = np.clip( + np.rint(region * (1 - alpha) + patch * alpha), 0, 255 + ).astype(np.uint8) + return out + + +def _read_track(track): + """The track record, from the dict a step handed on or its saved .json.""" + if isinstance(track, str): + import json + + from ..locations import validate_media_path + from ..security import ( + ALLOWED_JSON_EXTENSIONS, + validate_file_extension, + validate_json_size, + ) + + path = validate_media_path(track, None, "a track argument") + validate_file_extension(path, ALLOWED_JSON_EXTENSIONS) + validate_json_size(path) + with open(path, encoding="utf-8") as handle: + track = json.load(handle) + if not ( + isinstance(track, dict) + and isinstance(track.get("source"), dict) + and isinstance(track.get("frames"), list) + and "crop_frames" in track + and "pad_before" in track + ): + raise ValueError( + "paste_face_track needs 'track' as the track record crop_face_track " + "returned (previous_result:.track, or its saved .json)" + ) + return track + + +def paste_face_track(clip, repaired, track, feather=0.3, color_match=True): + """Task command: put repaired face crops back into the clip they came from. + + Args: + clip: The source video crop_face_track read - frames, an AudioVideo, + or the path or URL of a video file. Its audio, frame rate and + shots come through unchanged + repaired: The crops after a face-detail pass, still padded to the + 8n+1 count crop_face_track produced; any frame size + track: The `track` record crop_face_track returned, or its .json file + feather: Fraction of the paste's radius that fades out, 0 to 1 + color_match: Match each crop's mean colour to the source's inside the + mask before blending + Returns: + An AudioVideo: the source frames with the face pasted back where the + strength is above 0, and the source frame itself where it is 0 + """ + problems = paste_feather_problems(feather) + if problems: + raise ValueError("; ".join(message for _, message in problems)) + feather = float(feather) + track = _read_track(track) + + if isinstance(clip, str): + clip = load_audio_video(clip) + if isinstance(repaired, str): + repaired = load_audio_video(repaired) + frames = frames_as_pil_list(clip) + crops = frames_as_pil_list(repaired) + if not frames: + raise ValueError("paste_face_track needs a clip with at least one frame") + + source = track["source"] + width, height = frames[0].size + if source.get("frames") != len(frames) or len(track["frames"]) != len(frames): + raise ValueError( + f"paste_face_track: the track was made from a clip of " + f"{source.get('frames')} frames but this clip has {len(frames)} - " + f"pass the clip crop_face_track tracked" + ) + if (source.get("width"), source.get("height")) != (width, height): + raise ValueError( + f"paste_face_track: the track was made from a " + f"{source.get('width')}x{source.get('height')} clip but this clip " + f"is {width}x{height} - pass the clip crop_face_track tracked" + ) + needed = int(track["crop_frames"]) + if len(crops) < needed: + raise ValueError( + f"paste_face_track: 'repaired' has {len(crops)} frames but the " + f"track's padded crops number {needed} (pad_before " + f"{track['pad_before']}, {len(frames)} frames, pad_after " + f"{track.get('pad_after')}) - a repair pass must keep every crop" + ) + + before = int(track["pad_before"]) + out, pasted = [], 0 + for index, (frame, entry) in enumerate(zip(frames, track["frames"])): + strength = float(entry.get("strength") or 0.0) + square = entry.get("crop") + if strength <= 0 or not square: + out.append(frame) + continue + rgb = np.asarray(frame.convert("RGB")) + out.append( + Image.fromarray( + paste_crop( + rgb, + crops[before + index], + square, + min(strength, 1.0), + feather, + color_match, + ) + ) + ) + pasted += 1 + + emit_log( + f"paste_face_track: pasted into {pasted} of {len(frames)} frames " + f"(feather {feather}, color_match {bool(color_match)})" + ) + # Same frames, one for one, so the clip's sound and shots still hold + return AudioVideo( + out, + getattr(clip, "audio", None), + getattr(clip, "sample_rate", None), + fps=getattr(clip, "fps", None), + shots=carried_shots(clip), + ) diff --git a/dw/tasks/task.py b/dw/tasks/task.py index 78f64071..4ff64005 100644 --- a/dw/tasks/task.py +++ b/dw/tasks/task.py @@ -383,6 +383,42 @@ def _handle_crop_face_track(task, arguments, previous_pipelines): return crop_face_track(device=task.device_for(arguments), **arguments) +@register_command( + "paste_face_track", + implementation="dw.tasks.face_track.paste_face_track", + summary=( + "Blend repaired face crops back into the clip crop_face_track tracked, " + "feathered and scaled by each frame's strength, keeping its audio." + ), + parameter_descriptions={ + "clip": ( + "The source video crop_face_track tracked - a file path, an " + "asset:/output: reference, or an earlier step's video. Its audio, " + "frame rate and shots are kept." + ), + "repaired": ( + "The crops after a face-detail pass, still padded to the 8n+1 " + "count crop_face_track produced." + ), + "track": ( + "The track record crop_face_track returned - " + "previous_result:.track, or its saved .json." + ), + "feather": "Fraction of the paste's radius that fades out (0 to 1).", + "color_match": ( + "Match each crop's mean colour to the source inside the mask " + "before blending." + ), + }, +) +def _handle_paste_face_track(task, arguments, previous_pipelines): + """Blend repaired face crops back into the clip they were cut from""" + logger.debug("Pasting a face track") + from .face_track import paste_face_track + + return paste_face_track(**arguments) + + @register_command("mix_audio", implementation="dw.tasks.audio_utils.mix_audio") def _handle_mix_audio(task, arguments, previous_pipelines): """Layer audio tracks on top of one another, rather than end to end""" diff --git a/tests/test_shots.py b/tests/test_shots.py index be924596..441fe7eb 100644 --- a/tests/test_shots.py +++ b/tests/test_shots.py @@ -83,6 +83,7 @@ def audio_video(num_frames, level, fps=4, sample_rate=100): ("dw/output_extraction.py", "pair_audio_with_frames"): ("none", 1), ("dw/tasks/h3_latent_upscale.py", "decode_h3_latents"): ("none", 1), ("dw/tasks/face_track.py", "crop_face_track"): ("none", 1), + ("dw/tasks/face_track.py", "paste_face_track"): ("carries", 1), } _SHOTS_HELPER_BY_DECISION = { From 337f2d93acd7676b8c01cb7d107e43b19833a938 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 20:45:47 -0500 Subject: [PATCH 031/296] feat(tasks): #623 - paste_face_track unit tests and TASKS.md section Co-Authored-By: Claude Opus 5.5 --- docs/TASKS.md | 46 +++++++ tests/test_face_track.py | 252 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 298 insertions(+) diff --git a/docs/TASKS.md b/docs/TASKS.md index a310d7e5..25d19d91 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -550,6 +550,52 @@ refuses a `crop_size` that is not a multiple of 32, a `gate_zero` at or below `gate_full`, a `padding` out of range, a `detector_repo` that is not a repo id and a `detector_file` that is not a `.onnx` name, before anything downloads. +### paste_face_track + +Put the crops of a face-detail pass back into the clip they were cut from, +using the `track` record `crop_face_track` returned: + +```json +{ + "name": "pasted", + "task": { + "command": "paste_face_track", + "arguments": { + "clip": "asset:interview.mp4", + "repaired": "previous_result:face_repair", + "track": "previous_result:face_crops.track" + } + } +} +``` + +| Argument | Required | Description | +| -------- | -------- | ----------- | +| `clip` | Yes | The video `crop_face_track` tracked - frames, an audio+video pair, or the path or URL of a video file | +| `repaired` | Yes | The crops after a face-detail pass, still padded to the count `crop_face_track` produced; any frame size | +| `track` | Yes | The `track` record `crop_face_track` returned (`previous_result:.track`), or the path of its saved `.json`, such as an `output:` reference | +| `feather` | No | Fraction of the paste's radius that fades out, `0` to `1`. Default `0.3` | +| `color_match` | No | Shift each crop's mean colour onto the source's inside the mask before blending. Default `true` | + +The pad frames are dropped: the crop at `pad_before + i` goes back on source +frame `i`. Each crop is resized to the square it was cut from, then blended +over the frame with a radial feathered mask times that frame's strength. The +mask is the circle inscribed in the square; its outer `feather` fraction fades +from 1 to 0 along a smoothstep, so the square's corners and everything past +the circle keep the source. A square that ran past the frame's edge is +clipped to the frame. + +A frame with strength 0 (no face, or a face too large for the gate) is passed +through exactly. With `color_match`, the crop's mean colour is shifted onto the +source's inside the mask, so a repair pass that drifted in tone does not show +as a patch. + +It returns the source clip with its audio, frame rate and shots carried, the +same frames one for one. It refuses a track whose frame count or frame size +does not match the clip, a `repaired` with fewer frames than the track's +`crop_frames`, and a `track` that is not a `crop_face_track` record. +`validate_workflow` refuses a `feather` past `1` before the job runs. + ### video_frames The frames of a generated video, as one `(frames, height, width, channels)` diff --git a/tests/test_face_track.py b/tests/test_face_track.py index acd8089c..d93a9bb2 100644 --- a/tests/test_face_track.py +++ b/tests/test_face_track.py @@ -474,3 +474,255 @@ def test_the_task_result_saves_crops_as_video_and_track_as_one_json(): assert [os.path.splitext(n)[1] for n in names] == [".mp4", ".json"] with open(os.path.join(temp_dir, names[1])) as file: assert json.load(file) == {"a": {"b": 1}} + + +# ---- paste_face_track ---------------------------------------------------- + + +def gradient_frame(face=None): + """Smooth content (a gradient) with an optional red face square.""" + xs = np.linspace(20, 180, W, dtype=np.float32)[None, :] + ys = np.linspace(20, 120, H, dtype=np.float32)[:, None] + frame = np.stack([xs + 0 * ys, ys + 0 * xs, (xs + ys) / 2], axis=-1).astype( + np.uint8 + ) + if face is not None: + x, y, s = face + frame[y : y + s, x : x + s] = FACE + return frame + + +def tracked(frames, **clip_kwargs): + clip = clip_of(frames, **clip_kwargs) + result = ft.crop_face_track(clip) + return clip, result["crops"], result["track"] + + +def diffs(out, source_arrays): + return [ + np.abs(np.asarray(f).astype(int) - s.astype(int)) + for f, s in zip(out.frames, source_arrays) + ] + + +SQUARE = [60, 60, 100, 100] + + +def crop_of(region): + return Image.fromarray(region) + + +class TestFeatherMask: + def test_centre_one_corners_zero(self): + mask = ft.feather_mask(101, 0.3) + assert mask[50, 50] == pytest.approx(1.0) + for corner in ((0, 0), (0, -1), (-1, 0), (-1, -1)): + assert mask[corner] == 0 + + def test_non_increasing_along_radius(self): + mask = ft.feather_mask(101, 0.3) + row = mask[50, 50:] + assert np.all(np.diff(row) <= 1e-6) + diagonal = np.array([mask[50 + i, 50 + i] for i in range(51)]) + assert np.all(np.diff(diagonal) <= 1e-6) + + def test_symmetric(self): + mask = ft.feather_mask(64, 0.4) + assert np.allclose(mask, mask[::-1, :]) + assert np.allclose(mask, mask[:, ::-1]) + assert np.allclose(mask, mask.T) + + def test_zero_feather_is_a_hard_disc(self): + mask = ft.feather_mask(64, 0) + assert set(np.unique(mask)) == {0.0, 1.0} + assert mask[32, 32] == 1 and mask[0, 0] == 0 + + def test_inner_part_is_exactly_one(self): + side = 128 + mask = ft.feather_mask(side, 0.3) + radius = side / 2 + coords = np.arange(side) + 0.5 - radius + distance = np.hypot(coords[None, :], coords[:, None]) / radius + inner = distance <= 0.7 - 1e-6 + assert inner.any() + assert np.all(mask[inner] == 1.0) + assert mask[(distance > 0.75) & (distance < 0.99)].max() < 1.0 + + +class TestPasteCrop: + def source_and_region(self): + source = gradient_frame() + x, y, side, _ = SQUARE + return source, source[y : y + side, x : x + side].copy() + + def test_color_match_removes_a_constant_shift(self): + source, region = self.source_and_region() + shifted = region.copy() + shifted[..., 0] += 40 + out = ft.paste_crop(source, crop_of(shifted), SQUARE, 1.0, 0.3, True) + x, y, side, _ = SQUARE + mask = ft.feather_mask(side, 0.3) > 0 + diff = np.abs(out[y : y + side, x : x + side].astype(int) - region.astype(int)) + assert diff[mask].max() <= 2 + + def test_no_color_match_shows_the_shift(self): + source, region = self.source_and_region() + shifted = region.copy() + shifted[..., 0] += 40 + out = ft.paste_crop(source, crop_of(shifted), SQUARE, 1.0, 0.3, False) + centre = out[110, 110, 0].astype(int) - source[110, 110, 0].astype(int) + assert centre == 40 + + def test_altered_repair_only_around_the_face_without_a_seam(self): + source = gradient_frame() + white = np.full((100, 100, 3), 255, np.uint8) + out = ft.paste_crop(source, crop_of(white), SQUARE, 1.0, 0.3, False) + changed = np.any(out != source, axis=-1) + x, y, side, _ = SQUARE + # nothing outside the square moves + outside = changed.copy() + outside[y : y + side, x : x + side] = False + assert not outside.any() + # the corners of the square keep the source + assert not changed[y, x] and not changed[y + side - 1, x + side - 1] + # the centre is fully the repair + assert tuple(out[110, 110]) == (255, 255, 255) + # alpha ramps: no step between neighbours bigger than the ramp allows + step = np.abs(np.diff(out[110, x : x + side, 0].astype(int))) + assert step.max() <= 20 + + def test_strength_scales_the_blend(self): + source = gradient_frame() + white = np.full((100, 100, 3), 255, np.uint8) + out = ft.paste_crop(source, crop_of(white), SQUARE, 0.5, 0.3, False) + expected = (int(source[110, 110, 0]) + 255) / 2 + assert abs(int(out[110, 110, 0]) - expected) <= 1 + + +class TestPasteFaceTrack: + def frames(self, count=5): + return [gradient_frame((140 + i, 100, 16)) for i in range(count)] + + def test_identity_round_trip(self, stub_detector): + arrays = self.frames() + clip, crops, track = tracked(arrays) + assert track["frames"][0]["strength"] > 0 + out = ft.paste_face_track(clip, crops, track) + assert len(out.frames) == len(arrays) + assert all(f.size == (W, H) for f in out.frames) + d = diffs(out, arrays) + assert max(x.mean() for x in d) < 0.1 + assert max(x.max() for x in d) <= 32 + + def test_strength_zero_is_an_exact_passthrough(self, stub_detector): + arrays = self.frames() + clip, crops, track = tracked(arrays) + for entry in track["frames"]: + entry["strength"] = 0 + white = AudioVideo( + [Image.new("RGB", (512, 512), (255, 255, 255)) for _ in crops.frames], + None, + None, + ) + out = ft.paste_face_track(clip, white, track) + for frame, source in zip(out.frames, arrays): + assert np.array_equal(np.asarray(frame), source) + + def test_audio_fps_and_shots_are_carried(self, stub_detector): + arrays = self.frames() + shots = [ + {"name": "one", "start_frame": 0}, + {"name": "two", "start_frame": 3}, + ] + clip, crops, track = tracked(arrays, shots=shots, fps=24) + clip.audio = np.zeros((2, 48000), np.float32) + clip.sample_rate = 48000 + out = ft.paste_face_track(clip, crops, track) + assert np.array_equal(out.audio, clip.audio) + assert out.sample_rate == 48000 + assert out.fps == 24 + assert out.shots == shots + assert out.shots is not clip.shots + + def test_altered_repair_changes_the_face_only(self, stub_detector): + arrays = self.frames() + clip, crops, track = tracked(arrays) + white = AudioVideo( + [Image.new("RGB", f.size, (255, 255, 255)) for f in crops.frames], + None, + None, + ) + out = ft.paste_face_track(clip, white, track, color_match=False) + x, y, side, _ = (int(v) for v in track["frames"][0]["crop"]) + frame = np.asarray(out.frames[0]) + assert np.array_equal(frame[:10], arrays[0][:10]) + assert not np.array_equal( + frame[y + side // 2, x + side // 2], arrays[0][y + side // 2, x + side // 2] + ) + + def test_track_read_from_saved_json(self, stub_detector, tmp_path): + arrays = self.frames() + clip, crops, track = tracked(arrays) + path = tmp_path / "track.json" + path.write_text(json.dumps(track)) + from_dict = ft.paste_face_track(clip, crops, track) + from_file = ft.paste_face_track(clip, crops, str(path)) + for a, b in zip(from_dict.frames, from_file.frames): + assert np.array_equal(np.asarray(a), np.asarray(b)) + + def test_refusals(self, stub_detector): + arrays = self.frames() + clip, crops, track = tracked(arrays) + + with pytest.raises(ValueError, match="frames but this clip has"): + ft.paste_face_track(clip_of(arrays[:3]), crops, track) + + small = clip_of([a[:200, :300] for a in arrays]) + with pytest.raises(ValueError, match="clip but this clip is"): + ft.paste_face_track(small, crops, track) + + short = AudioVideo(crops.frames[:-1], None, None) + with pytest.raises(ValueError, match="must keep every crop"): + ft.paste_face_track(clip, short, track) + + with pytest.raises(ValueError, match="track record"): + ft.paste_face_track(clip, crops, {}) + + with pytest.raises(ValueError, match="feather"): + ft.paste_face_track(clip, crops, track, feather=1.5) + + +class TestPasteValidation: + @staticmethod + def paste_workflow(feather): + return { + "id": "x", + "steps": [ + { + "name": "s", + "task": { + "command": "paste_face_track", + "arguments": { + "clip": "v.mp4", + "repaired": "r.mp4", + "track": "t.json", + "feather": feather, + }, + }, + } + ], + } + + @pytest.mark.parametrize("feather", [1.5, -0.1]) + def test_bad_feather_is_refused(self, feather): + errors = task_argument_errors(self.paste_workflow(feather)) + assert any(e["path"] == "steps[0].task.arguments.feather" for e in errors) + + def test_good_feather_is_accepted(self): + assert task_argument_errors(self.paste_workflow(0.3)) == [] + + +def test_paste_command_is_registered(): + from dw.tasks.task import _COMMAND_REGISTRY + + assert "paste_face_track" in _COMMAND_REGISTRY From 431f8d63e6474b4184f8cd7c4a2d233edc4d3965 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 21:01:49 -0500 Subject: [PATCH 032/296] feat(tasks): #625 - analyze_beats: onset/DP beat tracking, RMS-peak fallback, anchor calibration Registry split into dw/tasks/registry.py so beats.py registers itself without growing task.py (back under the size ceiling). Co-Authored-By: Claude Opus 5.5 --- dw/dsp.py | 192 ++++++++++++++++++++++++++++++++ dw/task_domains.py | 121 ++++++++++++++++++++ dw/tasks/beats.py | 259 +++++++++++++++++++++++++++++++++++++++++++ dw/tasks/registry.py | 113 +++++++++++++++++++ dw/tasks/task.py | 108 +----------------- 5 files changed, 691 insertions(+), 102 deletions(-) create mode 100644 dw/tasks/beats.py create mode 100644 dw/tasks/registry.py diff --git a/dw/dsp.py b/dw/dsp.py index 530ad798..f8bc72a7 100644 --- a/dw/dsp.py +++ b/dw/dsp.py @@ -610,3 +610,195 @@ def biquad_coefficients(kind, cutoff_hz, q, sample_rate): numpy.array([b0, b1, b2], dtype=numpy.float64) / a0, numpy.array([a1, a2], dtype=numpy.float64) / a0, ) + + +# Beat tracking (#600). The onset envelope is spectral flux - the positive +# frame-to-frame rise of the log-compressed magnitude spectrum - on a 10 ms +# hop over a ~23 ms window; tempo is the autocorrelation peak of that +# envelope under a log-normal prior; the beats are Ellis's dynamic programme +# (2007), which trades onset strength against keeping each interval near the +# tempo's period +ONSET_HOP_SECONDS = 0.01 +ONSET_WINDOW_SECONDS = 0.023 +ONSET_LOG_GAIN = 1000.0 +# The tempo prior: centred on 120 BPM an octave wide, or on a caller's hint +# half an octave wide +TEMPO_PRIOR_BPM = 120.0 +TEMPO_PRIOR_OCTAVES = 1.0 +TEMPO_HINT_OCTAVES = 0.5 +# How strongly the programme holds an interval to the period: the penalty is +# BEAT_TIGHTNESS * log(interval / period)^2 +BEAT_TIGHTNESS = 100.0 +# An envelope is too flat to track when its autocorrelation at the tempo lag +# is under this fraction of its energy (noise, a held tone) or its peak is +# under this many times its mean (swells with no attack) +TRACKABLE_MIN_PERIODICITY = 0.2 +TRACKABLE_MIN_CREST = 3.0 +# A beat at the head or tail is trimmed while its onset is under this +# fraction of the RMS onset at the beats: the programme keeps stepping +# through an intro's silence at the period, onto nothing +BEAT_TRIM_FRACTION = 0.5 +# The RMS-peak fallback: 50 ms windows on the onset hop, peaks at least this +# prominent relative to the envelope's range +RMS_PEAK_WINDOW_SECONDS = 0.05 +RMS_PEAK_PROMINENCE = 0.1 + + +def mono_float64(waveform): + """A (channels, samples) or 1-D waveform as one float64 channel.""" + mono = numpy.asarray(waveform, dtype=numpy.float64) + return mono.mean(axis=0) if mono.ndim == 2 else mono + + +def onset_envelope(mono, sample_rate): + """(envelope, frames per second): spectral flux on a 10 ms hop. + + Frame k is centred on k / rate seconds, so a frame index divided by the + rate is a time in the track. + """ + hop = max(1, int(round(sample_rate * ONSET_HOP_SECONDS))) + window = 1 << int(math.ceil(math.log2(ONSET_WINDOW_SECONDS * sample_rate))) + window = max(window, hop + 1) + if mono.shape[0] < window: + return numpy.zeros(0), sample_rate / hop + _, _, spectrum = scipy.signal.stft( + mono, sample_rate, nperseg=window, noverlap=window - hop, padded=True + ) + magnitude = numpy.log1p(ONSET_LOG_GAIN * numpy.abs(spectrum)) + flux = numpy.maximum(0.0, numpy.diff(magnitude, axis=1)).mean(axis=0) + return numpy.concatenate([[0.0], flux]), sample_rate / hop + + +def estimate_tempo(envelope, rate, min_bpm, max_bpm, hint_bpm=None): + """(bpm, periodicity) from the envelope's autocorrelation in the range. + + `periodicity` is the normalized autocorrelation at the chosen lag, 0 to + 1; (None, 0.0) when the envelope holds no energy or the range no lag. + """ + centred = envelope - envelope.mean() if envelope.size else envelope + count = centred.shape[0] + if count < 2: + return None, 0.0 + correlation = scipy.signal.correlate(centred, centred, mode="full", method="fft") + correlation = correlation[count - 1 :] + if correlation[0] <= 0: + return None, 0.0 + correlation = correlation / correlation[0] + low = max(1, int(math.floor(rate * 60.0 / max_bpm))) + high = min(count - 2, int(math.ceil(rate * 60.0 / min_bpm))) + if high < low: + return None, 0.0 + lags = numpy.arange(low, high + 1) + centre = TEMPO_PRIOR_BPM if hint_bpm is None else hint_bpm + width = TEMPO_PRIOR_OCTAVES if hint_bpm is None else TEMPO_HINT_OCTAVES + prior = numpy.exp(-0.5 * (numpy.log2(60.0 * rate / lags / centre) / width) ** 2) + best = int(numpy.argmax(correlation[lags] * prior)) + lag = lags[best] + # Parabolic interpolation between the lags either side + before, at, after = correlation[lag - 1], correlation[lag], correlation[lag + 1] + curvature = before - 2.0 * at + after + shift = 0.5 * (before - after) / curvature if curvature < 0 else 0.0 + return 60.0 * rate / (lag + shift), float(max(at, 0.0)) + + +def is_trackable(envelope, periodicity): + """Whether an onset envelope has the attacks and the periodicity to track.""" + if not envelope.size or periodicity < TRACKABLE_MIN_PERIODICITY: + return False + mean = envelope.mean() + return mean > 0 and envelope.max() / mean >= TRACKABLE_MIN_CREST + + +def track_beats(envelope, rate, bpm, tightness=BEAT_TIGHTNESS): + """Beat frame indices by dynamic programming, leading and trailing beats + that land on next to nothing trimmed.""" + count = envelope.shape[0] + period = rate * 60.0 / bpm + spread = envelope.std() + if count == 0 or spread <= 0: + return numpy.zeros(0, dtype=int) + local = envelope / spread + score = local.copy() + backlink = numpy.full(count, -1, dtype=int) + steps = numpy.arange(max(1, int(round(period / 2))), int(round(2 * period)) + 1) + penalty = -tightness * numpy.log(steps / period) ** 2 + for frame in range(int(steps[0]), count): + reachable = steps <= frame + previous = frame - steps[reachable] + candidates = score[previous] + penalty[reachable] + best = int(numpy.argmax(candidates)) + if candidates[best] > 0: + score[frame] = local[frame] + candidates[best] + backlink[frame] = previous[best] + peaks = scipy.signal.argrelmax(score)[0] + if not peaks.size: + return numpy.zeros(0, dtype=int) + last = peaks[score[peaks] >= 0.5 * numpy.median(score[peaks])][-1] + beats = [int(last)] + while backlink[beats[-1]] >= 0: + beats.append(int(backlink[beats[-1]])) + beats = numpy.array(beats[::-1], dtype=int) + strength = envelope[beats] + floor = BEAT_TRIM_FRACTION * math.sqrt(float(numpy.mean(strength**2))) + kept = numpy.nonzero(strength >= floor)[0] + if not kept.size: + return numpy.zeros(0, dtype=int) + return beats[kept[0] : kept[-1] + 1] + + +def rms_peaks(mono, sample_rate, max_bpm): + """(peak times in seconds, peak level in dBFS) of a 50 ms RMS envelope, + peaks at least one max_bpm period apart - the fallback for a track with + no attacks to find onsets in.""" + hop = max(1, int(round(sample_rate * ONSET_HOP_SECONDS))) + window = max(hop, int(round(sample_rate * RMS_PEAK_WINDOW_SECONDS))) + if mono.shape[0] < window: + return numpy.zeros(0), SILENCE_DBFS + frames = numpy.lib.stride_tricks.sliding_window_view(mono, window)[::hop] + envelope = numpy.sqrt(numpy.mean(frames**2, axis=1)) + loudest = float(envelope.max()) + span = loudest - float(envelope.min()) + if span <= 0: + return numpy.zeros(0), dbfs(loudest, floor=SILENCE_DBFS) + peaks, _ = scipy.signal.find_peaks( + envelope, + distance=max(1, int(round(sample_rate / hop * 60.0 / max_bpm))), + prominence=RMS_PEAK_PROMINENCE * span, + ) + return (peaks * hop + window / 2.0) / sample_rate, dbfs(loudest, floor=SILENCE_DBFS) + + +def downbeat_phase(envelope, beat_frames, meter=4, margin=1.1): + """Which of `meter` beat positions (0 to meter-1) the bars start on: the + phase whose beats carry the most onset, when it beats the next by + `margin`; None when no phase stands out or there are under two bars.""" + if beat_frames.shape[0] < 2 * meter: + return None + strength = envelope[beat_frames] + means = sorted( + ((float(strength[phase::meter].mean()), phase) for phase in range(meter)), + reverse=True, + ) + if means[0][0] <= 0 or means[0][0] < margin * means[1][0]: + return None + return means[0][1] + + +def warp_times(times, knots_from, knots_to): + """Map times piecewise-linearly through (knots_from -> knots_to), the end + segments extended past the first and last knot; a single knot is a shift. + """ + times = numpy.asarray(times, dtype=numpy.float64) + knots_from = numpy.asarray(knots_from, dtype=numpy.float64) + knots_to = numpy.asarray(knots_to, dtype=numpy.float64) + if knots_from.shape[0] == 1: + return times + (knots_to[0] - knots_from[0]) + warped = numpy.interp(times, knots_from, knots_to) + for end, (a, b) in ( + (times < knots_from[0], (0, 1)), + (times > knots_from[-1], (-2, -1)), + ): + slope = (knots_to[b] - knots_to[a]) / (knots_from[b] - knots_from[a]) + anchor = 0 if a == 0 else -1 + warped[end] = knots_to[anchor] + (times[end] - knots_from[anchor]) * slope + return warped diff --git a/dw/task_domains.py b/dw/task_domains.py index b40fb21b..c07fa5bd 100644 --- a/dw/task_domains.py +++ b/dw/task_domains.py @@ -202,6 +202,12 @@ "paste_face_track": { "feather": NON_NEGATIVE, }, + "analyze_beats": { + "sample_rate": POSITIVE, + "tempo_bpm": POSITIVE, + "min_bpm": POSITIVE, + "max_bpm": POSITIVE, + }, } @@ -418,6 +424,120 @@ def face_track_errors(arguments): return errors +def beats_problems(min_bpm=None, max_bpm=None, anchors=None): + """[(argument, message)] for each analyze_beats rule these values break: + a tempo range that is empty, and anchors that are not one kind of mark + in time order. + + An anchor is a time in seconds, or {beat_index, seconds} placing the + beat at that index; a list holds one kind or the other. A value that is + a reference, or not a number, is skipped - the domain check or the run + owns that complaint. An anchor past the song's end is the run's to find: + only it has the song. + """ + problems = [] + low, high = as_number(min_bpm), as_number(max_bpm) + if low is not None and high is not None and low >= high: + problems.append( + ( + "min_bpm", + f"analyze_beats needs 'min_bpm' ({low:g}) below 'max_bpm' " + f"({high:g}) - the range is where the tempo is searched for", + ) + ) + if anchors is None or is_ref(DEFERRED, anchors): + return problems + if not isinstance(anchors, (list, tuple)): + return problems + [ + ( + "anchors", + "analyze_beats' 'anchors' is a list of times in seconds or of " + f"{{beat_index, seconds}} marks, not {type(anchors).__name__}", + ) + ] + kinds, marks = set(), [] + for index, anchor in enumerate(anchors): + if is_ref(DEFERRED, anchor): + marks.append(None) + continue + if isinstance(anchor, dict): + kinds.add("dict") + beat, seconds = anchor.get("beat_index"), anchor.get("seconds") + if ( + isinstance(beat, bool) + or not isinstance(beat, (int, float, str)) + or as_number(beat) is None + or as_number(beat) != int(as_number(beat)) + or as_number(beat) < 0 + ): + if not is_ref(DEFERRED, beat): + problems.append( + ( + "anchors", + f"analyze_beats' anchors[{index}] needs a whole " + f"'beat_index' at or above zero, not {beat!r}", + ) + ) + beat = None + else: + beat = as_number(beat) + else: + kinds.add("seconds") + beat, seconds = None, anchor + number = as_number(seconds) + if number is None or number < 0: + if not is_ref(DEFERRED, seconds): + problems.append( + ( + "anchors", + f"analyze_beats' anchors[{index}] needs a time in " + f"seconds at or above zero, not {seconds!r}", + ) + ) + number = None + marks.append((index, beat, number)) + if len(kinds) > 1: + problems.append( + ( + "anchors", + "analyze_beats' 'anchors' mixes bare times with " + "{beat_index, seconds} marks - give one kind or the other", + ) + ) + return problems + known = [mark for mark in marks if mark is not None] + for (_, beat_a, time_a), (index, beat_b, time_b) in zip(known, known[1:]): + if time_a is not None and time_b is not None and time_b <= time_a: + problems.append( + ( + "anchors", + f"analyze_beats' anchors[{index}] ({time_b:g} s) is not " + f"after the anchor before it ({time_a:g} s) - anchors are " + "given in time order, each at a different time", + ) + ) + elif beat_a is not None and beat_b is not None and beat_b <= beat_a: + problems.append( + ( + "anchors", + f"analyze_beats' anchors[{index}] places beat {beat_b:g} " + f"after beat {beat_a:g} - beat indexes rise with time", + ) + ) + return problems + + +def beats_errors(arguments): + """[(argument, message)] for the analyze_beats rules a literal workflow + can break before it runs (`beats_problems`).""" + literal = { + name: arguments.get(name) + for name in ("min_bpm", "max_bpm", "anchors") + if not is_ref(DEFERRED, arguments.get(name)) + } + return beats_problems(**literal) + + def in_domain(value, domain): """Whether a number satisfies a domain. Anything unmeasurable is True - a value this cannot read is not this check's to refuse.""" @@ -566,6 +686,7 @@ def task_argument_errors(workflow_definition, source_indices=None): "ingredients_grid": ingredients_grid_errors, "crop_face_track": face_track_errors, "paste_face_track": paste_face_track_errors, + "analyze_beats": beats_errors, }.get(command) if extra is not None: for key, message in extra(arguments): diff --git a/dw/tasks/beats.py b/dw/tasks/beats.py new file mode 100644 index 00000000..4b25dc9d --- /dev/null +++ b/dw/tasks/beats.py @@ -0,0 +1,259 @@ +"""analyze_beats: where a song's beats fall, for cutting picture to it (#600). + +A music video cuts on the beat, and a generated song (Music 3) carries no +tempo map. This reads one off the audio: an onset envelope, a tempo from its +autocorrelation, and the beats by dynamic programming (`dw/dsp.py`). A track +with no attacks to find - a pad, a swell, silence - falls back to the peaks +of its loudness, and says so. + +The caller can correct it with anchors, marks where a beat is known to be: +- two or more pull the detected beats through them piecewise-linearly, so a + detection that starts late or drifts lands on the marks; +- one alone shifts every beat by the same amount; +- one with `tempo_bpm` skips detection and lays an exact grid through it. + +It answers JSON and builds nothing: `beats` are the seconds a cut lands on. +""" + +import logging +import math +import types + +from .registry import register_command + +logger = logging.getLogger("dw") + +COMMAND = "analyze_beats" + +# A track whose loudest 50 ms is under this has nothing to track +SILENT_DBFS = -60.0 +# Seconds and BPM as reported +TIME_PLACES = 4 +BPM_PLACES = 2 + + +def _parse_anchors(anchors, duration): + """[(beat index or None, seconds)] for the anchors, refused when one is + past the song's end.""" + parsed = [] + for index, anchor in enumerate(anchors or ()): + if isinstance(anchor, dict): + beat, seconds = int(float(anchor["beat_index"])), float(anchor["seconds"]) + else: + beat, seconds = None, float(anchor) + if seconds > duration: + raise ValueError( + f"{COMMAND}: anchors[{index}] ({seconds:g} s) is past the song's " + f"end ({duration:.3f} s)" + ) + parsed.append((beat, seconds)) + return parsed + + +def _coerce_arguments(sample_rate, tempo_bpm, min_bpm, max_bpm, anchors): + from ..task_domains import beats_problems, check_arguments + from .audio_utils import coerce_number + + values = { + name: coerce_number(value, float, name, COMMAND) + for name, value in ( + ("sample_rate", sample_rate), + ("tempo_bpm", tempo_bpm), + ("min_bpm", min_bpm), + ("max_bpm", max_bpm), + ) + } + if values["sample_rate"] is not None: + values["sample_rate"] = int(values["sample_rate"]) + check_arguments(COMMAND, **values) + problems = beats_problems(values["min_bpm"], values["max_bpm"], anchors) + if problems: + raise ValueError("; ".join(message for _, message in problems)) + return types.SimpleNamespace(**values) + + +def _grid(anchor, tempo_bpm, duration): + """An exact grid at tempo_bpm through one anchor, within the song. + + A bare time is a beat the grid passes through, so it runs both ways from + it; a {beat_index, seconds} mark places beat 0 at seconds - index periods, + and the grid starts there. + """ + beat, seconds = anchor + period = 60.0 / tempo_bpm + if beat is None: + start = seconds - math.floor(seconds / period) * period + else: + start = seconds - beat * period + if start < -1e-9: + raise ValueError( + f"{COMMAND}: beat {beat} at {seconds:g} s puts beat 0 at " + f"{start:.3f} s, before the song starts, at {tempo_bpm:g} BPM" + ) + start = max(start, 0.0) + count = int(math.floor((duration - start) / period - 1e-9)) + 1 + return [start + k * period for k in range(max(count, 0))] + + +def _calibrate(beats, anchors, warnings): + """(warped beats, calibration) pulling the detected beats onto anchors.""" + import numpy + + from .. import dsp + + calibration = {"offset_s": 0.0, "drift": 0.0, "anchors_used": 0} + if not anchors: + return beats, calibration + if not len(beats): + warnings.append( + f"{COMMAND}: no beats were found, so the {len(anchors)} anchor(s) " + "were not used" + ) + return beats, calibration + knots_from, knots_to, used = [], [], set() + for beat, seconds in anchors: + if beat is None: + beat = int(numpy.argmin(numpy.abs(beats - seconds))) + elif beat >= len(beats): + raise ValueError( + f"{COMMAND}: an anchor names beat {beat}, and only {len(beats)} " + "were found (indexes count from 0)" + ) + if beat in used: + warnings.append( + f"{COMMAND}: the anchor at {seconds:g} s is nearest the same " + f"beat ({beats[beat]:.3f} s) as an earlier anchor and was not used" + ) + continue + used.add(beat) + knots_from.append(beats[beat]) + knots_to.append(seconds) + warped = dsp.warp_times(beats, knots_from, knots_to) + calibration = { + "offset_s": round(float(knots_to[0] - knots_from[0]), TIME_PLACES), + "drift": round( + float((knots_to[-1] - knots_from[-1]) - (knots_to[0] - knots_from[0])), + TIME_PLACES, + ), + "anchors_used": len(knots_from), + } + return warped, calibration + + +def _detect(mono, rate, args, warnings): + """(beat seconds, bpm, downbeat phase, method) detected from the audio.""" + import numpy + + from .. import dsp + + envelope, frame_rate = dsp.onset_envelope(mono, rate) + bpm, periodicity = dsp.estimate_tempo( + envelope, frame_rate, args.min_bpm, args.max_bpm, hint_bpm=args.tempo_bpm + ) + if bpm is not None and dsp.is_trackable(envelope, periodicity): + frames = dsp.track_beats(envelope, frame_rate, bpm) + if frames.shape[0] >= 2: + return ( + frames / frame_rate, + bpm, + dsp.downbeat_phase(envelope, frames), + "onset", + ) + peaks, loudest = dsp.rms_peaks(mono, rate, args.max_bpm) + if loudest < SILENT_DBFS: + warnings.append( + f"{COMMAND}: the track is silent (loudest {loudest:.1f} dBFS) - " + "no beats to find" + ) + return numpy.zeros(0), None, None, "rms_peaks" + if peaks.shape[0] < 2: + warnings.append( + f"{COMMAND}: the track has no clear onsets and too few loudness " + "peaks to place a beat" + ) + return numpy.zeros(0), None, None, "rms_peaks" + warnings.append( + f"{COMMAND}: the track has no clear onsets to track - these beats are " + "its loudness peaks, which follow swells rather than a pulse" + ) + return peaks, 60.0 / float(numpy.median(numpy.diff(peaks))), None, "rms_peaks" + + +def analyze_beats( + audio, sample_rate=None, tempo_bpm=None, anchors=None, min_bpm=60, max_bpm=200 +): + """Find a song's tempo and the seconds its beats fall on. + + Args: + audio: The song - a file, a generated audio result, or a video + carrying a soundtrack + sample_rate: The rate of a raw waveform; a file or result carries + its own + tempo_bpm: The tempo, when known. With exactly one anchor it lays an + exact grid and nothing is detected; otherwise it steers the + detection toward that tempo + anchors: Where beats are known to fall - a list of times in seconds, + or of {beat_index, seconds} marks placing a detected beat (from + 0) at a time. Two or more warp the detected beats through them; + one shifts them all; one with tempo_bpm lays a grid + min_bpm: The slowest tempo searched for + max_bpm: The fastest tempo searched for, above min_bpm + + Returns: + {bpm, beats, downbeat_phase, method, calibration, duration_seconds, + warnings}: `beats` ascend, in seconds; `method` is "onset", + "rms_peaks" (no clear onsets - loudness peaks instead) or "grid"; + `downbeat_phase` is which of the first four beats starts a bar, or + None when no beat stands out; `calibration` is the anchors' offset + and drift in seconds and how many were used + """ + import numpy + + from .. import dsp + from ..events import emit_warning + from .audio_utils import waveform_and_rate + + args = _coerce_arguments(sample_rate, tempo_bpm, min_bpm, max_bpm, anchors) + waveform, rate = waveform_and_rate(audio, args.sample_rate, COMMAND) + mono = dsp.mono_float64(waveform) + duration = mono.shape[0] / rate + parsed = _parse_anchors(anchors, duration) + warnings = [] + + if len(parsed) == 1 and args.tempo_bpm is not None: + beats = numpy.array(_grid(parsed[0], args.tempo_bpm, duration)) + bpm, phase, method = args.tempo_bpm, None, "grid" + calibration = {"offset_s": 0.0, "drift": 0.0, "anchors_used": 1} + else: + beats, bpm, phase, method = _detect(mono, rate, args, warnings) + beats, calibration = _calibrate(beats, parsed, warnings) + if calibration["anchors_used"]: + # A warp can carry an end beat past the song's edges; the phase + # counts from the first beat kept + dropped = int(numpy.count_nonzero(beats < 0)) + beats = beats[(beats >= 0) & (beats <= duration)] + if phase is not None: + phase = (phase - dropped) % 4 + if method == "onset" and len(beats) >= 2: + bpm = 60.0 / float(numpy.median(numpy.diff(beats))) + + for message in warnings: + emit_warning(message, kind="analyze_beats", command=COMMAND) + return { + "bpm": None if bpm is None else round(float(bpm), BPM_PLACES), + "beats": [round(float(time), TIME_PLACES) for time in beats], + "downbeat_phase": phase, + "method": method, + "calibration": calibration, + "duration_seconds": round(duration, TIME_PLACES), + "warnings": warnings, + } + + +@register_command( + COMMAND, implementation="dw.tasks.beats.analyze_beats", returns="json" +) +def _handle_analyze_beats(task, arguments, previous_pipelines): + """Find a song's tempo and the seconds its beats fall on""" + logger.debug("Analyzing beats") + return analyze_beats(**arguments) diff --git a/dw/tasks/registry.py b/dw/tasks/registry.py new file mode 100644 index 00000000..9e158ebf --- /dev/null +++ b/dw/tasks/registry.py @@ -0,0 +1,113 @@ +"""The task command registry: `register_command` and the two tables it fills. + +Split from `dw/tasks/task.py` so a task module can register its own handler +(`dw/tasks/beats.py`) without growing that file or importing it, which would +be a cycle - task.py imports this module and every self-registering task +module, and re-exports the tables under their old names. +""" + +import logging +from typing import Callable, Dict + +logger = logging.getLogger("dw") + + +# Command registry: maps command names to handler functions +_COMMAND_REGISTRY: Dict[str, Callable] = {} + +# What each command's arguments actually are. The handlers forward +# **arguments into an implementation function, so that function's signature +# is the command's argument schema - registering its dotted path here (a +# string, to preserve the lazy-import discipline) lets the introspection +# layer read the same signature the runtime calls. 'provided' names the +# parameters the dispatch supplies itself, which are not workflow arguments. +_COMMAND_INFO: Dict[str, dict] = {} + + +def register_command( + command_name: str, + implementation=None, + provided=(), + consumes_device=False, + returns="artifact", + summary=None, + parameter_descriptions=None, + assessment=False, +): + """ + Decorator to register a command handler function. + + Args: + command_name: The command name to register + implementation: Dotted path to the function whose signature defines + the command's arguments (None for a command that consumes a + free-form dict) + provided: Parameter names the dispatch supplies itself + consumes_device: True for a handler that calls task.device_for(arguments) + itself, to pick the accelerator a model-backed task runs on. False + (the default) is for a task that runs no model and forwards + arguments straight to its implementation - device is introspected + as a universal, always-safe-to-pass argument (dw/introspection.py), + so a non-consuming handler must drop it itself rather than let it + reach the implementation as an unexpected keyword argument (#185). + Some commands (e.g. gather_inputs) receive a non-dict argument + value, which never carries a device to drop + returns: "artifact" (the default - an image/video/audio/frames object + `Result.save` knows how to write) or "scalar" for a command whose + return value is a bare number with no file to save (`judge`). A + `result` block on a "scalar" command is refused in + `validation_errors` (dw/scalar_result_validation.py, #212) rather + than reaching `save_artifact` at run time, where a float has + nothing left identifying which command produced it + - or "json" for a command answering a JSON-safe dict (the + assessment probes, `attribute_voices`): its `result` may only be + `application/json`, since any other content type would explode + the dict key by key into files + summary: Overrides the command's `get_task` summary, which otherwise + reads the implementation function's docstring. For a command + whose handler dispatches its implementation per video frame + (`_per_frame`), that docstring describes the single-frame + function rather than the command a caller invokes - same reason + `_VIDEO_PROCESSOR_INFO` overrides `get_first_frame`/ + `get_last_frame` (#366, #383) + parameter_descriptions: Overrides one or more of the implementation's + per-parameter `get_task` descriptions by name, for the same + single-frame-vs-command reason as `summary` + assessment: True for an assessment probe (`dw/tasks/assess.py`) - a + command that measures a finished cut and says where to look, + listed in `list_tasks`' `assessment`. Declared, not inferred from + `returns="json"`: `attribute_voices` answers JSON too and is an + analysis of a song, not a check of a cut (#485) + + Returns: + Decorator function + """ + + def decorator(func: Callable) -> Callable: + if consumes_device: + handler = func + else: + + def handler(task, arguments, previous_pipelines, _func=func): + if isinstance(arguments, dict): + arguments.pop("device", None) + return _func(task, arguments, previous_pipelines) + + _COMMAND_REGISTRY[command_name] = handler + info = { + "kind": "command", + "implementation": implementation, + "provided": tuple(provided), + "returns": returns, + } + if assessment: + info["assessment"] = True + if summary: + info["summary"] = summary + if parameter_descriptions: + info["parameter_descriptions"] = dict(parameter_descriptions) + _COMMAND_INFO[command_name] = info + logger.debug(f"Registered command handler: {command_name}") + return func + + return decorator diff --git a/dw/tasks/task.py b/dw/tasks/task.py index 4ff64005..794b5a64 100644 --- a/dw/tasks/task.py +++ b/dw/tasks/task.py @@ -1,5 +1,4 @@ import logging -from typing import Callable, Dict from .. import resolve_device from ..events import emit_log @@ -14,6 +13,12 @@ get_dict_value, ) +# The registry lives in its own module so a task module can register its own +# handler (`beats`) without a cycle through this one; re-exported here under +# the names the rest of dw and the tests read +from .registry import _COMMAND_INFO, _COMMAND_REGISTRY, register_command # noqa: F401 +from . import beats # noqa: F401 - registers analyze_beats + # The model-backed handlers (upscale, restore_faces, segment, interpolate_frames, # image_to_text, text_generation, diffusion_upscale) are imported inside their # handlers - at module scope their transformers/model imports add seconds to @@ -22,107 +27,6 @@ logger = logging.getLogger("dw") -# Command registry: maps command names to handler functions -_COMMAND_REGISTRY: Dict[str, Callable] = {} - -# What each command's arguments actually are. The handlers forward -# **arguments into an implementation function, so that function's signature -# is the command's argument schema - registering its dotted path here (a -# string, to preserve the lazy-import discipline) lets the introspection -# layer read the same signature the runtime calls. 'provided' names the -# parameters the dispatch supplies itself, which are not workflow arguments. -_COMMAND_INFO: Dict[str, dict] = {} - - -def register_command( - command_name: str, - implementation=None, - provided=(), - consumes_device=False, - returns="artifact", - summary=None, - parameter_descriptions=None, - assessment=False, -): - """ - Decorator to register a command handler function. - - Args: - command_name: The command name to register - implementation: Dotted path to the function whose signature defines - the command's arguments (None for a command that consumes a - free-form dict) - provided: Parameter names the dispatch supplies itself - consumes_device: True for a handler that calls task.device_for(arguments) - itself, to pick the accelerator a model-backed task runs on. False - (the default) is for a task that runs no model and forwards - arguments straight to its implementation - device is introspected - as a universal, always-safe-to-pass argument (dw/introspection.py), - so a non-consuming handler must drop it itself rather than let it - reach the implementation as an unexpected keyword argument (#185). - Some commands (e.g. gather_inputs) receive a non-dict argument - value, which never carries a device to drop - returns: "artifact" (the default - an image/video/audio/frames object - `Result.save` knows how to write) or "scalar" for a command whose - return value is a bare number with no file to save (`judge`). A - `result` block on a "scalar" command is refused in - `validation_errors` (dw/scalar_result_validation.py, #212) rather - than reaching `save_artifact` at run time, where a float has - nothing left identifying which command produced it - - or "json" for a command answering a JSON-safe dict (the - assessment probes, `attribute_voices`): its `result` may only be - `application/json`, since any other content type would explode - the dict key by key into files - summary: Overrides the command's `get_task` summary, which otherwise - reads the implementation function's docstring. For a command - whose handler dispatches its implementation per video frame - (`_per_frame`), that docstring describes the single-frame - function rather than the command a caller invokes - same reason - `_VIDEO_PROCESSOR_INFO` overrides `get_first_frame`/ - `get_last_frame` (#366, #383) - parameter_descriptions: Overrides one or more of the implementation's - per-parameter `get_task` descriptions by name, for the same - single-frame-vs-command reason as `summary` - assessment: True for an assessment probe (`dw/tasks/assess.py`) - a - command that measures a finished cut and says where to look, - listed in `list_tasks`' `assessment`. Declared, not inferred from - `returns="json"`: `attribute_voices` answers JSON too and is an - analysis of a song, not a check of a cut (#485) - - Returns: - Decorator function - """ - - def decorator(func: Callable) -> Callable: - if consumes_device: - handler = func - else: - - def handler(task, arguments, previous_pipelines, _func=func): - if isinstance(arguments, dict): - arguments.pop("device", None) - return _func(task, arguments, previous_pipelines) - - _COMMAND_REGISTRY[command_name] = handler - info = { - "kind": "command", - "implementation": implementation, - "provided": tuple(provided), - "returns": returns, - } - if assessment: - info["assessment"] = True - if summary: - info["summary"] = summary - if parameter_descriptions: - info["parameter_descriptions"] = dict(parameter_descriptions) - _COMMAND_INFO[command_name] = info - logger.debug(f"Registered command handler: {command_name}") - return func - - return decorator - - def task_command_info(command_name): """Where a task command's argument schema lives: a dict with 'kind' ('command', 'image_processor' or 'video_processor'), 'implementation' From 45370207fae80e2c7590efc22d142ee4daa26371 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 21:05:02 -0500 Subject: [PATCH 033/296] feat(tasks): #625 - analyze_beats tests, TASKS.md section, Music 3 skill line Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 4 +- docs/TASKS.md | 57 +++++ dw/dsp.py | 3 +- plugins/dw/skills/minimax-music3/SKILL.md | 2 + tests/test_analyze_beats.py | 293 ++++++++++++++++++++++ tests/test_task_discovery.py | 10 + 6 files changed, 366 insertions(+), 3 deletions(-) create mode 100644 tests/test_analyze_beats.py diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 13600c38..7d6ab50a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -30,8 +30,8 @@ one to open. | On-demand and offload | `dw/pipeline_processors/placement.py`: `place_component`, `has_component_group_offload` | On-demand residency and `group_offload` cannot both be set on one component, and either one stops the whole pipeline from being moved to the device. | `tests/test_modular_pipeline.py::TestOnDemandResidency::test_group_offload_and_on_demand_together_are_rejected`, `tests/test_modular_pipeline.py::TestOffloadDevice::test_component_group_offload_is_not_moved_to_device` | | LoRAs | `dw/pipeline_processors/adapters.py`: `active_loras`, `load_loras` | A `loras` entry whose `model_name` is null is switched off. | `tests/test_lora_disable.py::TestLoadLoras::test_an_all_null_entry_loads_nothing` | | Step progress | `dw/pipeline_processors/progress.py`: `reported_progress_bars` | A pipeline with no step callback reports its denoise steps through its progress bars. | `tests/test_modular_progress.py::test_every_advance_of_the_bar_is_reported` | -| Adding a task | `dw/tasks/task.py`: `register_command` | A task is a function registered with `@register_command`, and the signature of its implementation is its argument schema. | `tests/test_task_discovery.py::TestDescribeTask::test_signature_becomes_the_schema` | -| Task argument domains | `dw/task_domains.py`: `check_arguments` | A numeric domain that a signature cannot express is declared in this table, and it is checked at validation and again at run time. A literal enum choice (`choice_errors`) and the `ingredients_grid` image count against `max_images` (`ingredients_grid_errors`) are refused at validation by the same module. So are `crop_face_track`'s cross-argument rules (`face_track_problems`, `face_track_errors`: crop size a multiple of 32, `gate_zero` above `gate_full`, padding and confidence limits), which the task checks again at run time, and `check_face_detector_source` vets its Hub repo and `.onnx` file before any download. | `tests/test_task_domains.py::TestTheStaticPass::test_a_negative_frame_count_is_refused_at_its_path`, `tests/test_task_domains.py::TestAtRunTime::test_slice_audio_refuses_a_negative_count`, `tests/test_ingredients_grid.py::TestStaticValidation::test_bad_layout_and_fit`, `tests/test_face_track.py::TestStaticValidation::test_bad_literal_is_an_error_at_its_argument`, `tests/test_face_track.py::TestCropFaceTrack::test_bad_arguments_are_refused_at_run_time`, `tests/test_face_track.py::TestDetectorSource::test_traversing_repo_is_refused` | +| Adding a task | `dw/tasks/registry.py`: `register_command` | A task is a function registered with `@register_command`, and the signature of its implementation is its argument schema. The registry lives in `dw/tasks/registry.py`; `dw/tasks/task.py` re-exports it, and a task module may register its own handler, as `dw/tasks/beats.py` does. | `tests/test_task_discovery.py::TestDescribeTask::test_signature_becomes_the_schema` | +| Task argument domains | `dw/task_domains.py`: `check_arguments` | A numeric domain that a signature cannot express is declared in this table, and it is checked at validation and again at run time. A literal enum choice (`choice_errors`) and the `ingredients_grid` image count against `max_images` (`ingredients_grid_errors`) are refused at validation by the same module. So are `crop_face_track`'s cross-argument rules (`face_track_problems`, `face_track_errors`: crop size a multiple of 32, `gate_zero` above `gate_full`, padding and confidence limits), which the task checks again at run time, as `analyze_beats`' are too (`beats_problems`, `beats_errors`: `min_bpm` below `max_bpm`, and anchors that are one kind per list, ascending, with whole beat indexes; `tests/test_analyze_beats.py`), and `check_face_detector_source` vets its Hub repo and `.onnx` file before any download. | `tests/test_task_domains.py::TestTheStaticPass::test_a_negative_frame_count_is_refused_at_its_path`, `tests/test_task_domains.py::TestAtRunTime::test_slice_audio_refuses_a_negative_count`, `tests/test_ingredients_grid.py::TestStaticValidation::test_bad_layout_and_fit`, `tests/test_face_track.py::TestStaticValidation::test_bad_literal_is_an_error_at_its_argument`, `tests/test_face_track.py::TestCropFaceTrack::test_bad_arguments_are_refused_at_run_time`, `tests/test_face_track.py::TestDetectorSource::test_traversing_repo_is_refused` | | Variables | `dw/variables.py`: `set_variables`, `argument_errors`, `resolve_variable_values` | `argument_errors` folds a caller's arguments in exactly as `set_variables` does when a run starts, so a bad name or value is refused before anything is queued. | `tests/test_variables.py::test_argument_errors_reports_a_dict_passed_for_a_string_variable` | | List variables from the CLI | `dw/run.py`, `dw/variables.py`: `get_value` | A `name=value` string given for a list variable is split on commas, so a list of objects can only be passed as JSON over the API or MCP. | `tests/test_variables.py::test_set_variables_list_default_splits_on_comma` | | `for_each` expansion | `dw/for_each.py`: `expand_for_each` | Each entry becomes an ordinary step named `@`, after substitution and before the reference check. | `tests/test_for_each.py::TestNaming::test_member_name_joins_with_at` | diff --git a/docs/TASKS.md b/docs/TASKS.md index 25d19d91..0ebe5556 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -1255,6 +1255,63 @@ scale as `rms_dbfs` (their powers sum to it), so the loudest band sits near `threshold_dbfs`. A silent track, or a band with no content at the track's sample rate, reads as `null` rather than `-inf`. +### analyze_beats + +Find where a song's beats fall, to cut picture to them - a Music 3 song +carries no tempo map, and a cut that lands on the beat needs one. An onset +envelope gives the tempo and the beats are tracked through it; the result is +JSON and nothing is built: + +```json +{ + "name": "beats", + "task": { + "command": "analyze_beats", + "arguments": { + "audio": "previous_result:song", + "anchors": [0.52, 31.9] + } + }, + "result": { "content_type": "application/json" } +} +``` + +| Argument | Required | Description | +| -------- | -------- | ----------- | +| `audio` | Yes | Path, `asset:`/`output:` reference of an audio or video file, a waveform from a previous step, or an earlier step's generated audio or video | +| `sample_rate` | With a waveform | Sample rate of a directly passed waveform (files carry their own) | +| `tempo_bpm` | No | The tempo, when known. With exactly one anchor it lays an exact grid and nothing is detected; otherwise it steers detection toward that tempo | +| `anchors` | No | Where beats are known to fall, ascending: a list of times in seconds, or of `{beat_index, seconds}` marks placing detected beat `beat_index` (from 0) at `seconds`. One kind per list | +| `min_bpm` / `max_bpm` | No | The tempo range searched (default `60` / `200`); `min_bpm` must be below `max_bpm` | + +Returns `{bpm, beats, downbeat_phase, method, calibration, duration_seconds, +warnings}`: `beats` ascend, in seconds. `method` is `onset` (tracked from the +onsets), `rms_peaks` or `grid`. `downbeat_phase` is which of the first four +beats starts a bar, `null` when none stands out. `calibration` is +`{offset_s, drift, anchors_used}`, the anchors' shift at the first mark and +how much it changes by the last. `warnings` repeats what was emitted. + +Anchors correct the detection three ways: + +- Two or more warp the detected beats piecewise-linearly through them, so a + detection that starts late or drifts lands on the marks. +- One alone shifts every beat by the same amount. +- One plus `tempo_bpm` lays an exact grid through it, with no detection: + `method` is `grid`. + +A bare-seconds anchor snaps to the nearest detected beat, so it corrects a +drift of under half a beat; where the detection is further off than that, use +`{beat_index, seconds}` marks, which name the beat instead of guessing it. + +A track with no clear onsets - a pad, a swell - falls back to the peaks of its +loudness (`method` `rms_peaks`, with a warning that they follow swells rather +than a pulse). A silent track returns empty `beats` and a warning. + +Refused: `min_bpm` at or above `max_bpm`, and malformed anchors (mixed kinds, +out of order, a non-whole `beat_index`), at `validate_workflow`; an anchor past +the song's end, or a `beat_index` past the detected count, at run start. +It returns JSON, so `result` may only be `application/json`. + ## Assessment Probes Three read-only commands measure a finished cut and say where to look - diff --git a/dw/dsp.py b/dw/dsp.py index f8bc72a7..3cffb2b5 100644 --- a/dw/dsp.py +++ b/dw/dsp.py @@ -3,7 +3,8 @@ Everything the audio tasks, the media probe and the chain's seam joins compute from a waveform lives here - level conversion, BS.1770 loudness and true peak, the limiter, the filters and envelope followers, the spectral -readings and the resampler. It emits no events and checks no arguments: a +readings, the resampler and the beat tracker (onset envelope, tempo +and beats). It emits no events and checks no arguments: a task decides what a measurement means and what to say about it, and this module only measures. diff --git a/plugins/dw/skills/minimax-music3/SKILL.md b/plugins/dw/skills/minimax-music3/SKILL.md index b5b0b57a..ca5c7928 100644 --- a/plugins/dw/skills/minimax-music3/SKILL.md +++ b/plugins/dw/skills/minimax-music3/SKILL.md @@ -173,6 +173,8 @@ Control" section. `url` (`list_gallery`, or the manifest's file name). 5. To use the track in a later workflow, `keep_output` makes it an `asset:`; to trim it in the same run, chain `templates/audio-trim-fade` on the output. + To cut picture to the song, `analyze_beats` on it returns its bpm and beat + times; with the tempo known, `tempo_bpm` plus one anchor lays an exact grid. 6. A run worth keeping: `references/keeping-a-run.md` saves it by name and exports it. diff --git a/tests/test_analyze_beats.py b/tests/test_analyze_beats.py new file mode 100644 index 00000000..5226d48d --- /dev/null +++ b/tests/test_analyze_beats.py @@ -0,0 +1,293 @@ +"""analyze_beats: onset tracking, the loudness fallback, anchors, validation.""" + +import json + +import numpy +import pytest + +from dw import dsp +from dw.task_domains import beats_problems, task_argument_errors +from dw.tasks.beats import analyze_beats + +SR = 44100 + + +def click(freq=1000.0, amplitude=1.0): + t = numpy.arange(int(0.01 * SR)) / SR + return amplitude * numpy.sin(2 * numpy.pi * freq * t) * numpy.exp(-t / 0.002) + + +def place(signal, seconds, burst): + start = int(round(seconds * SR)) + end = min(len(signal), start + len(burst)) + signal[start:end] += burst[: end - start] + + +def click_track(bpm, seconds=30.0, first=0.5): + signal = numpy.zeros(int(seconds * SR)) + period = 60.0 / bpm + times = numpy.arange(first, seconds - 0.02, period) + for time in times: + place(signal, time, click()) + return signal.astype(numpy.float32)[None, :], times + + +def run(waveform, **kwargs): + return analyze_beats(waveform, sample_rate=SR, **kwargs) + + +def assert_beats_match(beats, truth, tolerance=0.02): + beats = numpy.asarray(beats) + truth = numpy.asarray(truth) + for time in truth: + assert numpy.abs(beats - time).min() <= tolerance, f"true click {time}" + for time in beats: + assert numpy.abs(truth - time).min() <= tolerance, f"beat {time}" + + +@pytest.fixture(scope="module") +def track128(): + waveform, times = click_track(128) + return waveform, times, run(waveform) + + +class TestOnsetTracking: + @pytest.mark.parametrize("bpm", [90, 128]) + def test_click_tracks(self, bpm): + waveform, times = click_track(bpm) + result = run(waveform) + assert result["method"] == "onset" + assert abs(result["bpm"] - bpm) <= 1 + assert_beats_match(result["beats"], times) + assert all(b > a for a, b in zip(result["beats"], result["beats"][1:])) + + def test_swung_track_keeps_the_main_tempo(self): + bpm, seconds = 100, 30.0 + period = 60.0 / bpm + signal = numpy.zeros(int(seconds * SR)) + mains = numpy.arange(0.5, seconds - 0.1, period) + for time in mains: + place(signal, time, click()) + place(signal, time + period * 2 / 3, click(2000.0, 0.4)) + result = run(signal.astype(numpy.float32)[None, :]) + assert abs(result["bpm"] - 100) <= 3 + for time in result["beats"]: + assert numpy.abs(mains - time).min() <= 0.02 + + +class TestFallback: + def test_silence(self, monkeypatch): + seen = [] + monkeypatch.setattr( + "dw.events.emit_warning", lambda message, **data: seen.append(message) + ) + result = run(numpy.zeros((1, SR * 5), dtype=numpy.float32)) + assert result["beats"] == [] + assert result["bpm"] is None + assert result["method"] == "rms_peaks" + assert result["warnings"] + assert seen == result["warnings"] + + def test_swells_fall_back_to_loudness_peaks(self): + rng = numpy.random.default_rng(0) + t = numpy.arange(SR * 20) / SR + signal = rng.standard_normal(t.shape[0]) * ( + 0.5 - 0.5 * numpy.cos(2 * numpy.pi * t) + ) + result = run((0.3 * signal).astype(numpy.float32)[None, :]) + assert result["method"] == "rms_peaks" + assert abs(result["bpm"] - 60) <= 3 + assert result["warnings"] + + +class TestAnchors: + def test_one_anchor_and_tempo_lay_a_grid(self): + result = run( + numpy.zeros((1, SR * 10), dtype=numpy.float32), + tempo_bpm=128, + anchors=[{"beat_index": 0, "seconds": 0.5}], + ) + assert result["method"] == "grid" + assert result["bpm"] == 128 + beats = result["beats"] + assert beats[0] == 0.5 + assert numpy.allclose(numpy.diff(beats), 60 / 128, atol=1e-3) + + def test_a_bare_anchor_grid_runs_both_ways(self): + result = run( + numpy.zeros((1, SR * 10), dtype=numpy.float32), + tempo_bpm=120, + anchors=[3.0], + ) + assert result["method"] == "grid" + assert result["beats"][0] == 0.0 + assert 3.0 in result["beats"] + + def test_a_beat_index_putting_beat_zero_before_the_start_is_refused(self): + with pytest.raises(ValueError, match="before the song starts"): + run( + numpy.zeros((1, SR * 10), dtype=numpy.float32), + tempo_bpm=120, + anchors=[{"beat_index": 4, "seconds": 0.5}], + ) + + def _truth(self, track128): + _, _, base = track128 + truth = numpy.array(base["beats"]) * 1.005 + 0.05 + return base, truth + + def test_two_bare_anchors_correct_drift(self, track128): + waveform, _, _ = track128 + base, truth = self._truth(track128) + result = run(waveform, anchors=[float(truth[2]), float(truth[-3])]) + assert result["calibration"]["anchors_used"] == 2 + kept = numpy.array(result["beats"]) + assert numpy.abs(kept - truth[: len(kept)]).max() < 0.002 + expected = (truth[-3] - base["beats"][-3]) - (truth[2] - base["beats"][2]) + assert result["calibration"]["drift"] > 0 + assert result["calibration"]["drift"] == pytest.approx(expected, abs=2e-3) + + def test_two_indexed_anchors_correct_drift(self, track128): + waveform, _, _ = track128 + base, truth = self._truth(track128) + last = len(truth) - 3 + result = run( + waveform, + anchors=[ + {"beat_index": 2, "seconds": float(truth[2])}, + {"beat_index": last, "seconds": float(truth[last])}, + ], + ) + assert result["calibration"]["anchors_used"] == 2 + kept = numpy.array(result["beats"]) + assert numpy.abs(kept - truth[: len(kept)]).max() < 0.002 + assert result["calibration"]["drift"] > 0 + + def test_one_bare_anchor_shifts_every_beat(self, track128): + waveform, _, base = track128 + shifted = run(waveform, anchors=[base["beats"][4] + 0.1]) + assert shifted["calibration"]["anchors_used"] == 1 + deltas = numpy.array(shifted["beats"]) - numpy.array(base["beats"]) + assert numpy.allclose(deltas, 0.1, atol=2e-4) + + def test_an_anchor_past_the_end_is_refused(self): + waveform, _ = click_track(128, seconds=10) + with pytest.raises(ValueError, match="past the song's end"): + run(waveform, anchors=[11.0]) + + def test_a_beat_index_past_the_detected_count_is_refused(self): + waveform, _ = click_track(128, seconds=10) + with pytest.raises(ValueError, match="only"): + run(waveform, anchors=[{"beat_index": 500, "seconds": 5.0}]) + + def test_an_empty_range_is_refused(self): + with pytest.raises(ValueError, match="min_bpm"): + run(numpy.zeros((1, SR), dtype=numpy.float32), min_bpm=120, max_bpm=100) + + def test_mixed_and_unordered_anchors_are_refused(self): + silent = numpy.zeros((1, SR * 10), dtype=numpy.float32) + with pytest.raises(ValueError, match="mixes"): + run(silent, anchors=[1.0, {"beat_index": 2, "seconds": 3.0}]) + with pytest.raises(ValueError, match="not after"): + run(silent, anchors=[3.0, 1.0]) + + +def workflow_errors(arguments): + return task_argument_errors( + { + "id": "beats", + "steps": [ + { + "name": "beats", + "task": {"command": "analyze_beats", "arguments": arguments}, + "result": {"content_type": "application/json"}, + } + ], + } + ) + + +class TestStaticValidation: + @pytest.mark.parametrize( + "arguments", + [ + {"audio": "asset:a.wav", "min_bpm": 120, "max_bpm": 100}, + {"audio": "asset:a.wav", "anchors": [1.0, {"beat_index": 1, "seconds": 2}]}, + {"audio": "asset:a.wav", "anchors": [3.0, 1.0]}, + {"audio": "asset:a.wav", "anchors": [{"beat_index": -1, "seconds": 1}]}, + {"audio": "asset:a.wav", "anchors": [{"beat_index": 1.5, "seconds": 1}]}, + {"audio": "asset:a.wav", "tempo_bpm": 0}, + {"audio": "asset:a.wav", "tempo_bpm": -90}, + ], + ) + def test_bad_literals_are_reported(self, arguments): + assert workflow_errors(arguments) + + def test_references_are_not_complained_about(self): + assert ( + workflow_errors( + { + "audio": "asset:a.wav", + "anchors": "variable:marks", + "min_bpm": "variable:low", + "max_bpm": 100, + } + ) + == [] + ) + + def test_problems_names_each_argument(self): + problems = beats_problems(120, 100, [3.0, 1.0]) + assert {name for name, _ in problems} == {"min_bpm", "anchors"} + assert beats_problems(60, 200, [1.0, 2.0]) == [] + assert beats_problems("variable:x", 100, "variable:y") == [] + + +class TestRealPath: + def _file(self, tmp_path): + import soundfile + + waveform, times = click_track(128, seconds=12) + path = tmp_path / "clicks.wav" + soundfile.write(path, waveform[0], SR) + return path, times + + def test_a_wav_file(self, tmp_path): + path, times = self._file(tmp_path) + result = analyze_beats(str(path)) + assert result["method"] == "onset" + assert_beats_match(result["beats"], times) + + def test_the_registered_handler(self, tmp_path): + from dw.tasks.task import _COMMAND_REGISTRY + + path, _ = self._file(tmp_path) + result = _COMMAND_REGISTRY["analyze_beats"]( + None, {"audio": str(path), "device": "cpu"}, {} + ) + assert result["method"] == "onset" + assert abs(result["bpm"] - 128) <= 1 + + +class TestResultShape: + def test_json_serializable_with_plain_floats(self, track128): + _, _, result = track128 + json.dumps(result) + waveform, _, base = track128 + calibrated = run(waveform, anchors=[base["beats"][2], base["beats"][-3]]) + json.dumps(calibrated) + for value in calibrated["calibration"].values(): + assert type(value) in (float, int) + assert type(calibrated["calibration"]["offset_s"]) is float + assert type(calibrated["bpm"]) is float + assert all(type(b) is float for b in calibrated["beats"]) + + +class TestWarpTimes: + def test_two_knots_interpolate_and_extrapolate(self): + warped = dsp.warp_times([0.0, 1.0, 2.0, 3.0], [1.0, 2.0], [2.0, 4.0]) + assert warped.tolist() == pytest.approx([0.0, 2.0, 4.0, 6.0]) + + def test_one_knot_shifts(self): + warped = dsp.warp_times([0.0, 1.0, 2.0], [1.0], [1.5]) + assert warped.tolist() == pytest.approx([0.5, 1.5, 2.5]) diff --git a/tests/test_task_discovery.py b/tests/test_task_discovery.py index 71345651..6fe5a361 100644 --- a/tests/test_task_discovery.py +++ b/tests/test_task_discovery.py @@ -172,3 +172,13 @@ def test_named_parameters_win_over_the_nested_block(self): # the kwargs block says names = self.names(describe_task("text_generation")) assert names.count("device") == 1 + + +class TestAnalyzeBeatsRegistration: + def test_analyze_beats_is_registered_as_a_json_answer(self): + from dw.tasks.task import task_command_info + + info = task_command_info("analyze_beats") + assert info["implementation"] == "dw.tasks.beats.analyze_beats" + assert info["returns"] == "json" + assert "assessment" not in info From 1d05abf7f2dfafbcb8284121acb095f87c8096c8 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 21:22:49 -0500 Subject: [PATCH 034/296] feat(tasks): #625 - analyze_beats: beats to the ends, bpm folded into range, near-silence refused - C-F206: frame 0 counts as an onset from silence; end beats off the period are dropped and the ends re-extended onto prominent onsets - C-F207: tempo folded by octaves into min_bpm-max_bpm; bpm is always in range or null, with a warning where no octave fits - C-F208: a track under -40 dBFS returns no beats and a near-silent warning Co-Authored-By: Claude Opus 5.5 --- docs/TASKS.md | 12 ++- dw/dsp.py | 144 +++++++++++++++++++++++++++++------- dw/tasks/beats.py | 48 +++++++++--- tests/test_analyze_beats.py | 103 ++++++++++++++++++++++++++ 4 files changed, 271 insertions(+), 36 deletions(-) diff --git a/docs/TASKS.md b/docs/TASKS.md index 0ebe5556..b95a5870 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -1305,7 +1305,17 @@ drift of under half a beat; where the detection is further off than that, use A track with no clear onsets - a pad, a swell - falls back to the peaks of its loudness (`method` `rms_peaks`, with a warning that they follow swells rather -than a pulse). A silent track returns empty `beats` and a warning. +than a pulse). A silent track returns empty `beats` and a warning. So does a +near-silent one whose loudest 50 ms is under -40 dBFS - room tone, hiss, or a +song mixed far too low - since the log-compressed onset envelope would find a +pulse in it. Raise a real song's level first (`gain_audio`). + +`bpm` is always within `min_bpm`-`max_bpm`, or `null`. A pulse found outside the +range is folded by octaves into it (85.7 BPM searched at 140-200 reports 171.4, +beating every half pulse). Where no octave fits, the beats keep to a tempo in +the range and a warning names the pulse. Where the `rms_peaks` peaks have no +octave in the range, `bpm` is `null` with a warning. The beats run to the +song's ends: a hit at 0 s is a beat, and a noise bed starting at 0 s is not. Refused: `min_bpm` at or above `max_bpm`, and malformed anchors (mixed kinds, out of order, a non-whole `beat_index`), at `validate_workflow`; an anchor past diff --git a/dw/dsp.py b/dw/dsp.py index 3cffb2b5..59d467a5 100644 --- a/dw/dsp.py +++ b/dw/dsp.py @@ -627,6 +627,10 @@ def biquad_coefficients(kind, cutoff_hz, q, sample_rate): TEMPO_PRIOR_BPM = 120.0 TEMPO_PRIOR_OCTAVES = 1.0 TEMPO_HINT_OCTAVES = 0.5 +# When the caller's range holds no clear pulse, the pulse is looked for over +# this range and folded into the caller's by octaves - an 86 BPM song asked +# for at 140-200 is tracked at 172 +TEMPO_SEARCH_BPM = (30.0, 300.0) # How strongly the programme holds an interval to the period: the penalty is # BEAT_TIGHTNESS * log(interval / period)^2 BEAT_TIGHTNESS = 100.0 @@ -639,6 +643,17 @@ def biquad_coefficients(kind, cutoff_hz, q, sample_rate): # fraction of the RMS onset at the beats: the programme keeps stepping # through an intro's silence at the period, onto nothing BEAT_TRIM_FRACTION = 0.5 +# An end beat further than this fraction of the period (or two frames) off +# the period from its neighbour is dropped, and the ends re-extended one +# period at a time onto an onset within that distance: the programme's ends +# have a neighbour on one side only, and settle on a tone's release or a +# late transient instead of the beat +BEAT_EDGE_TOLERANCE = 0.06 +# ...onto an onset standing at least this fraction of the weaker beats' +# prominence (the 10th percentile's) above the envelope around it: a soft +# end beat a period from its neighbour counts, a bump in a noise bed +# after the music stops does not +BEAT_EXTEND_FRACTION = 0.75 # The RMS-peak fallback: 50 ms windows on the onset hop, peaks at least this # prominent relative to the envelope's range RMS_PEAK_WINDOW_SECONDS = 0.05 @@ -666,40 +681,74 @@ def onset_envelope(mono, sample_rate): mono, sample_rate, nperseg=window, noverlap=window - hop, padded=True ) magnitude = numpy.log1p(ONSET_LOG_GAIN * numpy.abs(spectrum)) - flux = numpy.maximum(0.0, numpy.diff(magnitude, axis=1)).mean(axis=0) - return numpy.concatenate([[0.0], flux]), sample_rate / hop + # Frame 0 rises from silence, so a hit at 0 s is an onset + flux = numpy.maximum(0.0, numpy.diff(magnitude, axis=1, prepend=0.0)) + return flux.mean(axis=0), sample_rate / hop -def estimate_tempo(envelope, rate, min_bpm, max_bpm, hint_bpm=None): - """(bpm, periodicity) from the envelope's autocorrelation in the range. +def fold_bpm(bpm, min_bpm, max_bpm, centre=TEMPO_PRIOR_BPM): + """bpm moved by octaves into [min_bpm, max_bpm], the octave nearest + `centre` when more than one fits; None when none does.""" + if bpm is None or bpm <= 0: + return None + if min_bpm <= bpm <= max_bpm: + return bpm + fits = [ + bpm * 2.0**octave + for octave in range(-4, 5) + if min_bpm <= bpm * 2.0**octave <= max_bpm + ] + return min(fits, key=lambda fit: abs(math.log2(fit / centre)), default=None) - `periodicity` is the normalized autocorrelation at the chosen lag, 0 to - 1; (None, 0.0) when the envelope holds no energy or the range no lag. + +def estimate_tempo(envelope, rate, min_bpm, max_bpm, hint_bpm=None): + """(bpm, periodicity, missed_pulse_bpm) from the envelope's autocorrelation. + + The tempo is searched for in the range first. When nothing there is + periodic enough to track, the pulse is searched for over + TEMPO_SEARCH_BPM and folded into the range by octaves. When no octave of + it fits, the range's best tempo is kept and `missed_pulse_bpm` is the + pulse it misses; otherwise that is None. `periodicity` is the normalized + autocorrelation at the pulse, 0 to 1. (None, 0.0, None) when the envelope + holds no energy or the range no lag; a bpm is always in the range. """ centred = envelope - envelope.mean() if envelope.size else envelope count = centred.shape[0] if count < 2: - return None, 0.0 + return None, 0.0, None correlation = scipy.signal.correlate(centred, centred, mode="full", method="fft") correlation = correlation[count - 1 :] if correlation[0] <= 0: - return None, 0.0 + return None, 0.0, None correlation = correlation / correlation[0] - low = max(1, int(math.floor(rate * 60.0 / max_bpm))) - high = min(count - 2, int(math.ceil(rate * 60.0 / min_bpm))) - if high < low: - return None, 0.0 - lags = numpy.arange(low, high + 1) centre = TEMPO_PRIOR_BPM if hint_bpm is None else hint_bpm width = TEMPO_PRIOR_OCTAVES if hint_bpm is None else TEMPO_HINT_OCTAVES - prior = numpy.exp(-0.5 * (numpy.log2(60.0 * rate / lags / centre) / width) ** 2) - best = int(numpy.argmax(correlation[lags] * prior)) - lag = lags[best] - # Parabolic interpolation between the lags either side - before, at, after = correlation[lag - 1], correlation[lag], correlation[lag + 1] - curvature = before - 2.0 * at + after - shift = 0.5 * (before - after) / curvature if curvature < 0 else 0.0 - return 60.0 * rate / (lag + shift), float(max(at, 0.0)) + + def best(low_bpm, high_bpm): + low = max(1, int(math.floor(rate * 60.0 / high_bpm))) + high = min(count - 2, int(math.ceil(rate * 60.0 / low_bpm))) + if high < low: + return None, 0.0 + lags = numpy.arange(low, high + 1) + prior = numpy.exp(-0.5 * (numpy.log2(60.0 * rate / lags / centre) / width) ** 2) + lag = lags[int(numpy.argmax(correlation[lags] * prior))] + # Parabolic interpolation between the lags either side + before, at, after = correlation[lag - 1], correlation[lag], correlation[lag + 1] + curvature = before - 2.0 * at + after + shift = 0.5 * (before - after) / curvature if curvature < 0 else 0.0 + bpm = min(max(60.0 * rate / (lag + shift), low_bpm), high_bpm) + return bpm, float(max(at, 0.0)) + + bpm, periodicity = best(min_bpm, max_bpm) + if bpm is None or periodicity >= TRACKABLE_MIN_PERIODICITY: + return bpm, periodicity, None + pulse, pulse_periodicity = best(*TEMPO_SEARCH_BPM) + if pulse is None or pulse_periodicity <= periodicity: + return bpm, periodicity, None + folded = fold_bpm(pulse, min_bpm, max_bpm, centre) + if folded is None: + return bpm, pulse_periodicity, pulse + return folded, pulse_periodicity, None def is_trackable(envelope, periodicity): @@ -712,13 +761,18 @@ def is_trackable(envelope, periodicity): def track_beats(envelope, rate, bpm, tightness=BEAT_TIGHTNESS): """Beat frame indices by dynamic programming, leading and trailing beats - that land on next to nothing trimmed.""" + that land on next to nothing trimmed and the ends settled on the period. + + Frame 0 is left out of the programme: it holds the track's rise from + silence whether or not a beat falls there, so only settling the ends + can put a beat on it, and only when it is a period from the next. + """ count = envelope.shape[0] period = rate * 60.0 / bpm - spread = envelope.std() - if count == 0 or spread <= 0: + if count < 2 or envelope[1:].std() <= 0: return numpy.zeros(0, dtype=int) - local = envelope / spread + local = envelope / envelope[1:].std() + local[0] = 0.0 score = local.copy() backlink = numpy.full(count, -1, dtype=int) steps = numpy.arange(max(1, int(round(period / 2))), int(round(2 * period)) + 1) @@ -744,7 +798,45 @@ def track_beats(envelope, rate, bpm, tightness=BEAT_TIGHTNESS): kept = numpy.nonzero(strength >= floor)[0] if not kept.size: return numpy.zeros(0, dtype=int) - return beats[kept[0] : kept[-1] + 1] + kept = beats[kept[0] : kept[-1] + 1] + floor = BEAT_EXTEND_FRACTION * float( + numpy.percentile([_prominence(envelope, frame, period) for frame in kept], 10) + ) + return _settle_ends(list(kept), envelope, period, floor) + + +def _prominence(envelope, frame, period): + """How far the envelope at frame stands above its median over the half + period either side.""" + half = max(1, int(round(period / 2))) + around = envelope[max(0, frame - half) : frame + half + 1] + return float(envelope[frame] - numpy.median(around)) + + +def _settle_ends(beats, envelope, period, floor): + """The beat frames with off-period end beats dropped, then the ends + extended a period at a time while an onset of at least `floor` + prominence is there.""" + tolerance = max(2.0, BEAT_EDGE_TOLERANCE * period) + while len(beats) >= 3 and abs(beats[-1] - beats[-2] - period) > tolerance: + beats.pop() + while len(beats) >= 3 and abs(beats[1] - beats[0] - period) > tolerance: + beats.pop(0) + reach = int(round(tolerance)) + + def onset_near(centre): + low = max(0, int(round(centre)) - reach) + high = min(envelope.shape[0], int(round(centre)) + reach + 1) + if low >= high: + return None + frame = low + int(numpy.argmax(envelope[low:high])) + return frame if _prominence(envelope, frame, period) >= floor else None + + while (frame := onset_near(beats[-1] + period)) is not None and frame > beats[-1]: + beats.append(frame) + while (frame := onset_near(beats[0] - period)) is not None and frame < beats[0]: + beats.insert(0, frame) + return numpy.array(beats, dtype=int) def rms_peaks(mono, sample_rate, max_bpm): diff --git a/dw/tasks/beats.py b/dw/tasks/beats.py index 4b25dc9d..f26d2b7c 100644 --- a/dw/tasks/beats.py +++ b/dw/tasks/beats.py @@ -27,6 +27,9 @@ # A track whose loudest 50 ms is under this has nothing to track SILENT_DBFS = -60.0 +# ...and one under this is near-silence - room tone, hiss - whose onset +# envelope, log-compressed, can still look like a pulse +QUIET_DBFS = -40.0 # Seconds and BPM as reported TIME_PLACES = 4 BPM_PLACES = 2 @@ -146,26 +149,42 @@ def _detect(mono, rate, args, warnings): from .. import dsp + peaks, loudest = dsp.rms_peaks(mono, rate, args.max_bpm) + # The onset envelope is log-compressed, so it finds the same pulse in a + # song at -120 dB as at full scale: the level is checked first + if loudest < SILENT_DBFS: + warnings.append( + f"{COMMAND}: the track is silent (loudest {loudest:.1f} dBFS) - " + "no beats to find" + ) + return numpy.zeros(0), None, None, "rms_peaks" + if loudest < QUIET_DBFS: + warnings.append( + f"{COMMAND}: the track is near-silent (loudest {loudest:.1f} dBFS, " + f"under {QUIET_DBFS:g}) - room tone or noise, no pulse to track; " + "raise its level first (gain_audio) if it is a real song" + ) + return numpy.zeros(0), None, None, "rms_peaks" + envelope, frame_rate = dsp.onset_envelope(mono, rate) - bpm, periodicity = dsp.estimate_tempo( + bpm, periodicity, missed = dsp.estimate_tempo( envelope, frame_rate, args.min_bpm, args.max_bpm, hint_bpm=args.tempo_bpm ) if bpm is not None and dsp.is_trackable(envelope, periodicity): frames = dsp.track_beats(envelope, frame_rate, bpm) if frames.shape[0] >= 2: + if missed is not None: + warnings.append( + f"{COMMAND}: the track's pulse is {missed:.1f} BPM, and no " + f"octave of it is within {args.min_bpm:g}-{args.max_bpm:g} " + f"BPM - these beats keep to {bpm:.1f} BPM, not the pulse" + ) return ( frames / frame_rate, bpm, dsp.downbeat_phase(envelope, frames), "onset", ) - peaks, loudest = dsp.rms_peaks(mono, rate, args.max_bpm) - if loudest < SILENT_DBFS: - warnings.append( - f"{COMMAND}: the track is silent (loudest {loudest:.1f} dBFS) - " - "no beats to find" - ) - return numpy.zeros(0), None, None, "rms_peaks" if peaks.shape[0] < 2: warnings.append( f"{COMMAND}: the track has no clear onsets and too few loudness " @@ -176,7 +195,15 @@ def _detect(mono, rate, args, warnings): f"{COMMAND}: the track has no clear onsets to track - these beats are " "its loudness peaks, which follow swells rather than a pulse" ) - return peaks, 60.0 / float(numpy.median(numpy.diff(peaks))), None, "rms_peaks" + rate_bpm = 60.0 / float(numpy.median(numpy.diff(peaks))) + folded = dsp.fold_bpm(rate_bpm, args.min_bpm, args.max_bpm) + if folded is None: + warnings.append( + f"{COMMAND}: the loudness peaks come at {rate_bpm:.1f} BPM, and no " + f"octave of that is within {args.min_bpm:g}-{args.max_bpm:g} BPM, " + "so no bpm is reported" + ) + return peaks, folded, None, "rms_peaks" def analyze_beats( @@ -236,6 +263,9 @@ def analyze_beats( phase = (phase - dropped) % 4 if method == "onset" and len(beats) >= 2: bpm = 60.0 / float(numpy.median(numpy.diff(beats))) + if not calibration["anchors_used"]: + # Frame rounding can carry the median a hair past the range + bpm = min(max(bpm, args.min_bpm), args.max_bpm) for message in warnings: emit_warning(message, kind="analyze_beats", command=COMMAND) diff --git a/tests/test_analyze_beats.py b/tests/test_analyze_beats.py index 5226d48d..32b2ffe1 100644 --- a/tests/test_analyze_beats.py +++ b/tests/test_analyze_beats.py @@ -75,7 +75,110 @@ def test_swung_track_keeps_the_main_tempo(self): assert numpy.abs(mains - time).min() <= 0.02 +def burst(name): + t = numpy.arange(int(0.05 * SR)) / SR + if name == "tone": + # A hard-gated tone: its release splatters a second, later onset + return 0.5 * numpy.sin(2 * numpy.pi * 880 * t) + return click() + + +def backbeat(bpm=85.71, seconds=30.0, first=0.3): + """Kick and snare alternating on the beat over a noise bed from 0 s - a + pulse strongest at the bar, with a weak kick at each end.""" + rng = numpy.random.default_rng(2) + signal = 0.05 * rng.standard_normal(int(seconds * SR)) + t = numpy.arange(int(0.2 * SR)) / SR + kick = numpy.sin(2 * numpy.pi * 60 * t) * numpy.exp(-t / 0.05) + snare = ( + 0.4 * rng.standard_normal(int(0.1 * SR)) * numpy.exp(-t[: int(0.1 * SR)] / 0.03) + ) + times = numpy.arange(first, seconds - 0.3, 60.0 / bpm) + for index, time in enumerate(times): + place(signal, time, kick if index % 2 == 0 else snare) + return signal.astype(numpy.float32)[None, :], times + + +class TestEdges: + """The first and last beats, which the programme sees from one side.""" + + @pytest.mark.parametrize("bpm", [100, 120, 128]) + @pytest.mark.parametrize("shape", ["click", "tone"]) + @pytest.mark.parametrize("first", [0.0, 0.5]) + def test_every_click_to_the_ends(self, bpm, shape, first): + signal = numpy.zeros(SR * 20) + times = numpy.arange(first, 20.0 - 0.02, 60.0 / bpm) + for time in times: + place(signal, time, burst(shape)) + result = run(signal.astype(numpy.float32)[None, :]) + assert result["method"] == "onset" + assert len(result["beats"]) == len(times) + assert_beats_match(result["beats"], times) + + def test_a_bed_from_zero_puts_no_beat_at_zero(self): + waveform, times = backbeat() + result = run(waveform) + assert result["beats"][0] == pytest.approx(times[0], abs=0.02) + assert_beats_match(result["beats"], times) + + +class TestRange: + def test_a_range_above_the_pulse_folds_it_up(self): + waveform, times = backbeat() + result = run(waveform, min_bpm=140, max_bpm=200) + assert result["method"] == "onset" + assert result["bpm"] == pytest.approx(171.4, abs=1) + assert result["warnings"] == [] + for time in times: + assert numpy.abs(numpy.array(result["beats"]) - time).min() <= 0.02 + + def test_a_range_missing_every_octave_keeps_to_the_range_and_says_so(self): + waveform, _ = backbeat() + result = run(waveform, min_bpm=119, max_bpm=120) + assert 119 <= result["bpm"] <= 120 + assert any("no octave" in warning for warning in result["warnings"]) + + def test_the_fallback_bpm_is_in_the_range(self): + rng = numpy.random.default_rng(0) + t = numpy.arange(SR * 20) / SR + signal = rng.standard_normal(t.shape[0]) * ( + 0.5 - 0.5 * numpy.cos(2 * numpy.pi * t) + ) + result = run( + (0.3 * signal).astype(numpy.float32)[None, :], min_bpm=100, max_bpm=140 + ) + assert result["method"] == "rms_peaks" + assert result["bpm"] == pytest.approx(120, abs=6) + result = run( + (0.3 * signal).astype(numpy.float32)[None, :], min_bpm=70, max_bpm=80 + ) + assert result["bpm"] is None + assert any("no octave" in warning for warning in result["warnings"]) + + def test_fold_bpm(self): + assert dsp.fold_bpm(85.7, 140, 200) == pytest.approx(171.4) + assert dsp.fold_bpm(85.7, 119, 120) is None + assert dsp.fold_bpm(240, 60, 200, centre=120) == 120 + + class TestFallback: + @pytest.mark.parametrize("gain_db", [-120, -50]) + def test_a_quiet_song_is_not_tracked(self, gain_db): + waveform, _ = backbeat() + scale = 10 ** (gain_db / 20) / numpy.abs(waveform).max() + result = run(waveform * scale) + assert result["beats"] == [] + assert result["bpm"] is None + assert result["method"] == "rms_peaks" + assert result["warnings"] + + def test_room_tone_is_not_tracked(self): + rng = numpy.random.default_rng(3) + tone = 10 ** (-50 / 20) * rng.standard_normal(SR * 20) + result = run(tone.astype(numpy.float32)[None, :]) + assert result["beats"] == [] + assert "near-silent" in result["warnings"][0] + def test_silence(self, monkeypatch): seen = [] monkeypatch.setattr( From ab250c446b4ef8ed359ff5ca198d00b9a3fd458d Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 21:33:22 -0500 Subject: [PATCH 035/296] feat(locations): #618 - URL scheme decision goes to security.validate_url _refuse_other_url keeps '://' as the test for 'this location is a URL' and asks validate_url whether its scheme is allowed, so the two allow-lists can't disagree (HTTPS:// was refused as an 'HTTPS' URL). location_errors sends such a URL through validate_media_url's host policy. Pinned by tests for HTTPS://. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- dw/locations.py | 19 ++++++++++++++----- tests/test_locations.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 7d6ab50a..2f55c782 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -120,7 +120,7 @@ one to open. | --- | --- | --- | --- | | Path validators | `dw/security.py`: `validate_path`, `validate_workflow_path`, `validate_output_path` | Filesystem access goes through a path validator that is given a base directory. | `tests/test_security.py::test_path_validation`, `.github/codeql/dw-security/DwPathInjection.ql` | | URL and argument validators | `dw/security.py`: `validate_url`, `sanitize_command_args` | A URL must be http or https, and a subprocess argument must not carry a shell metacharacter. | `tests/test_security.py::TestValidateUrl::test_dangerous_schemes_are_rejected`, `tests/test_security.py::TestSanitizeCommandArgs::test_every_shell_metacharacter_is_rejected` | -| Locations from a workflow | `dw/locations.py` | A local path in a workflow's media arguments is confined to a known root, and a URL must not name a host inside the deployment. | `tests/test_locations.py::TestMediaPathContainment::test_absolute_path_outside_every_root_is_refused`, `tests/test_security_ssrf.py::TestInternalAddressSpellings::test_a_name_that_resolves_inside` | +| Locations from a workflow | `dw/locations.py` | A local path in a workflow's media arguments is confined to a known root, a location written as a URL is one `security.validate_url` accepts, and a URL must not name a host inside the deployment. | `tests/test_locations.py::TestMediaPathContainment::test_absolute_path_outside_every_root_is_refused`, `tests/test_locations.py::TestMediaPathContainment::test_a_url_that_is_not_http_is_refused`, `tests/test_locations.py::TestMediaPathContainment::test_an_upper_case_https_url_is_not_refused_as_another_scheme`, `tests/test_security_ssrf.py::TestInternalAddressSpellings::test_a_name_that_resolves_inside` | | Trust gate | `dw/trust.py`: `require_trusted_dotted_name` | In an untrusted workflow (the default), a dotted type outside the allowlist is a validation error before anything is imported. | `tests/test_security_trust_gate.py::TestValidationRefusesBeforeImport::test_a_dotted_type_is_a_validation_error` | | CodeQL path-injection model | `.github/codeql/dw-security/DwPathSanitizers.qll`, `.github/workflows/codeql.yml` | The local pack models the `dw/security.py` validators as sanitizers, so a validator that moves is re-modelled in the same commit. | `.github/codeql/dw-security/DwPathInjection.ql` | | Archives never follow links | `dw/server/outputs.py`: `zip_download` | A gallery listing drops a symlink that leaves its root, and an archive skips one. | `tests/test_security_symlinks.py::TestOutputs::test_the_archive_route_does_not_follow_the_link`, `tests/test_security_symlinks.py::TestOutputs::test_the_gallery_listing_does_not_enumerate_the_link` | diff --git a/dw/locations.py b/dw/locations.py index d8cfc38b..dfd4fa2d 100644 --- a/dw/locations.py +++ b/dw/locations.py @@ -75,15 +75,23 @@ def _refuse_other_url(location, what): passed containment - a refusal at run time only because no such directory existed, and none at validation (#618). The rule `validate_model_name` applies to a model_name (#117), applied to every media location. + + "://" is what makes a location a URL; whether its scheme is allowed is + `validate_url`'s decision, so the two never disagree (`HTTPS://` is a + URL it accepts, not a refusal here). """ text = str(location) - if "://" in text and not is_http_url(text): - scheme = text.split("://", 1)[0] + if "://" not in text: + return + try: + validate_url(text) + except InvalidInputError as e: + scheme = urlparse(text).scheme or text.split("://", 1)[0] raise InvalidInputError( f"Refusing to read {what} at '{location}': it is a '{scheme}' URL, " - f"and only http(s) URLs are fetched. Name a local file by its path " - f"or with an 'asset:' reference." - ) + f"and only http(s) URLs are fetched ({e}). Name a local file by " + f"its path or with an 'asset:' reference." + ) from e def media_roots(base_dir=None): @@ -568,6 +576,7 @@ def _check(value, base_dir, what): validate_media_url(value, what) elif "://" in value: _refuse_other_url(value, what) + validate_media_url(value, what) elif os.path.isabs(value): validate_media_path(value, base_dir, what, require_exists=False) elif ".." in value.replace("\\", "/").split("/"): diff --git a/tests/test_locations.py b/tests/test_locations.py index 6e8bd6e7..43f4ad63 100644 --- a/tests/test_locations.py +++ b/tests/test_locations.py @@ -18,6 +18,7 @@ from dw.argument_media import fetch_image from dw.locations import ( + _refuse_other_url, contained_matches, location_errors, token_host_allowed, @@ -30,6 +31,7 @@ from dw.security import ( InvalidInputError, PathTraversalError, + validate_url, ) from dw.trust import TRUST_WORKFLOWS_ENV_VAR from dw.tasks.gather import gather_images @@ -120,6 +122,13 @@ def test_a_url_that_is_not_http_is_refused(self, location, workflow_dir): with pytest.raises(InvalidInputError, match="only http\\(s\\) URLs"): validate_media_path(location, workflow_dir, "an audio argument") + def test_an_upper_case_https_url_is_not_refused_as_another_scheme(self): + """The scheme decision is validate_url's: urlparse lowercases it, so + 'HTTPS://' is an http(s) URL there and must not be refused here as an + 'HTTPS' URL (#618 arch review).""" + assert validate_url("HTTPS://example.com/x.wav") + _refuse_other_url("HTTPS://example.com/x.wav", "an audio argument") + def test_fetch_image_refuses_an_out_of_root_absolute_path( self, untrusted, workflow_dir ): @@ -364,6 +373,30 @@ def test_a_file_url_is_an_error_with_its_path(self, untrusted, workflow_dir, key ] assert "'file' URL" in errors[0]["message"] + def test_an_upper_case_https_url_is_checked_as_a_url(self, untrusted, workflow_dir): + """Not refused for its scheme, and still sent through the host policy.""" + definition = { + "steps": [ + { + "name": "edit", + "pipeline": { + "arguments": {"hold_audio": "HTTPS://example.com/x.wav"} + }, + } + ] + } + with patch( + "dw.locations.socket.getaddrinfo", + return_value=[(None, None, None, "", ("93.184.216.34", 443))], + ): + assert location_errors(definition, base_dir=workflow_dir) == [] + with patch( + "dw.locations.socket.getaddrinfo", + return_value=[(None, None, None, "", ("127.0.0.1", 443))], + ): + errors = location_errors(definition, base_dir=workflow_dir) + assert "inside this deployment" in errors[0]["message"] + def test_a_loopback_url_is_an_error(self, untrusted, workflow_dir): definition = { "steps": [ From ba863a68583b3cdc7dd3fdcae11b2876604d12d8 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 22:32:38 -0500 Subject: [PATCH 036/296] feat(chain): #619 - match_audio slices a held track into each segment's hold_audio Co-Authored-By: Claude Opus 5.5 --- dw/pipeline_processors/chain.py | 74 +++++++++++------ tests/test_chain.py | 139 ++++++++++++++++++++++++++++++++ 2 files changed, 189 insertions(+), 24 deletions(-) diff --git a/dw/pipeline_processors/chain.py b/dw/pipeline_processors/chain.py index 50f94ed9..96016dba 100644 --- a/dw/pipeline_processors/chain.py +++ b/dw/pipeline_processors/chain.py @@ -14,10 +14,12 @@ Two ways to specify the length: - segments: N - run the pipeline N times as configured -- match_audio: true - derive the total frame count from the audio reference - in the step's arguments, slice that audio into frame-aligned per-segment - chunks, and mux the final video with the original, unsliced track - so the - soundtrack has no seams at all +- match_audio: true - derive the total frame count from the step's + `hold_audio` track, or else its one audio reference, slice that audio into + frame-aligned per-segment chunks, and mux the final video with the original, + unsliced track - so the soundtrack has no seams at all. A held track is + sliced into each segment's `hold_audio`, so every segment generates to its + own piece of it (dw/pipeline_processors/h3_blocks.py) The chain runs inside one cartesian iteration, so it composes with previous_result fan-out: three keyframes in, three chained videos out. @@ -43,6 +45,7 @@ from ..task_domains import frames_to_samples from ..tasks.joins import equal_power_crossfade_join from ..tasks.video_utils import extract_frame, frames_as_pil_list +from .h3_blocks import HOLD_AUDIO_INPUT, hold_audio_reference logger = logging.getLogger("dw") @@ -293,7 +296,10 @@ def run_chain(pipeline, chain_definition, arguments): # Step.run resolved any previous_result references in the chain's prompts, # which the chain block cannot express on its own config = ChainConfig( - chain_definition, arguments, getattr(pipeline, "chain_prompts", None) + chain_definition, + arguments, + getattr(pipeline, "chain_prompts", None), + getattr(pipeline, "base_dir", None), ) continuity = CONTINUITY_MODES[config.continuity](config) @@ -322,9 +328,12 @@ def run_chain(pipeline, chain_definition, arguments): if config.source_audio is not None: segment_arguments["num_frames"] = segment.num_frames - segment_arguments["references"] = _sliced_references( - config, segment, arguments["references"] - ) + if config.holds_audio: + segment_arguments[HOLD_AUDIO_INPUT] = _audio_slice(config, segment) + else: + segment_arguments["references"] = _sliced_references( + config, segment, arguments["references"] + ) if segment.index > 0: continuity.inject(segment_arguments, carry, config.segment_argument) @@ -438,7 +447,9 @@ def _trimmed_shots(shots, total_frames): class ChainConfig: """Validated chain settings plus the planned segments for one run.""" - def __init__(self, chain_definition, arguments, resolved_prompts=None): + def __init__( + self, chain_definition, arguments, resolved_prompts=None, base_dir=None + ): segments = chain_definition.get("segments", None) match_audio = bool(chain_definition.get("match_audio", False)) if (segments is not None) == match_audio: @@ -471,10 +482,11 @@ def __init__(self, chain_definition, arguments, resolved_prompts=None): self.source_audio = None self.source_rate = None self.audio_reference = None + self.holds_audio = False self.total_frames = None if match_audio: - self._plan_from_audio(arguments) + self._plan_from_audio(arguments, base_dir) else: segments = int(segments) if not 1 <= segments <= MAX_SEGMENTS: @@ -494,8 +506,9 @@ def __init__(self, chain_definition, arguments, resolved_prompts=None): for index in range(segments) ] - def _plan_from_audio(self, arguments): - """Derive the segment plan from the audio reference's duration.""" + def _plan_from_audio(self, arguments, base_dir=None): + """Derive the segment plan from the matched track's duration - the + step's `hold_audio` when it names one, else its audio reference.""" if self.fps is None: raise ValueError( "A match_audio chain needs the frame rate - set 'fps' on the " @@ -509,12 +522,19 @@ def _plan_from_audio(self, arguments): "as the per-segment length" ) - reference = _find_audio_reference(arguments) + held = arguments.get(HOLD_AUDIO_INPUT) + if held is not None: + # Read once here, sliced per segment - each slice is a reference + # already built, which the segment's own hold takes as it is + reference = hold_audio_reference(held, base_dir) + self.holds_audio = True + else: + reference = _find_audio_reference(arguments) self.audio_reference = reference self.source_audio = as_channels_samples(reference.audio) self.source_rate = reference.sample_rate if self.source_rate is None: - raise ValueError("The chain's audio reference has no sample rate") + raise ValueError("The chain's matched audio has no sample rate") total_samples = self.source_audio.shape[1] self.total_frames = max(1, round(total_samples / self.source_rate * self.fps)) @@ -628,12 +648,13 @@ def _resolve_fps(chain_definition, arguments): def _find_audio_reference(arguments): - """The single audio reference a match_audio chain slices per segment.""" + """The single audio reference a match_audio chain with no `hold_audio` + slices per segment.""" references = arguments.get("references", None) if not isinstance(references, list): raise ValueError( - "A match_audio chain needs a 'references' argument holding the " - "audio reference to match" + "A match_audio chain needs a 'hold_audio' track, or a 'references' " + "argument holding the audio reference to match" ) audio_references = [ @@ -649,19 +670,24 @@ def _find_audio_reference(arguments): return audio_references[0] +def _audio_slice(config, segment): + """The segment's piece of the matched track, as a new reference of the + matched one's type - the original is never touched.""" + start = frames_to_samples(segment.audio_start_frame, config.fps, config.source_rate) + length = frames_to_samples(segment.num_frames, config.fps, config.source_rate) + piece = slice_samples(config.source_audio, start, length) + return type(config.audio_reference)( + audio=torch.from_numpy(piece), sample_rate=config.source_rate + ) + + def _sliced_references(config, segment, references): """A copy of the references list with the segment's audio slice swapped in. The original list and reference objects are never touched - iteration arguments share nested values, so they must not be mutated in place. """ - start = frames_to_samples(segment.audio_start_frame, config.fps, config.source_rate) - length = frames_to_samples(segment.num_frames, config.fps, config.source_rate) - piece = slice_samples(config.source_audio, start, length) - - sliced = type(config.audio_reference)( - audio=torch.from_numpy(piece), sample_rate=config.source_rate - ) + sliced = _audio_slice(config, segment) return [ sliced if reference is config.audio_reference else reference for reference in references diff --git a/tests/test_chain.py b/tests/test_chain.py index 763d7632..cebbebfd 100644 --- a/tests/test_chain.py +++ b/tests/test_chain.py @@ -7,6 +7,7 @@ keyed videos/audio/sampling_rate. No GPU is involved. """ +import os from dataclasses import dataclass from types import SimpleNamespace @@ -392,6 +393,144 @@ def test_a_reference_without_a_sample_rate_raises(self): run_chain(pipeline, self.chain(), arguments) +class TestMatchAudioHold: + """A match_audio chain whose step names `hold_audio` slices that track into + each segment's own `hold_audio` - the references are left alone (#619).""" + + @pytest.fixture(autouse=True) + def reference_type(self): + references = pytest.importorskip( + "diffusers.modular_pipelines.minimax_h3.references" + ) + self.reference = references.MiniMaxH3AudioReference + + def make_arguments(self, held, num_frames=8): + return { + "prompt": "test", + "num_frames": num_frames, + "references": [FakeImageReference(solid_frame((9, 9, 9)))], + "hold_audio": held, + } + + def chain(self, **overrides): + return { + "match_audio": True, + "fps": 4, + "trim_frames": 2, + "segment_argument": "references", + } | overrides + + def ramp(self, samples=500): + """A track whose every sample says where it came from.""" + return torch.arange(samples, dtype=torch.float32).repeat(2, 1) + + def test_each_segment_holds_its_frame_aligned_slice(self): + held = self.reference(audio=self.ramp(), sample_rate=100) + pipeline = FakePipeline(modular_output) + + result = run_chain(pipeline, self.chain(), self.make_arguments(held)) + + # 5s at 4 fps -> 20 frames: segments of 8 frames, 2 trimmed per seam + assert len(result.frames) == 20 + assert len(pipeline.calls) == 3 + slices = [call["hold_audio"] for call in pipeline.calls] + assert all(isinstance(piece, self.reference) for piece in slices) + assert all(piece is not held for piece in slices) + assert all(piece.sample_rate == 100 for piece in slices) + # 8 frames at 4 fps and 100Hz -> 200 samples, starting at output + # frames 0, 6 and 12 - each later slice backs up by the 2-frame trim + assert [int(piece.audio[0, 0]) for piece in slices] == [0, 150, 300] + assert all(piece.audio.shape == (2, 200) for piece in slices) + + def test_no_audio_reference_is_added_or_needed(self): + held = self.reference(audio=self.ramp(), sample_rate=100) + pipeline = FakePipeline(modular_output) + + run_chain(pipeline, self.chain(), self.make_arguments(held)) + + for call in pipeline.calls: + assert not [r for r in call["references"] if r.kind == "audio"] + # the workflow's own image, then the carried frame beside it + assert len(pipeline.calls[0]["references"]) == 1 + assert len(pipeline.calls[1]["references"]) == 2 + + def test_the_final_audio_is_the_original_held_track(self): + track = self.ramp() + held = self.reference(audio=track, sample_rate=100) + pipeline = FakePipeline(modular_output) + + result = run_chain(pipeline, self.chain(), self.make_arguments(held)) + + assert result.sample_rate == 100 + assert torch.equal(torch.as_tensor(result.audio), track) + + def test_the_held_reference_is_never_mutated(self): + track = self.ramp() + held = self.reference(audio=track, sample_rate=100) + arguments = self.make_arguments(held) + pipeline = FakePipeline(modular_output) + + run_chain(pipeline, self.chain(), arguments) + + assert arguments["hold_audio"] is held + assert torch.equal(held.audio, self.ramp()) + + def test_a_track_artifact_is_held_too(self): + from dw.media_types import AudioTrack + + pipeline = FakePipeline(modular_output) + + result = run_chain( + pipeline, self.chain(), self.make_arguments(AudioTrack(self.ramp(), 100)) + ) + + assert len(result.frames) == 20 + assert all( + isinstance(call["hold_audio"], self.reference) for call in pipeline.calls + ) + + def test_a_held_file_is_read_once_against_the_workflow_directory( + self, tmp_path, monkeypatch + ): + (tmp_path / "song.wav").write_bytes(b"") + opened = [] + + def from_file(cls, location): + opened.append(location) + return cls(audio=self.ramp(), sample_rate=100) + + monkeypatch.setattr(self.reference, "from_file", classmethod(from_file)) + pipeline = FakePipeline(modular_output) + pipeline.base_dir = str(tmp_path) + + run_chain(pipeline, self.chain(), self.make_arguments("song.wav")) + + assert opened == [os.path.realpath(tmp_path / "song.wav")] + assert len(pipeline.calls) == 3 + + def test_a_held_value_that_is_not_audio_raises(self): + pipeline = FakePipeline(modular_output) + + with pytest.raises(ValueError, match="hold_audio"): + run_chain(pipeline, self.chain(), self.make_arguments("cover.png")) + + assert pipeline.calls == [] + + def test_an_audio_reference_beside_the_hold_is_passed_through(self): + # The hold is what is matched; a voice reference beside it is the + # caller's, carried to every segment as it is + voice = FakeAudioReference(torch.zeros(2, 50), 100) + held = self.reference(audio=self.ramp(), sample_rate=100) + arguments = self.make_arguments(held) + arguments["references"].append(voice) + pipeline = FakePipeline(modular_output) + + run_chain(pipeline, self.chain(), arguments) + + for call in pipeline.calls: + assert [r for r in call["references"] if r.kind == "audio"] == [voice] + + class TestSaveSegments: """Incremental segment saving - segments spill to disk as they complete.""" From bfc8ed67528db60f2ec856cbfefc99ac49ea07e3 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 5 Oct 2026 22:36:25 -0500 Subject: [PATCH 037/296] feat(templates): #619 - music-video and the match_audio chains hold their track instead of referencing it Co-Authored-By: Claude Opus 5.5 --- tests/test_for_each.py | 11 ++-- tests/test_h3_hold_audio.py | 54 +++++++++++++++++++ .../minimax/chain-matched-and-aligned.json | 11 ++-- .../minimax/chain-matched-to-audio.json | 9 ++-- workflows/templates/minimax/music-video.json | 19 +++---- 5 files changed, 74 insertions(+), 30 deletions(-) diff --git a/tests/test_for_each.py b/tests/test_for_each.py index 1b3109cf..f4ec73fa 100644 --- a/tests/test_for_each.py +++ b/tests/test_for_each.py @@ -657,13 +657,14 @@ def test_each_slice_starts_where_its_entry_says(self): ] assert starts == [0, 124, 248, 372] - def test_each_shot_reads_its_own_slice_and_the_one_portrait(self): + def test_each_shot_holds_its_own_slice_and_reads_the_one_portrait(self): + """The slice is the shot's held soundtrack, not an audio reference (#619).""" got = steps_by_name(self.expanded()) for key in self.KEYS: - references = got[f"shot@{key}"]["pipeline"]["arguments"]["references"] - assert [r["from_previous_result"] for r in references] == [ - "draw_singer", - f"slice@{key}", + arguments = got[f"shot@{key}"]["pipeline"]["arguments"] + assert arguments["hold_audio"] == f"previous_result:slice@{key}" + assert [r["from_previous_result"] for r in arguments["references"]] == [ + "draw_singer" ] def test_each_shot_carries_its_entry_s_prompt(self): diff --git a/tests/test_h3_hold_audio.py b/tests/test_h3_hold_audio.py index c836e8a2..aa0685ad 100644 --- a/tests/test_h3_hold_audio.py +++ b/tests/test_h3_hold_audio.py @@ -427,6 +427,60 @@ def template(*parts): return json.load(f) +HELD_TEMPLATES = { + "chain-matched-to-audio.json": "variable:voice", + "chain-matched-and-aligned.json": "variable:voice", + "music-video.json": "previous_result:slice", +} + + +def h3_steps(definition): + return [ + step + for step in definition["steps"] + if step.get("pipeline", {}) + .get("from_pretrained_arguments", {}) + .get("model_name") + == "MiniMaxAI/MiniMax-H3" + ] + + +class TestHeldTemplates: + """The templates that follow supplied audio hold it rather than pass it as an + audio reference (#619, D4: hold replaces the reference, no both-arm).""" + + @pytest.mark.parametrize("name", sorted(HELD_TEMPLATES)) + def test_the_h3_step_holds_the_track(self, name): + steps = h3_steps(template("minimax", name)) + + assert [step["pipeline"]["arguments"]["hold_audio"] for step in steps] == [ + HELD_TEMPLATES[name] + ] + + @pytest.mark.parametrize("name", sorted(HELD_TEMPLATES)) + def test_no_audio_reference_is_left_beside_the_hold(self, name): + text = json.dumps(template("minimax", name)) + + assert "MiniMaxH3AudioReference" not in text + assert "audio_reference_type" not in text + # Ref2VA labels references in order; with no audio reference there is + # no