fix(mcp): #553 - fill silent concat_videos inputs, load a string pair_audio video

concat_videos was skipping a silent input among videos where at least
one other carried audio, contributing zero duration to the joined
track and shifting every later shot's audio early. It now fills that
input's own length with silence, matching join_into_song's rule (#513)
that a shot with no track is silence for its length rather than a gap.
Requires fps to size the silence, as every other length-dependent path
here already does; every catalog template calling concat_videos
already passes it. When no input carries audio, output audio still
stays None.

pair_audio already loaded a string 'audio' path via load_audio, but
not a string 'video' - it now loads a video path/URL/{"location": ...}
dict the same way concat_videos loads one of its own inputs, confined
the same way audio's own path already is.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

Author: dkackman

dw/tasks/concat_videos.py

@@ -11,6 +11,8 @@
 import logging
 import os
 
+import numpy
+
 from ..events import emit_log, emit_warning
 from ..result import AudioVideo
 from ..shots import measured_num_samples, nested_shots, shot_record, trimmed_shots
@@ -226,6 +228,18 @@ def concat_videos(
     # long shows up here as the samples it actually took (#378)
     shots = []
 
+    # Whether the joined track carries anything at all - decides whether a
+    # silent input among these gets silence of its own length or is skipped
+    # outright, matching join_into_song's rule that a shot with no track is
+    # silence for its length rather than a gap that shifts everything after
+    # it early (#513, #553)
+    has_audio = any(waveform is not None for waveform in waveforms)
+    silence_channels = (
+        next(waveform.shape[0] for waveform in waveforms if waveform is not None)
+        if has_audio
+        else None
+    )
+
     for index, (video, clip) in enumerate(zip(videos, clips)):
         head_trim = trim_frames if index > 0 else 0
         start_frame = len(frames)
@@ -236,7 +250,7 @@ def concat_videos(
             video_shots = nested_shots(
                 trimmed_shots(inner, head_trim),
                 start_frame,
-                start_sample if waveforms[index] is not None else None,
+                start_sample if has_audio else None,
                 getattr(video, "sample_rate", None),
                 sample_rate,
             )
@@ -258,13 +272,31 @@ def concat_videos(
             shot["source_index"] = index
         shots.extend(video_shots)
 
-        if waveforms[index] is None:
+        if not has_audio:
             continue
 
-        waveform = waveforms[index]
+        if waveforms[index] is None:
+            if fps is None:
+                raise ValueError(
+                    "concat_videos needs 'fps' to fill silence for a video "
+                    "with no audio track of its own"
+                )
+            waveform = numpy.zeros(
+                (silence_channels, frames_to_samples(len(clip), fps, sample_rate)),
+                dtype=numpy.float32,
+            )
+            emit_log(
+                f"concat_videos: {names[index]} carries no audio - filled with "
+                "silence",
+                command="concat_videos",
+                video=names[index],
+            )
+        else:
+            waveform = waveforms[index]
+
         if audio is None:
             audio = waveform
-            audio_native_rate = video.sample_rate
+            audio_native_rate = getattr(video, "sample_rate", None)
             continue
 
         if head_trim > 0 and fps is None:
@@ -294,7 +326,7 @@ def concat_videos(
                 crossfade_ms,
                 seam_fade_ms,
             )
-        audio_native_rate = video.sample_rate
+        audio_native_rate = getattr(video, "sample_rate", None)
 
     # The rate the caller declared, else the rate the first input carries -
     # either beats the result's 8 fps default (#84)

dw/tasks/pair_audio.py

@@ -214,8 +214,13 @@ def pair_audio(video, audio, sample_rate=None, fps=None, fit=None):
 
     Args:
         video: The frames - a frame list, a frame array or tensor (or a
-            pipeline's batch of one of those, unwrapped), or an
-            AudioVideo whose own soundtrack is replaced by this one; the
+            pipeline's batch of one of those, unwrapped), an AudioVideo whose
+            own soundtrack is replaced by this one, or the path or URL of a
+            video file (or a {"location": ...} dict wrapping either - the
+            same idiom `get_last_frame(video=...)` and a pipeline's `image`
+            argument accept, #510) - loaded the same way `concat_videos`
+            loads one of its inputs, confined the same way `audio`'s own
+            path is (#553); the
             frames' own rate is carried through to the output, so set
             `result.fps` only to override it (a loaded file brings its rate
             along; frames that carry none are written at 8 fps)
@@ -268,6 +273,14 @@ def pair_audio(video, audio, sample_rate=None, fps=None, fit=None):
 
         audio = _Loaded(*load_audio(audio))
     waveform = getattr(audio, "audio", audio)
+
+    from .video_utils import is_video_location, load_audio_video
+
+    if is_video_location(video):
+        # A path/URL naming a video an earlier run wrote - confined the same
+        # way concat_videos loads one of its own inputs (#553)
+        video = load_audio_video(video)
+
     if waveform is None:
         raise ValueError(
             "pair_audio needs an audio track - the video it was given carries none"

tests/test_concat_videos.py

@@ -66,12 +66,15 @@ def test_untrimmed_audio_concatenates_whole(self):
         assert result.audio.shape == (2, 400)
 
     def test_mixed_inputs_keep_the_audio_that_exists(self):
+        # #553: the silent first video now fills its own length with
+        # silence rather than being skipped, which is why it needs fps -
+        # the same requirement trimming or crossfading audio already has
         videos = [frames(4), audio_video(8, 1)]
 
-        result = concat_videos(videos)
+        result = concat_videos(videos, fps=4)
 
         assert len(result.frames) == 12
-        assert result.audio.shape == (2, 200)
+        assert result.audio.shape == (2, 100 + 200)
 
     def test_mismatched_sample_rates_are_resampled_to_the_highest(self, caplog):
         """#108: a 24 kHz voice clip paired onto a 32 kHz generation used to
@@ -417,6 +420,56 @@ def test_a_location_dict_is_loaded_with_its_audio(self, tmp_path):
         assert result.audio.shape == (2, 8 / 4 * 8000)
 
 
+class TestSilentInputAmongSoundedOnes:
+    """#553: a silent input contributed zero duration to the joined track,
+    shifting every later shot's audio early instead of the gap it actually
+    left in the picture."""
+
+    def test_a_silent_middle_shot_does_not_shift_the_third_shots_audio(self):
+        sound, silent, more_sound = (
+            audio_video(8, 0.5),
+            frames(8),
+            audio_video(8, 0.25),
+        )
+
+        result = concat_videos([sound, silent, more_sound], fps=4)
+
+        # Each shot is 8 frames @ 4fps @ 100Hz = 200 samples; the silent shot
+        # must still occupy its own 200 samples rather than contributing none
+        assert result.audio.shape == (2, 600)
+        third = result.shots[-1]
+        assert third["start_sample"] == 400
+        assert third["start_frame"] == 16
+
+    def test_shots_stay_contiguous_across_the_silent_gap(self):
+        sound, silent, more_sound = (
+            audio_video(8, 0.5),
+            frames(8),
+            audio_video(8, 0.25),
+        )
+
+        result = concat_videos([sound, silent, more_sound], fps=4)
+
+        for earlier, later in zip(result.shots, result.shots[1:]):
+            assert (
+                earlier["start_sample"] + earlier["num_samples"]
+                == later["start_sample"]
+            )
+
+    def test_a_silent_first_shot_is_also_filled(self):
+        silent, sound = frames(8), audio_video(8, 0.5)
+
+        result = concat_videos([silent, sound], fps=4)
+
+        assert result.audio.shape == (2, 400)
+        assert result.shots[-1]["start_sample"] == 200
+
+    def test_no_input_carrying_audio_still_leaves_audio_none(self):
+        result = concat_videos([frames(4), frames(4)], fps=4)
+
+        assert result.audio is None
+
+
 class TestLevelMatching:
     """Independently generated shots land at whatever level the model chose,
     and cutting a -2.6 dBFS shot against a -12.6 dBFS one is audible as a
@@ -470,7 +523,10 @@ def test_a_silent_shot_is_left_alone(self):
         assert float(numpy.abs(result.audio[:, :100]).max()) == 0.0
 
     def test_a_shot_with_no_soundtrack_joins_as_before(self):
-        result = concat_videos([frames(4), audio_video(4, 0.5)], match_levels="rms")
+        # #553: fps is now required to fill the silent shot's own length
+        result = concat_videos(
+            [frames(4), audio_video(4, 0.5)], match_levels="rms", fps=4
+        )
 
         assert result.audio is not None
         assert len(result.frames) == 8

tests/test_pair_audio.py

@@ -11,6 +11,7 @@
 import numpy
 import pytest
 import torch
+from PIL import Image
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
@@ -99,6 +100,44 @@ def test_raises_when_no_sample_rate_can_be_established():
         pair_audio(["frame"], _waveform())
 
 
+def test_loads_a_string_video_path_the_same_way_concat_videos_does(tmp_path):
+    """#553: 'audio' already loaded a string path via load_audio; 'video' did
+    not, so a workflow naming its video by an asset:/output: path (resolved
+    to a plain string by the time the task runs) failed rather than loading
+    it the way concat_videos loads one of its own inputs."""
+    from diffusers.utils.export_utils import encode_video
+
+    path = tmp_path / "shot.mp4"
+    encode_video(
+        [Image.new("RGB", (16, 16)) for _ in range(4)],
+        fps=4,
+        output_path=str(path),
+    )
+
+    paired = pair_audio(str(path), _waveform(samples=40), sample_rate=24000)
+
+    assert isinstance(paired, AudioVideo)
+    assert len(paired.frames) == 4
+    assert paired.audio.shape == (2, 40)
+
+
+def test_loads_a_location_dict_video(tmp_path):
+    """The same {"location": ...} idiom concat_videos and get_last_frame accept."""
+    from diffusers.utils.export_utils import encode_video
+
+    path = tmp_path / "shot.mp4"
+    encode_video(
+        [Image.new("RGB", (16, 16)) for _ in range(4)],
+        fps=4,
+        output_path=str(path),
+    )
+
+    paired = pair_audio({"location": str(path)}, _waveform(samples=40), sample_rate=24000)
+
+    assert isinstance(paired, AudioVideo)
+    assert len(paired.frames) == 4
+
+
 def test_registered_as_a_task_command():
     from dw.tasks.task import _COMMAND_REGISTRY