fix(video): stop re-encoding the soundtrack kept by a trim

docs/src/content/docs/Users Guide/05.Video Generation/introduction.mdx

@@ -203,7 +203,7 @@ Extension is available on all three families. LTX-2's continues the soundtrack a
 
 ### Quality drift across iterations
 
-Each iteration's source is itself a generation output, so artifacts compound — by the 4th or 5th extension you may see softening or color drift. That drift is in the *new* footage: the segments already in the initial video are carried into the result without being re-encoded, picture and sound, apart from up to two seconds of picture at each cut — before the join, and at a trim's start. A trimmed initial video's soundtrack is re-encoded once. Extending a clip whose frame rate differs from the model's (MiniMax H3 always runs at 24 fps) re-encodes it in full, once, as does extending an uploaded video or one made by an earlier version of Invoke. Mitigations:
+Each iteration's source is itself a generation output, so artifacts compound — by the 4th or 5th extension you may see softening or color drift. That drift is in the *new* footage: the segments already in the initial video are carried into the result without being re-encoded, picture and sound, apart from up to two seconds of picture and about a tenth of a second of sound at each cut — before the join, and at each end of a trim. Extending a clip whose frame rate differs from the model's (MiniMax H3 always runs at 24 fps) re-encodes it in full, once, as does extending an uploaded video or one made by an earlier version of Invoke. Mitigations:
 
 1. **Trim the bridge point a few frames back from the end** — boundary frames are the most artifact-prone.
 2. **Refresh a bridge frame with a low-strength img2img pass** (SDXL or FLUX at ~0.2 strength) before using it as a first frame for the next segment.

invokeai/app/invocations/video_concat.py

@@ -52,10 +52,10 @@
 from invokeai.app.services.shared.invocation_context import InvocationContext
 from invokeai.app.util.aac_splice import (
     AAC_FRAME_SAMPLES,
+    EDGE_FRAMES,
     PREROLL_FRAMES,
-    encoder_aac_layout,
-    mux_spliced_aac,
-    read_aac_layout,
+    splice_soundtrack,
+    spliceable_aac_layout,
 )
 from invokeai.app.util.h264_splice import (
     MIN_ENCODED_FRAMES,
@@ -67,7 +67,6 @@
     read_h264_layout,
 )
 from invokeai.app.util.video_audio import (
-    AudioExtractionError,
     extract_audio_pcm,
     mux_audio_into_video,
     resample_linear,
@@ -455,15 +454,12 @@ def _mux_spliced_audio(
         transition — provided that clip plays at the output rate, so its audio is not
         retimed, and sets the soundtrack's sample rate. Returns False, leaving the caller to
         encode the whole soundtrack, when the clip's track is not one this app's encoder
-        writes, nothing whole-frame precedes the transition, or the result's head does not
-        decode back to exactly the samples it should hold.
+        writes, too little precedes the transition, or the splice does not verify.
         """
         if first_fps is None or first_fps != output_fps:
             return False
-        layout, encoder = read_aac_layout(first_path), encoder_aac_layout(rate)
-        if layout is None or encoder is None or layout.sample_rate != rate:
-            return False
-        if (layout.decoder_config, layout.priming) != (encoder.decoder_config, encoder.priming):
+        layout = spliceable_aac_layout(first_path, rate)
+        if layout is None:
             return False
         # The soundtrack holds the clip's own decoded samples only where its audio was mapped
         # one-to-one: the clip's span at its native rate must be exactly its span on the timeline.
@@ -475,66 +471,31 @@ def _mux_spliced_audio(
             layout.packet_count * AAC_FRAME_SAMPLES - layout.priming,
             pcm.shape[1],
         )
-        preroll = PREROLL_FRAMES * AAC_FRAME_SAMPLES
-        # Keep the cut PREROLL_FRAMES clear of the first changed sample (see aac_splice).
-        head_packets = (unchanged + layout.priming) // AAC_FRAME_SAMPLES - PREROLL_FRAMES
-        cut = head_packets * AAC_FRAME_SAMPLES - layout.priming
-        if cut < preroll:
-            return False
-        try:
-            seam_reference = mux_spliced_aac(
-                video_path,
-                first_path,
-                head_packets,
-                layout.priming,
-                pcm[:, cut - preroll :],
-                rate,
-                pcm.shape[1],
-                out_path,
-                context.util.is_canceled,
-            )
-            muxed = extract_audio_pcm(out_path)
-        except (SpliceError, AudioExtractionError) as e:
-            context.logger.info(f"Re-encoding the whole soundtrack: it cannot be stream-copied ({e})")
-            return False
-        reason = self._spliced_audio_defect(muxed, pcm, rate, cut, seam_reference)
+        # Keep the cut PREROLL_FRAMES clear of the first changed sample, and a full encoded tail
+        # after it (see aac_splice).
+        head_packets = min(
+            (unchanged + layout.priming) // AAC_FRAME_SAMPLES - PREROLL_FRAMES,
+            (pcm.shape[1] - EDGE_FRAMES * AAC_FRAME_SAMPLES + layout.priming) // AAC_FRAME_SAMPLES,
+        )
+        reason = splice_soundtrack(
+            video_path,
+            first_path,
+            layout.priming,
+            0,
+            head_packets,
+            -layout.priming,
+            pcm,
+            rate,
+            out_path,
+            context.util.is_canceled,
+        )
         if reason is not None:
             context.logger.info(f"Re-encoding the whole soundtrack: {reason}")
             return False
+        cut = head_packets * AAC_FRAME_SAMPLES - layout.priming
         context.logger.info(f"Stream-copied the first {cut} audio samples of the first clip without re-encoding")
         return True
 
-    @staticmethod
-    def _spliced_audio_defect(
-        muxed: Optional[tuple[np.ndarray, int]],
-        pcm: np.ndarray,
-        rate: int,
-        cut: int,
-        seam_reference: np.ndarray,
-    ) -> Optional[str]:
-        """Why the spliced soundtrack cannot be used, or None.
-
-        The copied head must decode back to exactly the samples it stands for, and the
-        seam — the frames where copied and new packets overlap — must be no noisier than an
-        ordinary encode of the same samples (``seam_reference``). Whether the overlap
-        cancels depends on the encoder's window decisions around the content there, so it is
-        measured, not assumed; and it is measured against the seam's own content, since the
-        audio after it may be silence that codes with no error at all.
-        """
-        if muxed is None or muxed[1] != rate or muxed[0].shape[1] < pcm.shape[1]:
-            return "the spliced soundtrack did not decode to its full length"
-        decoded = muxed[0]
-        if not np.array_equal(decoded[:, :cut], pcm[:, :cut]):
-            return "the copied audio did not decode back exactly"
-        seam = slice(cut, cut + seam_reference.shape[1])
-        if seam_reference.shape[1] != PREROLL_FRAMES * AAC_FRAME_SAMPLES or seam.stop > pcm.shape[1]:
-            return "the new audio does not cover the seam"
-        seam_error = float(np.sqrt(np.mean((decoded[:, seam] - pcm[:, seam]) ** 2)))
-        coding_error = float(np.sqrt(np.mean((seam_reference - pcm[:, seam]) ** 2)))
-        if seam_error > 2.0 * coding_error + 1e-3:
-            return "the seam between copied and new audio is not clean"
-        return None
-
     def _estimate_transition_memory(self, width: int, height: int) -> int:
         if self.transition == "cut" or self.transition_frames == 0:
             return 0

invokeai/app/invocations/video_frame_extract_range.py

@@ -11,7 +11,9 @@
 through this node (as video extension does with an untrimmed source) costs no
 quality. A trim of a source this app encoded stream-copies the keyframe-aligned
 run of frames inside the range and encodes only the partial keyframe intervals
-at either end (see ``h264_splice``). Anything else is re-encoded in full.
+at either end (see ``h264_splice``), and likewise copies the source's AAC
+packets for the audio inside the range, encoding only a short lead-in and tail
+(see ``aac_splice``). Anything else is re-encoded in full.
 
 If the source carries an audio track, the matching slice of it is carried
 through (and retimed alongside the video when the output fps differs from
@@ -48,6 +50,12 @@
 from invokeai.app.invocations.primitives import VideoOutput
 from invokeai.app.services.session_processor.session_processor_common import CanceledException
 from invokeai.app.services.shared.invocation_context import InvocationContext
+from invokeai.app.util.aac_splice import (
+    AAC_FRAME_SAMPLES,
+    EDGE_FRAMES,
+    splice_soundtrack,
+    spliceable_aac_layout,
+)
 from invokeai.app.util.h264_splice import (
     MIN_ENCODED_FRAMES,
     H264Layout,
@@ -281,14 +289,17 @@ def invoke(self, context: InvocationContext) -> ExtractVideoRangeOutput:
             if audio is not None:
                 pcm, rate = audio
                 context.util.signal_progress("Muxing audio")
-                wav_tmp = tempfile.NamedTemporaryFile(prefix="invokeai_video_range_", suffix=".wav", delete=False)
-                wav_tmp.close()
-                wav_path = Path(wav_tmp.name)
-                write_stereo_wav(wav_path, pcm, rate)
                 mux_tmp = tempfile.NamedTemporaryFile(prefix="invokeai_video_range_", suffix=".mp4", delete=False)
                 mux_tmp.close()
                 muxed_path = Path(mux_tmp.name)
-                mux_audio_into_video(tmp_path, wav_path, muxed_path)
+                if not self._mux_spliced_audio(
+                    context, video_path, start, end, source_fps, output_fps, pcm, rate, tmp_path, muxed_path
+                ):
+                    wav_tmp = tempfile.NamedTemporaryFile(prefix="invokeai_video_range_", suffix=".wav", delete=False)
+                    wav_tmp.close()
+                    wav_path = Path(wav_tmp.name)
+                    write_stereo_wav(wav_path, pcm, rate)
+                    mux_audio_into_video(tmp_path, wav_path, muxed_path)
                 source_path = muxed_path
 
             out_duration = num_frames / output_fps
@@ -466,6 +477,68 @@ def _save_unchanged(
             end_frame=n_frames - 1,
         )
 
+    def _mux_spliced_audio(
+        self,
+        context: InvocationContext,
+        video_path: Path,
+        start: int,
+        end: int,
+        source_fps: Optional[float],
+        output_fps: float,
+        pcm: np.ndarray,
+        rate: int,
+        trimmed_video: Path,
+        out_path: Path,
+    ) -> bool:
+        """Mux ``pcm`` onto ``trimmed_video``, stream-copying the source's own AAC packets
+        for the part of the range they hold whole (see ``aac_splice``).
+
+        ``pcm`` is the source's decoded audio over the range, sample for sample, when the
+        range plays at the source's own rate and its span maps one-to-one onto the output
+        timeline; the source's packets are then copied for every output frame that lies
+        inside both the range and the source's real audio, and only the partial frames at
+        each end are encoded. Returns False, leaving the caller to encode the whole
+        soundtrack, when the source's track is not one this app's encoder writes, the range
+        is too short, or the splice does not verify.
+        """
+        if source_fps is None or source_fps != output_fps:
+            return False
+        window_start = round(start * rate / source_fps)
+        window_end = round((end + 1) * rate / source_fps)
+        if window_end - window_start != pcm.shape[1]:
+            return False
+        layout = spliceable_aac_layout(video_path, rate)
+        if layout is None:
+            return False
+        frame, priming = AAC_FRAME_SAMPLES, layout.priming
+        # Packet k's output frame is decoded samples [k * frame - priming, (k + 1) * frame - priming).
+        if window_start == 0:
+            first, copied_at = 0, -priming
+        else:
+            # The first frame wholly inside the range that leaves a full lead-in before it.
+            first = -(-(window_start + priming + EDGE_FRAMES * frame) // frame)
+            copied_at = first * frame - priming - window_start
+        # The last packet that leaves a full encoded tail after the run (see aac_splice); past the
+        # source's own audio the tail is the silence the timeline was padded with.
+        end_packet = min(layout.packet_count, (window_end - EDGE_FRAMES * frame + priming) // frame)
+        reason = splice_soundtrack(
+            trimmed_video,
+            video_path,
+            priming,
+            first,
+            end_packet,
+            copied_at,
+            pcm,
+            rate,
+            out_path,
+            context.util.is_canceled,
+        )
+        if reason is not None:
+            context.logger.info(f"Re-encoding the whole soundtrack: {reason}")
+            return False
+        context.logger.info(f"Stream-copied audio packets {first}-{end_packet - 1} of the source without re-encoding")
+        return True
+
     def _build_audio_track(
         self,
         context: InvocationContext,
@@ -511,7 +584,9 @@ def _build_audio_track(
             # Audio track shorter than the video (or the range lies past its end):
             # keep temporal alignment by padding the shortfall with silence.
             window = np.pad(window, ((0, 0), (0, (window_end - window_start) - window.shape[1])))
-        target = round((end - start + 1) / output_fps * rate)
+        # At the source's own rate the window maps sample for sample; rounding the span on its
+        # own could come out one off the window and interpolate the whole of it.
+        target = window.shape[1] if output_fps == eff_fps else round((end - start + 1) / output_fps * rate)
         return resample_linear(window, target), rate
 
     @staticmethod