diff --git a/docs/src/content/docs/Users Guide/05.Video Generation/introduction.mdx b/docs/src/content/docs/Users Guide/05.Video Generation/introduction.mdx index 3cf8efa3d2b..5b870c8039a 100644 --- a/docs/src/content/docs/Users Guide/05.Video Generation/introduction.mdx +++ b/docs/src/content/docs/Users Guide/05.Video Generation/introduction.mdx @@ -203,7 +203,7 @@ Extension is available on all three families. LTX-2's continues the soundtrack a ### Quality drift across iterations -Each iteration's source is itself a generation output, so artifacts compound — by the 4th or 5th extension you may see softening or color drift. That drift is in the *new* footage: the segments already in the initial video are carried into the result without being re-encoded, picture and sound, apart from up to two seconds of picture at each cut — before the join, and at a trim's start. A trimmed initial video's soundtrack is re-encoded once. Extending a clip whose frame rate differs from the model's (MiniMax H3 always runs at 24 fps) re-encodes it in full, once, as does extending an uploaded video or one made by an earlier version of Invoke. Mitigations: +Each iteration's source is itself a generation output, so artifacts compound — by the 4th or 5th extension you may see softening or color drift. That drift is in the *new* footage: the segments already in the initial video are carried into the result without being re-encoded, picture and sound, apart from up to two seconds of picture and about a tenth of a second of sound at each cut — before the join, and at each end of a trim. Extending a clip whose frame rate differs from the model's (MiniMax H3 always runs at 24 fps) re-encodes it in full, once, as does extending an uploaded video or one made by an earlier version of Invoke. Mitigations: 1. **Trim the bridge point a few frames back from the end** — boundary frames are the most artifact-prone. 2. **Refresh a bridge frame with a low-strength img2img pass** (SDXL or FLUX at ~0.2 strength) before using it as a first frame for the next segment. diff --git a/invokeai/app/invocations/video_concat.py b/invokeai/app/invocations/video_concat.py index 0701a488b09..cb4dfe5126b 100644 --- a/invokeai/app/invocations/video_concat.py +++ b/invokeai/app/invocations/video_concat.py @@ -52,10 +52,10 @@ from invokeai.app.services.shared.invocation_context import InvocationContext from invokeai.app.util.aac_splice import ( AAC_FRAME_SAMPLES, + EDGE_FRAMES, PREROLL_FRAMES, - encoder_aac_layout, - mux_spliced_aac, - read_aac_layout, + splice_soundtrack, + spliceable_aac_layout, ) from invokeai.app.util.h264_splice import ( MIN_ENCODED_FRAMES, @@ -67,7 +67,6 @@ read_h264_layout, ) from invokeai.app.util.video_audio import ( - AudioExtractionError, extract_audio_pcm, mux_audio_into_video, resample_linear, @@ -455,15 +454,12 @@ def _mux_spliced_audio( transition — provided that clip plays at the output rate, so its audio is not retimed, and sets the soundtrack's sample rate. Returns False, leaving the caller to encode the whole soundtrack, when the clip's track is not one this app's encoder - writes, nothing whole-frame precedes the transition, or the result's head does not - decode back to exactly the samples it should hold. + writes, too little precedes the transition, or the splice does not verify. """ if first_fps is None or first_fps != output_fps: return False - layout, encoder = read_aac_layout(first_path), encoder_aac_layout(rate) - if layout is None or encoder is None or layout.sample_rate != rate: - return False - if (layout.decoder_config, layout.priming) != (encoder.decoder_config, encoder.priming): + layout = spliceable_aac_layout(first_path, rate) + if layout is None: return False # The soundtrack holds the clip's own decoded samples only where its audio was mapped # one-to-one: the clip's span at its native rate must be exactly its span on the timeline. @@ -475,66 +471,31 @@ def _mux_spliced_audio( layout.packet_count * AAC_FRAME_SAMPLES - layout.priming, pcm.shape[1], ) - preroll = PREROLL_FRAMES * AAC_FRAME_SAMPLES - # Keep the cut PREROLL_FRAMES clear of the first changed sample (see aac_splice). - head_packets = (unchanged + layout.priming) // AAC_FRAME_SAMPLES - PREROLL_FRAMES - cut = head_packets * AAC_FRAME_SAMPLES - layout.priming - if cut < preroll: - return False - try: - seam_reference = mux_spliced_aac( - video_path, - first_path, - head_packets, - layout.priming, - pcm[:, cut - preroll :], - rate, - pcm.shape[1], - out_path, - context.util.is_canceled, - ) - muxed = extract_audio_pcm(out_path) - except (SpliceError, AudioExtractionError) as e: - context.logger.info(f"Re-encoding the whole soundtrack: it cannot be stream-copied ({e})") - return False - reason = self._spliced_audio_defect(muxed, pcm, rate, cut, seam_reference) + # Keep the cut PREROLL_FRAMES clear of the first changed sample, and a full encoded tail + # after it (see aac_splice). + head_packets = min( + (unchanged + layout.priming) // AAC_FRAME_SAMPLES - PREROLL_FRAMES, + (pcm.shape[1] - EDGE_FRAMES * AAC_FRAME_SAMPLES + layout.priming) // AAC_FRAME_SAMPLES, + ) + reason = splice_soundtrack( + video_path, + first_path, + layout.priming, + 0, + head_packets, + -layout.priming, + pcm, + rate, + out_path, + context.util.is_canceled, + ) if reason is not None: context.logger.info(f"Re-encoding the whole soundtrack: {reason}") return False + cut = head_packets * AAC_FRAME_SAMPLES - layout.priming context.logger.info(f"Stream-copied the first {cut} audio samples of the first clip without re-encoding") return True - @staticmethod - def _spliced_audio_defect( - muxed: Optional[tuple[np.ndarray, int]], - pcm: np.ndarray, - rate: int, - cut: int, - seam_reference: np.ndarray, - ) -> Optional[str]: - """Why the spliced soundtrack cannot be used, or None. - - The copied head must decode back to exactly the samples it stands for, and the - seam — the frames where copied and new packets overlap — must be no noisier than an - ordinary encode of the same samples (``seam_reference``). Whether the overlap - cancels depends on the encoder's window decisions around the content there, so it is - measured, not assumed; and it is measured against the seam's own content, since the - audio after it may be silence that codes with no error at all. - """ - if muxed is None or muxed[1] != rate or muxed[0].shape[1] < pcm.shape[1]: - return "the spliced soundtrack did not decode to its full length" - decoded = muxed[0] - if not np.array_equal(decoded[:, :cut], pcm[:, :cut]): - return "the copied audio did not decode back exactly" - seam = slice(cut, cut + seam_reference.shape[1]) - if seam_reference.shape[1] != PREROLL_FRAMES * AAC_FRAME_SAMPLES or seam.stop > pcm.shape[1]: - return "the new audio does not cover the seam" - seam_error = float(np.sqrt(np.mean((decoded[:, seam] - pcm[:, seam]) ** 2))) - coding_error = float(np.sqrt(np.mean((seam_reference - pcm[:, seam]) ** 2))) - if seam_error > 2.0 * coding_error + 1e-3: - return "the seam between copied and new audio is not clean" - return None - def _estimate_transition_memory(self, width: int, height: int) -> int: if self.transition == "cut" or self.transition_frames == 0: return 0 diff --git a/invokeai/app/invocations/video_frame_extract_range.py b/invokeai/app/invocations/video_frame_extract_range.py index fd798478c8b..b71a9e919b8 100644 --- a/invokeai/app/invocations/video_frame_extract_range.py +++ b/invokeai/app/invocations/video_frame_extract_range.py @@ -11,7 +11,9 @@ through this node (as video extension does with an untrimmed source) costs no quality. A trim of a source this app encoded stream-copies the keyframe-aligned run of frames inside the range and encodes only the partial keyframe intervals -at either end (see ``h264_splice``). Anything else is re-encoded in full. +at either end (see ``h264_splice``), and likewise copies the source's AAC +packets for the audio inside the range, encoding only a short lead-in and tail +(see ``aac_splice``). Anything else is re-encoded in full. If the source carries an audio track, the matching slice of it is carried through (and retimed alongside the video when the output fps differs from @@ -48,6 +50,12 @@ from invokeai.app.invocations.primitives import VideoOutput from invokeai.app.services.session_processor.session_processor_common import CanceledException from invokeai.app.services.shared.invocation_context import InvocationContext +from invokeai.app.util.aac_splice import ( + AAC_FRAME_SAMPLES, + EDGE_FRAMES, + splice_soundtrack, + spliceable_aac_layout, +) from invokeai.app.util.h264_splice import ( MIN_ENCODED_FRAMES, H264Layout, @@ -281,14 +289,17 @@ def invoke(self, context: InvocationContext) -> ExtractVideoRangeOutput: if audio is not None: pcm, rate = audio context.util.signal_progress("Muxing audio") - wav_tmp = tempfile.NamedTemporaryFile(prefix="invokeai_video_range_", suffix=".wav", delete=False) - wav_tmp.close() - wav_path = Path(wav_tmp.name) - write_stereo_wav(wav_path, pcm, rate) mux_tmp = tempfile.NamedTemporaryFile(prefix="invokeai_video_range_", suffix=".mp4", delete=False) mux_tmp.close() muxed_path = Path(mux_tmp.name) - mux_audio_into_video(tmp_path, wav_path, muxed_path) + if not self._mux_spliced_audio( + context, video_path, start, end, source_fps, output_fps, pcm, rate, tmp_path, muxed_path + ): + wav_tmp = tempfile.NamedTemporaryFile(prefix="invokeai_video_range_", suffix=".wav", delete=False) + wav_tmp.close() + wav_path = Path(wav_tmp.name) + write_stereo_wav(wav_path, pcm, rate) + mux_audio_into_video(tmp_path, wav_path, muxed_path) source_path = muxed_path out_duration = num_frames / output_fps @@ -466,6 +477,68 @@ def _save_unchanged( end_frame=n_frames - 1, ) + def _mux_spliced_audio( + self, + context: InvocationContext, + video_path: Path, + start: int, + end: int, + source_fps: Optional[float], + output_fps: float, + pcm: np.ndarray, + rate: int, + trimmed_video: Path, + out_path: Path, + ) -> bool: + """Mux ``pcm`` onto ``trimmed_video``, stream-copying the source's own AAC packets + for the part of the range they hold whole (see ``aac_splice``). + + ``pcm`` is the source's decoded audio over the range, sample for sample, when the + range plays at the source's own rate and its span maps one-to-one onto the output + timeline; the source's packets are then copied for every output frame that lies + inside both the range and the source's real audio, and only the partial frames at + each end are encoded. Returns False, leaving the caller to encode the whole + soundtrack, when the source's track is not one this app's encoder writes, the range + is too short, or the splice does not verify. + """ + if source_fps is None or source_fps != output_fps: + return False + window_start = round(start * rate / source_fps) + window_end = round((end + 1) * rate / source_fps) + if window_end - window_start != pcm.shape[1]: + return False + layout = spliceable_aac_layout(video_path, rate) + if layout is None: + return False + frame, priming = AAC_FRAME_SAMPLES, layout.priming + # Packet k's output frame is decoded samples [k * frame - priming, (k + 1) * frame - priming). + if window_start == 0: + first, copied_at = 0, -priming + else: + # The first frame wholly inside the range that leaves a full lead-in before it. + first = -(-(window_start + priming + EDGE_FRAMES * frame) // frame) + copied_at = first * frame - priming - window_start + # The last packet that leaves a full encoded tail after the run (see aac_splice); past the + # source's own audio the tail is the silence the timeline was padded with. + end_packet = min(layout.packet_count, (window_end - EDGE_FRAMES * frame + priming) // frame) + reason = splice_soundtrack( + trimmed_video, + video_path, + priming, + first, + end_packet, + copied_at, + pcm, + rate, + out_path, + context.util.is_canceled, + ) + if reason is not None: + context.logger.info(f"Re-encoding the whole soundtrack: {reason}") + return False + context.logger.info(f"Stream-copied audio packets {first}-{end_packet - 1} of the source without re-encoding") + return True + def _build_audio_track( self, context: InvocationContext, @@ -511,7 +584,9 @@ def _build_audio_track( # Audio track shorter than the video (or the range lies past its end): # keep temporal alignment by padding the shortfall with silence. window = np.pad(window, ((0, 0), (0, (window_end - window_start) - window.shape[1]))) - target = round((end - start + 1) / output_fps * rate) + # At the source's own rate the window maps sample for sample; rounding the span on its + # own could come out one off the window and interpolate the whole of it. + target = window.shape[1] if output_fps == eff_fps else round((end - start + 1) / output_fps * rate) return resample_linear(window, target), rate @staticmethod diff --git a/invokeai/app/util/aac_splice.py b/invokeai/app/util/aac_splice.py index ddb8b412219..e31d140a029 100644 --- a/invokeai/app/util/aac_splice.py +++ b/invokeai/app/util/aac_splice.py @@ -10,9 +10,11 @@ packet after them blends with the last copied one. That blend cancels cleanly only when both packets were coded from the same signal across their shared window, so two things keep the seam at coding-noise level rather than a glitch: the new encode starts -``PREROLL_FRAMES`` frames before the cut (one is not enough — the encoder's first frames -after a cold start make different window decisions), and callers keep the cut that many -frames clear of anything the joined soundtrack changes. Callers also verify the seam. +``PREROLL_FRAMES`` frames before the cut (the encoder takes several frames after a cold +start to settle its window and bit-allocation decisions: one frame of pre-roll leaves +glitches, two still leaves an occasional 40 ms burst on music), and callers keep the cut +that many frames clear of anything the joined soundtrack changes. Callers also verify the +seam. Packets are joined as ADTS, which carries no timing, so two things the MP4 encodes in its edit list are restored afterwards: the encoder priming the source's first packet @@ -34,12 +36,16 @@ from invokeai.app.util.h264_splice import SpliceError, run_ffmpeg from invokeai.app.util.mp4_metadata import find_box, iter_boxes -from invokeai.app.util.video_audio import AAC_ENCODE_ARGS, extract_audio_pcm +from invokeai.app.util.video_audio import AAC_ENCODE_ARGS, AudioExtractionError, extract_audio_pcm AAC_FRAME_SAMPLES = 1024 # Frames of already-known audio the new encode starts with before the cut, and the clearance # the cut keeps from the first changed sample. -PREROLL_FRAMES = 2 +PREROLL_FRAMES = 4 +# The fewest whole frames an encoded lead-in or tail holds beside the copied run: the frame +# it shares with the run plus PREROLL_FRAMES more, so the seam is far from the encoder's cold +# start (a lead-in) or its end-of-stream flush (a tail). +EDGE_FRAMES = PREROLL_FRAMES + 1 _MAX_TOP_LEVEL_BOXES = 1024 _MAX_PACKETS = 1 << 24 # AudioSampleEntry fields before its child boxes (SampleEntry 8 + version-0 audio header 20). @@ -57,7 +63,9 @@ class AacLayout(NamedTuple): sample_rate: int """The media timescale, which for AAC is the sample rate.""" priming: int - """Leading samples the edit list skips: the encoder delay held by the first packet.""" + """Leading samples the edit list skips. For a file the encoder wrote directly this is its + priming; a splice that opens on a lead-in skips the lead-in's padding too. Either way + packet ``k`` decodes to presented samples ``[k * 1024 - priming, (k + 1) * 1024 - priming)``.""" decoder_config: bytes """The ``AudioSpecificConfig`` (object type, sample rate, channels), without the backward-compatible "no SBR" extension ffmpeg appends to its own encodes but cannot @@ -232,6 +240,18 @@ def _probe_encoder_layout(sample_rate: int) -> Optional[AacLayout]: return read_aac_layout(probe) +def spliceable_aac_layout(path: Path, sample_rate: int) -> Optional[AacLayout]: + """``path``'s audio layout if its packets can be spliced into a soundtrack this app + encodes at ``sample_rate``: the same AudioSpecificConfig as its encoder, and a leading edit + that skips at least that encoder's priming (anything less would be another encoder's).""" + layout, encoder = read_aac_layout(path), encoder_aac_layout(sample_rate) + if layout is None or encoder is None or layout.sample_rate != sample_rate: + return None + if layout.decoder_config != encoder.decoder_config or layout.priming < encoder.priming: + return None + return layout + + def _adts_frames(data: bytes) -> list[bytes]: frames: list[bytes] = [] position = 0 @@ -247,9 +267,11 @@ def _adts_frames(data: bytes) -> list[bytes]: return frames -def _set_track_duration(path: Path, handler: bytes, samples: int, sample_rate: int) -> None: - """Trim the ``handler`` track's presentation to ``samples`` via its edit list, and keep - ``tkhd`` and ``mvhd`` durations consistent with it. All fields are rewritten in place.""" +def _set_audio_edit(path: Path, media_time: int, samples: int, sample_rate: int) -> None: + """Make the audio track present ``samples`` samples starting ``media_time`` samples into + its media, via its edit list, and keep ``tkhd`` and ``mvhd`` durations consistent with it. + All fields are rewritten in place.""" + handler = b"soun" with open(path, "r+b") as fh: fh.seek(0, 2) moov = find_box(fh, 0, fh.tell(), b"moov", max_boxes=_MAX_TOP_LEVEL_BOXES) @@ -282,9 +304,9 @@ def _set_track_duration(path: Path, handler: bytes, samples: int, sample_rate: i if elst is None: raise SpliceError(f"{path.name}'s {handler.decode()} track has no edit list") fh.seek(elst[0]) - elst_field = ">Q" if fh.read(1)[0] == 1 else ">I" + wide_edit = fh.read(1)[0] == 1 fh.seek(elst[0] + 8) - fh.write(struct.pack(elst_field, duration)) + fh.write(struct.pack(">Qq" if wide_edit else ">Ii", duration, media_time)) fh.seek(tkhd_duration_at) fh.write(struct.pack(field, duration)) patched = True @@ -296,79 +318,163 @@ def _set_track_duration(path: Path, handler: bytes, samples: int, sample_rate: i fh.write(struct.pack(">Q" if mvhd_version == 1 else ">I", max(track_durations))) -def mux_spliced_aac( +def _encode( + pcm: np.ndarray, sample_rate: int, work: Path, name: str, is_canceled: Optional[Callable[[], bool]] +) -> tuple[list[bytes], np.ndarray]: + """Encode ``pcm`` as the app does; returns its ADTS frames, priming frames included, and + their standalone decode (which, knowing nothing of priming, starts with the priming).""" + from invokeai.app.util.video_encoding import write_stereo_wav + + wav, adts = work / f"{name}.wav", work / f"{name}.aac" + write_stereo_wav(wav, pcm, sample_rate) + run_ffmpeg(["-i", str(wav), *AAC_ENCODE_ARGS, "-f", "adts", str(adts)], "encoding the new audio", is_canceled) + decoded = extract_audio_pcm(adts) + if decoded is None: + raise SpliceError("the new audio decoded to nothing") + return _adts_frames(adts.read_bytes()), decoded[0] + + +def _seam_defect(decoded: np.ndarray, timeline: np.ndarray, start: int, reference: np.ndarray) -> Optional[str]: + """Whether the seam ``decoded[:, start:]`` is noisier than an ordinary encode of the same + samples (``reference``). Whether copied and new packets overlap cleanly depends on the + encoder's window decisions around the content there, so it is measured, not assumed — and + against the seam's own content, since the audio either side may be silence that codes with + no error at all.""" + length = min(reference.shape[1], timeline.shape[1] - start) + if start < 0 or length <= 0: + return "the new audio does not cover a seam" + seam, reference = slice(start, start + length), reference[:, :length] + seam_error = float(np.sqrt(np.mean((decoded[:, seam] - timeline[:, seam]) ** 2))) + coding_error = float(np.sqrt(np.mean((reference - timeline[:, seam]) ** 2))) + if seam_error > 2.0 * coding_error + 1e-3: + return "a seam between copied and new audio is not clean" + return None + + +def splice_soundtrack( video_path: Path, source: Path, - head_packets: int, - priming: int, - tail_pcm: np.ndarray, + source_skip: int, + first: int, + end: int, + copied_at: int, + timeline: np.ndarray, sample_rate: int, - total_samples: int, out_path: Path, is_canceled: Optional[Callable[[], bool]] = None, -) -> np.ndarray: - """Write ``out_path`` = ``video_path``'s video plus a soundtrack spliced from ``source``. - - The soundtrack is ``source``'s first ``head_packets`` AAC packets, stream-copied — - ``head_packets * 1024 - priming`` samples after priming — followed by ``tail_pcm`` - encoded from ``PREROLL_FRAMES`` frames *before* that cut (``tail_pcm`` must start - there), presented for ``total_samples`` samples in all. - - Returns the new encode's own decode of the ``PREROLL_FRAMES`` frames after the cut: - the coding noise this content gets from an ordinary encode, which is what a clean - seam over the same samples should be measured against. +) -> Optional[str]: + """Write ``out_path`` = ``video_path``'s video with ``timeline`` as its soundtrack, + stream-copying ``source``'s AAC packets ``[first, end)`` for the part of it they hold. + + ``timeline`` (``(2, n)`` float PCM) is the soundtrack to present; packet ``first``'s + output frame begins at ``timeline[:, copied_at]``, and the copied packets must decode to + the timeline's samples there. ``source_skip`` is the source's leading edit (its layout's + ``priming``). ``first == 0`` copies the source's own start, leading edit included + (``copied_at == -source_skip``); otherwise the frames before the copied run are + encoded as a lead-in. The frames after it are encoded from ``PREROLL_FRAMES`` frames + before its end. Both seams are verified, as is every copied frame not on a seam, which + must decode bit-exact. + + Returns None on success, or why the soundtrack could not be spliced — ``out_path`` is + then unusable and the caller encodes the whole soundtrack instead. """ - from invokeai.app.util.video_encoding import write_stereo_wav - - if priming % AAC_FRAME_SAMPLES: - raise SpliceError(f"priming of {priming} samples is not a whole AAC frame") - with tempfile.TemporaryDirectory(prefix="invokeai_aac_splice_") as work: - head, tail_wav, tail = Path(work) / "head.aac", Path(work) / "tail.wav", Path(work) / "tail.aac" - run_ffmpeg( - ["-i", str(source), "-map", "0:a:0", "-c", "copy", "-frames:a", str(head_packets), "-f", "adts", str(head)], - f"copying the first {head_packets} audio packets of {source.name}", - is_canceled, - ) - write_stereo_wav(tail_wav, tail_pcm, sample_rate) - run_ffmpeg( - ["-i", str(tail_wav), *AAC_ENCODE_ARGS, "-f", "adts", str(tail)], "encoding the new audio", is_canceled - ) - head_frames = _adts_frames(head.read_bytes()) - if len(head_frames) != head_packets: - raise SpliceError(f"copied {len(head_frames)} audio packets, expected {head_packets}") - # Drop the new encode's priming packets and its pre-roll. - dropped = priming // AAC_FRAME_SAMPLES + PREROLL_FRAMES - tail_frames = _adts_frames(tail.read_bytes())[dropped:] - decoded = extract_audio_pcm(tail) - if decoded is None: - raise SpliceError("the new audio decoded to nothing") - # An ADTS decode knows nothing of priming, so it starts with the priming frame. - seam_start = priming + PREROLL_FRAMES * AAC_FRAME_SAMPLES - seam_reference = decoded[0][:, seam_start : seam_start + PREROLL_FRAMES * AAC_FRAME_SAMPLES] - joined = Path(work) / "joined.aac" - joined.write_bytes(b"".join(head_frames + tail_frames)) - run_ffmpeg( - [ - "-i", - str(video_path), - # The first copied packet still holds the source's priming; presenting it from - # -priming makes ffmpeg write the leading edit that skips it. - "-itsoffset", - repr(-priming / sample_rate), - "-i", - str(joined), - "-map", - "0:v:0", - "-map", - "1:a:0", - "-c", - "copy", - "-f", - "mp4", - str(out_path), - ], - "muxing the spliced audio", - is_canceled, - ) - _set_track_duration(out_path, b"soun", total_samples, sample_rate) - return seam_reference + encoder = encoder_aac_layout(sample_rate) + if encoder is None or encoder.priming % AAC_FRAME_SAMPLES: + return "the AAC encoder's priming is not a whole number of frames" + frame, priming = AAC_FRAME_SAMPLES, encoder.priming + priming_frames = priming // frame + copied_end = copied_at + (end - first) * frame + if (first == 0) != (copied_at == -source_skip) or end - first < PREROLL_FRAMES: + return "the copied packets do not fit the soundtrack" + if first > 0 and copied_at < EDGE_FRAMES * frame: + return "the copied packets do not fit the soundtrack" + if copied_end < timeline.shape[1] and timeline.shape[1] - copied_end < EDGE_FRAMES * frame: + return "the copied packets do not fit the soundtrack" + if copied_end > timeline.shape[1] or copied_end - PREROLL_FRAMES * frame < 0: + return "the copied packets do not fit the soundtrack" + try: + with tempfile.TemporaryDirectory(prefix="invokeai_aac_splice_") as work: + work_dir = Path(work) + # The source's packets as ADTS, sliced by index: an ADTS frame per packet. + source_adts = work_dir / "source.aac" + run_ffmpeg( + ["-i", str(source), "-map", "0:a:0", "-c", "copy", "-f", "adts", str(source_adts)], + f"copying the audio packets of {source.name}", + is_canceled, + ) + source_frames = _adts_frames(source_adts.read_bytes()) + if len(source_frames) < end: + return f"{source.name} holds {len(source_frames)} audio packets, not {end}" + frames: list[bytes] = [] + seams: list[tuple[int, np.ndarray]] = [] + exact_from = 0 + if first == 0: + # The stream opens with the source's own first packet, and keeps its leading edit. + media_time = source_skip + else: + # A lead-in encode up to the copied run, padded at its start so its frames fall on + # the copied packets' grid, and run on past the seam so its last kept packet was + # coded with the real signal in the window it shares with the first copied one. + pad = (-copied_at) % frame + lead_frames = (pad + copied_at) // frame + lead_input = np.concatenate( + [ + np.zeros((2, pad), dtype=np.float32), + timeline[:, : min(timeline.shape[1], copied_at + PREROLL_FRAMES * frame)], + ], + axis=1, + ) + lead, lead_decoded = _encode(lead_input, sample_rate, work_dir, "lead", is_canceled) + frames += lead[: priming_frames + lead_frames] + media_time = priming + pad + seam_at = copied_at - frame + seams.append((seam_at, lead_decoded[:, priming + pad + seam_at :][:, : 2 * frame])) + exact_from = copied_at + frame + frames += source_frames[first:end] + if copied_end < timeline.shape[1]: + tail, tail_decoded = _encode( + timeline[:, copied_end - PREROLL_FRAMES * frame :], sample_rate, work_dir, "tail", is_canceled + ) + frames += tail[priming_frames + PREROLL_FRAMES :] + seams.append((copied_end, tail_decoded[:, priming + PREROLL_FRAMES * frame :][:, : 2 * frame])) + joined = work_dir / "joined.aac" + joined.write_bytes(b"".join(frames)) + run_ffmpeg( + [ + "-i", + str(video_path), + # Presenting the audio from -media_time makes ffmpeg write a leading edit, whose + # fields are then set exactly below. + "-itsoffset", + repr(-media_time / sample_rate), + "-i", + str(joined), + "-map", + "0:v:0", + "-map", + "1:a:0", + "-c", + "copy", + "-f", + "mp4", + str(out_path), + ], + "muxing the spliced audio", + is_canceled, + ) + _set_audio_edit(out_path, media_time, timeline.shape[1], sample_rate) + muxed = extract_audio_pcm(out_path) + except (SpliceError, AudioExtractionError, OSError) as e: + return f"it cannot be stream-copied ({e})" + + if muxed is None or muxed[1] != sample_rate or muxed[0].shape[1] < timeline.shape[1]: + return "the spliced soundtrack did not decode to its full length" + decoded = muxed[0] + interior = slice(max(exact_from, 0), copied_end) + if not np.array_equal(decoded[:, interior], timeline[:, interior]): + return "the copied audio did not decode back exactly" + for start, reference in seams: + defect = _seam_defect(decoded, timeline, start, reference) + if defect is not None: + return defect + return None diff --git a/invokeai/app/util/video_audio.py b/invokeai/app/util/video_audio.py index cd7ed1d3fbf..eca1db28281 100644 --- a/invokeai/app/util/video_audio.py +++ b/invokeai/app/util/video_audio.py @@ -24,7 +24,11 @@ import numpy as np # How every soundtrack the app writes is encoded; ``aac_splice`` matches a spliced tail to it. -AAC_ENCODE_ARGS = ["-c:a", "aac"] +# Perceptual noise substitution is off: the decoder fills those bands from a random generator +# whose state carries across packets, so a freshly encoded lead-in that used it would change +# how the stream-copied packets after it decode, and a splice could not verify them exact. +AAC_ENCODER_OPTIONS = ["-aac_pns", "0"] +AAC_ENCODE_ARGS = ["-c:a", "aac", *AAC_ENCODER_OPTIONS] def _ffmpeg_exe() -> str: diff --git a/invokeai/app/util/video_encoding.py b/invokeai/app/util/video_encoding.py index 91fcb1e80ee..906fcf590f1 100644 --- a/invokeai/app/util/video_encoding.py +++ b/invokeai/app/util/video_encoding.py @@ -36,6 +36,8 @@ import imageio.v2 as iio2 import numpy as np +from invokeai.app.util.video_audio import AAC_ENCODER_OPTIONS + CRF = 18 KEYFRAME_INTERVAL_SECONDS = 2.0 @@ -57,6 +59,8 @@ def make_mp4_writer(path: Path | str, fps: float, audio_path: Path | str | None raise FileNotFoundError(f"Audio file for muxing does not exist: {audio_path}") kwargs["audio_path"] = str(audio_path) kwargs["audio_codec"] = audio_codec + # Soundtracks share the WAV-mux path's encoder options so either can be spliced onto the other. + audio_options = AAC_ENCODER_OPTIONS if audio_path is not None and audio_codec == "aac" else [] keyframe_interval = max(1, round(fps * KEYFRAME_INTERVAL_SECONDS)) return iio2.get_writer( str(path), @@ -66,7 +70,7 @@ def make_mp4_writer(path: Path | str, fps: float, audio_path: Path | str | None codec="libx264", macro_block_size=1, quality=None, - output_params=["-crf", str(CRF), "-g", str(keyframe_interval)], + output_params=["-crf", str(CRF), "-g", str(keyframe_interval), *audio_options], **kwargs, ) diff --git a/invokeai/app/util/video_ingest.py b/invokeai/app/util/video_ingest.py index 29b51c4c35a..a37a8c8e571 100644 --- a/invokeai/app/util/video_ingest.py +++ b/invokeai/app/util/video_ingest.py @@ -35,6 +35,8 @@ from pathlib import Path from typing import Literal, Optional +from invokeai.app.util.video_audio import AAC_ENCODER_OPTIONS + # Hard wall-clock bound for a single ffmpeg child (probe or convert). Matches the bound # used for audio extraction: generous enough for a worst-case ~1 GB 4K transcode on a # modest CPU, small enough that a hostile file cannot camp an upload slot forever. @@ -123,7 +125,7 @@ def _audio_args(probe: MediaProbe) -> list[str]: return [] if probe.audio_codec == "aac": return ["-c:a", "copy"] - return ["-c:a", "aac", "-b:a", "192k"] + return ["-c:a", "aac", "-b:a", "192k", *AAC_ENCODER_OPTIONS] def plan_ingest(probe: MediaProbe) -> IngestAction: diff --git a/tests/app/invocations/test_video_concat_splice.py b/tests/app/invocations/test_video_concat_splice.py index 6336aa08263..b108ff82d6b 100644 --- a/tests/app/invocations/test_video_concat_splice.py +++ b/tests/app/invocations/test_video_concat_splice.py @@ -385,7 +385,7 @@ def test_first_clip_with_presentation_tags_is_re_encoded(tmp_path, tag): assert len(_decode(out)) == 138 -@pytest.mark.parametrize("stage", ["copy_frames", "mux_spliced_aac"]) +@pytest.mark.parametrize("stage", ["copy_frames", "splice_soundtrack"]) def test_cancellation_during_a_splice_saves_nothing(tmp_path, stage): """Cancelling while a splice stage runs stops it — the stage itself raises, not just the pre-save check — and nothing is saved.""" diff --git a/tests/app/invocations/test_video_frame_extract_range_audio_splice.py b/tests/app/invocations/test_video_frame_extract_range_audio_splice.py new file mode 100644 index 00000000000..d33d649ab8f --- /dev/null +++ b/tests/app/invocations/test_video_frame_extract_range_audio_splice.py @@ -0,0 +1,222 @@ +"""A trimmed source's soundtrack must not be re-encoded where the trim keeps it. + +Extend Video trims its source before joining a continuation onto it. The trim used to +decode and re-encode the whole kept soundtrack, so every extension of a trimmed source +cost the earlier audio one more AAC generation even though its video was stream-copied. +The node now copies the source's AAC packets for every output frame wholly inside the +range and encodes only a lead-in and a tail. These tests pin that the copied audio comes +through bit-exact, that the seams are no worse than a full re-encode, that the soundtrack +ends with the video, and that a source the splice cannot vouch for is re-encoded instead. + +Real ffmpeg (the imageio-ffmpeg bundled binary) encodes and decodes throughout. +""" + +import struct +from pathlib import Path +from unittest.mock import MagicMock, patch + +import imageio.v2 as iio2 +import numpy as np +import pytest + +from invokeai.app.invocations.fields import VideoField +from invokeai.app.invocations.video_concat import VideoConcatInvocation +from invokeai.app.invocations.video_frame_extract_range import ExtractVideoRangeInvocation +from invokeai.app.util.aac_splice import EDGE_FRAMES, PREROLL_FRAMES, splice_soundtrack, spliceable_aac_layout +from invokeai.app.util.mp4_metadata import find_box, write_mp4_tags +from invokeai.app.util.video_audio import extract_audio_pcm +from invokeai.app.util.video_encoding import make_mp4_writer, write_stereo_wav + +FPS = 16.0 +RATE = 32000 +SIZE = 64 +# How far into a trimmed soundtrack its encoded lead-in and its seam reach, and how far from its +# end the tail's: the copied run starts within a frame after EDGE_FRAMES of lead-in, and ends at +# least EDGE_FRAMES (and less than one more) before the end. +LEAD_IN_LIMIT = (EDGE_FRAMES + 2) * 1024 +TAIL_LIMIT = (EDGE_FRAMES + 1) * 1024 + + +def _signal(kind: str, n_frames: int, seed: int) -> np.ndarray: + n = round(n_frames / FPS * RATE) + t = np.arange(n) / RATE + if kind == "tone": + mono = 0.4 * np.sin(2 * np.pi * (440 + 110 * seed) * t) + else: + # Broadband noise is where the encoder would use perceptual noise substitution. + mono = 0.25 * np.random.default_rng(seed).standard_normal(n) + return np.stack([mono, 0.9 * mono]).astype(np.float32) + + +def _clip(dirpath: Path, name: str, n_frames: int, kind: str, seed: int) -> Path: + path, wav = dirpath / f"{name}.mp4", dirpath / f"{name}.wav" + write_stereo_wav(wav, _signal(kind, n_frames, seed), RATE) + writer = make_mp4_writer(path, FPS, audio_path=wav) + rng = np.random.default_rng(seed) + for _ in range(n_frames): + writer.append_data(rng.integers(0, 256, (SIZE, SIZE, 3), dtype=np.uint8)) + writer.close() + return path + + +def _save_to(saved: Path): + def _save(source_path: Path, **_: object) -> MagicMock: + # The stream-copy remux save() runs to embed the record. + write_mp4_tags(source_path, saved, {"invokeai_metadata": "{}"}) + return MagicMock() + + return _save + + +def _trim(dirpath: Path, source: Path, start: int, end: int, name: str = "trimmed") -> tuple[Path, MagicMock]: + saved = dirpath / f"{name}.mp4" + context = MagicMock() + context.videos.get_path.return_value = source + context.util.is_canceled.return_value = False + context.videos.save.side_effect = _save_to(saved) + node = ExtractVideoRangeInvocation(video=VideoField(video_name=source.stem), start_frame=start, end_frame=end) + base = MagicMock(video=VideoField(video_name=name), width=SIZE, height=SIZE, num_frames=1, fps=FPS, duration=1.0) + with patch("invokeai.app.invocations.video_frame_extract_range.VideoOutput.build", return_value=base): + node.invoke(context) + return saved, context + + +def _logged(context: MagicMock) -> str: + return "\n".join(str(call.args[0]) for call in context.logger.info.call_args_list) + + +def _movie_duration(path: Path) -> float: + with open(path, "rb") as fh: + fh.seek(0, 2) + moov = find_box(fh, 0, fh.tell(), b"moov") + assert moov is not None + mvhd = find_box(fh, *moov, b"mvhd") + assert mvhd is not None + fh.seek(mvhd[0]) + wide = fh.read(1)[0] == 1 + fh.seek(mvhd[0] + (20 if wide else 12)) + timescale, duration = struct.unpack(">IQ" if wide else ">II", fh.read(12 if wide else 8)) + return duration / timescale + + +def _window(source: Path, start: int, end: int) -> np.ndarray: + decoded = extract_audio_pcm(source) + assert decoded is not None + return decoded[0][:, round(start * RATE / FPS) : round((end + 1) * RATE / FPS)] + + +@pytest.mark.parametrize("kind", ["tone", "noise"]) +@pytest.mark.parametrize("start,end", [(20, 90), (0, 80), (33, 99)]) +def test_trim_copies_the_source_audio_it_keeps(tmp_path, monkeypatch, kind, start, end): + source = _clip(tmp_path, "source", 100, kind, seed=1) + trimmed, context = _trim(tmp_path, source, start, end) + assert "audio packets" in _logged(context) + + want = _window(source, start, end) + got = extract_audio_pcm(trimmed) + assert got is not None + n = end - start + 1 + # The soundtrack ends with the video, and decodes to whole frames past it. + assert _movie_duration(trimmed) == pytest.approx(n / FPS, abs=0.002) + assert 0 <= got[0].shape[1] - round(n * RATE / FPS) < 1024 + # Every frame away from the lead-in and tail seams is the source's own audio, bit for bit. + interior = slice(LEAD_IN_LIMIT, want.shape[1] - TAIL_LIMIT) + assert np.array_equal(got[0][:, interior], want[:, interior]) + + # The seams are no worse than re-encoding the whole range. + monkeypatch.setattr(ExtractVideoRangeInvocation, "_mux_spliced_audio", lambda *args, **kwargs: False) + full, _ = _trim(tmp_path, source, start, end, name="full") + reference = extract_audio_pcm(full) + assert reference is not None + m = min(got[0].shape[1], reference[0].shape[1], want.shape[1]) + spliced_error = np.abs(got[0][:, :m] - want[:, :m]) + full_error = np.abs(reference[0][:, :m] - want[:, :m]) + for seam in (slice(0, LEAD_IN_LIMIT), slice(want.shape[1] - TAIL_LIMIT, m)): + assert ( + float(np.sqrt(np.mean(spliced_error[:, seam] ** 2))) + <= 2 * float(np.sqrt(np.mean(full_error[:, seam] ** 2))) + 1e-3 + ) + + +def test_copied_audio_that_does_not_decode_back_is_rejected(tmp_path): + """The splice trusts nothing it has not decoded: if the source's packets do not reproduce + the soundtrack's samples where they are copied (here, a timeline nudged by one LSB), the + splice is refused and the caller re-encodes.""" + source = _clip(tmp_path, "source", 100, "tone", seed=1) + video = tmp_path / "video.mp4" + writer = make_mp4_writer(video, FPS) + for _ in range(71): + writer.append_data(np.zeros((SIZE, SIZE, 3), dtype=np.uint8)) + writer.close() + layout = spliceable_aac_layout(source, RATE) + assert layout is not None + timeline = _window(source, 20, 90) + window_start = round(20 * RATE / FPS) + first = -(-(window_start + layout.priming + EDGE_FRAMES * 1024) // 1024) + copied_at = first * 1024 - layout.priming - window_start + end = (round(91 * RATE / FPS) - EDGE_FRAMES * 1024 + layout.priming) // 1024 + args = (video, source, layout.priming, first, end, copied_at) + + assert splice_soundtrack(*args, timeline, RATE, tmp_path / "ok.mp4") is None + nudged = timeline.copy() + nudged[:, timeline.shape[1] // 2] += 1 / 32768 + assert ( + splice_soundtrack(*args, nudged, RATE, tmp_path / "bad.mp4") == "the copied audio did not decode back exactly" + ) + + +def test_extending_a_trimmed_source_keeps_its_audio(tmp_path): + """The Extend graph: trim the source, then join a continuation onto the trim.""" + source = _clip(tmp_path, "source", 100, "tone", seed=1) + trimmed, trim_context = _trim(tmp_path, source, 20, 95) + assert "audio packets" in _logged(trim_context) + continuation = _clip(tmp_path, "continuation", 40, "tone", seed=3) + + joined = tmp_path / "joined.mp4" + by_name = {"trimmed": trimmed, "continuation": continuation} + context = MagicMock() + context.videos.get_path.side_effect = lambda name: by_name[name] + context.util.is_canceled.return_value = False + context.videos.save.side_effect = _save_to(joined) + node = VideoConcatInvocation( + videos=[VideoField(video_name="trimmed"), VideoField(video_name="continuation")], + transition="crossfade", + transition_frames=2, + ) + with patch("invokeai.app.invocations.video_concat.VideoOutput.build", return_value=MagicMock()): + node.invoke(context) + assert "audio samples of the first clip without re-encoding" in _logged(context) + + want = _window(source, 20, 95) + got = extract_audio_pcm(joined) + assert got is not None + # The source's audio inside the trim, clear of the trim's lead-in and the join, is unchanged. + interior = slice(LEAD_IN_LIMIT, want.shape[1] - round(2 * RATE / FPS) - (PREROLL_FRAMES + 2) * 1024) + assert np.array_equal(got[0][:, interior], want[:, interior]) + reader = iio2.get_reader(str(joined), format="FFMPEG") + try: + assert sum(1 for _ in reader) == 76 + 40 - 2 + finally: + reader.close() + + +def test_trim_splices_when_frames_hold_fractional_samples(tmp_path): + """At 44.1 kHz and 24 fps a frame is 1837.5 samples: rounding the trimmed span on its own + came out one off the window, which interpolated the whole soundtrack and ruled out a copy.""" + rate, fps = 44100, 24.0 + n = round(110 / fps * rate) + tone = (0.4 * np.sin(2 * np.pi * 523.0 * np.arange(n) / rate)).astype(np.float32) + wav, source = tmp_path / "source.wav", tmp_path / "source.mp4" + write_stereo_wav(wav, np.stack([tone, tone]), rate) + writer = make_mp4_writer(source, fps, audio_path=wav) + for i in range(110): + writer.append_data(np.full((SIZE, SIZE, 3), i, dtype=np.uint8)) + writer.close() + + trimmed, context = _trim(tmp_path, source, 7, 100) + assert "audio packets" in _logged(context) + decoded, got = extract_audio_pcm(source), extract_audio_pcm(trimmed) + assert decoded is not None and got is not None + want = decoded[0][:, round(7 * rate / fps) : round(101 * rate / fps)] + interior = slice(LEAD_IN_LIMIT, want.shape[1] - TAIL_LIMIT) + assert np.array_equal(got[0][:, interior], want[:, interior])