From 73de5e0a5fff3e719594edb4cde38de48cc570b3 Mon Sep 17 00:00:00 2001 From: JPPhoto Date: Wed, 14 Oct 2026 13:07:20 -0500 Subject: [PATCH 1/2] feat(video): add GOP-aware smart video concatenation Preserve compatible H.264 segments and fall back to full encoding when needed. Verify smart-render bridge values against independent transition math and full encoding. --- .../05.Video Generation/video-workflows.mdx | 2 + invokeai/app/invocations/video_concat.py | 383 +++++++++----- invokeai/app/util/h264_splice.py | 64 ++- invokeai/app/util/video_smart_concat.py | 398 +++++++++++++++ tests/app/invocations/test_video_concat.py | 1 + .../invocations/test_video_concat_splice.py | 481 +++++++++++++++++- tests/app/util/test_h264_splice.py | 4 + tests/app/util/test_video_smart_concat.py | 165 ++++++ 8 files changed, 1363 insertions(+), 135 deletions(-) create mode 100644 invokeai/app/util/video_smart_concat.py create mode 100644 tests/app/util/test_video_smart_concat.py diff --git a/docs/src/content/docs/Users Guide/05.Video Generation/video-workflows.mdx b/docs/src/content/docs/Users Guide/05.Video Generation/video-workflows.mdx index ef81fc732ca..99438094ee0 100644 --- a/docs/src/content/docs/Users Guide/05.Video Generation/video-workflows.mdx +++ b/docs/src/content/docs/Users Guide/05.Video Generation/video-workflows.mdx @@ -128,6 +128,8 @@ This variant takes **two inputs** — the source video and a destination image: Because the endpoint is pinned to your image, it's ideal for bridging a clip to a known next shot or landing on a specific final frame. ::: +The **Concatenate Videos** node smart-renders compatible H.264 MP4 clips by preserving compressed video outside transition regions. Clips must share dimensions and frame rate and use compatible encoding settings. Other inputs, or a failed smart-render validation, automatically use the regular full-encoding path. Set the node's **Smart Render** input to `false` to always use full encoding. + --- ## Bundled MiniMax H3 workflows diff --git a/invokeai/app/invocations/video_concat.py b/invokeai/app/invocations/video_concat.py index 0d7a48957aa..894c7a9e941 100644 --- a/invokeai/app/invocations/video_concat.py +++ b/invokeai/app/invocations/video_concat.py @@ -4,22 +4,19 @@ clips into this node to glue them into one longer video. The transition options hide the seam between independently-denoised clips. -Implementation uses imageio (FFMPEG plugin) for both decode and encode, matching -``wan_latents_to_video`` and ``video_thumbnails`` — so we can read our own -output without surprises. Frames stream from the decoders into the encoder one -at a time, buffering only the transition windows, so peak memory stays +The full-encode fallback uses the existing imageio frame pipeline; smart bridges stream +RGB frames to the bundled FFmpeg encoder. Frames stream one at a time, buffering only +the transition windows, so peak memory stays O(transition_frames) even when the inputs are long uploads (the upload cap admits files whose decoded frames would run to tens of gigabytes). -The first clip is not re-encoded when it can be avoided: if it was written by this -app's own encoder at the output canvas and frame rate, everything before its last -usable keyframe is stream-copied, and only the remainder — its tail, the transitions -and the later clips — is encoded and joined on (see ``h264_splice``). Repeatedly -extending a video therefore stops degrading its earlier segments, which otherwise go -through one more lossy generation per extension. Any other first clip is re-encoded -in full, as before. The soundtrack is treated the same way: the first clip's own AAC -packets are kept up to its outgoing transition and only the rest is encoded (see -``aac_splice``). +When ``smart_render`` is enabled, compatible clips keep their compressed H.264 +samples outside transition bridges. Each bridge spans from a verified IDR boundary in +the outgoing clip to one in the incoming clip, and the complete timeline is assembled +from copied samples and continuously encoded bridges (see ``video_smart_concat`` and +``h264_splice``). Unsupported streams fall back to the full-encoding path. The +soundtrack is treated as before: compatible AAC packets from the first clip are kept +up to its outgoing transition and only the rest is encoded (see ``aac_splice``). The AUDIO path is a deliberate exception to that bound: soundtracks buffer in full (float32 stereo PCM of the emitted timeline, roughly 23 MB per minute at @@ -32,8 +29,8 @@ import math import tempfile +import threading from collections import deque -from itertools import islice from pathlib import Path from typing import Callable, Iterable, Iterator, Literal, Optional @@ -58,20 +55,25 @@ spliceable_aac_layout, ) from invokeai.app.util.h264_splice import ( - MIN_ENCODED_FRAMES, - H264Layout, SpliceError, concat_stream_copy, copy_frames, probes_as, read_h264_layout, + run_ffmpeg, ) from invokeai.app.util.video_audio import ( extract_audio_pcm, mux_audio_into_video, resample_linear, ) -from invokeai.app.util.video_encoding import make_mp4_writer, write_stereo_wav, writer_sample_entry +from invokeai.app.util.video_encoding import make_mp4_writer, write_stereo_wav +from invokeai.app.util.video_smart_concat import ( + CopySegment, + SmartRenderPlan, + encode_rgb_frames, + try_build_smart_render_plan, +) from invokeai.app.util.video_thumbnails import iter_video_frames, probe_video TransitionMode = Literal["cut", "crossfade", "fade_through_black"] @@ -127,7 +129,7 @@ def _fade_through_black(a_tail: list[np.ndarray], b_head: list[np.ndarray]) -> I title="Concatenate Videos", tags=["video", "concat", "transition"], category="video", - version="1.2.0", + version="1.3.0", classification=Classification.Prototype, ) class VideoConcatInvocation(BaseInvocation, WithMetadata, WithBoard): @@ -192,6 +194,14 @@ class VideoConcatInvocation(BaseInvocation, WithMetadata, WithBoard): ), ) + smart_render: bool = InputField( + default=True, + description=( + "Preserve original video encoding outside transitions when possible. Automatically falls back to full " + "encoding for incompatible inputs." + ), + ) + def invoke(self, context: InvocationContext) -> VideoOutput: if len(self.videos) < 2: raise ValueError("video_concat requires at least two input videos.") @@ -229,18 +239,55 @@ def invoke(self, context: InvocationContext) -> VideoOutput: with tempfile.TemporaryDirectory(prefix="invokeai_video_concat_") as work: work_dir = Path(work) tmp_path = work_dir / "joined.mp4" - joined = None - splice = self._plan_splice(paths[0], probes[0][3], (width, height), output_fps) - if splice is not None: - keyframe, layout = splice - rest_path = work_dir / "rest.mp4" - rest = self._encode_joined(context, paths, (width, height), output_fps, rest_path, skip=keyframe) - if self._splice( - context, paths[0], keyframe, layout, rest_path, rest[0][0], output_fps, work_dir, tmp_path - ): - joined = rest + joined: Optional[tuple[list[int], int]] = None + if self.smart_render: + if context.util.is_canceled(): + raise CanceledException + tail_need, head_need = self._transition_windows() + plan, reason = try_build_smart_render_plan( + paths=paths, + probes=probes, + canvas=(width, height), + output_fps=output_fps, + transition=self.transition, + transition_frames=self.transition_frames, + tail_need=tail_need, + head_need=head_need, + is_canceled=context.util.is_canceled, + ) + if plan is not None: + context.logger.info( + f"Smart rendering selected: {plan.output_frames} output frames, " + f"copying {plan.copied_frames}, encoding {plan.encoded_frames} across " + f"{plan.bridge_count} bridge(s)" + ) + context.util.signal_progress( + f"Smart rendering: copying {plan.copied_frames} frames, " + f"encoding {plan.encoded_frames} in {plan.bridge_count} bridge(s)" + ) + smart_path = work_dir / "smart_joined.mp4" + try: + self._execute_smart_render( + context, paths, (width, height), output_fps, plan, work_dir, smart_path + ) + joined = (list(plan.frame_counts), plan.output_frames) + tmp_path = smart_path + except CanceledException: + raise + except (SpliceError, TimeoutError) as e: + if context.util.is_canceled(): + raise CanceledException from e + smart_path.unlink(missing_ok=True) + reason = str(e) + context.logger.info(f"Smart rendering failed; falling back to full video encoding: {reason}") + context.util.signal_progress(f"Smart rendering failed; using full video encoding: {reason}") + else: + context.logger.info(f"Smart rendering unavailable; falling back to full video encoding: {reason}") + context.util.signal_progress(f"Smart rendering unavailable; using full video encoding: {reason}") + else: + context.logger.info("Smart rendering disabled; using full video encoding") if joined is None: - joined = self._encode_joined(context, paths, (width, height), output_fps, tmp_path, skip=0) + joined = self._encode_joined(context, paths, (width, height), output_fps, tmp_path) frame_counts, num_frames = joined return self._save_joined(context, paths, probes, tmp_path, work_dir, frame_counts, num_frames, output_fps) @@ -312,30 +359,199 @@ def _transition_windows(self) -> tuple[int, int]: return tail_need, self.transition_frames - tail_need return 0, 0 - def _plan_splice( - self, first_path: Path, first_fps: Optional[float], canvas: tuple[int, int], output_fps: float - ) -> Optional[tuple[int, H264Layout]]: - """Pick the keyframe of the first clip up to which it can be stream-copied, or None. + def _execute_smart_render( + self, + context: InvocationContext, + paths: list[Path], + canvas: tuple[int, int], + output_fps: float, + plan: SmartRenderPlan, + work_dir: Path, + out_path: Path, + ) -> None: + """Build validated copy and bridge parts, then join them on one MP4 timeline.""" + parts: list[Path] = [] + part_counts: list[int] = [] + target_entry = plan.layouts[0].sample_entry + bridge_index = 0 + for segment_index, segment in enumerate(plan.segments): + if context.util.is_canceled(): + raise CanceledException + if isinstance(segment, CopySegment): + expected = segment.end - segment.start + source = paths[segment.clip] + if segment.start == 0 and segment.end == plan.layouts[segment.clip].sample_count: + part = source + else: + part = work_dir / f"copy_{segment_index}.mp4" + context.util.signal_progress( + f"Smart rendering: copying clip {segment.clip + 1}, frames {segment.start}-{segment.end - 1}" + ) + copy_frames( + source, + segment.start, + segment.end, + plan.layouts[segment.clip].sample_count, + part, + context.util.is_canceled, + ) + self._validate_smart_part(part, expected, target_entry, output_fps, context.util.is_canceled) + if segment.clip == 0 and segment.start == 0: + context.logger.info( + f"Stream-copied the first {expected} frames of the first clip without re-encoding" + ) + context.logger.info( + f"Stream-copied {expected} frames from clip {segment.clip + 1} " + f"(source frames {segment.start}-{segment.end - 1})" + ) + else: + bridge_index += 1 + context.util.signal_progress( + f"Smart rendering transition {bridge_index}/{plan.bridge_count} " + f"between clips {segment.left_clip + 1} and {segment.right_clip + 1}" + ) + left_path = work_dir / f"bridge_{bridge_index}_left.mp4" + right_path = work_dir / f"bridge_{bridge_index}_right.mp4" + copy_frames( + paths[segment.left_clip], + segment.left_start, + plan.layouts[segment.left_clip].sample_count, + plan.layouts[segment.left_clip].sample_count, + left_path, + context.util.is_canceled, + ) + copy_frames( + paths[segment.right_clip], + 0, + segment.right_end, + plan.layouts[segment.right_clip].sample_count, + right_path, + context.util.is_canceled, + ) + left_expected = plan.layouts[segment.left_clip].sample_count - segment.left_start + self._validate_smart_part(left_path, left_expected, target_entry, output_fps, context.util.is_canceled) + self._validate_smart_part( + right_path, segment.right_end, target_entry, output_fps, context.util.is_canceled + ) - The keyframe must leave the clip's transition tail, and at least ``MIN_ENCODED_FRAMES`` - so the re-encoded remainder keeps the copied head's timeline, after it. The clip must - have a constant frame duration and already play at the - output rate — a stream copy cannot retime — and carry the decoder configuration this - node's encoder writes, or the re-encoded remainder could not share its track. - """ - if first_fps is None or not math.isclose(first_fps, output_fps, rel_tol=1e-6): - return None - layout = read_h264_layout(first_path) + bridge_path = work_dir / f"bridge_{bridge_index}.mp4" + encoded_source_counts: list[int] = [] + abort_event = threading.Event() + + def decoder_cancel(event: threading.Event = abort_event) -> bool: + return event.is_set() or context.util.is_canceled() + + left_frames = iter_video_frames(left_path, is_canceled=decoder_cancel) + right_frames = iter_video_frames(right_path, is_canceled=decoder_cancel) + try: + source_frames = self._iter_joined_frames( + [_iter_frames_on_canvas(left_frames, canvas), _iter_frames_on_canvas(right_frames, canvas)], + is_canceled=decoder_cancel, + frame_counts=encoded_source_counts, + ) + encoded = encode_rgb_frames( + source_frames, + bridge_path, + canvas, + output_fps, + segment.output_frames, + # Only a user cancellation is fatal. Internal encoder aborts also set + # ``abort_event`` so the frame decoder can stop, then fall back safely. + is_canceled=context.util.is_canceled, + abort_event=abort_event, + ) + finally: + left_frames.close() + right_frames.close() + if encoded != segment.output_frames or encoded_source_counts != [left_expected, segment.right_end]: + raise SpliceError( + f"transition {bridge_index} emitted {encoded} frames from source counts {encoded_source_counts}; " + f"expected {segment.output_frames}" + ) + self._validate_smart_part( + bridge_path, segment.output_frames, target_entry, output_fps, context.util.is_canceled + ) + parts.append(bridge_path) + part_counts.append(segment.output_frames) + continue + parts.append(part) + part_counts.append(expected) + + if not parts or sum(part_counts) != plan.output_frames: + raise SpliceError("the planned smart-render parts do not cover the expected output timeline") + context.util.signal_progress("Assembling smart-rendered video") + concat_stream_copy(parts, out_path, context.util.is_canceled) + joined = self._validate_smart_part( + out_path, plan.output_frames, target_entry, output_fps, context.util.is_canceled + ) + boundaries: set[tuple[int, int]] = set() + cumulative = 0 + for count in part_counts[:-1]: + cumulative += count + boundaries.add((cumulative, cumulative)) + actual_access_points = {(point.sample_index, point.frame_index) for point in joined.access_points} + if not boundaries.issubset(actual_access_points): + raise SpliceError("the joined MP4 does not retain independent IDR decoding at every segment boundary") + for _, frame_index in sorted(boundaries): + if context.util.is_canceled(): + raise CanceledException + run_ffmpeg( + [ + "-ss", + format(frame_index / output_fps, ".12g"), + "-i", + str(out_path), + "-map", + "0:v:0", + "-frames:v", + "1", + "-f", + "null", + "-", + ], + f"validating splice boundary at frame {frame_index}", + context.util.is_canceled, + ) + if context.util.is_canceled(): + raise CanceledException + + def _validate_smart_part( + self, + path: Path, + expected_frames: int, + sample_entry: bytes, + fps: float, + is_canceled: Optional[Callable[[], bool]] = None, + ): + """Check the sample table, entry, cadence, start IDR and bounded container probe.""" + if is_canceled is not None and is_canceled(): + raise CanceledException + try: + layout = read_h264_layout(path) + except OSError as e: + raise SpliceError(f"could not inspect smart-render part {path.name}: {e}") from e + if is_canceled is not None and is_canceled(): + raise CanceledException if ( layout is None + or layout.sample_count != expected_frames + or layout.sample_entry != sample_entry or layout.frame_duration is None - or layout.sample_entry != writer_sample_entry(*canvas, output_fps) + or not math.isclose(layout.time_scale / layout.frame_duration, fps, rel_tol=1e-6) + or not any(point.sample_index == 0 and point.frame_index == 0 for point in layout.access_points) ): - return None - tail_need, _ = self._transition_windows() - last_usable = layout.sample_count - max(tail_need, MIN_ENCODED_FRAMES) - keyframes = [k for k in layout.sync_samples if 0 < k <= last_usable] - return (keyframes[-1], layout) if keyframes else None + raise SpliceError(f"smart-render part {path.name} failed frame, codec, cadence, or IDR validation") + try: + timeline_valid = probes_as(path, fps, expected_frames) + except (OSError, TimeoutError) as e: + if is_canceled is not None and is_canceled(): + raise CanceledException from e + raise SpliceError(f"could not validate smart-render timing for {path.name}: {e}") from e + if is_canceled is not None and is_canceled(): + raise CanceledException + if not timeline_valid: + raise SpliceError(f"smart-render part {path.name} failed frame, codec, cadence, or IDR validation") + return layout def _encode_joined( self, @@ -344,14 +560,8 @@ def _encode_joined( canvas: tuple[int, int], output_fps: float, out_path: Path, - skip: int, ) -> tuple[list[int], int]: - """Encode the joined timeline, minus the first clip's first ``skip`` frames, to ``out_path``. - - Returns ``(frame_counts, num_frames)`` for the WHOLE timeline — the skipped frames - are counted as part of the first clip and of the output — so the audio is laid out - the same whether or not the head was stream-copied. - """ + """Encode every joined frame to ``out_path`` as the compatibility fallback.""" # Frames stream from the decoders straight into the encoder; only the # transition windows are buffered. See _iter_joined_frames. writer = make_mp4_writer(out_path, output_fps) @@ -362,10 +572,7 @@ def _encode_joined( # Every clip goes through the canvas guard, including the first: frames already # on it pass straight through, so nothing can reach the writer at a size the # file is not being encoded at. - clip_iters = [ - _iter_frames_on_canvas(islice(frames, skip, None) if i == 0 else frames, canvas) - for i, frames in enumerate(decoders) - ] + clip_iters = [_iter_frames_on_canvas(frames, canvas) for frames in decoders] for frame in self._iter_joined_frames( clip_iters, is_canceled=context.util.is_canceled, frame_counts=frame_counts ): @@ -377,63 +584,7 @@ def _encode_joined( # fallback re-encode, would otherwise wait behind. for frames in decoders: frames.close() - frame_counts[0] += skip - return frame_counts, num_frames + skip - - def _splice( - self, - context: InvocationContext, - first_path: Path, - keyframe: int, - layout: H264Layout, - rest_path: Path, - first_clip_frames: int, - output_fps: float, - work_dir: Path, - out_path: Path, - ) -> bool: - """Join the first clip's frames before ``keyframe``, stream-copied, onto ``rest_path``. - - Returns False, leaving the caller to re-encode everything, when the two cannot be - joined losslessly: the remainder was encoded with a different decoder configuration - than the first clip carries (an upload, or a file from an older encoder setting), or - the first clip decoded to a different frame count than its sample table lists, which - would put the copied head and the skipped frames out of step. - """ - rest = read_h264_layout(rest_path) - if first_clip_frames != layout.sample_count: - reason = f"it decoded to {first_clip_frames} frames but lists {layout.sample_count}" - elif rest is None or rest.sample_entry != layout.sample_entry or rest.sync_samples[:1] != (0,): - reason = "its encoding differs from this node's" - else: - reason = None - if reason is None: - head_path = work_dir / "head.mp4" - try: - copy_frames(first_path, 0, keyframe, layout.sample_count, head_path, context.util.is_canceled) - head = read_h264_layout(head_path) - # ffmpeg can derive presentation boxes (e.g. a pasp from a tkhd display size) while - # copying, so the copied head must keep the source's own sample entry. - if head is None or head.sample_count != keyframe or head.sample_entry != layout.sample_entry: - reason = "copying its head did not yield the expected frames" - else: - concat_stream_copy([head_path, rest_path], out_path, context.util.is_canceled) - joined = read_h264_layout(out_path) - if ( - joined is None - or joined.sample_entry != layout.sample_entry - or joined.sample_count != keyframe + rest.sample_count - or joined.frame_duration != layout.frame_duration - or not probes_as(out_path, output_fps, joined.sample_count) - ): - reason = "the joined file does not hold the expected frames on one timeline" - except SpliceError as e: - reason = str(e) - if reason is not None: - context.logger.info(f"Re-encoding the first clip in full: it cannot be stream-copied ({reason})") - return False - context.logger.info(f"Stream-copied the first {keyframe} frames of the first clip without re-encoding") - return True + return frame_counts, num_frames def _mux_spliced_audio( self, diff --git a/invokeai/app/util/h264_splice.py b/invokeai/app/util/h264_splice.py index af35c267a2e..a8ede70a937 100644 --- a/invokeai/app/util/h264_splice.py +++ b/invokeai/app/util/h264_splice.py @@ -77,6 +77,15 @@ _IDENTITY_MATRIX = (0x10000, 0, 0, 0, 0x10000, 0, 0, 0, 0x40000000) +class H264AccessPoint(NamedTuple): + """An IDR sample and its exact position on both the packet and presentation timelines.""" + + sample_index: int + frame_index: int + dts: int + pts: int + + class H264Layout(NamedTuple): """What a splice needs to know about an MP4's first video track.""" @@ -93,6 +102,10 @@ class H264Layout(NamedTuple): Two tracks can share a joined track only if these bytes match.""" frame_duration: Optional[int] """Every sample's duration in media ticks, or None if they are not all equal.""" + time_scale: int + """The media clock rate in ticks per second.""" + access_points: tuple[H264AccessPoint, ...] + """IDR points with sample and presentation indices, DTS and PTS; empty unless timing is verified CFR.""" class SpliceError(RuntimeError): @@ -148,7 +161,7 @@ def read_h264_layout(path: Path) -> Optional[H264Layout]: fh, find_box(fh, *edts, b"elst", max_boxes=_MAX_CHILD_BOXES), stbl, movie_timescale, media_timescale ): return None - return _read_video_stbl(fh, *stbl) + return _read_video_stbl(fh, *stbl, media_timescale) return None @@ -328,7 +341,7 @@ def _is_idr(fh: BinaryIO, offset: int, size: int, length_size: int) -> bool: return False -def _read_video_stbl(fh: BinaryIO, stbl_start: int, stbl_end: int) -> Optional[H264Layout]: +def _read_video_stbl(fh: BinaryIO, stbl_start: int, stbl_end: int, time_scale: int) -> Optional[H264Layout]: stbl = (stbl_start, stbl_end) stsd = find_box(fh, *stbl, b"stsd", max_boxes=_MAX_CHILD_BOXES) stsz = find_box(fh, *stbl, b"stsz", max_boxes=_MAX_CHILD_BOXES) @@ -407,16 +420,61 @@ def _read_video_stbl(fh: BinaryIO, stbl_start: int, stbl_end: int) -> Optional[H ) runs = _read_stts(fh, stbl) - if runs is None: + if runs is None or sum(count for count, _ in runs) != sample_count or any(delta <= 0 for _, delta in runs): return None deltas = {delta for _, delta in runs} frame_duration = deltas.pop() if len(deltas) == 1 else None + access_points: tuple[H264AccessPoint, ...] = () + if frame_duration is not None: + offsets = _read_composition_offsets(fh, stbl, sample_count) + if offsets is not None: + dts = np.arange(sample_count, dtype=np.int64) * frame_duration + pts = dts + offsets + sorted_pts = np.sort(pts) + if sample_count == 1 or np.all(np.diff(sorted_pts) == frame_duration): + access_points = tuple( + H264AccessPoint( + sample_index=k, + frame_index=int(np.searchsorted(sorted_pts, pts[k])), + dts=int(dts[k]), + pts=int(pts[k]), + ) + for k in idr_samples + ) return H264Layout( sample_count=sample_count, sync_samples=idr_samples, decoder_config=decoder_config, sample_entry=sample_entry, frame_duration=frame_duration, + time_scale=time_scale, + access_points=access_points, + ) + + +def _read_composition_offsets(fh: BinaryIO, stbl: tuple[int, int], sample_count: int) -> Optional[np.ndarray]: + """Expand the bounded ``ctts`` runs, or return zero offsets when the table is absent.""" + ctts = find_box(fh, *stbl, b"ctts", max_boxes=_MAX_CHILD_BOXES) + if ctts is None: + return np.zeros(sample_count, dtype=np.int64) + if ctts[0] + 8 > ctts[1]: + return None + fh.seek(ctts[0]) + version = fh.read(1)[0] + if version not in (0, 1): + return None + fh.seek(ctts[0] + 4) + (run_count,) = struct.unpack(">I", fh.read(4)) + if not 0 < run_count <= _MAX_SAMPLES or ctts[0] + 8 + 8 * run_count > ctts[1]: + return None + fh.seek(ctts[0] + 8) + fmt = ">II" if version == 0 else ">Ii" + runs = [struct.unpack(fmt, fh.read(8)) for _ in range(run_count)] + if any(count == 0 for count, _ in runs) or sum(count for count, _ in runs) != sample_count: + return None + return np.repeat( + np.fromiter((offset for _, offset in runs), dtype=np.int64, count=run_count), + np.fromiter((count for count, _ in runs), dtype=np.int64, count=run_count), ) diff --git a/invokeai/app/util/video_smart_concat.py b/invokeai/app/util/video_smart_concat.py new file mode 100644 index 00000000000..9ddbd00e0e4 --- /dev/null +++ b/invokeai/app/util/video_smart_concat.py @@ -0,0 +1,398 @@ +"""GOP-aligned planning and bounded bridge encoding for ``video_concat``.""" + +import math +import subprocess +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Iterable, Optional, TypeAlias + +import imageio_ffmpeg +import numpy as np + +from invokeai.app.services.session_processor.session_processor_common import CanceledException +from invokeai.app.util.h264_splice import MIN_ENCODED_FRAMES, H264Layout, SpliceError, read_h264_layout +from invokeai.app.util.video_encoding import CRF, KEYFRAME_INTERVAL_SECONDS, writer_sample_entry + +Probe = tuple[int, int, Optional[float], Optional[float]] + + +@dataclass(frozen=True) +class CopySegment: + """A source range preserved as compressed H.264 samples.""" + + clip: int + start: int + end: int + + +@dataclass(frozen=True) +class BridgeSegment: + """A single continuously encoded range joining two adjacent source clips.""" + + left_clip: int + right_clip: int + left_start: int + right_end: int + output_frames: int + + +SmartSegment: TypeAlias = CopySegment | BridgeSegment + + +@dataclass(frozen=True) +class SmartRenderPlan: + """A complete, frame-counted plan for one concatenated output timeline.""" + + layouts: tuple[H264Layout, ...] + segments: tuple[SmartSegment, ...] + frame_counts: tuple[int, ...] + output_frames: int + copied_frames: int + encoded_frames: int + bridge_count: int + + +def try_build_smart_render_plan( + paths: list[Path], + probes: list[Probe], + canvas: tuple[int, int], + output_fps: float, + transition: str, + transition_frames: int, + tail_need: int, + head_need: int, + is_canceled: Optional[Callable[[], bool]] = None, +) -> tuple[Optional[SmartRenderPlan], str]: + """Build a conservative all-input plan. Unsupported layouts return a reason for fallback.""" + if is_canceled is not None and is_canceled(): + raise CanceledException + if len(paths) != len(probes): + raise ValueError("Smart-render preflight requires one probe result per input path") + if any((width, height) != canvas for width, height, _, _ in probes): + return None, "one or more clips need resizing" + if any(fps is None or not math.isclose(fps, output_fps, rel_tol=1e-6) for _, _, _, fps in probes): + return None, "one or more clip frame rates do not match the output rate" + layouts: list[H264Layout] = [] + for index, path in enumerate(paths): + if is_canceled is not None and is_canceled(): + raise CanceledException + layout = read_h264_layout(path) + if is_canceled is not None and is_canceled(): + raise CanceledException + if layout is None or layout.frame_duration is None or not layout.access_points: + return None, f"clip {index + 1} is not a verified compatible CFR H.264 MP4" + layouts.append(layout) + + target_entry = writer_sample_entry(*canvas, output_fps) + if is_canceled is not None and is_canceled(): + raise CanceledException + if target_entry is None: + return None, "the output encoder sample entry could not be verified" + for index, (layout, (_, _, duration, _)) in enumerate(zip(layouts, probes, strict=True)): + if layout.sample_entry != target_entry or not math.isclose( + layout.time_scale / layout.frame_duration, output_fps, rel_tol=1e-6 + ): + return None, f"clip {index + 1} is not a verified compatible CFR H.264 MP4" + if not any(point.sample_index == 0 and point.frame_index == 0 for point in layout.access_points): + return None, f"clip {index + 1} does not begin at a verified IDR access point" + expected_duration = layout.sample_count / output_fps + if duration is None or abs(duration - expected_duration) > 0.5 / output_fps + 0.01: + return None, f"clip {index + 1} duration does not match its exact frame timing" + + counts = [layout.sample_count for layout in layouts] + if head_need + tail_need and any( + count < (head_need if i else 0) + (tail_need if i < len(counts) - 1 else 0) for i, count in enumerate(counts) + ): + return None, "one or more clips are too short for the requested transition" + + if tail_need == 0 and head_need == 0: + segments: list[SmartSegment] = [CopySegment(i, 0, count) for i, count in enumerate(counts)] + copied = sum(counts) + output_frames = copied + if any(count < MIN_ENCODED_FRAMES for count in counts): + return None, "a cut segment is too short to keep one continuous packet timeline" + return ( + SmartRenderPlan(tuple(layouts), tuple(segments), tuple(counts), output_frames, copied, 0, 0), + "eligible", + ) + + boundaries: list[tuple[int, int, int]] = [] + encoded_counts: list[int] = [] + for i in range(len(counts) - 1): + left = _safe_points(layouts[i]) + right = _safe_points(layouts[i + 1]) + left_limit = counts[i] - tail_need + left_candidates = [(sample, frame) for sample, frame in left if frame <= left_limit] + right_candidates = [(sample, frame) for sample, frame in right if frame >= head_need] + if not left_candidates: + left_candidates = [(0, 0)] + if not right_candidates or right_candidates[-1][1] != counts[i + 1]: + # The source edge is always a safe endpoint: no packet follows it to depend on. + right_candidates.append((counts[i + 1], counts[i + 1])) + left_candidates = sorted(set(left_candidates), key=lambda pair: pair[1], reverse=True) + right_candidates = sorted(set(right_candidates), key=lambda pair: pair[1]) + selected: Optional[tuple[int, int, int]] = None + for left_sample, left_frame in left_candidates: + for right_sample, right_frame in right_candidates: + encoded = counts[i] - left_frame + right_frame + if transition == "crossfade": + encoded -= transition_frames + if encoded >= MIN_ENCODED_FRAMES: + selected = (left_sample, right_sample, encoded) + break + if selected is not None: + break + if selected is None: + return None, f"transition {i + 1} cannot form a valid encoded bridge" + boundaries.append(selected) + encoded_counts.append(selected[2]) + + segments = [] + first_start = boundaries[0][0] + if first_start: + segments.append(CopySegment(0, 0, first_start)) + for i, (left_start, right_end, encoded) in enumerate(boundaries): + segments.append(BridgeSegment(i, i + 1, left_start, right_end, encoded)) + if i + 1 < len(boundaries): + next_start = boundaries[i + 1][0] + if right_end > next_start: + return None, f"the bridge ranges overlap around clip {i + 2}" + if right_end < next_start: + segments.append(CopySegment(i + 1, right_end, next_start)) + last_end = boundaries[-1][1] + if last_end < counts[-1]: + segments.append(CopySegment(len(counts) - 1, last_end, counts[-1])) + + copy_segments = [segment for segment in segments if isinstance(segment, CopySegment)] + if any(segment.end - segment.start < MIN_ENCODED_FRAMES for segment in copy_segments): + return None, "a copied interval is too short to preserve a continuous packet timeline" + copied = sum(segment.end - segment.start for segment in copy_segments) + encoded = sum(encoded_counts) + output_frames = sum(counts) - (transition_frames * (len(counts) - 1) if transition == "crossfade" else 0) + if copied == 0 or copied + encoded != output_frames: + return None, "the plan does not preserve an exact output frame partition" + return ( + SmartRenderPlan( + tuple(layouts), tuple(segments), tuple(counts), output_frames, copied, encoded, len(boundaries) + ), + "eligible", + ) + + +def _safe_points(layout: H264Layout) -> list[tuple[int, int]]: + """Return only IDRs whose packet and presentation indices identify the same frame.""" + return [ + (point.sample_index, point.frame_index) + for point in layout.access_points + if point.sample_index == point.frame_index + ] + + +def encode_rgb_frames( + frames: Iterable[np.ndarray], + output_path: Path, + canvas: tuple[int, int], + fps: float, + expected_frames: int, + is_canceled: Optional[Callable[[], bool]] = None, + abort_event: Optional[threading.Event] = None, + timeout: float = 600, +) -> int: + """Stream RGB frames to bounded, cancellable bundled FFmpeg H.264 encoding.""" + width, height = canvas + command = [ + imageio_ffmpeg.get_ffmpeg_exe(), + "-y", + "-loglevel", + "error", + "-f", + "rawvideo", + "-pixel_format", + "rgb24", + "-video_size", + f"{width}x{height}", + "-framerate", + format(fps, ".12g"), + "-i", + "pipe:0", + "-an", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-crf", + str(CRF), + "-g", + str(max(1, round(fps * KEYFRAME_INTERVAL_SECONDS))), + "-frames:v", + str(expected_frames), + str(output_path), + ] + if expected_frames < MIN_ENCODED_FRAMES: + raise ValueError(f"A smart-render bridge must contain at least {MIN_ENCODED_FRAMES} frames") + if is_canceled is not None and is_canceled(): + raise CanceledException + count = 0 + failures: list[BaseException] = [] + stopped = abort_event or threading.Event() + stderr_limit = 64 * 1024 + stderr_tail = bytearray() + stderr_lock = threading.Lock() + process: Optional[subprocess.Popen[bytes]] = None + feeder: Optional[threading.Thread] = None + stderr_reader: Optional[threading.Thread] = None + canceled = False + timed_out = False + + try: + try: + process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + bufsize=0, + ) + except OSError as e: + raise SpliceError(f"could not start ffmpeg to encode a transition bridge: {e}") from e + + def read_stderr() -> None: + assert process is not None and process.stderr is not None + while chunk := process.stderr.read(8192): + with stderr_lock: + stderr_tail.extend(chunk) + if len(stderr_tail) > stderr_limit: + del stderr_tail[: len(stderr_tail) - stderr_limit] + + def feed() -> None: + nonlocal count + try: + assert process is not None and process.stdin is not None + for frame in frames: + if stopped.is_set() or (is_canceled is not None and is_canceled()): + raise RuntimeError("bridge frame input stopped") + if count >= expected_frames: + raise SpliceError(f"bridge input exceeded its planned {expected_frames} frames") + if frame.shape != (height, width, 3) or frame.dtype != np.uint8: + raise ValueError( + f"Expected uint8 RGB frames at {width}x{height}, got {frame.shape} {frame.dtype}" + ) + data = memoryview(np.ascontiguousarray(frame)).cast("B") + while data: + written = process.stdin.write(data) + if written is None or written == 0: + raise BrokenPipeError("ffmpeg stopped reading bridge frames") + data = data[written:] + count += 1 + except BaseException as e: + failures.append(e) + finally: + try: + if process is not None and process.stdin is not None: + process.stdin.close() + except OSError: + pass + + stderr_reader = threading.Thread(target=read_stderr, name="invokeai-video-bridge-stderr", daemon=True) + feeder = threading.Thread(target=feed, name="invokeai-video-bridge-feed", daemon=True) + try: + stderr_reader.start() + feeder.start() + except RuntimeError as e: + raise SpliceError(f"could not start bridge worker thread: {e}") from e + + deadline = time.monotonic() + timeout + while process.poll() is None: + if is_canceled is not None and is_canceled(): + canceled = True + stopped.set() + break + if time.monotonic() >= deadline: + timed_out = True + stopped.set() + break + time.sleep(0.05) + if canceled or timed_out: + process.kill() + elif process.returncode is not None and process.returncode != 0: + stopped.set() + process.wait() + feeder.join(timeout=35) + if feeder.is_alive(): + stopped.set() + process.kill() + process.wait() + feeder.join(timeout=5) + stderr_reader.join(timeout=5) + except BaseException: + output_path.unlink(missing_ok=True) + raise + finally: + # Any exception after Popen must stop the child and both workers before it escapes. + stopped.set() + if process is not None: + try: + if process.poll() is None: + process.kill() + except BaseException: + pass + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + try: + process.kill() + except BaseException: + pass + process.wait() + except BaseException: + pass + for worker in (feeder, stderr_reader): + if worker is not None and worker.is_alive(): + worker.join(timeout=5) + if process is not None: + for stream in (process.stdin, process.stderr): + if stream is not None: + try: + stream.close() + except OSError: + pass + + diagnostic = bytes(stderr_tail).decode("utf-8", errors="replace").strip() + if feeder is not None and feeder.is_alive(): + output_path.unlink(missing_ok=True) + raise SpliceError("the bridge encoder did not stop after its input was closed") + if stderr_reader is not None and stderr_reader.is_alive(): + output_path.unlink(missing_ok=True) + raise SpliceError("the bridge encoder diagnostic reader did not stop") + if canceled or (is_canceled is not None and is_canceled()): + output_path.unlink(missing_ok=True) + raise CanceledException + if timed_out: + output_path.unlink(missing_ok=True) + raise SpliceError("ffmpeg timed out encoding a transition bridge") + assert process is not None + if process.returncode != 0: + output_path.unlink(missing_ok=True) + raise SpliceError(f"ffmpeg failed encoding a transition bridge: {diagnostic[-500:]}") + if failures: + failure = failures[0] + if isinstance(failure, CanceledException): + output_path.unlink(missing_ok=True) + raise failure + if isinstance(failure, BrokenPipeError): + output_path.unlink(missing_ok=True) + raise SpliceError(f"ffmpeg closed the transition bridge input early: {diagnostic[-500:]}") from failure + if isinstance(failure, ValueError): + output_path.unlink(missing_ok=True) + raise SpliceError(f"could not decode or prepare transition frames: {failure}") from failure + if isinstance(failure, SpliceError): + output_path.unlink(missing_ok=True) + raise failure + output_path.unlink(missing_ok=True) + raise failure + if count != expected_frames: + output_path.unlink(missing_ok=True) + raise SpliceError(f"bridge input contained {count} frames; expected {expected_frames}") + return count diff --git a/tests/app/invocations/test_video_concat.py b/tests/app/invocations/test_video_concat.py index 60dea24d53f..8345149adf3 100644 --- a/tests/app/invocations/test_video_concat.py +++ b/tests/app/invocations/test_video_concat.py @@ -270,6 +270,7 @@ def frames(path: Path, **_: object) -> Iterator[np.ndarray]: patch("invokeai.app.invocations.video_concat.extract_audio_pcm", return_value=None), # The inputs are not real files, so there is no sample table to splice from. patch("invokeai.app.invocations.video_concat.read_h264_layout", return_value=None), + patch("invokeai.app.util.video_smart_concat.read_h264_layout", return_value=None), patch("invokeai.app.invocations.video_concat.VideoOutput.build", return_value=MagicMock()), ): invocation.invoke(context) diff --git a/tests/app/invocations/test_video_concat_splice.py b/tests/app/invocations/test_video_concat_splice.py index 2bb2a80f37e..e89e84161de 100644 --- a/tests/app/invocations/test_video_concat_splice.py +++ b/tests/app/invocations/test_video_concat_splice.py @@ -1,16 +1,11 @@ -"""Joining onto a clip this app encoded must not re-encode that clip's head. +"""Verify GOP-aware smart concatenation and the inherited H.264/AAC splice contracts. -Extend Video joins each continuation onto the video it extends, so a video extended N -times used to carry its first segment through N lossy re-encodes, with artifacts -compounding in the earliest footage. The concat node now stream-copies the first clip up -to its last usable keyframe and encodes only the rest. These tests pin that the copied -frames come through bit-exact (also across repeated extensions), that the joined -timeline has the same length and audio as a full re-encode would, and that inputs which -cannot be spliced losslessly still join correctly by re-encoding. - -Real ffmpeg (the imageio-ffmpeg bundled binary) encodes and decodes throughout. +The smart-render cases assert that copied picture data bypasses the video encoder, while +the bridge cases use the existing full-timeline blending and audio behavior. Real FFmpeg +(the imageio-ffmpeg bundled binary) encodes and decodes throughout. """ +import hashlib import importlib import shutil import struct @@ -77,6 +72,23 @@ def _encode(path: Path, frames: list[np.ndarray], tone: float | None = None, leg return path +def _encode_open_gop(path: Path, frames: list[np.ndarray]) -> Path: + writer = iio2.get_writer( + str(path), + format="FFMPEG", + mode="I", + fps=FPS, + codec="libx264", + macro_block_size=1, + quality=None, + output_params=["-crf", "18", "-g", "32", "-x264-params", "open-gop=1"], + ) + for frame in frames: + writer.append_data(frame) + writer.close() + return path + + def _decode(path: Path) -> list[np.ndarray]: reader = iio2.get_reader(str(path), format="FFMPEG") try: @@ -106,6 +118,66 @@ def _dominant_freq(pcm: np.ndarray, rate: int) -> float: return float(np.fft.rfftfreq(pcm.shape[1], 1.0 / rate)[int(np.argmax(spectrum))]) +def _sample_payload_hashes(path: Path, start: int, end: int) -> list[str]: + """Hash coded picture NALs, ignoring legal parameter-set and framing changes.""" + from invokeai.app.util import h264_splice + from invokeai.app.util.mp4_metadata import find_box, iter_boxes + + with path.open("rb") as stream: + stream.seek(0, 2) + moov = find_box(stream, 0, stream.tell(), b"moov") + assert moov is not None + stbl = None + for box_type, trak_start, trak_end in iter_boxes(stream, *moov): + if box_type != b"trak": + continue + mdia = find_box(stream, trak_start, trak_end, b"mdia") + if mdia is None: + continue + hdlr = find_box(stream, *mdia, b"hdlr") + if hdlr is None: + continue + stream.seek(hdlr[0] + 8) + if stream.read(4) != b"vide": + continue + minf = find_box(stream, *mdia, b"minf") + stbl = find_box(stream, *minf, b"stbl") if minf is not None else None + break + assert stbl is not None + stsz = find_box(stream, *stbl, b"stsz") + assert stsz is not None + stream.seek(stsz[0] + 8) + sample_count = int.from_bytes(stream.read(4), "big") + stream.seek(0, 2) + file_size = stream.tell() + sizes = h264_splice._sample_sizes(stream, stsz, sample_count, file_size) + assert sizes is not None + locations = h264_splice._sample_locations(stream, stbl, sizes, tuple(range(start, end)), file_size) + assert locations is not None + layout = read_h264_layout(path) + assert layout is not None + length_size = (layout.decoder_config[4] & 0x03) + 1 + hashes = [] + for offset, size in locations: + stream.seek(offset) + sample = stream.read(size) + digest = hashlib.sha256() + position = 0 + has_picture = False + while position + length_size < len(sample): + nal_size = int.from_bytes(sample[position : position + length_size], "big") + position += length_size + nal = sample[position : position + nal_size] + assert len(nal) == nal_size + if nal and (nal[0] & 0x1F) in range(1, 6): + digest.update(nal) + has_picture = True + position += nal_size + assert position == len(sample) and has_picture + hashes.append(digest.hexdigest()) + return hashes + + def _concat(tmp_path: Path, inputs: list[Path], out_name: str, **fields: object) -> tuple[Path, MagicMock]: """Run the node on real files; returns the saved MP4 and the mocked context.""" by_name = {p.stem: p for p in inputs} @@ -147,8 +219,8 @@ def test_first_clip_head_is_copied_bit_exact(tmp_path, transition, transition_fr out, context = _concat(tmp_path, [a, b], "out", transition=transition, transition_frames=transition_frames) tail = {"crossfade": transition_frames, "fade_through_black": transition_frames // 2}.get(transition, 0) - keyframe = _last_usable_keyframe(a, tail) a_frames, out_frames = _decode(a), _decode(out) + keyframe = len(a_frames) if transition == "cut" else _last_usable_keyframe(a, tail) assert len(out_frames) == expected_len assert all(np.array_equal(x, y) for x, y in zip(a_frames[:keyframe], out_frames[:keyframe], strict=True)) assert f"Stream-copied the first {keyframe} frames" in _logged(context) @@ -388,11 +460,12 @@ def test_first_clip_with_presentation_tags_is_re_encoded(tmp_path, tag): @pytest.mark.parametrize( "stage", [ - "invokeai.app.invocations.video_concat.copy_frames", - "invokeai.app.invocations.video_concat.splice_soundtrack", + ("invokeai.app.invocations.video_concat.copy_frames", "crossfade"), + ("invokeai.app.invocations.video_concat.concat_stream_copy", "cut"), + ("invokeai.app.invocations.video_concat.splice_soundtrack", "cut"), # The audio decodes: an input's soundtrack, and the splice's own validation decodes. - "invokeai.app.invocations.video_concat.extract_audio_pcm", - "invokeai.app.util.aac_splice.extract_audio_pcm", + ("invokeai.app.invocations.video_concat.extract_audio_pcm", "cut"), + ("invokeai.app.util.aac_splice.extract_audio_pcm", "cut"), ], ) def test_cancellation_during_a_splice_saves_nothing(tmp_path, stage): @@ -402,6 +475,7 @@ def test_cancellation_during_a_splice_saves_nothing(tmp_path, stage): b = _encode(tmp_path / "b.mp4", _frames(40, seed=2), tone=660.0) canceled = [False] raised: list[BaseException] = [] + stage, transition = stage module, name = stage.rsplit(".", 1) real = getattr(importlib.import_module(module), name) @@ -416,7 +490,11 @@ def cancel_then_run(*args, **kwargs): context = MagicMock() context.videos.get_path.side_effect = lambda name: {"a": a, "b": b}[name] context.util.is_canceled.side_effect = lambda: canceled[0] - node = VideoConcatInvocation(videos=[VideoField(video_name="a"), VideoField(video_name="b")], transition="cut") + node = VideoConcatInvocation( + videos=[VideoField(video_name="a"), VideoField(video_name="b")], + transition=transition, + transition_frames=2 if transition == "crossfade" else 0, + ) with patch(stage, side_effect=cancel_then_run): with pytest.raises(CanceledException): node.invoke(context) @@ -441,3 +519,374 @@ def test_tkhd_display_size_does_not_leak_onto_the_join(tmp_path): layout = read_h264_layout(out) assert layout is not None and b"pasp" not in layout.sample_entry assert len(_decode(out)) == 140 + + +def test_smart_render_false_disables_video_packet_copy(tmp_path): + a = _encode(tmp_path / "a.mp4", _frames(100, seed=1)) + b = _encode(tmp_path / "b.mp4", _frames(40, seed=2)) + + out, context = _concat(tmp_path, [a, b], "full", transition="cut", transition_frames=0, smart_render=False) + + assert len(_decode(out)) == 140 + assert "Stream-copied" not in _logged(context) + + +def test_smart_render_defaults_to_true(): + node = VideoConcatInvocation(videos=[VideoField(video_name="a"), VideoField(video_name="b")]) + assert node.smart_render is True + + +def test_smart_render_preserves_safe_regions_on_both_sides_of_transition(tmp_path, monkeypatch): + a = _encode(tmp_path / "a.mp4", _frames(100, seed=21)) + b = _encode(tmp_path / "b.mp4", _frames(100, seed=22)) + a_layout, b_layout = read_h264_layout(a), read_h264_layout(b) + assert a_layout is not None and b_layout is not None + a_boundary = max(k for k in a_layout.sync_samples if k <= a_layout.sample_count - 2) + b_boundary = min(k for k in b_layout.sync_samples if k >= 2) + bridge_frames = a_layout.sample_count - a_boundary + b_boundary - 2 + encoded_counts: list[int] = [] + video_concat = __import__("invokeai.app.invocations.video_concat", fromlist=["encode_rgb_frames"]) + encode_frames = video_concat.encode_rgb_frames + + def count_encoded_frames(frames, *args, **kwargs): + count = 0 + + def counted_frames(): + nonlocal count + for frame in frames: + count += 1 + yield frame + + encoded = encode_frames(counted_frames(), *args, **kwargs) + encoded_counts.append(count) + return encoded + + monkeypatch.setattr(video_concat, "encode_rgb_frames", count_encoded_frames) + out, context = _concat(tmp_path, [a, b], "smart", transition="crossfade", transition_frames=2) + + a_frames, b_frames, output_frames = _decode(a), _decode(b), _decode(out) + expected_frames = len(a_frames) + len(b_frames) - 2 + assert len(output_frames) == expected_frames + assert _sample_payload_hashes(a, 0, a_boundary) == _sample_payload_hashes(out, 0, a_boundary) + b_copy_frames = len(b_frames) - b_boundary + assert _sample_payload_hashes(b, b_boundary, len(b_frames)) == _sample_payload_hashes( + out, expected_frames - b_copy_frames, expected_frames + ) + assert all(np.array_equal(x, y) for x, y in zip(a_frames[:a_boundary], output_frames[:a_boundary], strict=True)) + assert all( + np.array_equal(x, y) + for x, y in zip(b_frames[b_boundary:], output_frames[-(len(b_frames) - b_boundary) :], strict=True) + ) + copied_frames = a_boundary + len(b_frames) - b_boundary + assert encoded_counts == [bridge_frames] + assert f"copying {copied_frames}, encoding {bridge_frames} across 1 bridge" in _logged(context) + + +@pytest.mark.parametrize( + "transition,transition_frames", + [("crossfade", 8), ("fade_through_black", 4), ("fade_through_black", 5)], + ids=["crossfade", "fade-even", "fade-odd"], +) +def test_smart_render_transition_values_match_full_encoding(tmp_path, transition, transition_frames): + a = _encode(tmp_path / "a.mp4", _frames(100, seed=71)) + b = _encode(tmp_path / "b.mp4", _frames(100, seed=72)) + + smart, smart_context = _concat( + tmp_path, [a, b], "smart", transition=transition, transition_frames=transition_frames + ) + full, _ = _concat( + tmp_path, + [a, b], + "full", + transition=transition, + transition_frames=transition_frames, + smart_render=False, + ) + + smart_log = _logged(smart_context) + assert "Smart rendering selected" in smart_log + assert "falling back to full video encoding" not in smart_log + assert "Stream-copied" in smart_log + a_frames, b_frames = _decode(a), _decode(b) + smart_frames, full_frames = _decode(smart), _decode(full) + tail_frames = transition_frames if transition == "crossfade" else transition_frames // 2 + head_frames = transition_frames if transition == "crossfade" else transition_frames - tail_frames + transition_start = 100 - tail_frames + transition_end = transition_start + transition_frames + assert len(smart_frames) == len(full_frames) + + if transition == "crossfade": + expected_frames = [ + np.clip( + a_frames[100 - tail_frames + i].astype(np.float32) * (1 - (i + 1) / (transition_frames + 1)) + + b_frames[i].astype(np.float32) * ((i + 1) / (transition_frames + 1)), + 0, + 255, + ).astype(np.uint8) + for i in range(transition_frames) + ] + else: + expected_frames = [ + np.clip( + a_frames[100 - tail_frames + i].astype(np.float32) * (1 - (i + 1) / (tail_frames + 1)), 0, 255 + ).astype(np.uint8) + for i in range(tail_frames) + ] + [ + np.clip( + b_frames[i].astype(np.float32) * ((i + 1) / (head_frames + 1)), + 0, + 255, + ).astype(np.uint8) + for i in range(head_frames) + ] + + # The smart bridge has different GOP context, so allow codec noise while requiring the + # decoded transition to match both the existing full-encode path and independent math. + full_encode_errors = [ + float(np.abs(actual.astype(np.int16) - expected.astype(np.int16)).mean()) + for actual, expected in zip( + full_frames[transition_start:transition_end], + expected_frames, + strict=True, + ) + ] + smart_render_errors = [ + float(np.abs(actual.astype(np.int16) - expected.astype(np.int16)).mean()) + for actual, expected in zip( + smart_frames[transition_start:transition_end], + full_frames[transition_start:transition_end], + strict=True, + ) + ] + assert len(full_encode_errors) == len(smart_render_errors) == transition_frames + assert max(full_encode_errors) < 8.0 + assert max(smart_render_errors) < 8.0 + + +def test_smart_render_copies_intermediate_clip_gap_between_bridges(tmp_path): + sources = [ + _encode(tmp_path / f"{name}.mp4", _frames(100, seed=seed)) + for name, seed in zip("abc", (31, 32, 33), strict=True) + ] + layouts = [read_h264_layout(path) for path in sources] + assert all(layout is not None for layout in layouts) + a0 = max(k for k in layouts[0].sync_samples if k <= 98) + b1 = min(k for k in layouts[1].sync_samples if k >= 2) + a1 = max(k for k in layouts[1].sync_samples if k <= 98) + b2 = min(k for k in layouts[2].sync_samples if k >= 2) + out, context = _concat(tmp_path, sources, "three", transition="crossfade", transition_frames=2) + + output_frames = _decode(out) + first_bridge_frames = 100 - a0 + b1 - 2 + copied_middle_start = a0 + first_bridge_frames + assert len(output_frames) == 3 * 100 - 2 * 2 + middle_copy_frames = a1 - b1 + third_copy_frames = 100 - b2 + assert _sample_payload_hashes(sources[0], 0, a0) == _sample_payload_hashes(out, 0, a0) + assert _sample_payload_hashes(sources[1], b1, a1) == _sample_payload_hashes( + out, copied_middle_start, copied_middle_start + middle_copy_frames + ) + assert _sample_payload_hashes(sources[2], b2, 100) == _sample_payload_hashes( + out, len(output_frames) - third_copy_frames, len(output_frames) + ) + assert all( + np.array_equal(source, output) + for source, output in zip( + _decode(sources[1])[b1:a1], + output_frames[copied_middle_start : copied_middle_start + a1 - b1], + strict=True, + ) + ) + assert f"{a1 - b1} frames from clip 2" in _logged(context) + assert all( + np.array_equal(source, output) + for source, output in zip(_decode(sources[2])[b2:], output_frames[-third_copy_frames:], strict=True) + ) + assert f"{third_copy_frames} frames from clip 3" in _logged(context) + + +@pytest.mark.parametrize("transition_frames", [4, 5], ids=["even", "odd"]) +def test_smart_render_fade_through_black_keeps_asymmetric_transition_counts(tmp_path, transition_frames): + a = _encode(tmp_path / "a.mp4", _frames(100, seed=41)) + b = _encode(tmp_path / "b.mp4", _frames(100, seed=42)) + a_layout, b_layout = read_h264_layout(a), read_h264_layout(b) + assert a_layout is not None and b_layout is not None + tail = transition_frames // 2 + head = transition_frames - tail + a_boundary = max(k for k in a_layout.sync_samples if k <= 100 - tail) + b_boundary = min(k for k in b_layout.sync_samples if k >= head) + encoded = 100 - a_boundary + b_boundary + out, context = _concat( + tmp_path, + [a, b], + f"fade_{transition_frames}", + transition="fade_through_black", + transition_frames=transition_frames, + ) + + output_frames = _decode(out) + assert len(output_frames) == 200 + assert all(np.array_equal(x, y) for x, y in zip(_decode(a)[:a_boundary], output_frames[:a_boundary], strict=True)) + assert all( + np.array_equal(x, y) for x, y in zip(_decode(b)[b_boundary:], output_frames[-(100 - b_boundary) :], strict=True) + ) + assert f"encoding {encoded} across 1 bridge" in _logged(context) + + +@pytest.mark.parametrize( + "transition,transition_frames", + [("cut", 8), ("crossfade", 0), ("fade_through_black", 0)], +) +def test_smart_render_copy_only_joins_do_not_encode_frames(tmp_path, monkeypatch, transition, transition_frames): + a = _encode(tmp_path / "a.mp4", _frames(100, seed=51)) + b = _encode(tmp_path / "b.mp4", _frames(40, seed=52)) + + def unexpected_encode(*args, **kwargs): + raise AssertionError("copy-only smart rendering must not invoke a video encoder") + + monkeypatch.setattr("invokeai.app.invocations.video_concat.encode_rgb_frames", unexpected_encode) + out, context = _concat( + tmp_path, + [a, b], + "copy_only", + transition=transition, + transition_frames=transition_frames, + ) + + output_frames = _decode(out) + assert len(output_frames) == 140 + assert all(np.array_equal(x, y) for x, y in zip(_decode(a), output_frames[:100], strict=True)) + assert all(np.array_equal(x, y) for x, y in zip(_decode(b), output_frames[100:], strict=True)) + assert "copying 140, encoding 0 across 0 bridge" in _logged(context) + + +def test_failed_smart_output_validation_falls_back_before_saving(tmp_path, monkeypatch): + from invokeai.app.util.h264_splice import SpliceError + + a = _encode(tmp_path / "a.mp4", _frames(100, seed=61)) + b = _encode(tmp_path / "b.mp4", _frames(100, seed=62)) + validate = VideoConcatInvocation._validate_smart_part + + def fail_joined_output(self, path, *args, **kwargs): + if path.name == "smart_joined.mp4": + raise SpliceError("test rejected assembled timestamps") + return validate(self, path, *args, **kwargs) + + monkeypatch.setattr(VideoConcatInvocation, "_validate_smart_part", fail_joined_output) + out, context = _concat(tmp_path, [a, b], "validated", transition="crossfade", transition_frames=2) + + assert len(_decode(out)) == 198 + assert "falling back to full video encoding: test rejected assembled timestamps" in _logged(context) + assert context.videos.save.call_count == 1 + + +@pytest.mark.parametrize("open_side", ["left", "right"]) +def test_smart_render_extends_bridge_to_source_edge_when_no_interior_idr(tmp_path, open_side): + normal = _encode(tmp_path / "normal.mp4", _frames(100, seed=71)) + open_gop = _encode_open_gop(tmp_path / "open.mp4", _frames(100, seed=72)) + assert read_h264_layout(open_gop).sync_samples == (0,) + a, b = (open_gop, normal) if open_side == "left" else (normal, open_gop) + a_layout, b_layout = read_h264_layout(a), read_h264_layout(b) + assert a_layout is not None and b_layout is not None + a_boundary = 0 if open_side == "left" else max(k for k in a_layout.sync_samples if k <= 98) + b_boundary = min(k for k in b_layout.sync_samples if k >= 2) if open_side == "left" else 100 + encoded = 100 - a_boundary + b_boundary - 2 + copied = a_boundary + 100 - b_boundary + + out, context = _concat(tmp_path, [a, b], "edge", transition="crossfade", transition_frames=2) + + output_frames = _decode(out) + assert len(output_frames) == 198 + if a_boundary: + assert all( + np.array_equal(x, y) for x, y in zip(_decode(a)[:a_boundary], output_frames[:a_boundary], strict=True) + ) + if b_boundary < 100: + assert all( + np.array_equal(x, y) + for x, y in zip(_decode(b)[b_boundary:], output_frames[-(100 - b_boundary) :], strict=True) + ) + assert f"copying {copied}, encoding {encoded} across 1 bridge" in _logged(context) + + +def test_overlapping_intermediate_bridges_fall_back_to_full_encoding(tmp_path): + clips = [ + _encode(tmp_path / "a.mp4", _frames(100, seed=81)), + _encode(tmp_path / "b.mp4", _frames(40, seed=82)), + _encode(tmp_path / "c.mp4", _frames(100, seed=83)), + ] + out, context = _concat(tmp_path, clips, "overlap", transition="crossfade", transition_frames=16) + + assert len(_decode(out)) == 100 + 40 + 100 - 2 * 16 + assert "overlap around clip 2" in _logged(context) + assert "falling back to full video encoding" in _logged(context) + + +def test_bridge_ffmpeg_failure_falls_back_to_full_encoding(tmp_path, monkeypatch): + from invokeai.app.util.h264_splice import SpliceError + + a = _encode(tmp_path / "a.mp4", _frames(100, seed=91)) + b = _encode(tmp_path / "b.mp4", _frames(100, seed=92)) + + def fail_bridge(*args, **kwargs): + raise SpliceError("test bridge encoder failure") + + monkeypatch.setattr("invokeai.app.invocations.video_concat.encode_rgb_frames", fail_bridge) + out, context = _concat(tmp_path, [a, b], "ffmpeg_failure", transition="crossfade", transition_frames=2) + + assert len(_decode(out)) == 198 + assert "falling back to full video encoding: test bridge encoder failure" in _logged(context) + + +def test_bridge_timeout_falls_back_to_full_encoding(tmp_path, monkeypatch): + a = _encode(tmp_path / "a.mp4", _frames(100, seed=101)) + b = _encode(tmp_path / "b.mp4", _frames(100, seed=102)) + + def timeout_bridge(*args, **kwargs): + raise TimeoutError("test bridge decoder timeout") + + monkeypatch.setattr("invokeai.app.invocations.video_concat.encode_rgb_frames", timeout_bridge) + out, context = _concat(tmp_path, [a, b], "bridge_timeout", transition="crossfade", transition_frames=2) + + assert len(_decode(out)) == 198 + assert "falling back to full video encoding: test bridge decoder timeout" in _logged(context) + + +def test_internal_bridge_abort_is_not_reported_as_user_cancellation(tmp_path, monkeypatch): + from invokeai.app.util.h264_splice import SpliceError + + a = _encode(tmp_path / "a.mp4", _frames(100, seed=111)) + b = _encode(tmp_path / "b.mp4", _frames(100, seed=112)) + + def fail_after_internal_abort(frames, output_path, canvas, fps, expected_frames, is_canceled, abort_event): + del frames, output_path, canvas, fps, expected_frames + abort_event.set() + assert not is_canceled(), "an internal encoder abort must not appear as user cancellation" + raise SpliceError("simulated bridge encoder timeout") + + monkeypatch.setattr("invokeai.app.invocations.video_concat.encode_rgb_frames", fail_after_internal_abort) + out, context = _concat(tmp_path, [a, b], "internal_abort", transition="crossfade", transition_frames=2) + + assert len(_decode(out)) == 198 + assert "falling back to full video encoding: simulated bridge encoder timeout" in _logged(context) + + +def test_bridge_source_decode_failure_falls_back_to_full_encoding(tmp_path, monkeypatch): + a = _encode(tmp_path / "a.mp4", _frames(100, seed=121)) + b = _encode(tmp_path / "b.mp4", _frames(100, seed=122)) + module = __import__("invokeai.app.invocations.video_concat", fromlist=["encode_rgb_frames"]) + encode = module.encode_rgb_frames + + def decode_failure(frames, output_path, canvas, fps, expected_frames, **kwargs): + def broken_frames(): + yield next(iter(frames)) + raise ValueError("simulated bridge decoder failure") + + return encode(broken_frames(), output_path, canvas, fps, expected_frames, **kwargs) + + monkeypatch.setattr(module, "encode_rgb_frames", decode_failure) + out, context = _concat(tmp_path, [a, b], "decoder_failure", transition="crossfade", transition_frames=2) + + assert len(_decode(out)) == 198 + assert "falling back to full video encoding" in _logged(context) + assert "simulated bridge decoder failure" in _logged(context) diff --git a/tests/app/util/test_h264_splice.py b/tests/app/util/test_h264_splice.py index 1ad8ae25753..f95ac0415f1 100644 --- a/tests/app/util/test_h264_splice.py +++ b/tests/app/util/test_h264_splice.py @@ -62,6 +62,10 @@ def test_own_output_lists_exactly_the_decoded_frames(tmp_path: Path) -> None: assert layout.sample_count == _decoded_count(path) == 70 assert layout.sync_samples[0] == 0 and len(layout.sync_samples) > 1 assert layout.sample_entry == writer_sample_entry(96, 64, FPS) + assert layout.time_scale / layout.frame_duration == FPS + assert [point.sample_index for point in layout.access_points] == list(layout.sync_samples) + assert [point.frame_index for point in layout.access_points] == list(layout.sync_samples) + assert all(point.pts - point.dts == 2 * layout.frame_duration for point in layout.access_points) def test_edit_list_trim_is_refused(tmp_path: Path) -> None: diff --git a/tests/app/util/test_video_smart_concat.py b/tests/app/util/test_video_smart_concat.py new file mode 100644 index 00000000000..5dd014711f6 --- /dev/null +++ b/tests/app/util/test_video_smart_concat.py @@ -0,0 +1,165 @@ +"""Bounded bridge encoding and timeline planning for smart video concatenation.""" + +import io +import threading +from pathlib import Path +from unittest.mock import patch + +import numpy as np +import pytest + +from invokeai.app.services.session_processor.session_processor_common import CanceledException +from invokeai.app.util.h264_splice import SpliceError +from invokeai.app.util.video_smart_concat import encode_rgb_frames + + +def test_bridge_encoder_cancellation_reaps_ffmpeg_and_discards_partial_output(tmp_path: Path) -> None: + output = tmp_path / "canceled.mp4" + canceled = [False] + frame = np.zeros((64, 64, 3), dtype=np.uint8) + + def frames(): + yield frame + canceled[0] = True + yield frame + + with pytest.raises(CanceledException): + encode_rgb_frames(frames(), output, (64, 64), 16.0, 8, is_canceled=lambda: canceled[0]) + + assert not output.exists() + + +def test_bridge_encoder_timeout_stops_a_blocked_frame_source(tmp_path: Path) -> None: + output = tmp_path / "timed-out.mp4" + stop = threading.Event() + frame = np.zeros((64, 64, 3), dtype=np.uint8) + + def frames(): + stop.wait() + yield frame + + with pytest.raises(SpliceError, match="timed out"): + encode_rgb_frames(frames(), output, (64, 64), 16.0, 8, abort_event=stop, timeout=0.05) + + assert stop.is_set() + assert not output.exists() + + +def test_bridge_encoder_reports_process_start_failure(tmp_path: Path) -> None: + output = tmp_path / "no-ffmpeg.mp4" + with patch("invokeai.app.util.video_smart_concat.subprocess.Popen", side_effect=OSError("not found")): + with pytest.raises(SpliceError, match="could not start ffmpeg"): + encode_rgb_frames([], output, (64, 64), 16.0, 8) + + assert not output.exists() + + +def test_bridge_encoder_does_not_treat_internal_abort_as_user_cancellation(tmp_path: Path) -> None: + output = tmp_path / "internal-abort.mp4" + abort = threading.Event() + user_canceled = threading.Event() + frame = np.zeros((64, 64, 3), dtype=np.uint8) + + def frames(): + abort.wait() + yield frame + + with pytest.raises(SpliceError, match="timed out"): + encode_rgb_frames( + frames(), + output, + (64, 64), + 16.0, + 8, + is_canceled=user_canceled.is_set, + abort_event=abort, + timeout=0.05, + ) + + assert abort.is_set() + assert not output.exists() + + +def test_bridge_source_value_error_is_a_fallback_eligible_splice_error(tmp_path: Path) -> None: + output = tmp_path / "decoder-error.mp4" + frame = np.zeros((64, 64, 3), dtype=np.uint8) + + def frames(): + yield frame + raise ValueError("source decoder failed") + + with pytest.raises(SpliceError, match="source decoder failed"): + encode_rgb_frames(frames(), output, (64, 64), 16.0, 8) + + assert not output.exists() + + +class _FakeProcess: + def __init__(self) -> None: + self.stdin = io.BytesIO() + self.stderr = io.BytesIO(b"bounded diagnostic test") + self.returncode = None + self.waited = False + + def poll(self): + return self.returncode + + def kill(self) -> None: + self.returncode = -9 + + def wait(self, timeout=None): + del timeout + self.waited = True + if self.returncode is None: + self.returncode = 0 + return self.returncode + + +def test_bridge_encoder_reaps_process_if_worker_thread_cannot_start(tmp_path: Path, monkeypatch) -> None: + output = tmp_path / "worker-start-failure.mp4" + process = _FakeProcess() + monkeypatch.setattr("invokeai.app.util.video_smart_concat.subprocess.Popen", lambda *args, **kwargs: process) + monkeypatch.setattr( + "invokeai.app.util.video_smart_concat.threading.Thread.start", + lambda self: (_ for _ in ()).throw(RuntimeError("thread limit")), + ) + + with pytest.raises(SpliceError, match="could not start bridge worker"): + encode_rgb_frames([], output, (64, 64), 16.0, 8) + + assert process.waited + assert process.returncode == -9 + assert not output.exists() + + +def test_bridge_encoder_reaps_process_and_feeder_if_cancel_callback_raises(tmp_path: Path, monkeypatch) -> None: + output = tmp_path / "callback-failure.mp4" + process = _FakeProcess() + monkeypatch.setattr("invokeai.app.util.video_smart_concat.subprocess.Popen", lambda *args, **kwargs: process) + source_started = threading.Event() + abort = threading.Event() + checks = 0 + + def frames(): + source_started.set() + abort.wait() + if not abort.is_set(): + yield np.zeros((64, 64, 3), dtype=np.uint8) + + def callback() -> bool: + nonlocal checks + checks += 1 + if checks == 1: + return False + assert source_started.wait(timeout=1) + raise RuntimeError("cancellation callback failed") + + with pytest.raises(RuntimeError, match="cancellation callback failed"): + encode_rgb_frames(frames(), output, (64, 64), 16.0, 8, is_canceled=callback, abort_event=abort) + + assert process.waited + assert process.returncode == -9 + assert not output.exists() + assert not any( + thread.name == "invokeai-video-bridge-feed" and thread.is_alive() for thread in threading.enumerate() + ) From 05cbb98ce5a0db7eb7d2dba087bce9bc1e72cd14 Mon Sep 17 00:00:00 2001 From: JPPhoto Date: Wed, 14 Oct 2026 20:20:18 -0500 Subject: [PATCH 2/2] fix(video): sync smart-render contracts Regenerate OpenAPI and TypeScript schemas for video_concat 1.3.0. Update bundled workflows with the version and default-enabled input. --- .../default_workflows/Extend Video - LTX-2.json | 8 +++++++- .../default_workflows/Extend Video - MiniMax H3.json | 8 +++++++- ...d Video - Wan 2.2 Lightning w_ Concept LoRAs.json | 8 +++++++- .../Extend Video - Wan 2.2 Lightning.json | 8 +++++++- .../Extend Video to Image - LTX-2.json | 8 +++++++- .../Extend Video to Image - Wan 2.2 Lightning.json | 8 +++++++- invokeai/frontend/api/openapi.json | 12 +++++++++++- invokeai/frontend/api/schema.ts | 6 ++++++ 8 files changed, 59 insertions(+), 7 deletions(-) diff --git a/invokeai/app/services/workflow_records/default_workflows/Extend Video - LTX-2.json b/invokeai/app/services/workflow_records/default_workflows/Extend Video - LTX-2.json index 9ade5ebb47a..8c672763de6 100644 --- a/invokeai/app/services/workflow_records/default_workflows/Extend Video - LTX-2.json +++ b/invokeai/app/services/workflow_records/default_workflows/Extend Video - LTX-2.json @@ -458,7 +458,7 @@ "type": "invocation", "data": { "id": "video_output", - "version": "1.2.0", + "version": "1.3.0", "nodePack": "invokeai", "label": "", "notes": "Joins source + extension. `transition_frames` arrives over an edge from context_frames -- the crossfade must consume exactly the held frames or they play twice.", @@ -493,6 +493,12 @@ "label": "", "description": "", "value": "match_first" + }, + "smart_render": { + "name": "smart_render", + "label": "", + "description": "", + "value": true } }, "isOpen": true, diff --git a/invokeai/app/services/workflow_records/default_workflows/Extend Video - MiniMax H3.json b/invokeai/app/services/workflow_records/default_workflows/Extend Video - MiniMax H3.json index fd92249f9c5..4dbce27faec 100644 --- a/invokeai/app/services/workflow_records/default_workflows/Extend Video - MiniMax H3.json +++ b/invokeai/app/services/workflow_records/default_workflows/Extend Video - MiniMax H3.json @@ -628,7 +628,7 @@ "type": "invocation", "data": { "id": "4b32c22b-679d-4d5e-a167-c3309265fcd9", - "version": "1.2.0", + "version": "1.3.0", "nodePack": "invokeai", "label": "", "notes": "Joins the trimmed source and the generated continuation, cross-fading over the hand-off frame so the seam is not visible.", @@ -657,6 +657,12 @@ "label": "", "description": "", "value": "match_first" + }, + "smart_render": { + "name": "smart_render", + "label": "", + "description": "", + "value": true } }, "isOpen": true, diff --git a/invokeai/app/services/workflow_records/default_workflows/Extend Video - Wan 2.2 Lightning w_ Concept LoRAs.json b/invokeai/app/services/workflow_records/default_workflows/Extend Video - Wan 2.2 Lightning w_ Concept LoRAs.json index c5c783c9fb6..dca5f9b4af6 100644 --- a/invokeai/app/services/workflow_records/default_workflows/Extend Video - Wan 2.2 Lightning w_ Concept LoRAs.json +++ b/invokeai/app/services/workflow_records/default_workflows/Extend Video - Wan 2.2 Lightning w_ Concept LoRAs.json @@ -570,7 +570,7 @@ "type": "invocation", "data": { "id": "4b32c22b-679d-4d5e-a167-c3309265fcd9", - "version": "1.2.0", + "version": "1.3.0", "nodePack": "invokeai", "label": "", "notes": "Joins the source clip and the generated continuation. Size Mismatch is 'match_first', so the generated clip is resampled onto the source clip's canvas -- the model renders onto its own fixed canvas family and will rarely match your footage, and without this the join fails outright. Your source keeps its own resolution; the generated segment is the part that gets rescaled.", @@ -615,6 +615,12 @@ "label": "", "description": "", "value": "match_first" + }, + "smart_render": { + "name": "smart_render", + "label": "", + "description": "", + "value": true } }, "dynamicInputTemplates": {}, diff --git a/invokeai/app/services/workflow_records/default_workflows/Extend Video - Wan 2.2 Lightning.json b/invokeai/app/services/workflow_records/default_workflows/Extend Video - Wan 2.2 Lightning.json index ed115aa6b76..78521df385e 100644 --- a/invokeai/app/services/workflow_records/default_workflows/Extend Video - Wan 2.2 Lightning.json +++ b/invokeai/app/services/workflow_records/default_workflows/Extend Video - Wan 2.2 Lightning.json @@ -564,7 +564,7 @@ "type": "invocation", "data": { "id": "4b32c22b-679d-4d5e-a167-c3309265fcd9", - "version": "1.2.0", + "version": "1.3.0", "nodePack": "invokeai", "label": "", "notes": "Joins the source clip and the generated continuation. Size Mismatch is 'match_first', so the generated clip is resampled onto the source clip's canvas -- the model renders onto its own fixed canvas family and will rarely match your footage, and without this the join fails outright. Your source keeps its own resolution; the generated segment is the part that gets rescaled.", @@ -609,6 +609,12 @@ "label": "", "description": "", "value": "match_first" + }, + "smart_render": { + "name": "smart_render", + "label": "", + "description": "", + "value": true } }, "isOpen": true, diff --git a/invokeai/app/services/workflow_records/default_workflows/Extend Video to Image - LTX-2.json b/invokeai/app/services/workflow_records/default_workflows/Extend Video to Image - LTX-2.json index 701b2d342e5..5a721fc99bd 100644 --- a/invokeai/app/services/workflow_records/default_workflows/Extend Video to Image - LTX-2.json +++ b/invokeai/app/services/workflow_records/default_workflows/Extend Video to Image - LTX-2.json @@ -462,7 +462,7 @@ "type": "invocation", "data": { "id": "video_output", - "version": "1.2.0", + "version": "1.3.0", "nodePack": "invokeai", "label": "", "notes": "Joins source + extension. `transition_frames` arrives over an edge from context_frames -- the crossfade must consume exactly the held frames or they play twice.", @@ -497,6 +497,12 @@ "label": "", "description": "", "value": "match_first" + }, + "smart_render": { + "name": "smart_render", + "label": "", + "description": "", + "value": true } }, "isOpen": true, diff --git a/invokeai/app/services/workflow_records/default_workflows/Extend Video to Image - Wan 2.2 Lightning.json b/invokeai/app/services/workflow_records/default_workflows/Extend Video to Image - Wan 2.2 Lightning.json index 6c58d083d11..e464845cbf7 100644 --- a/invokeai/app/services/workflow_records/default_workflows/Extend Video to Image - Wan 2.2 Lightning.json +++ b/invokeai/app/services/workflow_records/default_workflows/Extend Video to Image - Wan 2.2 Lightning.json @@ -1062,7 +1062,7 @@ "type": "invocation", "data": { "id": "4b32c22b-679d-4d5e-a167-c3309265fcd9", - "version": "1.2.0", + "version": "1.3.0", "nodePack": "invokeai", "label": "", "notes": "Joins the source clip and the generated continuation. Size Mismatch is 'match_first', so the generated clip is resampled onto the source clip's canvas -- the model renders onto its own fixed canvas family and will rarely match your footage, and without this the join fails outright. Your source keeps its own resolution; the generated segment is the part that gets rescaled.", @@ -1107,6 +1107,12 @@ "label": "", "description": "", "value": "match_first" + }, + "smart_render": { + "name": "smart_render", + "label": "", + "description": "", + "value": true } }, "isOpen": true, diff --git a/invokeai/frontend/api/openapi.json b/invokeai/frontend/api/openapi.json index a78b1e299b2..059dae38ac9 100644 --- a/invokeai/frontend/api/openapi.json +++ b/invokeai/frontend/api/openapi.json @@ -110925,6 +110925,16 @@ "title": "Size Mismatch", "type": "string" }, + "smart_render": { + "default": true, + "description": "Preserve original video encoding outside transitions when possible. Automatically falls back to full encoding for incompatible inputs.", + "field_kind": "input", + "input": "any", + "orig_default": true, + "orig_required": false, + "title": "Smart Render", + "type": "boolean" + }, "type": { "const": "video_concat", "default": "video_concat", @@ -110937,7 +110947,7 @@ "tags": ["video", "concat", "transition"], "title": "Concatenate Videos", "type": "object", - "version": "1.2.0", + "version": "1.3.0", "output": { "$ref": "#/components/schemas/VideoOutput" } diff --git a/invokeai/frontend/api/schema.ts b/invokeai/frontend/api/schema.ts index 1c635d6f3c3..70da0658f3f 100644 --- a/invokeai/frontend/api/schema.ts +++ b/invokeai/frontend/api/schema.ts @@ -49522,6 +49522,12 @@ export type components = { * @enum {string} */ size_mismatch?: "error" | "match_first"; + /** + * Smart Render + * @description Preserve original video encoding outside transitions when possible. Automatically falls back to full encoding for incompatible inputs. + * @default true + */ + smart_render?: boolean; /** * type * @default video_concat