From 76fd3cc0c3d73089b0760c3501bff0cd43365dc9 Mon Sep 17 00:00:00 2001 From: Santhi Prakash Date: Thu, 13 Aug 2026 05:16:04 +0000 Subject: [PATCH] fix(engine): warn when a video input declares alpha but decodes opaque - Problem: a video input whose alpha_mode=1 tag outlives its alpha plane (a remux drops the BlockAdditional sidecar while keeping the tag) composites as a solid rectangle, and nothing tells the user their file, not the renderer, is the problem (#3220 / #3226). - Fix: during extraction preflight, when an input declares alpha (hasAlpha) and its codec can carry alpha, sample the decoded alpha plane (up to 3 frames at 8x8 rgba, bounded 30s, ~768-byte ceiling). If uniformly opaque, emit a non-blocking stderr warning naming the file with the re-export remedy. Inconclusive probes stay silent; opaque full-frame backgrounds are legitimate and never fail the render. - Verification: 8 new unit tests (byte logic + message); engine suite 1483/1487 with 4 pre-existing failures on clean main (missing hdr-regression PNG fixture in this clone); typecheck/lint/format clean; real-fixture checks: lying-tag WebM warns, genuine-alpha WebM stays silent, both agree with ffmpeg alphaextract. Signed-off-by: Santhi Prakash --- .../src/services/videoFrameExtractor.ts | 26 ++++ .../engine/src/utils/alphaPlaneProbe.test.ts | 60 ++++++++ packages/engine/src/utils/alphaPlaneProbe.ts | 130 ++++++++++++++++++ 3 files changed, 216 insertions(+) create mode 100644 packages/engine/src/utils/alphaPlaneProbe.test.ts create mode 100644 packages/engine/src/utils/alphaPlaneProbe.ts diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index 1ecc62d4eb..eb12117374 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -23,6 +23,7 @@ import { extractMediaMetadata, type VideoMetadata, } from "../utils/ffprobe.js"; +import { inputAlphaOpaqueWarning, probeInputAlphaPlane } from "../utils/alphaPlaneProbe.js"; import { analyzeCompositionHdr, isHdrColorSpace as isHdrColorSpaceUtil, @@ -1555,6 +1556,31 @@ export async function extractAllVideoFrames( const sdrToHdrTransfers: Array = resolvedVideos.map(() => undefined); breakdown.hdrProbeMs = Date.now() - phase2ProbeStart; + // Phase 2a: warn when a video input declares alpha but its decoded alpha + // plane is uniformly opaque. `alpha_mode=1` is container metadata that can + // outlive the alpha it describes — a remux can drop the BlockAdditional + // sidecar while keeping the tag — so a file can promise transparency it no + // longer contains and then composite as a solid rectangle with no way for + // the user to know their file, not the renderer, is the problem + // (heygen-com/hyperframes#3220 / #3226). Warning only, never an error: an + // opaque video used as a full-frame background is legitimate, and an + // inconclusive probe stays silent. + const alphaWarnedSrcs = new Set(); + if (resolvedVideos.length > 0) { + await Promise.all( + resolvedVideos.map(async ({ video, videoPath }, index) => { + if (signal?.aborted) return; + const metadata = videoMetadata[index]; + if (!metadata?.hasAlpha || !codecMayHaveAlpha(metadata.videoCodec)) return; + if (alphaWarnedSrcs.has(video.src)) return; + const opaque = await probeInputAlphaPlane(videoPath); + if (opaque !== true) return; + alphaWarnedSrcs.add(video.src); + process.stderr.write(inputAlphaOpaqueWarning(video.src)); + }), + ); + } + const hdrPreflightStart = Date.now(); const hdrInfo = analyzeCompositionHdr(videoColorSpaces); // Track entries the HDR preflight validated as non-extractable so they can diff --git a/packages/engine/src/utils/alphaPlaneProbe.test.ts b/packages/engine/src/utils/alphaPlaneProbe.test.ts new file mode 100644 index 0000000000..4d5b1f0946 --- /dev/null +++ b/packages/engine/src/utils/alphaPlaneProbe.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; +import { inputAlphaOpaqueWarning, sampledRgbaAlphaIsFullyOpaque } from "./alphaPlaneProbe.js"; + +const BYTES_PER_FRAME = 8 * 8 * 4; // 256 — one 8x8 rgba frame + +/** Build a raw rgba sample where the alpha byte of every pixel is `alpha`. */ +function sample(alpha: number, frameCount: number): Buffer { + const buf = Buffer.alloc(BYTES_PER_FRAME * frameCount, 0); + for (let i = 3; i < buf.length; i += 4) { + buf[i] = alpha; + } + return buf; +} + +describe("sampledRgbaAlphaIsFullyOpaque", () => { + it("returns true when every alpha byte in a single frame is 255", () => { + expect(sampledRgbaAlphaIsFullyOpaque(sample(255, 1))).toBe(true); + }); + + it("returns true across a multi-frame sample (2 and 3 frames)", () => { + expect(sampledRgbaAlphaIsFullyOpaque(sample(255, 2))).toBe(true); + expect(sampledRgbaAlphaIsFullyOpaque(sample(255, 3))).toBe(true); + }); + + it("returns false when any pixel shows full transparency", () => { + const buf = sample(255, 1); + buf[3] = 0; // first pixel's alpha + expect(sampledRgbaAlphaIsFullyOpaque(buf)).toBe(false); + }); + + it("returns false when any pixel shows partial transparency", () => { + const buf = sample(255, 2); + buf[3 + 4 * 10] = 254; // partial alpha on the 11th pixel + expect(sampledRgbaAlphaIsFullyOpaque(buf)).toBe(false); + }); + + it("returns undefined for an empty sample (inconclusive, not a warning)", () => { + expect(sampledRgbaAlphaIsFullyOpaque(Buffer.alloc(0))).toBeUndefined(); + }); + + it("returns undefined for a byte count that is not a whole-frame multiple", () => { + expect(sampledRgbaAlphaIsFullyOpaque(Buffer.alloc(BYTES_PER_FRAME - 1))).toBeUndefined(); + }); + + it("returns undefined for an oversized sample beyond the 3-frame ceiling", () => { + expect(sampledRgbaAlphaIsFullyOpaque(Buffer.alloc(BYTES_PER_FRAME * 4))).toBeUndefined(); + }); +}); + +describe("inputAlphaOpaqueWarning", () => { + it("names the offending file and carries the re-export remedy", () => { + const line = inputAlphaOpaqueWarning("avatar.webm"); + expect(line).toContain('src="avatar.webm"'); + expect(line).toContain("declares an alpha channel"); + expect(line).toContain("decodes fully opaque"); + expect(line).toContain("yuva420p"); + expect(line).toContain("alpha sidecar"); + expect(line.endsWith("\n")).toBe(true); + }); +}); diff --git a/packages/engine/src/utils/alphaPlaneProbe.ts b/packages/engine/src/utils/alphaPlaneProbe.ts new file mode 100644 index 0000000000..f08edd0ce2 --- /dev/null +++ b/packages/engine/src/utils/alphaPlaneProbe.ts @@ -0,0 +1,130 @@ +/** + * Alpha-plane probe for video INPUTS. + * + * `alpha_mode=1` (or an alpha-capable `pix_fmt`) is metadata that can outlive + * the alpha it describes: a remux can keep the tag while dropping the + * BlockAdditional sidecar that carries the VP9 alpha plane, so a file can + * keep promising transparency it no longer contains. Such an input renders + * as a solid opaque rectangle over whatever is beneath it, and nothing tells + * the user their file — not the renderer — is the problem. + * + * This module answers one question about a video input: it declares alpha, + * but does the decoded alpha plane come out uniformly opaque? The answer is + * surfaced as a warning only, never an error — an opaque video used as a + * full-frame background is perfectly legitimate, so this can never fail a + * render. + * + * The sibling CLI-side check (`packages/cli/src/utils/webmAlphaCheck.ts`) + * verifies the same property on the render OUTPUT after a WebM encode; this + * probe is the input-side counterpart, implemented in the engine so it runs + * for every render path (local CLI, Docker, cloud) at the point where all + * input metadata is already in hand. + */ + +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { runFfmpeg } from "./runFfmpeg.js"; + +/** + * Bytes per sampled frame at 8x8 rgba: 8 * 8 * 4 = 256. `-frames:v 3` samples + * AT MOST 3 frames — a legitimate 1-frame WebM (a still) yields 256 bytes and + * a 2-frame yields 512, both valid opaque samples that must be evaluated. + * Mirrors the byte accounting in the CLI's `webmAlphaCheck.sampledAlphaIsFullyOpaque`. + */ +const BYTES_PER_SAMPLE_FRAME = 8 * 8 * 4; +const MAX_SAMPLE_BYTES = BYTES_PER_SAMPLE_FRAME * 3; + +/** + * Decode a raw rgba sample buffer and decide whether the alpha plane is + * uniformly opaque. Pure — exported so tests exercise the shipped byte logic + * directly without spawning ffmpeg. + * + * Returns: + * - `true` — every alpha byte across all sampled frames is 255; + * - `false` — any pixel shows partial or full transparency; + * - `undefined` — the sample is malformed (empty, oversized, or not a + * positive whole-frame byte count). An inconclusive probe is not a warning + * trigger. + */ +export function sampledRgbaAlphaIsFullyOpaque(buf: Buffer): boolean | undefined { + if ( + buf.length === 0 || + buf.length > MAX_SAMPLE_BYTES || + buf.length % BYTES_PER_SAMPLE_FRAME !== 0 + ) { + return undefined; + } + for (let i = 3; i < buf.length; i += 4) { + if (buf[i] !== 255) return false; + } + return true; +} + +/** + * Sample a video input's alpha plane and report whether it decodes fully + * opaque. Runs ffmpeg with the libvpx-vp9 input decoder forced (the default + * decoder silently discards VP9 alpha), sampling up to 3 frames at 8x8 rgba — + * the same command the CLI's post-render WebM check uses. The decision to + * sample a few frames rather than the first frame only is deliberate: it + * catches a clip that is opaque at the head and transparent later, at a fixed + * ~768-byte ceiling. + * + * Best-effort and non-blocking: any failure (missing binary, decode error, + * malformed output) returns `undefined` — a diagnostic can never take down a + * render. + */ +export async function probeInputAlphaPlane(videoPath: string): Promise { + const probeDir = mkdtempSync(join(tmpdir(), "hf-alpha-probe-")); + const samplePath = join(probeDir, "alpha.raw"); + try { + const result = await runFfmpeg( + [ + "-v", + "error", + "-c:v", + "libvpx-vp9", + "-i", + videoPath, + "-frames:v", + "3", + "-vf", + "scale=8:8", + "-pix_fmt", + "rgba", + "-f", + "rawvideo", + samplePath, + ], + { timeout: 30_000 }, + ); + if (!result.success) return undefined; + let buf: Buffer; + try { + buf = readFileSync(samplePath); + } catch { + return undefined; + } + return sampledRgbaAlphaIsFullyOpaque(buf); + } catch { + return undefined; + } finally { + rmSync(probeDir, { recursive: true, force: true }); + } +} + +/** + * The stderr warning line written when a video input declares alpha but its + * decoded alpha plane is uniformly opaque. Matches the existing + * `[hyperframes:render] WARNING:` convention in videoFrameExtractor (missing + * src, unwritable cache dir) so the same surface carries the same shape. + * Pure — exported for tests. + */ +export function inputAlphaOpaqueWarning(src: string): string { + return ( + `[hyperframes:render] WARNING: video src="${src}" declares an alpha channel ` + + "but decodes fully opaque. Transparency will not composite. If it should " + + "be transparent, re-export with `-pix_fmt yuva420p` and avoid remuxing " + + "afterward, which can drop the alpha sidecar while keeping the tag.\n" + ); +}