From 557137aeb8043a1f8516a38268faa5f484314bf2 Mon Sep 17 00:00:00 2001 From: Sandeepa Nadahalli <1698507+snadahalli@users.noreply.github.com> Date: Wed, 26 Aug 2026 14:02:01 +0530 Subject: [PATCH] fix(pipeline): link the byte-stream capsfilter after h264parse, not before MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Starting an AirPlay cast on macOS failed immediately, before any network traffic: Selected encoder "vtenc_h264" VideoToolbox H.264 (Hardware) hw=true ERROR AirPlay casting failed e=GStreamer error: Failed to link elements 'vtenc_h264-1' and 'capsfilter1' Capture and encoder probing both worked; the pipeline would not assemble. `vtenc_h264` advertises `stream-format: avc` on its src pad as a fixed string, not a list, so it cannot emit byte-stream at all. The AirPlay chain demanded byte-stream directly from the encoder: encoder -> capsfilter(byte-stream) -> h264parse -> appsink leaving an empty caps intersection. h264parse is the element that converts avc to byte-stream, so it has to come first. `x264enc` offers `{ avc, byte-stream }`, which is why the same chain links on Linux and why this went unnoticed — the Miracast pipeline already uses the correct shape (`encoder -> h264parse -> mpegtsmux`), which is why Miracast works. Reduced, the old order reproduces the failure exactly and the new one does not: $ gst-launch-1.0 videotestsrc ! vtenc_h264 \ ! 'video/x-h264,profile=high,stream-format=byte-stream' ! h264parse ! fakesink WARNING: erroneous pipeline: could not link vtenc_h264-0 to h264parse0, vtenc_h264-0 can't handle caps video/x-h264, profile=high, stream-format=byte-stream $ gst-launch-1.0 videotestsrc ! vtenc_h264 ! h264parse \ ! 'video/x-h264,stream-format=byte-stream,alignment=au' ! fakesink (clean) Also drops the `profile=high` constraint. `vtenc_h264` has no profile property and does not advertise the field, so pinning it would just move the negotiation failure one element downstream. `sender_pipeline.rs` (the WebRTC path) carried the same over-constraint and is fixed alongside, though it has no callers so this is latent rather than observed. It needs no h264parse: `rtph264pay` accepts avc and byte-stream alike, so constraining alignment only lets the two negotiate. --- .../src/airplay_sender_pipeline.rs | 21 ++++++++++++++----- .../openplay-pipeline/src/sender_pipeline.rs | 10 +++++++-- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/crates/openplay-pipeline/src/airplay_sender_pipeline.rs b/crates/openplay-pipeline/src/airplay_sender_pipeline.rs index 77ceadd..a360f29 100644 --- a/crates/openplay-pipeline/src/airplay_sender_pipeline.rs +++ b/crates/openplay-pipeline/src/airplay_sender_pipeline.rs @@ -13,7 +13,7 @@ use crate::PipelineError; /// Captures the screen, encodes to H.264, and outputs raw NALUs via `appsink` /// for the AirPlay mirror stream protocol. /// -/// Pipeline: [capture src] → capsfilter(fps) → queue → encoder → capsfilter(h264,byte-stream) → h264parse → appsink +/// Pipeline: [capture src] → capsfilter(fps) → queue → encoder → h264parse → capsfilter(h264,byte-stream) → appsink pub struct AirPlaySenderPipeline { pipeline: gst::Pipeline, appsink: gst_app::AppSink, @@ -64,13 +64,24 @@ impl AirPlaySenderPipeline { })?; configure_encoder(&encoder, encoder_type, bitrate_kbps); - // H.264 output caps (byte-stream for raw NALUs) + // H.264 output caps (byte-stream for raw NALUs). + // + // This is linked *after* h264parse, not before it. `vtenc_h264` offers + // only `stream-format=avc` on its src pad, so demanding byte-stream + // straight out of the encoder leaves an empty caps intersection and the + // link fails outright on macOS. h264parse is the element that converts + // avc to byte-stream. `x264enc` advertises both formats, which is why + // the old order linked fine on Linux and hid this. + // + // `profile` is deliberately not constrained: `vtenc_h264` has no profile + // property and does not advertise the field, so pinning it here would + // just move the negotiation failure one element downstream. let h264_caps = gst::ElementFactory::make("capsfilter") .property( "caps", gst::Caps::builder("video/x-h264") - .field("profile", "high") .field("stream-format", "byte-stream") + .field("alignment", "au") .build(), ) .build() @@ -102,8 +113,8 @@ impl AirPlaySenderPipeline { &capsfilter, &video_queue, &encoder, - &h264_caps, &h264parse, + &h264_caps, appsink.upcast_ref(), ]) .map_err(|e| PipelineError::Gstreamer(format!("Failed to add elements: {e}")))?; @@ -113,8 +124,8 @@ impl AirPlaySenderPipeline { &capsfilter, &video_queue, &encoder, - &h264_caps, &h264parse, + &h264_caps, appsink.upcast_ref(), ]) .map_err(|e| PipelineError::Gstreamer(format!("Failed to link elements: {e}")))?; diff --git a/crates/openplay-pipeline/src/sender_pipeline.rs b/crates/openplay-pipeline/src/sender_pipeline.rs index e459665..22f4286 100644 --- a/crates/openplay-pipeline/src/sender_pipeline.rs +++ b/crates/openplay-pipeline/src/sender_pipeline.rs @@ -55,12 +55,18 @@ impl SenderPipeline { })?; configure_encoder(&encoder, encoder_type, bitrate_kbps); + // Constrain alignment only. `vtenc_h264` offers just + // `stream-format=avc` and advertises no `profile` field, so requiring + // byte-stream or high profile straight out of the encoder makes this + // link fail on macOS — the same defect fixed in the AirPlay pipeline. + // Unlike that one this chain has no h264parse to convert with, but it + // does not need one: `rtph264pay` accepts avc and byte-stream alike, so + // letting the two negotiate is both correct and simpler. let h264_caps = gst::ElementFactory::make("capsfilter") .property( "caps", gst::Caps::builder("video/x-h264") - .field("profile", "high") - .field("stream-format", "byte-stream") + .field("alignment", "au") .build(), ) .build()