Skip to content

perf(rendering): convert IOSurface frames straight into the layer frame texture #2601

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
richiemcilroy merged 4 commits into from
Oct 18, 2026
Merged

perf(rendering): convert IOSurface frames straight into the layer frame texture #2601

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6c9d51f
perf(rendering): convert IOSurface frames straight into the layer fra…
richiemcilroy Oct 14, 2026
555b566
perf(rendering): size the camera converter only when a fallback conve…
richiemcilroy Oct 14, 2026
6af13b0
perf(rendering): skip converter pre-sizing for exports on macOS
richiemcilroy Oct 14, 2026
2f41dab
perf(rendering): start the YUV converter textures minimal on macOS
richiemcilroy Oct 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions crates/rendering/src/composite_frame.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -422,6 +422,14 @@ impl CompositeVideoFramePipeline {
}

pub fn create_frame_texture(device: &wgpu::Device, width: u32, height: u32) -> wgpu::Texture {
#[cfg(target_os = "macos")]
let usage = wgpu::TextureUsages::TEXTURE_BINDING
| wgpu::TextureUsages::STORAGE_BINDING
| wgpu::TextureUsages::COPY_DST;
#[cfg(not(target_os = "macos"))]
let usage = wgpu::TextureUsages::TEXTURE_BINDING
| wgpu::TextureUsages::RENDER_ATTACHMENT
| wgpu::TextureUsages::COPY_DST;
device.create_texture(
&(wgpu::TextureDescriptor {
size: wgpu::Extent3d {
Expand All @@ -433,9 +441,7 @@ impl CompositeVideoFramePipeline {
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Rgba8Unorm,
usage: wgpu::TextureUsages::TEXTURE_BINDING
| wgpu::TextureUsages::RENDER_ATTACHMENT
| wgpu::TextureUsages::COPY_DST,
usage,
label: Some("Frame Composite texture"),
view_formats: &[],
}),
Expand Down
121 changes: 73 additions & 48 deletions crates/rendering/src/layers/camera.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ use crate::{
yuv_converter::{YuvConverterPipelines, YuvToRgbaConverter},
};

#[cfg(target_os = "macos")]
use crate::yuv_converter::RetainedImageBuf;

pub struct CameraLayer {
frame_textures: [wgpu::Texture; 2],
frame_texture_views: [wgpu::TextureView; 2],
Expand All @@ -20,6 +23,8 @@ pub struct CameraLayer {
last_recording_time: Option<f32>,
last_frame_storage: Option<DecodedFrameStorageIdentity>,
yuv_converter: YuvToRgbaConverter,
#[cfg(target_os = "macos")]
frame_sources: [Option<RetainedImageBuf>; 2],
blur_bind_group: Option<wgpu::BindGroup>,
blur_active: bool,
blur_cache: Option<BlurCacheEntry>,
Expand Down Expand Up @@ -87,6 +92,8 @@ impl CameraLayer {
last_recording_time: None,
last_frame_storage: None,
yuv_converter,
#[cfg(target_os = "macos")]
frame_sources: [None, None],
blur_bind_group: None,
blur_active: false,
blur_cache: None,
Expand Down Expand Up @@ -436,6 +443,9 @@ impl CameraLayer {
));
}

#[cfg(target_os = "macos")]
let mut frame_source = None;

match format {
PixelFormat::Rgba => {
let frame_data_bytes = camera_frame.data();
Expand All @@ -462,57 +472,70 @@ impl CameraLayer {
);
}
PixelFormat::Nv12 => {
if let Err(e) = self.yuv_converter.prepare_for_dimensions(
device,
frame_size.x,
frame_size.y,
) {
tracing::warn!(error = %e, "YUV converter prepare failed");
return;
}

#[cfg(target_os = "macos")]
let iosurface_converted = camera_frame
.with_image_buf(|image_buf| {
self.yuv_converter
.convert_nv12_from_iosurface_to_encoder(device, encoder, image_buf)
.is_ok()
})
.unwrap_or(false);
// Submitted on its own ahead of `encoder`, so queue order
// still has the conversion land before the copy below.
#[cfg(target_os = "linux")]
let iosurface_converted = camera_frame.cuda_nv12().is_some_and(|cuda| {
self.yuv_converter
.convert_nv12_cuda(device, queue, cuda)
.inspect_err(|error| {
tracing::warn!(%error, "CUDA camera frame conversion failed")
let converted_into_frame = {
frame_source = camera_frame
.with_image_buf(|image_buf| {
self.yuv_converter
.convert_nv12_from_iosurface_into(
device,
encoder,
image_buf,
&self.frame_texture_views[next_texture],
)
.ok()
})
.is_ok()
});
#[cfg(not(any(target_os = "macos", target_os = "linux")))]
let iosurface_converted = false;
.flatten();
frame_source.is_some()
};
#[cfg(not(target_os = "macos"))]
let converted_into_frame = false;

if !converted_into_frame {
if let Err(e) = self.yuv_converter.prepare_for_dimensions(
device,
frame_size.x,
frame_size.y,
) {
tracing::warn!(error = %e, "YUV converter prepare failed");
return;
}

if iosurface_converted && self.yuv_converter.output_texture().is_some() {
self.copy_from_yuv_output_to_encoder(encoder, next_texture, frame_size);
} else if let (Some(y_data), Some(uv_data)) =
(camera_frame.y_plane(), camera_frame.uv_plane())
&& self
.yuv_converter
.convert_nv12_to_encoder(
device,
queue,
encoder,
y_data,
uv_data,
frame_size.x,
frame_size.y,
camera_frame.y_stride(),
camera_frame.uv_stride(),
)
.is_ok()
{
self.copy_from_yuv_output_to_encoder(encoder, next_texture, frame_size);
// Submitted on its own ahead of `encoder`, so queue order
// still has the conversion land before the copy below.
#[cfg(target_os = "linux")]
let iosurface_converted = camera_frame.cuda_nv12().is_some_and(|cuda| {
self.yuv_converter
.convert_nv12_cuda(device, queue, cuda)
.inspect_err(|error| {
tracing::warn!(%error, "CUDA camera frame conversion failed")
})
.is_ok()
});
#[cfg(not(target_os = "linux"))]
let iosurface_converted = false;

if iosurface_converted && self.yuv_converter.output_texture().is_some() {
self.copy_from_yuv_output_to_encoder(encoder, next_texture, frame_size);
} else if let (Some(y_data), Some(uv_data)) =
(camera_frame.y_plane(), camera_frame.uv_plane())
&& self
.yuv_converter
.convert_nv12_to_encoder(
device,
queue,
encoder,
y_data,
uv_data,
frame_size.x,
frame_size.y,
camera_frame.y_stride(),
camera_frame.uv_stride(),
)
.is_ok()
{
self.copy_from_yuv_output_to_encoder(encoder, next_texture, frame_size);
}
}
}
PixelFormat::Yuv420p => {
Expand Down Expand Up @@ -550,6 +573,8 @@ impl CameraLayer {
}
}

#[cfg(target_os = "macos")]
RetainedImageBuf::replace(queue, &mut self.frame_sources[next_texture], frame_source);
self.last_recording_time = Some(recording_time);
self.last_frame_storage = Some(frame_storage);
self.current_texture = next_texture;
Expand Down
50 changes: 39 additions & 11 deletions crates/rendering/src/layers/display.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ use crate::{
yuv_converter::{YuvConverterPipelines, YuvToRgbaConverter},
};

#[cfg(target_os = "macos")]
use crate::yuv_converter::RetainedImageBuf;

struct PendingTextureCopy {
width: u32,
height: u32,
Expand All @@ -25,6 +28,8 @@ pub struct DisplayLayer {
last_recording_time: Option<f32>,
last_frame_storage: Option<DecodedFrameStorageIdentity>,
yuv_converter: YuvToRgbaConverter,
#[cfg(target_os = "macos")]
frame_sources: [Option<RetainedImageBuf>; 2],
pending_copy: Option<PendingTextureCopy>,
prefer_cpu_conversion: bool,
has_valid_frame: bool,
Expand Down Expand Up @@ -85,6 +90,8 @@ impl DisplayLayer {
last_recording_time: None,
last_frame_storage: None,
yuv_converter,
#[cfg(target_os = "macos")]
frame_sources: [None, None],
pending_copy: None,
prefer_cpu_conversion,
has_valid_frame: false,
Expand Down Expand Up @@ -615,6 +622,9 @@ impl DisplayLayer {
));
}

#[cfg(target_os = "macos")]
let mut frame_source = None;

frame_uploaded = match format {
PixelFormat::Rgba => {
let frame_data = screen_frame.data();
Expand Down Expand Up @@ -766,15 +776,23 @@ impl DisplayLayer {
#[cfg(not(target_os = "windows"))]
if !self.prefer_cpu_conversion {
#[cfg(target_os = "macos")]
let iosurface_converted = screen_frame
.with_image_buf(|image_buf| {
self.yuv_converter
.convert_nv12_from_iosurface_to_encoder(
device, encoder, image_buf,
)
.is_ok()
})
.unwrap_or(false);
let converted_into_frame = {
frame_source = screen_frame
.with_image_buf(|image_buf| {
self.yuv_converter
.convert_nv12_from_iosurface_into(
device,
encoder,
image_buf,
&self.frame_texture_views[next_texture],
)
.ok()
})
.flatten();
frame_source.is_some()
};
#[cfg(not(target_os = "macos"))]
let converted_into_frame = false;
// Submitted ahead of `encoder`, so the conversion
// lands before the pending copy recorded below.
#[cfg(target_os = "linux")]
Expand All @@ -786,10 +804,14 @@ impl DisplayLayer {
)
.is_ok()
});
#[cfg(not(any(target_os = "macos", target_os = "linux")))]
#[cfg(not(target_os = "linux"))]
let iosurface_converted = false;

if iosurface_converted && self.yuv_converter.output_texture().is_some() {
if converted_into_frame {
true
} else if iosurface_converted
&& self.yuv_converter.output_texture().is_some()
{
self.pending_copy = Some(PendingTextureCopy {
width: source_size.x,
height: source_size.y,
Expand Down Expand Up @@ -923,6 +945,12 @@ impl DisplayLayer {
};

if frame_uploaded {
#[cfg(target_os = "macos")]
RetainedImageBuf::replace(
queue,
&mut self.frame_sources[next_texture],
frame_source,
);
self.last_recording_time = Some(current_recording_time);
self.last_frame_storage = Some(frame_storage);
self.current_texture = next_texture;
Expand Down
5 changes: 5 additions & 0 deletions crates/rendering/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7947,6 +7947,11 @@ impl RendererLayers {
camera_width: Option<u32>,
camera_height: Option<u32>,
) {
// macOS converts IOSurface frames into the layer textures; a fallback
// conversion grows the converter textures on first use.
if cfg!(target_os = "macos") {
return;
}
tracing::info!(
screen_width = screen_width,
screen_height = screen_height,
Expand Down
Loading
Loading