diff --git a/crates/rendering/src/composite_frame.rs b/crates/rendering/src/composite_frame.rs index ebe9063cf3b..ccf19e628ba 100644 --- a/crates/rendering/src/composite_frame.rs +++ b/crates/rendering/src/composite_frame.rs @@ -422,6 +422,14 @@ impl CompositeVideoFramePipeline { } pub fn create_frame_texture(device: &wgpu::Device, width: u32, height: u32) -> wgpu::Texture { + #[cfg(target_os = "macos")] + let usage = wgpu::TextureUsages::TEXTURE_BINDING + | wgpu::TextureUsages::STORAGE_BINDING + | wgpu::TextureUsages::COPY_DST; + #[cfg(not(target_os = "macos"))] + let usage = wgpu::TextureUsages::TEXTURE_BINDING + | wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::COPY_DST; device.create_texture( &(wgpu::TextureDescriptor { size: wgpu::Extent3d { @@ -433,9 +441,7 @@ impl CompositeVideoFramePipeline { sample_count: 1, dimension: wgpu::TextureDimension::D2, format: wgpu::TextureFormat::Rgba8Unorm, - usage: wgpu::TextureUsages::TEXTURE_BINDING - | wgpu::TextureUsages::RENDER_ATTACHMENT - | wgpu::TextureUsages::COPY_DST, + usage, label: Some("Frame Composite texture"), view_formats: &[], }), diff --git a/crates/rendering/src/layers/camera.rs b/crates/rendering/src/layers/camera.rs index 9fca2068dd4..f2d3c7a2462 100644 --- a/crates/rendering/src/layers/camera.rs +++ b/crates/rendering/src/layers/camera.rs @@ -9,6 +9,9 @@ use crate::{ yuv_converter::{YuvConverterPipelines, YuvToRgbaConverter}, }; +#[cfg(target_os = "macos")] +use crate::yuv_converter::RetainedImageBuf; + pub struct CameraLayer { frame_textures: [wgpu::Texture; 2], frame_texture_views: [wgpu::TextureView; 2], @@ -20,6 +23,8 @@ pub struct CameraLayer { last_recording_time: Option, last_frame_storage: Option, yuv_converter: YuvToRgbaConverter, + #[cfg(target_os = "macos")] + frame_sources: [Option; 2], blur_bind_group: Option, blur_active: bool, blur_cache: Option, @@ -87,6 +92,8 @@ impl CameraLayer { last_recording_time: None, last_frame_storage: None, yuv_converter, + #[cfg(target_os = "macos")] + frame_sources: [None, None], blur_bind_group: None, blur_active: false, blur_cache: None, @@ -436,6 +443,9 @@ impl CameraLayer { )); } + #[cfg(target_os = "macos")] + let mut frame_source = None; + match format { PixelFormat::Rgba => { let frame_data_bytes = camera_frame.data(); @@ -462,57 +472,70 @@ impl CameraLayer { ); } PixelFormat::Nv12 => { - if let Err(e) = self.yuv_converter.prepare_for_dimensions( - device, - frame_size.x, - frame_size.y, - ) { - tracing::warn!(error = %e, "YUV converter prepare failed"); - return; - } - #[cfg(target_os = "macos")] - let iosurface_converted = camera_frame - .with_image_buf(|image_buf| { - self.yuv_converter - .convert_nv12_from_iosurface_to_encoder(device, encoder, image_buf) - .is_ok() - }) - .unwrap_or(false); - // Submitted on its own ahead of `encoder`, so queue order - // still has the conversion land before the copy below. - #[cfg(target_os = "linux")] - let iosurface_converted = camera_frame.cuda_nv12().is_some_and(|cuda| { - self.yuv_converter - .convert_nv12_cuda(device, queue, cuda) - .inspect_err(|error| { - tracing::warn!(%error, "CUDA camera frame conversion failed") + let converted_into_frame = { + frame_source = camera_frame + .with_image_buf(|image_buf| { + self.yuv_converter + .convert_nv12_from_iosurface_into( + device, + encoder, + image_buf, + &self.frame_texture_views[next_texture], + ) + .ok() }) - .is_ok() - }); - #[cfg(not(any(target_os = "macos", target_os = "linux")))] - let iosurface_converted = false; + .flatten(); + frame_source.is_some() + }; + #[cfg(not(target_os = "macos"))] + let converted_into_frame = false; + + if !converted_into_frame { + if let Err(e) = self.yuv_converter.prepare_for_dimensions( + device, + frame_size.x, + frame_size.y, + ) { + tracing::warn!(error = %e, "YUV converter prepare failed"); + return; + } - if iosurface_converted && self.yuv_converter.output_texture().is_some() { - self.copy_from_yuv_output_to_encoder(encoder, next_texture, frame_size); - } else if let (Some(y_data), Some(uv_data)) = - (camera_frame.y_plane(), camera_frame.uv_plane()) - && self - .yuv_converter - .convert_nv12_to_encoder( - device, - queue, - encoder, - y_data, - uv_data, - frame_size.x, - frame_size.y, - camera_frame.y_stride(), - camera_frame.uv_stride(), - ) - .is_ok() - { - self.copy_from_yuv_output_to_encoder(encoder, next_texture, frame_size); + // Submitted on its own ahead of `encoder`, so queue order + // still has the conversion land before the copy below. + #[cfg(target_os = "linux")] + let iosurface_converted = camera_frame.cuda_nv12().is_some_and(|cuda| { + self.yuv_converter + .convert_nv12_cuda(device, queue, cuda) + .inspect_err(|error| { + tracing::warn!(%error, "CUDA camera frame conversion failed") + }) + .is_ok() + }); + #[cfg(not(target_os = "linux"))] + let iosurface_converted = false; + + if iosurface_converted && self.yuv_converter.output_texture().is_some() { + self.copy_from_yuv_output_to_encoder(encoder, next_texture, frame_size); + } else if let (Some(y_data), Some(uv_data)) = + (camera_frame.y_plane(), camera_frame.uv_plane()) + && self + .yuv_converter + .convert_nv12_to_encoder( + device, + queue, + encoder, + y_data, + uv_data, + frame_size.x, + frame_size.y, + camera_frame.y_stride(), + camera_frame.uv_stride(), + ) + .is_ok() + { + self.copy_from_yuv_output_to_encoder(encoder, next_texture, frame_size); + } } } PixelFormat::Yuv420p => { @@ -550,6 +573,8 @@ impl CameraLayer { } } + #[cfg(target_os = "macos")] + RetainedImageBuf::replace(queue, &mut self.frame_sources[next_texture], frame_source); self.last_recording_time = Some(recording_time); self.last_frame_storage = Some(frame_storage); self.current_texture = next_texture; diff --git a/crates/rendering/src/layers/display.rs b/crates/rendering/src/layers/display.rs index 21f1f97860c..1b8ec3c520e 100644 --- a/crates/rendering/src/layers/display.rs +++ b/crates/rendering/src/layers/display.rs @@ -9,6 +9,9 @@ use crate::{ yuv_converter::{YuvConverterPipelines, YuvToRgbaConverter}, }; +#[cfg(target_os = "macos")] +use crate::yuv_converter::RetainedImageBuf; + struct PendingTextureCopy { width: u32, height: u32, @@ -25,6 +28,8 @@ pub struct DisplayLayer { last_recording_time: Option, last_frame_storage: Option, yuv_converter: YuvToRgbaConverter, + #[cfg(target_os = "macos")] + frame_sources: [Option; 2], pending_copy: Option, prefer_cpu_conversion: bool, has_valid_frame: bool, @@ -85,6 +90,8 @@ impl DisplayLayer { last_recording_time: None, last_frame_storage: None, yuv_converter, + #[cfg(target_os = "macos")] + frame_sources: [None, None], pending_copy: None, prefer_cpu_conversion, has_valid_frame: false, @@ -615,6 +622,9 @@ impl DisplayLayer { )); } + #[cfg(target_os = "macos")] + let mut frame_source = None; + frame_uploaded = match format { PixelFormat::Rgba => { let frame_data = screen_frame.data(); @@ -766,15 +776,23 @@ impl DisplayLayer { #[cfg(not(target_os = "windows"))] if !self.prefer_cpu_conversion { #[cfg(target_os = "macos")] - let iosurface_converted = screen_frame - .with_image_buf(|image_buf| { - self.yuv_converter - .convert_nv12_from_iosurface_to_encoder( - device, encoder, image_buf, - ) - .is_ok() - }) - .unwrap_or(false); + let converted_into_frame = { + frame_source = screen_frame + .with_image_buf(|image_buf| { + self.yuv_converter + .convert_nv12_from_iosurface_into( + device, + encoder, + image_buf, + &self.frame_texture_views[next_texture], + ) + .ok() + }) + .flatten(); + frame_source.is_some() + }; + #[cfg(not(target_os = "macos"))] + let converted_into_frame = false; // Submitted ahead of `encoder`, so the conversion // lands before the pending copy recorded below. #[cfg(target_os = "linux")] @@ -786,10 +804,14 @@ impl DisplayLayer { ) .is_ok() }); - #[cfg(not(any(target_os = "macos", target_os = "linux")))] + #[cfg(not(target_os = "linux"))] let iosurface_converted = false; - if iosurface_converted && self.yuv_converter.output_texture().is_some() { + if converted_into_frame { + true + } else if iosurface_converted + && self.yuv_converter.output_texture().is_some() + { self.pending_copy = Some(PendingTextureCopy { width: source_size.x, height: source_size.y, @@ -923,6 +945,12 @@ impl DisplayLayer { }; if frame_uploaded { + #[cfg(target_os = "macos")] + RetainedImageBuf::replace( + queue, + &mut self.frame_sources[next_texture], + frame_source, + ); self.last_recording_time = Some(current_recording_time); self.last_frame_storage = Some(frame_storage); self.current_texture = next_texture; diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index 8e95cd3b623..dfe7030cf43 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -7947,6 +7947,11 @@ impl RendererLayers { camera_width: Option, camera_height: Option, ) { + // macOS converts IOSurface frames into the layer textures; a fallback + // conversion grows the converter textures on first use. + if cfg!(target_os = "macos") { + return; + } tracing::info!( screen_width = screen_width, screen_height = screen_height, diff --git a/crates/rendering/src/yuv_converter.rs b/crates/rendering/src/yuv_converter.rs index 7f31d0ab72d..ce29ac62541 100644 --- a/crates/rendering/src/yuv_converter.rs +++ b/crates/rendering/src/yuv_converter.rs @@ -102,7 +102,15 @@ fn upload_plane_with_stride( const MAX_TEXTURE_WIDTH: u32 = 7680; const MAX_TEXTURE_HEIGHT: u32 = 4320; +// On macOS, IOSurface frames convert straight into the layer textures, so the +// converter's own textures start minimal and only grow for a fallback path. +#[cfg(target_os = "macos")] +const INITIAL_TEXTURE_WIDTH: u32 = TEXTURE_SIZE_PADDING; +#[cfg(target_os = "macos")] +const INITIAL_TEXTURE_HEIGHT: u32 = TEXTURE_SIZE_PADDING; +#[cfg(not(target_os = "macos"))] const INITIAL_TEXTURE_WIDTH: u32 = 1920; +#[cfg(not(target_os = "macos"))] const INITIAL_TEXTURE_HEIGHT: u32 = 1080; const TEXTURE_SIZE_PADDING: u32 = 64; @@ -478,10 +486,29 @@ impl YuvConverterPipelines { const RETAINED_IOSURFACE_SOURCES: usize = 4; #[cfg(target_os = "macos")] -struct RetainedImageBuf { +#[must_use] +pub struct RetainedImageBuf { _image_buf: cidre::arc::R, } +#[cfg(target_os = "macos")] +impl RetainedImageBuf { + fn new(image_buf: &cv::ImageBuf) -> Self { + Self { + _image_buf: image_buf.retained(), + } + } + + /// Stores `source` in `slot`. The work that sampled the buffer it replaces + /// has already been submitted, so that buffer is released once the queue + /// has finished it. + pub(crate) fn replace(queue: &wgpu::Queue, slot: &mut Option, source: Option) { + if let Some(previous) = std::mem::replace(slot, source) { + queue.on_submitted_work_done(move || drop(previous)); + } + } +} + // SAFETY: the buffer is only retained and released, never read, and // CoreVideo reference counting is thread-safe. #[cfg(target_os = "macos")] @@ -1129,9 +1156,9 @@ impl YuvToRgbaConverter { return Err(IOSurfaceTextureError::NoMetalDevice.into()); } - let io_surface = image_buf - .io_surf() - .ok_or(IOSurfaceTextureError::NoIOSurface)?; + if image_buf.io_surf().is_none() { + return Err(IOSurfaceTextureError::NoIOSurface.into()); + } let width = image_buf.width() as u32; let height = image_buf.height() as u32; @@ -1141,7 +1168,56 @@ impl YuvToRgbaConverter { self.ensure_texture_size(device, effective_width, effective_height); self.swap_output_buffer(); - let cache = self.iosurface_cache.as_ref().unwrap(); + self.encode_nv12_from_iosurface(device, encoder, image_buf, self.current_output_view())?; + + // The Metal textures alias the IOSurface without retaining the pixel + // buffer, and the decoder recycles released buffers for new pictures. + // Hold the source until enough later conversions have started that + // the caller has waited for the GPU work that sampled it. + self.iosurface_sources[self.next_iosurface_source] = Some(RetainedImageBuf::new(image_buf)); + self.next_iosurface_source = (self.next_iosurface_source + 1) % RETAINED_IOSURFACE_SOURCES; + + Ok(self.current_output_view()) + } + + #[cfg(target_os = "macos")] + pub fn convert_nv12_from_iosurface_into( + &self, + device: &wgpu::Device, + encoder: &mut wgpu::CommandEncoder, + image_buf: &cv::ImageBuf, + target: &wgpu::TextureView, + ) -> Result { + validate_dimensions( + image_buf.width() as u32, + image_buf.height() as u32, + self.gpu_max_texture_size, + )?; + + self.encode_nv12_from_iosurface(device, encoder, image_buf, target)?; + + Ok(RetainedImageBuf::new(image_buf)) + } + + #[cfg(target_os = "macos")] + fn encode_nv12_from_iosurface( + &self, + device: &wgpu::Device, + encoder: &mut wgpu::CommandEncoder, + image_buf: &cv::ImageBuf, + output_view: &wgpu::TextureView, + ) -> Result<(), YuvConversionError> { + let io_surface = image_buf + .io_surf() + .ok_or(IOSurfaceTextureError::NoIOSurface)?; + + let width = image_buf.width() as u32; + let height = image_buf.height() as u32; + + let cache = self + .iosurface_cache + .as_ref() + .ok_or(IOSurfaceTextureError::NoMetalDevice)?; let y_metal_texture = cache.create_y_texture(io_surface, width, height)?; let uv_metal_texture = cache.create_uv_texture(io_surface, width, height)?; @@ -1180,7 +1256,7 @@ impl YuvToRgbaConverter { }, wgpu::BindGroupEntry { binding: 2, - resource: wgpu::BindingResource::TextureView(self.current_output_view()), + resource: wgpu::BindingResource::TextureView(output_view), }, ], }); @@ -1195,16 +1271,7 @@ impl YuvToRgbaConverter { compute_pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1); } - // The Metal textures alias the IOSurface without retaining the pixel - // buffer, and the decoder recycles released buffers for new pictures. - // Hold the source until enough later conversions have started that - // the caller has waited for the GPU work that sampled it. - self.iosurface_sources[self.next_iosurface_source] = Some(RetainedImageBuf { - _image_buf: image_buf.retained(), - }); - self.next_iosurface_source = (self.next_iosurface_source + 1) % RETAINED_IOSURFACE_SOURCES; - - Ok(self.current_output_view()) + Ok(()) } #[allow(clippy::too_many_arguments)]