wip

Author: kbalt

media-video/capture/src/wayland/mod.rs

@@ -16,7 +16,7 @@ pub use ashpd::{
 #[derive(Debug)]
 pub struct ScreenCaptureOptions {
     /// Embed the cursor in the video
-    pub embed_cursor: bool,
+    pub show_cursor: bool,
 
     /// Which sources to captures
     pub source_types: BitFlags<SourceType>,
@@ -31,7 +31,7 @@ pub struct ScreenCaptureOptions {
 impl Default for ScreenCaptureOptions {
     fn default() -> Self {
         Self {
-            embed_cursor: true,
+            show_cursor: true,
             source_types: SourceType::all(),
             persist_mode: PersistMode::DoNot,
             pipewire: PipewireOptions::default(),
@@ -42,16 +42,30 @@ impl Default for ScreenCaptureOptions {
 #[derive(Debug, Clone)]
 pub struct PipewireOptions {
     /// Maximum framerate to negotiate in the pipewire stream
+    ///
+    /// > Note: This does not guarantee that the framerate is exceeded and a proper frame limit can only be achieved by
+    /// >       blocking the frame callback.
     pub max_framerate: u32,
 
+    /// Set the supported pixel formats, only they will be negotiated
+    pub pixel_formats: Vec<PixelFormat>,
+
     /// Configure usage of DMA buffers
     pub dma_usage: Option<DmaUsageOptions>,
 }
 
 impl Default for PipewireOptions {
     fn default() -> Self {
-        Self {
+        PipewireOptions {
             max_framerate: 30,
+            pixel_formats: vec![
+                PixelFormat::NV12,
+                PixelFormat::I420,
+                PixelFormat::RGBA(RgbaSwizzle::RGBA),
+                PixelFormat::RGBA(RgbaSwizzle::BGRA),
+                PixelFormat::RGBA(RgbaSwizzle::ARGB),
+                PixelFormat::RGBA(RgbaSwizzle::ABGR),
+            ],
             dma_usage: None,
         }
     }
@@ -75,7 +89,7 @@ pub struct DmaUsageOptions {
     pub supported_modifier: Vec<u64>,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone, Copy)]
 pub enum PixelFormat {
     /// 2 Plane YUV with 4:2:0 subsampling
     NV12,
@@ -85,7 +99,7 @@ pub enum PixelFormat {
     RGBA(RgbaSwizzle),
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone, Copy)]
 pub enum RgbaSwizzle {
     RGBA,
     BGRA,
@@ -181,6 +195,8 @@ impl StreamHandle {
 
 #[derive(Debug, thiserror::Error)]
 pub enum StartCaptureError {
+    #[error("Config contains an empty list of pixel formats")]
+    NoPixelFormats,
     #[error(transparent)]
     DesktopPortal(#[from] ashpd::Error),
     #[error("no streams were selected")]
@@ -208,11 +224,15 @@ async fn start_screen_capture_boxed(
     options: ScreenCaptureOptions,
     on_frame: Box<dyn FnMut(CapturedFrame) -> bool + Send>,
 ) -> Result<StreamHandle, StartCaptureError> {
+    if options.pipewire.pixel_formats.is_empty() {
+        return Err(StartCaptureError::NoPixelFormats);
+    }
+
     let proxy = Screencast::new().await?;
 
     let session = proxy.create_session().await?;
 
-    let cursor_mode = if options.embed_cursor {
+    let cursor_mode = if options.show_cursor {
         CursorMode::Embedded
     } else {
         CursorMode::Hidden

media-video/capture/src/wayland/stream.rs

@@ -1,6 +1,6 @@
 use crate::wayland::{
     CapturedDmaBuffer, CapturedDmaBufferSync, CapturedFrame, CapturedFrameBuffer,
-    CapturedFrameFormat, PipewireOptions, RgbaSwizzle,
+    CapturedFrameFormat, PipewireOptions, PixelFormat, RgbaSwizzle,
 };
 use pipewire::{
     context::ContextRc,
@@ -52,6 +52,13 @@ struct UserStreamState {
 impl UserStreamState {
     fn handle_state_changed(&mut self, _stream: &Stream, old: StreamState, new: StreamState) {
         log::debug!("stream changed: {old:?} -> {new:?}");
+
+        if old == StreamState::Streaming
+            && matches!(new, StreamState::Unconnected | StreamState::Error(..))
+            && let Some(main_loop) = self.main_loop.upgrade()
+        {
+            main_loop.quit();
+        }
     }
 
     fn handle_param_changed(&mut self, stream: &Stream, id: u32, param: Option<&Pod>) {
@@ -543,15 +550,18 @@ fn build_stream(
 
     // Add the format params with the video drm modifier property first if dma buffers are to be used
     if let Some(dma_usage) = options.dma_usage {
-        let mut format_params = format_params(options.max_framerate);
+        let mut format_params = format_params(&options.pixel_formats, options.max_framerate);
         format_params
             .properties
             .push(drm_modifier_property(&dma_usage.supported_modifier));
         connect_params.push(serialize_object(format_params));
     }
 
     // Add format without video drm modifier property
-    connect_params.push(serialize_object(format_params(options.max_framerate)));
+    connect_params.push(serialize_object(format_params(
+        &options.pixel_formats,
+        options.max_framerate,
+    )));
 
     let mut connect_params: SmallVec<[&Pod; 2]> = connect_params
         .iter()
@@ -569,31 +579,44 @@ fn build_stream(
 }
 
 /// Build the video format capabilities which will be used to negotiate a video stream with pipewire
-fn format_params(max_framerate: u32) -> Object {
+fn format_params(pixel_formats: &[PixelFormat], max_framerate: u32) -> Object {
+    fn map(p: PixelFormat) -> &'static [VideoFormat] {
+        match p {
+            PixelFormat::NV12 => &[VideoFormat::NV12],
+            PixelFormat::I420 => &[VideoFormat::I420],
+            PixelFormat::RGBA(rgba_swizzle) => match rgba_swizzle {
+                RgbaSwizzle::RGBA => &[VideoFormat::RGBA, VideoFormat::RGBx],
+                RgbaSwizzle::BGRA => &[VideoFormat::BGRA, VideoFormat::BGRx],
+                RgbaSwizzle::ARGB => &[VideoFormat::ARGB, VideoFormat::xRGB],
+                RgbaSwizzle::ABGR => &[VideoFormat::ABGR, VideoFormat::xBGR],
+            },
+        }
+    }
+
+    let video_formats = Value::Choice(ChoiceValue::Id(Choice(
+        ChoiceFlags::empty(),
+        ChoiceEnum::Enum {
+            default: Id(map(pixel_formats[0])[0].0),
+            alternatives: pixel_formats
+                .iter()
+                .flat_map(|p| map(*p).iter().copied())
+                .map(|video_format| Id(video_format.as_raw()))
+                .collect(),
+        },
+    )));
+
+    let video_formats_property = Property {
+        key: FormatProperties::VideoFormat.as_raw(),
+        flags: PropertyFlags::empty(),
+        value: video_formats,
+    };
+
     object!(
         SpaTypes::ObjectParamFormat,
         ParamType::EnumFormat,
         property!(FormatProperties::MediaType, Id, MediaType::Video),
         property!(FormatProperties::MediaSubtype, Id, MediaSubtype::Raw),
-        property!(
-            FormatProperties::VideoFormat,
-            Choice,
-            Enum,
-            Id,
-            // Default
-            VideoFormat::NV12,
-            // Alternatives
-            VideoFormat::NV12,
-            VideoFormat::I420,
-            VideoFormat::RGBA,
-            VideoFormat::BGRA,
-            VideoFormat::ARGB,
-            VideoFormat::ABGR,
-            VideoFormat::RGBx,
-            VideoFormat::BGRx,
-            VideoFormat::xRGB,
-            VideoFormat::xBGR,
-        ),
+        video_formats_property,
         property!(
             FormatProperties::VideoSize,
             Choice,

media-video/h264/tests/vaenc_pw_capture.rs

@@ -1,5 +1,6 @@
 use capture::wayland::{
-    BitFlag, CapturedFrameBuffer, PersistMode, PipewireOptions, ScreenCaptureOptions, SourceType,
+    BitFlag, CapturedFrameBuffer, PersistMode, PipewireOptions, PixelFormat, RgbaSwizzle,
+    ScreenCaptureOptions, SourceType,
 };
 use ezk_h264::{
     Level, Profile,
@@ -30,11 +31,12 @@ async fn va_encode_memory_inner() {
     let (tx, mut rx) = mpsc::channel(8);
 
     let options = ScreenCaptureOptions {
-        embed_cursor: true,
+        show_cursor: true,
         source_types: SourceType::all(),
         persist_mode: PersistMode::DoNot,
         pipewire: PipewireOptions {
             max_framerate: 30,
+            pixel_formats: vec![PixelFormat::RGBA(RgbaSwizzle::BGRA)],
             dma_usage: None,
         },
     };

media-video/h264/tests/vkenc_pw_capture.rs

@@ -1,6 +1,6 @@
 use capture::wayland::{
     BitFlag, CapturedDmaBufferSync, CapturedFrameBuffer, CapturedFrameFormat, DmaUsageOptions,
-    PersistMode, PipewireOptions, ScreenCaptureOptions, SourceType,
+    PersistMode, PipewireOptions, PixelFormat, RgbaSwizzle, ScreenCaptureOptions, SourceType,
 };
 use ezk_h264::{
     Level, Profile,
@@ -48,11 +48,12 @@ async fn vk_encode_dma_inner() {
     let (tx, mut rx) = mpsc::channel(8);
 
     let options = ScreenCaptureOptions {
-        embed_cursor: true,
+        show_cursor: true,
         source_types: SourceType::all(),
         persist_mode: PersistMode::DoNot,
         pipewire: PipewireOptions {
             max_framerate: 30,
+            pixel_formats: vec![PixelFormat::RGBA(RgbaSwizzle::BGRA)],
             dma_usage: Some(DmaUsageOptions {
                 request_sync_obj: true,
                 num_buffers: 16,
@@ -257,11 +258,12 @@ async fn vk_encode_memory_inner() {
     let (tx, mut rx) = mpsc::channel(8);
 
     let options = ScreenCaptureOptions {
-        embed_cursor: true,
+        show_cursor: true,
         source_types: SourceType::all(),
         persist_mode: PersistMode::DoNot,
         pipewire: PipewireOptions {
             max_framerate: 30,
+            pixel_formats: vec![PixelFormat::RGBA(RgbaSwizzle::BGRA)],
             dma_usage: None,
         },
     };

media-video/vulkan/Cargo.toml

@@ -7,7 +7,6 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
-bitflags = "2"
 log = "0.4"
 ash = "0.38.0"
 naga = { version = "27", features = ["wgsl-in", "spv-out"] }
@@ -27,7 +26,6 @@ anyhow = "1"
 
 [dev-dependencies]
 image = { version = "0.25", default-features = false, features = ["png"] }
-env_logger = "0.11"
 
 [lints]
 workspace = true

media-video/vulkan/src/device.rs

@@ -121,10 +121,12 @@ impl Device {
             };
 
             // Query all available queues families
-            let queue_family_properties = vk_adapter
-                .shared_instance()
-                .raw_instance()
-                .get_physical_device_queue_family_properties(vk_adapter.raw_physical_device());
+            let queue_family_properties = dbg!(
+                vk_adapter
+                    .shared_instance()
+                    .raw_instance()
+                    .get_physical_device_queue_family_properties(vk_adapter.raw_physical_device())
+            );
 
             let mut separate_encode_queue_family_index = None;
 

media/rtp/Cargo.toml

@@ -14,4 +14,3 @@ workspace = true
 bytes = "1"
 rtcp-types = "0.2"
 rtp-types = "0.1"
-thiserror = "2"