fix: correct Hyprland targeted screenshots (#48)

* fix: correct Hyprland targeted screenshots

* fix: clip offscreen window crops

* fix: reject screenshots for unresolved targets

Author: LikelyLucid

src/server.rs

@@ -295,14 +295,34 @@ impl ComputerUseLinux {
         let max_depth = params.max_depth.unwrap_or(12).min(12);
         let include_screenshot = params.include_screenshot.unwrap_or(true);
         let screenshot_options = params.screenshot_options();
+        let screenshot_target_requested = params.window_target().has_target();
         let app_filter = self
             .resolve_accessibility_app_filter(&params, window_context.as_ref())
             .await;
         let (screenshot, screenshot_error) = if include_screenshot {
-            match capture_screenshot_raw()
-                .await
-                .and_then(|raw| prepare_screenshot_payload(raw, screenshot_options))
-            {
+            let result = capture_screenshot_raw().await.and_then(|raw| {
+                if let Some(window) = window_context.as_ref() {
+                    let bounds = window.bounds.as_ref().ok_or_else(|| {
+                        anyhow::anyhow!(
+                            "targeted screenshot requires window bounds; refusing to return the full desktop"
+                        )
+                    })?;
+                    prepare_app_state_screenshot(
+                        raw,
+                        Some(bounds),
+                        screenshot_target_requested,
+                        screenshot_options,
+                    )
+                } else {
+                    prepare_app_state_screenshot(
+                        raw,
+                        None,
+                        screenshot_target_requested,
+                        screenshot_options,
+                    )
+                }
+            });
+            match result {
                 Ok(capture) => (Some(capture), None),
                 Err(error) => (None, Some(format!("{error:#}"))),
             }
@@ -1934,9 +1954,9 @@ struct ActionOutput {
 impl ComputerUseLinux {
     fn is_wayland_session(&self) -> bool {
         crate::diagnostics::hydrate_session_bus_env();
-        env::var("XDG_SESSION_TYPE")
-            .ok()
-            .is_some_and(|value| value.eq_ignore_ascii_case("wayland"))
+        let session_type = env::var("XDG_SESSION_TYPE").ok();
+        let wayland_display = env::var("WAYLAND_DISPLAY").ok();
+        session_is_wayland(session_type.as_deref(), wayland_display.as_deref())
     }
 
     // The Wayland remote-desktop portal is now a *fallback* for input: when a
@@ -2975,6 +2995,56 @@ fn data_url_payload(data_url: &str) -> String {
         .to_string()
 }
 
+fn session_is_wayland(session_type: Option<&str>, wayland_display: Option<&str>) -> bool {
+    match session_type {
+        Some(value) => value.eq_ignore_ascii_case("wayland"),
+        None => wayland_display.is_some_and(|value| !value.is_empty()),
+    }
+}
+
+fn prepare_app_state_screenshot(
+    mut raw: RawScreenshotCapture,
+    bounds: Option<&crate::windowing::WindowBounds>,
+    target_requested: bool,
+    options: ScreenshotPayloadOptions,
+) -> Result<ScreenshotCapture> {
+    if target_requested && bounds.is_none() {
+        anyhow::bail!(
+            "targeted screenshot requires a resolved window; refusing to return the full desktop"
+        );
+    }
+    if let Some(bounds) = bounds {
+        let (mut x, mut y, mut width, mut height) = window_crop_rect(bounds).ok_or_else(|| {
+            anyhow::anyhow!(
+                "targeted screenshot has unusable window bounds; refusing to return the full desktop"
+            )
+        })?;
+        if x < 0 {
+            width = width.saturating_sub(x.unsigned_abs());
+            x = 0;
+        }
+        if y < 0 {
+            height = height.saturating_sub(y.unsigned_abs());
+            y = 0;
+        }
+        if width == 0 || height == 0 {
+            anyhow::bail!(
+                "targeted screenshot window is outside the captured desktop; refusing to return the full desktop"
+            );
+        }
+        let (bytes, width, height) = crop_png(&raw.bytes, x, y, width, height)
+            .map_err(|error| anyhow::anyhow!("targeted screenshot crop failed: {error}"))?;
+        raw = RawScreenshotCapture {
+            mime_type: raw.mime_type,
+            bytes,
+            source: raw.source,
+            width,
+            height,
+        };
+    }
+    prepare_screenshot_payload(raw, options)
+}
+
 /// Convert a window's bounds into a crop rectangle, if it has a usable origin
 /// and non-zero size.
 fn window_crop_rect(bounds: &crate::windowing::WindowBounds) -> Option<(i32, i32, u32, u32)> {
@@ -3876,6 +3946,95 @@ mod tests {
         out
     }
 
+    #[test]
+    fn targeted_app_state_crops_before_screenshot_payload_resize() {
+        let raw = RawScreenshotCapture {
+            mime_type: "image/png".to_string(),
+            bytes: solid_png(400, 200),
+            source: "test".to_string(),
+            width: 400,
+            height: 200,
+        };
+        let bounds = WindowBounds {
+            x: Some(50),
+            y: Some(20),
+            width: 200,
+            height: 100,
+        };
+        let capture = prepare_app_state_screenshot(
+            raw,
+            Some(&bounds),
+            true,
+            ScreenshotPayloadOptions {
+                max_width: Some(100),
+                max_height: Some(100),
+                max_bytes: Some(1024 * 1024),
+                ..Default::default()
+            },
+        )
+        .unwrap();
+
+        assert_eq!(
+            (capture.coordinate_width, capture.coordinate_height),
+            (200, 100)
+        );
+        assert_eq!((capture.width, capture.height), (100, 50));
+    }
+
+    #[test]
+    fn unresolved_app_state_target_refuses_full_desktop_screenshot() {
+        let raw = RawScreenshotCapture {
+            mime_type: "image/png".to_string(),
+            bytes: solid_png(400, 200),
+            source: "test".to_string(),
+            width: 400,
+            height: 200,
+        };
+
+        let error =
+            prepare_app_state_screenshot(raw, None, true, ScreenshotPayloadOptions::default())
+                .unwrap_err();
+
+        assert!(error.to_string().contains("requires a resolved window"));
+    }
+
+    #[test]
+    fn targeted_app_state_crops_only_visible_part_of_offscreen_window() {
+        let raw = RawScreenshotCapture {
+            mime_type: "image/png".to_string(),
+            bytes: solid_png(400, 200),
+            source: "test".to_string(),
+            width: 400,
+            height: 200,
+        };
+        let bounds = WindowBounds {
+            x: Some(-50),
+            y: Some(-40),
+            width: 100,
+            height: 100,
+        };
+
+        let capture = prepare_app_state_screenshot(
+            raw,
+            Some(&bounds),
+            true,
+            ScreenshotPayloadOptions::default(),
+        )
+        .unwrap();
+
+        assert_eq!(
+            (capture.coordinate_width, capture.coordinate_height),
+            (50, 60)
+        );
+    }
+
+    #[test]
+    fn wayland_display_is_enough_to_select_portal_fallback() {
+        assert!(session_is_wayland(None, Some("wayland-1")));
+        assert!(session_is_wayland(Some("wayland"), None));
+        assert!(!session_is_wayland(Some("x11"), None));
+    }
+
     #[test]
     fn window_crop_happens_before_screenshot_payload_resize() {
         let (cropped, width, height) = crop_png(&solid_png(400, 200), 50, 20, 200, 100).unwrap();

src/windowing/backends/hyprland.rs

@@ -64,13 +64,57 @@ pub fn list_windows() -> Result<Vec<WindowInfo>> {
         );
     }
 
-    parse_hyprland_clients(&String::from_utf8_lossy(&output.stdout))
+    let clients_json = String::from_utf8_lossy(&output.stdout);
+    let monitors_output = hyprctl_output(&["monitors", "-j"]).ok();
+    match monitors_output.filter(|output| output.status.success()) {
+        Some(monitors) => parse_hyprland_clients_with_monitors(&clients_json, &monitors.stdout),
+        None => parse_hyprland_clients(&clients_json),
+    }
+}
+
+fn parse_hyprland_clients_with_monitors(
+    clients_json: &str,
+    monitors_json: &[u8],
+) -> Result<Vec<WindowInfo>> {
+    let monitors: Vec<HyprlandMonitor> = serde_json::from_slice(monitors_json)
+        .context("failed to parse hyprctl monitors -j output")?;
+    let monitors = monitors
+        .into_iter()
+        .map(|monitor| (monitor.id, monitor))
+        .collect::<std::collections::HashMap<_, _>>();
+    let mut clients: Vec<HyprlandClient> =
+        serde_json::from_str(clients_json).context("failed to parse hyprctl clients -j output")?;
+    for client in &mut clients {
+        let Some(monitor) = client.monitor.and_then(|id| monitors.get(&id)) else {
+            continue;
+        };
+        if let Some(at) = client.at.as_mut() {
+            at[0] = scale_i32(at[0] - monitor.x, monitor.scale);
+            at[1] = scale_i32(at[1] - monitor.y, monitor.scale);
+        }
+        if let Some(size) = client.size.as_mut() {
+            size[0] = scale_u32(size[0], monitor.scale);
+            size[1] = scale_u32(size[1], monitor.scale);
+        }
+    }
+    windows_from_hyprland_clients(clients)
 }
 
 pub(crate) fn parse_hyprland_clients(json: &str) -> Result<Vec<WindowInfo>> {
     let clients: Vec<HyprlandClient> =
         serde_json::from_str(json).context("failed to parse hyprctl clients -j output")?;
+    windows_from_hyprland_clients(clients)
+}
 
+fn scale_i32(value: i32, scale: f64) -> i32 {
+    (f64::from(value) * scale).round() as i32
+}
+
+fn scale_u32(value: u32, scale: f64) -> u32 {
+    (f64::from(value) * scale).round() as u32
+}
+
+fn windows_from_hyprland_clients(clients: Vec<HyprlandClient>) -> Result<Vec<WindowInfo>> {
     let mut windows = clients
         .into_iter()
         .filter(|client| client.mapped.unwrap_or(true))
@@ -199,13 +243,22 @@ struct HyprlandInstanceCandidate {
     modified: SystemTime,
 }
 
+#[derive(Debug, Deserialize)]
+struct HyprlandMonitor {
+    id: i32,
+    x: i32,
+    y: i32,
+    scale: f64,
+}
+
 #[derive(Debug, Deserialize)]
 struct HyprlandClient {
     address: String,
     mapped: Option<bool>,
     hidden: Option<bool>,
     at: Option<[i32; 2]>,
     size: Option<[u32; 2]>,
+    monitor: Option<i32>,
     workspace: Option<HyprlandWorkspace>,
     #[serde(rename = "class")]
     class_name: Option<String>,
@@ -271,6 +324,28 @@ mod tests {
     use super::*;
     use std::time::Duration;
 
+    #[test]
+    fn rebases_global_window_coordinates_to_screenshot_space() {
+        let clients = r#"[{
+            "address":"0x1234",
+            "mapped":true,
+            "at":[4714,1494],
+            "size":[931,1124],
+            "monitor":0,
+            "class":"com.mitchellh.ghostty",
+            "title":"Ghostty"
+        }]"#;
+        let monitors = br#"[
+            {"id":0,"x":3747,"y":1440,"scale":1.8},
+            {"id":1,"x":5667,"y":0,"scale":2.0}
+        ]"#;
+        let windows = parse_hyprland_clients_with_monitors(clients, monitors).unwrap();
+
+        let bounds = windows[0].bounds.as_ref().unwrap();
+        assert_eq!((bounds.x, bounds.y), (Some(1741), Some(97)));
+        assert_eq!((bounds.width, bounds.height), (1676, 2023));
+    }
+
     #[test]
     fn builds_hyprland_055_lua_focus_dispatch() {
         assert_eq!(

src/windowing/target.rs

@@ -3,7 +3,7 @@ use crate::windowing::types::{WindowFocusResult, WindowInfo, WindowTarget};
 use anyhow::{bail, Result};
 use tokio::time::{sleep, Duration};
 
-const FOCUS_VERIFY_ATTEMPTS: usize = 6;
+const FOCUS_VERIFY_ATTEMPTS: usize = 21;
 const FOCUS_VERIFY_DELAY: Duration = Duration::from_millis(50);
 
 pub async fn list_windows() -> Result<Vec<WindowInfo>> {
@@ -371,3 +371,14 @@ fn same_optional_string(left: &Option<String>, right: &Option<String>) -> bool {
         _ => false,
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn focus_verification_allows_workspace_transition_latency() {
+        let verification_budget = FOCUS_VERIFY_DELAY * (FOCUS_VERIFY_ATTEMPTS - 1) as u32;
+        assert!(verification_budget >= Duration::from_secs(1));
+    }
+}