freedak/MyAiDesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update: 将子项目从 submodule 转为完整内容

Browse Source 
- 移除 GovAI, nomifun-tauri, 算力盒子 的 submodule 引用
- 添加所有子项目的完整源代码
- 保留原始 .git 为 .git.bak 备份
This commit is contained in:
freedak
2026-07-04 19:20:46 +08:00
parent 54d6465fa7
commit f7a720204a
3360 changed files with 802660 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nomifun-tauri/crates/agent/nomi-computer/Cargo.toml
+51
View File
@@ -0,0 +1,51 @@
[package]
name = "nomi-computer"
description = "Computer-use tool for Nomi (screenshot, mouse/keyboard synthesis, window control)"
version.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
nomi-types.workspace = true
nomi-protocol.workspace = true
nomi-config.workspace = true
nomi-tools.workspace = true
nomi-a11y.workspace = true
tracing.workspace = true
tokio.workspace = true
serde_json.workspace = true
async-trait.workspace = true
xcap.workspace = true
enigo.workspace = true
image.workspace = true
base64.workspace = true
# Reliable app/URL/file launch via the OS shell. The workspace enables the
# `shellexecute-on-windows` feature so this uses ShellExecuteExW on Windows
# instead of the `cmd /c start` fallback (whose window-title quirk pops the
# "Windows cannot find 'X'" dialog). macOS/Linux use `open`/`xdg-open`.
open.workspace = true
# macOS TCC permission probing (Accessibility / Screen Recording): proactive
# status + prompt via AXIsProcessTrusted(WithOptions) and CG*ScreenCaptureAccess.
[target.'cfg(target_os = "macos")'.dependencies]
core-foundation = "0.10"
# Windows: correct absolute mouse actuation across the whole virtual desktop
# (enigo 0.6's Coordinate::Abs normalizes against the PRIMARY monitor only and
# omits MOUSEEVENTF_VIRTUALDESK, mis-projecting multi-monitor / negative-origin
# targets) and real window activation (SetForegroundWindow). 0.61 matches the
# version nomi-a11y already pins.
[target.'cfg(target_os = "windows")'.dependencies]
windows = { version = "0.61", features = [
"Win32_Foundation",
"Win32_UI_WindowsAndMessaging",
"Win32_UI_Input_KeyboardAndMouse",
"Win32_System_Threading",
] }
[dev-dependencies]
# test-util enables tokio::test(start_paused) so wait-clamp tests run instantly.
tokio = { workspace = true, features = ["test-util"] }
nomifun-tauri/crates/agent/nomi-computer/examples/appresolve.rs
+26
View File
@@ -0,0 +1,26 @@
//! Real-machine check for Start-Menu app-name resolution (the `launch` action's
//! fix for "找不到" on bare app names like "QQ音乐").
//!
//! Run: cargo run -p nomi-computer --example appresolve -- "QQ音乐" qqmusic 网易云音乐 notepad
//! Prints the resolved launch target for each name WITHOUT launching anything.
#[cfg(target_os = "windows")]
fn main() {
let names: Vec<String> = std::env::args().skip(1).collect();
let names = if names.is_empty() {
vec!["QQ音乐".to_string(), "qqmusic".to_string(), "notepad".to_string()]
} else {
names
};
for name in names {
match nomi_computer::launch::resolve_app_for_diagnostics(&name) {
Some(t) => println!("{name:?} -> {t:?}"),
None => println!("{name:?} -> (not found in Start Menu)"),
}
}
}
#[cfg(not(target_os = "windows"))]
fn main() {
eprintln!("appresolve is a Windows-only example.");
}
nomifun-tauri/crates/agent/nomi-computer/src/fallback_backend.rs
+230
View File
@@ -0,0 +1,230 @@
//! Platform-neutral fallback backend: window enumeration and best-effort
//! focusing via xcap (cross-platform). This is NOT the Windows-OS backend —
//! the file was historically named `windows.rs`, which collided conceptually
//! with the future per-OS UI-Automation submodule; it is renamed to make clear
//! it is the generic xcap/enigo fallback used on every platform.
//!
//! xcap 0.9 has no focus/activate API on any platform, so `focus_window`
//! falls back to clicking the window's center (which raises and focuses it
//! on macOS and most Linux WMs). Window x/y/width/height from xcap are in
//! the same logical coordinate space enigo uses on macOS.
//!
//! Real window activation (macOS NSRunningApplication / Windows UIA SetFocus /
//! Linux per-WM) is designed in the cross-platform computer-use spec and will
//! replace the click-to-raise fallback as the a11y engine lands.
use xcap::Window;
// The click-to-raise focus fallback (and only that) uses synthetic input; on
// Windows we activate the real window via Win32 instead, so the input crate is
// not referenced there.
#[cfg(not(target_os = "windows"))]
use crate::input;
/// A snapshot of one window's metadata.
#[derive(Debug, Clone)]
pub struct WindowInfo {
pub id: u32,
pub title: String,
pub app_name: String,
pub x: i32,
pub y: i32,
pub width: u32,
pub height: u32,
pub is_focused: bool,
}
/// Enumerate windows (front-to-back z order). Blocking: call from
/// spawn_blocking.
pub fn list_windows() -> Result<Vec<WindowInfo>, String> {
let windows = Window::all().map_err(|e| format!("Failed to enumerate windows: {e}"))?;
let mut infos = Vec::with_capacity(windows.len());
for w in &windows {
// Skip windows whose metadata cannot be read instead of failing the
// whole listing.
let Ok(id) = w.id() else { continue };
infos.push(WindowInfo {
id,
title: w.title().unwrap_or_default(),
app_name: w.app_name().unwrap_or_default(),
x: w.x().unwrap_or(0),
y: w.y().unwrap_or(0),
width: w.width().unwrap_or(0),
height: w.height().unwrap_or(0),
is_focused: w.is_focused().unwrap_or(false),
});
}
Ok(infos)
}
/// Render a window list as human-readable text for the LLM.
pub fn format_window_list(windows: &[WindowInfo]) -> String {
if windows.is_empty() {
return "No windows found.".to_string();
}
let mut out = String::from("Windows (front to back):\n");
for w in windows {
let focus = if w.is_focused { " [focused]" } else { "" };
out.push_str(&format!(
"- id={} app={:?} title={:?} at ({}, {}) size {}x{}{}\n",
w.id, w.app_name, w.title, w.x, w.y, w.width, w.height, focus
));
}
out
}
/// Find a window by id. Blocking: call from spawn_blocking.
pub fn find_window(window_id: u32) -> Result<WindowInfo, String> {
let windows = list_windows()?;
windows
.into_iter()
.find(|w| w.id == window_id)
.ok_or_else(|| {
format!(
"Window {window_id} not found. Use the list_windows action to get current ids."
)
})
}
/// Bring a window to the foreground.
///
/// On Windows this uses the real activation API (`SetForegroundWindow` on the
/// exact target HWND, restoring it if minimized, with a foreground-lock
/// workaround) so subsequent `type`/`key` input lands in the intended window.
/// On macOS / Linux, where xcap exposes no activate API and a center click
/// reliably raises and focuses the window, it falls back to click-to-raise.
pub async fn focus_window(window_id: u32) -> Result<String, String> {
let info = tokio::task::spawn_blocking(move || find_window(window_id))
.await
.map_err(|e| format!("Window lookup task failed: {e}"))??;
#[cfg(target_os = "windows")]
{
tokio::task::spawn_blocking(move || set_foreground_window(window_id))
.await
.map_err(|e| format!("Focus task failed: {e}"))??;
Ok(format!(
"Activated window {window_id} ({:?} — {:?}) via the Windows foreground API. \
Take a screenshot to verify.",
info.app_name, info.title
))
}
#[cfg(not(target_os = "windows"))]
{
if info.width == 0 || info.height == 0 {
return Err(format!(
"Window {window_id} ({:?}) has zero size; it may be minimized or hidden. \
Cannot focus it by clicking.",
info.title
));
}
let cx = info.x + info.width as i32 / 2;
let cy = info.y + info.height as i32 / 2;
input::click(cx, cy, enigo::Button::Left, 1).await?;
Ok(format!(
"Clicked the center of window {window_id} ({:?} — {:?}) at ({cx}, {cy}) to focus it. \
Note: the platform exposes no direct focus API, so this is a click-to-raise fallback; \
the click may interact with whatever is at the window center. \
Take a screenshot to verify the result.",
info.app_name, info.title
))
}
}
/// Windows: activate the exact target window (xcap window ids are HWNDs).
/// Restores it if minimized, briefly attaches our input queue to the current
/// foreground thread so the OS honors the activation (the foreground-stealing
/// lock otherwise silently no-ops), and reports a clear error if Windows still
/// refuses (e.g. an elevated/higher-integrity target).
#[cfg(target_os = "windows")]
fn set_foreground_window(window_id: u32) -> Result<(), String> {
use core::ffi::c_void;
use windows::Win32::Foundation::HWND;
use windows::Win32::System::Threading::{AttachThreadInput, GetCurrentThreadId};
use windows::Win32::UI::WindowsAndMessaging::{
GetForegroundWindow, GetWindowThreadProcessId, IsIconic, IsWindow, SW_RESTORE, SW_SHOW,
SetForegroundWindow, ShowWindow,
};
let hwnd = HWND(window_id as usize as *mut c_void);
// SAFETY: window-management calls on a possibly-stale handle. `IsWindow`
// guards validity first, and every call below fails (rather than UB) on an
// invalid HWND.
unsafe {
if !IsWindow(Some(hwnd)).as_bool() {
return Err(format!(
"Window {window_id} no longer exists. Use list_windows to get current ids."
));
}
if IsIconic(hwnd).as_bool() {
let _ = ShowWindow(hwnd, SW_RESTORE);
} else {
let _ = ShowWindow(hwnd, SW_SHOW);
}
// Windows only honors SetForegroundWindow from the thread owning the
// current foreground window, so briefly share its input state.
let cur = GetCurrentThreadId();
let fg = GetForegroundWindow();
let fg_tid = if fg.0.is_null() {
0
} else {
GetWindowThreadProcessId(fg, None)
};
let attached =
fg_tid != 0 && fg_tid != cur && AttachThreadInput(cur, fg_tid, true).as_bool();
let ok = SetForegroundWindow(hwnd).as_bool();
if attached {
let _ = AttachThreadInput(cur, fg_tid, false);
}
if ok {
Ok(())
} else {
Err(format!(
"Windows refused to bring window {window_id} to the foreground (foreground \
activation is OS-restricted; the target may be elevated / higher-integrity than \
this app, or another app holds the foreground lock). Try a pixel click instead."
))
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn format_empty_window_list() {
assert_eq!(format_window_list(&[]), "No windows found.");
}
#[test]
fn format_window_list_includes_fields() {
let windows = vec![WindowInfo {
id: 42,
title: "Inbox".to_string(),
app_name: "Mail".to_string(),
x: 10,
y: 20,
width: 800,
height: 600,
is_focused: true,
}];
let text = format_window_list(&windows);
assert!(text.contains("id=42"));
assert!(text.contains("Mail"));
assert!(text.contains("Inbox"));
assert!(text.contains("800x600"));
assert!(text.contains("[focused]"));
}
// Requires a real window server session.
#[test]
#[ignore]
fn list_windows_real() {
let windows = list_windows().expect("should enumerate windows");
// There is at least a desktop-level window in a real session.
assert!(!windows.is_empty());
}
}
nomifun-tauri/crates/agent/nomi-computer/src/input.rs
+410
View File
@@ -0,0 +1,410 @@
//! Input synthesis via enigo.
//!
//! Enigo handles are not `Send`, so each operation constructs a fresh Enigo
//! inside `tokio::task::spawn_blocking` and the whole blocking task is
//! wrapped in a 10s timeout. Coordinates passed in here are already absolute
//! screen coordinates (mapped from screenshot space by the caller).
use std::time::Duration;
use enigo::{Axis, Button, Direction, Enigo, Keyboard, Mouse, Settings};
// `Coordinate::Abs` is only used on the non-Windows actuation path; Windows
// moves the cursor via SendInput over the virtual desktop instead (see
// `move_abs`).
#[cfg(not(target_os = "windows"))]
use enigo::Coordinate;
use crate::permissions;
const INPUT_TIMEOUT: Duration = Duration::from_secs(10);
/// Pause between press and release (and between repeated clicks) so target
/// apps register distinct events.
const CLICK_PAUSE: Duration = Duration::from_millis(20);
/// Scroll direction accepted by the `scroll` action.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ScrollDirection {
Up,
Down,
Left,
Right,
}
impl ScrollDirection {
pub fn parse(s: &str) -> Result<Self, String> {
match s {
"up" => Ok(Self::Up),
"down" => Ok(Self::Down),
"left" => Ok(Self::Left),
"right" => Ok(Self::Right),
other => Err(format!(
"Unknown scroll direction {other:?}. Use one of: up, down, left, right."
)),
}
}
}
/// Map an absolute global virtual-desktop screen coordinate into the 0..=65535
/// normalized range that `SendInput` expects with
/// `MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK`, relative to the
/// virtual-screen rectangle `(v_left, v_top, v_width, v_height)` reported by
/// `GetSystemMetrics(SM_*VIRTUALSCREEN)`. The endpoints `v_left` and
/// `v_left + v_width - 1` map to 0 and 65535 respectively (round-to-nearest);
/// coordinates outside the desktop clamp into range.
///
/// This is the Windows-only fix for enigo 0.6's `Coordinate::Abs`, which
/// normalizes against the PRIMARY monitor (`GetSystemMetrics(SM_CXSCREEN)`) and
/// omits `MOUSEEVENTF_VIRTUALDESK`, so any target on a secondary monitor — or a
/// monitor whose virtual-desktop origin is negative/non-zero — is mis-projected
/// onto the primary display.
#[cfg(target_os = "windows")]
fn normalize_to_virtual_desktop(
x: i32,
y: i32,
v_left: i32,
v_top: i32,
v_width: i32,
v_height: i32,
) -> (i32, i32) {
// Map [origin, origin + extent - 1] onto [0, 65535] (round-to-nearest),
// clamped so out-of-desktop inputs never escape the range.
fn axis(coord: i32, origin: i32, extent: i32) -> i32 {
let span = (extent as i64) - 1;
if span <= 0 {
return 0;
}
let rel = (coord as i64 - origin as i64).max(0);
let n = (rel * 65535 + span / 2) / span;
n.clamp(0, 65535) as i32
}
(axis(x, v_left, v_width), axis(y, v_top, v_height))
}
/// Move the cursor to an absolute global screen coordinate (the space produced
/// by `to_screen()` / xcap monitor origins).
///
/// On Windows we bypass enigo's `Coordinate::Abs` — it normalizes against the
/// primary monitor only and omits `MOUSEEVENTF_VIRTUALDESK`, so multi-monitor
/// and negative/non-zero-origin targets land on the wrong display — and emit a
/// `SendInput` move across the whole virtual desktop. On macOS / Linux enigo
/// already actuates global coordinates correctly, so its path is unchanged.
fn move_abs(enigo: &mut Enigo, x: i32, y: i32) -> Result<(), String> {
#[cfg(target_os = "windows")]
{
let _ = enigo;
move_abs_windows(x, y)
}
#[cfg(not(target_os = "windows"))]
{
enigo.move_mouse(x, y, Coordinate::Abs).map_err(input_err)
}
}
/// Windows absolute cursor move over the entire virtual desktop via `SendInput`
/// (`MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK`),
/// normalized against `GetSystemMetrics(SM_*VIRTUALSCREEN)`.
#[cfg(target_os = "windows")]
fn move_abs_windows(x: i32, y: i32) -> Result<(), String> {
use windows::Win32::UI::Input::KeyboardAndMouse::{
INPUT, INPUT_0, INPUT_MOUSE, MOUSEEVENTF_ABSOLUTE, MOUSEEVENTF_MOVE,
MOUSEEVENTF_VIRTUALDESK, MOUSEINPUT, SendInput,
};
use windows::Win32::UI::WindowsAndMessaging::{
GetSystemMetrics, SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN, SM_XVIRTUALSCREEN,
SM_YVIRTUALSCREEN,
};
// SAFETY: GetSystemMetrics reads global display metrics; no preconditions.
let (v_left, v_top, v_width, v_height) = unsafe {
(
GetSystemMetrics(SM_XVIRTUALSCREEN),
GetSystemMetrics(SM_YVIRTUALSCREEN),
GetSystemMetrics(SM_CXVIRTUALSCREEN),
GetSystemMetrics(SM_CYVIRTUALSCREEN),
)
};
if v_width <= 0 || v_height <= 0 {
return Err(
"Could not read the Windows virtual-screen dimensions for absolute \
cursor positioning."
.to_string(),
);
}
let (nx, ny) = normalize_to_virtual_desktop(x, y, v_left, v_top, v_width, v_height);
let input = INPUT {
r#type: INPUT_MOUSE,
Anonymous: INPUT_0 {
mi: MOUSEINPUT {
dx: nx,
dy: ny,
mouseData: 0,
dwFlags: MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK,
time: 0,
dwExtraInfo: 0,
},
},
};
// SAFETY: a single well-formed INPUT value; cbsize matches its size.
let sent = unsafe { SendInput(&[input], std::mem::size_of::<INPUT>() as i32) };
if sent == 1 {
Ok(())
} else {
Err(
"Windows refused the synthetic mouse move (SendInput inserted no \
events; input may be blocked by a higher-integrity window or the \
secure desktop)."
.to_string(),
)
}
}
fn new_enigo() -> Result<Enigo, String> {
let settings = Settings {
// Never block the agent on an interactive permission prompt.
open_prompt_to_get_permissions: false,
..Settings::default()
};
Enigo::new(&settings).map_err(|e| {
format!(
"Failed to initialize input synthesis: {e}. {}",
permissions::accessibility_hint_detailed()
)
})
}
/// Run an input operation on a fresh Enigo instance inside spawn_blocking,
/// bounded by a 10s timeout.
async fn with_enigo<T, F>(op: F) -> Result<T, String>
where
T: Send + 'static,
F: FnOnce(&mut Enigo) -> Result<T, String> + Send + 'static,
{
let handle = tokio::task::spawn_blocking(move || {
let mut enigo = new_enigo()?;
op(&mut enigo)
});
match tokio::time::timeout(INPUT_TIMEOUT, handle).await {
Ok(Ok(result)) => result,
Ok(Err(join_err)) => Err(format!("Input task failed: {join_err}")),
Err(_) => Err(format!(
"Input operation timed out after {}s. The system may be blocking \
synthetic input. {}",
INPUT_TIMEOUT.as_secs(),
permissions::accessibility_hint_detailed()
)),
}
}
fn input_err(e: enigo::InputError) -> String {
format!(
"Input synthesis failed: {e}. {}",
permissions::accessibility_hint()
)
}
/// Move the cursor to absolute screen coordinates.
pub async fn mouse_move(x: i32, y: i32) -> Result<(), String> {
with_enigo(move |enigo| move_abs(enigo, x, y)).await
}
/// Click `button` `count` times at absolute screen coordinates.
pub async fn click(x: i32, y: i32, button: Button, count: u32) -> Result<(), String> {
with_enigo(move |enigo| {
move_abs(enigo, x, y)?;
for i in 0..count {
if i > 0 {
std::thread::sleep(CLICK_PAUSE);
}
enigo.button(button, Direction::Click).map_err(input_err)?;
}
Ok(())
})
.await
}
/// Press at (start), drag to (end), release. Includes intermediate moves so
/// apps that track motion register the drag.
pub async fn drag(start_x: i32, start_y: i32, end_x: i32, end_y: i32) -> Result<(), String> {
with_enigo(move |enigo| {
move_abs(enigo, start_x, start_y)?;
enigo
.button(Button::Left, Direction::Press)
.map_err(input_err)?;
std::thread::sleep(CLICK_PAUSE);
// A few intermediate steps make drags more reliable than a teleport.
const STEPS: i32 = 8;
for i in 1..=STEPS {
let ix = start_x + (end_x - start_x) * i / STEPS;
let iy = start_y + (end_y - start_y) * i / STEPS;
move_abs(enigo, ix, iy)?;
std::thread::sleep(Duration::from_millis(10));
}
enigo
.button(Button::Left, Direction::Release)
.map_err(input_err)?;
Ok(())
})
.await
}
/// Type a unicode string (layout-independent).
pub async fn type_text(text: String) -> Result<(), String> {
with_enigo(move |enigo| enigo.text(&text).map_err(input_err)).await
}
/// Press a key combo: press front-to-back, release back-to-front.
pub async fn key_combo(keys: Vec<enigo::Key>) -> Result<(), String> {
with_enigo(move |enigo| {
let mut pressed: Vec<enigo::Key> = Vec::with_capacity(keys.len());
for key in &keys {
if let Err(e) = enigo.key(*key, Direction::Press) {
// Release anything already held before bailing out.
for held in pressed.iter().rev() {
let _ = enigo.key(*held, Direction::Release);
}
return Err(input_err(e));
}
pressed.push(*key);
}
std::thread::sleep(CLICK_PAUSE);
let mut result = Ok(());
for key in pressed.iter().rev() {
if let Err(e) = enigo.key(*key, Direction::Release) {
result = Err(input_err(e));
}
}
result
})
.await
}
/// Scroll by `amount` wheel clicks in `direction` (optionally moving the
/// cursor to (x, y) first so the scroll lands on the right surface).
pub async fn scroll(
at: Option<(i32, i32)>,
direction: ScrollDirection,
amount: i32,
) -> Result<(), String> {
with_enigo(move |enigo| {
if let Some((x, y)) = at {
move_abs(enigo, x, y)?;
}
let (axis, length) = match direction {
ScrollDirection::Up => (Axis::Vertical, -amount),
ScrollDirection::Down => (Axis::Vertical, amount),
ScrollDirection::Left => (Axis::Horizontal, -amount),
ScrollDirection::Right => (Axis::Horizontal, amount),
};
enigo.scroll(length, axis).map_err(input_err)
})
.await
}
/// Current cursor location in absolute screen coordinates.
pub async fn cursor_position() -> Result<(i32, i32), String> {
with_enigo(|enigo| enigo.location().map_err(input_err)).await
}
/// Size (width, height) of the main display in enigo's coordinate system.
/// Blocking variant for use inside other spawn_blocking sections.
pub fn main_display_size_blocking() -> Result<(i32, i32), String> {
let enigo = new_enigo()?;
enigo.main_display().map_err(input_err)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn scroll_direction_parses_all_variants() {
assert_eq!(ScrollDirection::parse("up").unwrap(), ScrollDirection::Up);
assert_eq!(
ScrollDirection::parse("down").unwrap(),
ScrollDirection::Down
);
assert_eq!(
ScrollDirection::parse("left").unwrap(),
ScrollDirection::Left
);
assert_eq!(
ScrollDirection::parse("right").unwrap(),
ScrollDirection::Right
);
}
#[test]
fn scroll_direction_unknown_is_error() {
let err = ScrollDirection::parse("diagonal").unwrap_err();
assert!(err.contains("diagonal"));
}
// --- virtual-desktop coordinate normalization (Windows actuation fix) ---
#[cfg(target_os = "windows")]
#[test]
fn vd_center_of_single_primary_maps_to_midrange() {
// One 1920x1080 monitor at the virtual-desktop origin.
let (nx, ny) = normalize_to_virtual_desktop(960, 540, 0, 0, 1920, 1080);
assert!((32000..=33500).contains(&nx), "nx={nx}");
assert!((32000..=33500).contains(&ny), "ny={ny}");
}
#[cfg(target_os = "windows")]
#[test]
fn vd_point_on_secondary_monitor_maps_to_upper_range() {
// Two 1920-wide monitors side by side; target is the centre of the RIGHT
// (secondary) monitor. The fixed mapping must land in the upper half of
// the 0..65535 range — enigo's primary-only normalization would divide
// 2880 by the primary width (1920) and overflow past 65535 onto the
// primary display.
let (nx, _) = normalize_to_virtual_desktop(2880, 540, 0, 0, 3840, 1080);
assert!(nx > 40000 && nx <= 65535, "nx={nx}");
}
#[cfg(target_os = "windows")]
#[test]
fn vd_point_on_negative_origin_monitor_maps_to_lower_range() {
// A monitor to the LEFT of the primary (negative virtual-desktop origin).
// Target is the centre of that left monitor; it must map to the lower
// half — enigo would produce a negative normalized value (off-screen).
let (nx, _) = normalize_to_virtual_desktop(-960, 540, -1920, 0, 3840, 1080);
assert!(nx > 10000 && nx < 25000, "nx={nx}");
}
#[cfg(target_os = "windows")]
#[test]
fn vd_endpoints_map_to_full_range() {
// Left/top edge -> 0, right/bottom edge -> 65535, with a non-zero origin.
assert_eq!(normalize_to_virtual_desktop(100, 50, 100, 50, 1920, 1080).0, 0);
assert_eq!(
normalize_to_virtual_desktop(100 + 1920 - 1, 50, 100, 50, 1920, 1080).0,
65535
);
}
#[cfg(target_os = "windows")]
#[test]
fn vd_out_of_desktop_clamps() {
// Beyond the right/below the left edge -> clamped, never out of [0,65535].
assert_eq!(normalize_to_virtual_desktop(99999, 0, 0, 0, 1920, 1080).0, 65535);
assert_eq!(normalize_to_virtual_desktop(-99999, 0, 0, 0, 1920, 1080).0, 0);
}
// Requires a real input device and (on macOS) Accessibility permission.
#[tokio::test]
#[ignore]
async fn cursor_position_real() {
let (x, y) = cursor_position().await.expect("should read cursor");
assert!(x >= -20_000 && x <= 20_000);
assert!(y >= -20_000 && y <= 20_000);
}
#[tokio::test]
#[ignore]
async fn mouse_move_real() {
mouse_move(10, 10).await.expect("should move cursor");
}
}
nomifun-tauri/crates/agent/nomi-computer/src/keys.rs
+240
View File
@@ -0,0 +1,240 @@
//! Parse xdotool-style key combos ("cmd+shift+t") into enigo keys.
//!
//! Aliases are case-insensitive. Single characters map to `Key::Unicode`;
//! ASCII letters are lowercased because shift is expressed as an explicit
//! modifier, not via capitalization.
use enigo::Key;
/// Parse a "+"-separated key combo into the keys to press, in input order.
/// The caller presses them front-to-back and releases back-to-front.
pub fn parse_key_combo(combo: &str) -> Result<Vec<Key>, String> {
let trimmed = combo.trim();
if trimmed.is_empty() {
return Err("Key combo is empty. Provide e.g. \"enter\" or \"cmd+shift+t\".".to_string());
}
let mut keys = Vec::new();
for token in trimmed.split('+') {
let token = token.trim();
if token.is_empty() {
return Err(format!(
"Malformed key combo {combo:?}: empty segment between '+' separators."
));
}
keys.push(parse_single_key(token)?);
}
Ok(keys)
}
/// Parse one token (a named key alias or a single character).
fn parse_single_key(token: &str) -> Result<Key, String> {
let lower = token.to_ascii_lowercase();
let key = match lower.as_str() {
// Modifiers.
//
// `cmd`/`command` is the macOS-idiomatic "primary" accelerator. Models
// trained on macOS habitually emit "cmd+c"/"cmd+a"/"cmd+shift+t" for
// copy/select-all/reopen-tab. On macOS that is the Command key (Meta);
// on Windows/Linux the equivalent accelerator is Control. Mapping `cmd`
// to Meta everywhere is wrong off macOS — enigo's `Key::Meta` is the
// Win/Super key there, so "cmd+c" would fire Win+C (Copilot) instead of
// copy. Remap `cmd`/`command` per-platform.
//
// `super`/`win`/`meta` stay Meta on every platform: a model that asks
// for those explicitly means the OS/Super key, not the accelerator.
#[cfg(target_os = "macos")]
"cmd" | "command" => Key::Meta,
#[cfg(not(target_os = "macos"))]
"cmd" | "command" => Key::Control,
"super" | "win" | "meta" => Key::Meta,
"ctrl" | "control" => Key::Control,
"alt" | "option" | "opt" => Key::Alt,
"shift" => Key::Shift,
// Whitespace / editing
"enter" | "return" => Key::Return,
"esc" | "escape" => Key::Escape,
"tab" => Key::Tab,
"space" => Key::Space,
"backspace" => Key::Backspace,
"delete" | "del" => Key::Delete,
// Navigation
"up" => Key::UpArrow,
"down" => Key::DownArrow,
"left" => Key::LeftArrow,
"right" => Key::RightArrow,
"home" => Key::Home,
"end" => Key::End,
"pageup" | "page_up" | "pgup" => Key::PageUp,
"pagedown" | "page_down" | "pgdn" => Key::PageDown,
// Function keys
"f1" => Key::F1,
"f2" => Key::F2,
"f3" => Key::F3,
"f4" => Key::F4,
"f5" => Key::F5,
"f6" => Key::F6,
"f7" => Key::F7,
"f8" => Key::F8,
"f9" => Key::F9,
"f10" => Key::F10,
"f11" => Key::F11,
"f12" => Key::F12,
_ => {
let mut chars = token.chars();
match (chars.next(), chars.next()) {
(Some(c), None) => Key::Unicode(c.to_ascii_lowercase()),
_ => {
return Err(format!(
"Unknown key {token:?}. Use a single character or one of: \
cmd, ctrl, alt, shift, enter, esc, tab, space, backspace, \
delete, up, down, left, right, home, end, pageup, pagedown, f1-f12."
));
}
}
}
};
Ok(key)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn single_named_key() {
assert_eq!(parse_key_combo("enter").unwrap(), vec![Key::Return]);
assert_eq!(parse_key_combo("return").unwrap(), vec![Key::Return]);
assert_eq!(parse_key_combo("esc").unwrap(), vec![Key::Escape]);
assert_eq!(parse_key_combo("escape").unwrap(), vec![Key::Escape]);
assert_eq!(parse_key_combo("tab").unwrap(), vec![Key::Tab]);
assert_eq!(parse_key_combo("space").unwrap(), vec![Key::Space]);
assert_eq!(parse_key_combo("backspace").unwrap(), vec![Key::Backspace]);
assert_eq!(parse_key_combo("delete").unwrap(), vec![Key::Delete]);
}
#[test]
fn arrow_and_navigation_keys() {
assert_eq!(parse_key_combo("up").unwrap(), vec![Key::UpArrow]);
assert_eq!(parse_key_combo("down").unwrap(), vec![Key::DownArrow]);
assert_eq!(parse_key_combo("left").unwrap(), vec![Key::LeftArrow]);
assert_eq!(parse_key_combo("right").unwrap(), vec![Key::RightArrow]);
assert_eq!(parse_key_combo("home").unwrap(), vec![Key::Home]);
assert_eq!(parse_key_combo("end").unwrap(), vec![Key::End]);
assert_eq!(parse_key_combo("pageup").unwrap(), vec![Key::PageUp]);
assert_eq!(parse_key_combo("pagedown").unwrap(), vec![Key::PageDown]);
}
#[test]
fn modifier_aliases() {
// `cmd`/`command` is the macOS primary accelerator; on macOS it is Meta,
// elsewhere it is Control (enigo's Meta is Win/Super off macOS).
#[cfg(target_os = "macos")]
let cmd_key = Key::Meta;
#[cfg(not(target_os = "macos"))]
let cmd_key = Key::Control;
for alias in ["cmd", "command"] {
assert_eq!(parse_key_combo(alias).unwrap(), vec![cmd_key], "{alias}");
}
// `super`/`win`/`meta` are the OS/Super key on every platform.
for alias in ["super", "win", "meta"] {
assert_eq!(parse_key_combo(alias).unwrap(), vec![Key::Meta], "{alias}");
}
for alias in ["ctrl", "control"] {
assert_eq!(
parse_key_combo(alias).unwrap(),
vec![Key::Control],
"{alias}"
);
}
for alias in ["alt", "option", "opt"] {
assert_eq!(parse_key_combo(alias).unwrap(), vec![Key::Alt], "{alias}");
}
assert_eq!(parse_key_combo("shift").unwrap(), vec![Key::Shift]);
}
#[test]
fn function_keys() {
assert_eq!(parse_key_combo("f1").unwrap(), vec![Key::F1]);
assert_eq!(parse_key_combo("F5").unwrap(), vec![Key::F5]);
assert_eq!(parse_key_combo("f12").unwrap(), vec![Key::F12]);
assert!(parse_key_combo("f13").is_err());
}
#[test]
fn single_character_key() {
assert_eq!(parse_key_combo("a").unwrap(), vec![Key::Unicode('a')]);
assert_eq!(parse_key_combo("/").unwrap(), vec![Key::Unicode('/')]);
assert_eq!(parse_key_combo("0").unwrap(), vec![Key::Unicode('0')]);
}
#[test]
fn uppercase_single_char_is_lowercased() {
// Shift is an explicit modifier; "T" alone means the 't' key.
assert_eq!(parse_key_combo("T").unwrap(), vec![Key::Unicode('t')]);
}
#[test]
fn combo_preserves_modifier_order() {
// `cmd` resolves per-platform (Meta on macOS, Control elsewhere).
#[cfg(target_os = "macos")]
let cmd_key = Key::Meta;
#[cfg(not(target_os = "macos"))]
let cmd_key = Key::Control;
assert_eq!(
parse_key_combo("cmd+shift+t").unwrap(),
vec![cmd_key, Key::Shift, Key::Unicode('t')]
);
assert_eq!(
parse_key_combo("shift+cmd+t").unwrap(),
vec![Key::Shift, cmd_key, Key::Unicode('t')]
);
}
#[test]
fn combo_case_insensitive_aliases() {
#[cfg(target_os = "macos")]
let cmd_key = Key::Meta;
#[cfg(not(target_os = "macos"))]
let cmd_key = Key::Control;
assert_eq!(
parse_key_combo("CMD+SHIFT+Enter").unwrap(),
vec![cmd_key, Key::Shift, Key::Return]
);
assert_eq!(
parse_key_combo("Ctrl+Alt+Delete").unwrap(),
vec![Key::Control, Key::Alt, Key::Delete]
);
}
#[test]
fn combo_with_whitespace_around_tokens() {
#[cfg(target_os = "macos")]
let cmd_key = Key::Meta;
#[cfg(not(target_os = "macos"))]
let cmd_key = Key::Control;
assert_eq!(
parse_key_combo(" cmd + t ").unwrap(),
vec![cmd_key, Key::Unicode('t')]
);
}
#[test]
fn empty_string_is_error() {
assert!(parse_key_combo("").is_err());
assert!(parse_key_combo(" ").is_err());
}
#[test]
fn empty_segment_is_error() {
assert!(parse_key_combo("cmd+").is_err());
assert!(parse_key_combo("+t").is_err());
assert!(parse_key_combo("cmd++t").is_err());
}
#[test]
fn unknown_key_is_error_and_names_token() {
let err = parse_key_combo("cmd+bogus").unwrap_err();
assert!(err.contains("bogus"), "error should name the token: {err}");
}
}
nomifun-tauri/crates/agent/nomi-computer/src/launch.rs
+420
View File
@@ -0,0 +1,420 @@
//! Reliable app / URL / file launch via the OS shell.
//!
//! On Windows this does two things `cmd /c start` and a naive `ShellExecute`
//! cannot:
//! 1. **Resolve an application by name.** A bare display name like "QQ音乐" or
//! "notepad" is not a file/URL/registered-app, so `ShellExecute("QQ音乐")`
//! fails with ERROR_CANCELLED (1223) after popping the "Windows can't find"
//! chooser. We instead resolve the name through the Start Menu
//! (`Get-StartApps`, fuzzy-matched), turn the resulting AppID into a real
//! launch target (a resolved `.exe` path, or `shell:AppsFolder\<AUMID>` for
//! packaged apps), and ShellExecute THAT — which works.
//! 2. **Open URLs/files** via `ShellExecuteExW` (the `open` crate with the
//! workspace's `shellexecute-on-windows` feature), the same path a
//! double-click uses — no `cmd /c start` window-title quirk, no dialog.
//!
//! macOS/Linux fall through to `open` / `xdg-open` (which already resolve apps by
//! name well enough); the Start-Menu resolution is Windows-only.
/// Reject the degenerate targets that make the Windows shell pop a "cannot find"
/// dialog: empty/whitespace, or a string that is nothing but path separators
/// (`\`, `\\`, `//`) — the exact shape behind the `\` dialog.
pub fn validate_launch_target(target: &str) -> Result<(), String> {
let t = target.trim();
if t.is_empty() {
return Err("launch target is empty".to_string());
}
if t.chars().all(|c| c == '\\' || c == '/') {
return Err(format!(
"launch target {target:?} is just path separators — give a URL (https://…), a file or \
folder path, or an application name"
));
}
Ok(())
}
/// True if `target` looks like a URL or registered protocol (`https://…`,
/// `mailto:…`, `microsoft-edge:…`) — those ShellExecute directly and must not be
/// run through Start-Menu app resolution. A drive-letter path (`C:\…`) is NOT a
/// URL (single-letter scheme is rejected).
fn is_url(target: &str) -> bool {
if target.contains("://") {
return true;
}
match target.find(':') {
Some(i) if i > 1 => target[..i]
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '+' || c == '-' || c == '.'),
_ => false,
}
}
/// Launch `target` (a URL, file/folder path, or application name) reliably via
/// the OS shell, optionally opening it WITH a specific `app`. Detached. Returns
/// a human-readable success message.
pub async fn launch(target: &str, app: Option<&str>) -> Result<String, String> {
validate_launch_target(target)?;
let target = target.to_string();
let app = app.map(|s| s.to_string());
tokio::task::spawn_blocking(move || launch_blocking(&target, app.as_deref()))
.await
.unwrap_or_else(|e| Err(format!("launch task failed: {e}")))
}
/// Blocking launch (runs on a `spawn_blocking` thread): resolve the target if it
/// is a bare Windows app name, then ShellExecute it.
fn launch_blocking(target: &str, app: Option<&str>) -> Result<String, String> {
// Opening a target WITH a specific app (e.g. a URL in a chosen browser):
// the app handler resolves it; no Start-Menu lookup.
if let Some(a) = app {
return open::with_detached(target, a)
.map(|()| format!("Opened {target:?} with {a:?}."))
.map_err(|e| format!("failed to open {target:?} with {a:?}: {e}"));
}
// Windows: resolve a bare application NAME through the Start Menu before
// ShellExecuting it. URLs and existing paths skip resolution (they open
// directly).
#[cfg(target_os = "windows")]
{
if !is_url(target) && !std::path::Path::new(target).exists() {
if let Some(resolved) = resolve_start_app(target) {
return open::that_detached(&resolved)
.map(|()| format!("Launched {target:?} (resolved via the Start Menu to {resolved:?})."))
.map_err(|e| {
format!("found {target:?} in the Start Menu ({resolved:?}) but failed to launch it: {e}")
});
}
// Not in the Start Menu — best-effort raw open, with a clear error so
// the model knows to use the exact name / a full path instead of
// retrying the same string.
return open::that_detached(target)
.map(|()| format!("Opened {target:?}."))
.map_err(|e| {
format!(
"could not find an application named {target:?} in the Start Menu, and the \
OS could not open it directly ({e}). Use the app's exact Start-menu name, \
or a full path to its .exe."
)
});
}
}
// URL / existing path (and all non-Windows launches).
open::that_detached(target)
.map(|()| format!("Opened {target:?}."))
.map_err(|e| format!("failed to open {target:?}: {e}"))
}
// ---- Windows Start-Menu application resolution --------------------------
/// Resolve a bare application name to a ShellExecute-able launch target via the
/// Start Menu. `None` when no Start-Menu app matches.
#[cfg(target_os = "windows")]
fn resolve_start_app(name: &str) -> Option<String> {
let apps = get_start_apps();
let app_id = match_app(&apps, name)?;
Some(app_id_to_launch_target(&app_id))
}
/// Diagnostic: resolve an application name to its launch target WITHOUT launching
/// it (Windows only). Used by the `appresolve` example to verify Start-Menu
/// resolution end-to-end on a real machine. `None` if nothing matches.
#[cfg(target_os = "windows")]
pub fn resolve_app_for_diagnostics(name: &str) -> Option<String> {
resolve_start_app(name)
}
/// Enumerate Start-Menu apps as `(display_name, app_id)` via PowerShell
/// `Get-StartApps`. Uses `-EncodedCommand` (base64 UTF-16LE) to avoid all
/// argument-quoting pitfalls, and `CREATE_NO_WINDOW` so no console flashes.
/// Returns empty on any failure (caller falls back to a raw open).
#[cfg(target_os = "windows")]
fn get_start_apps() -> Vec<(String, String)> {
use base64::Engine;
use std::os::windows::process::CommandExt;
const CREATE_NO_WINDOW: u32 = 0x0800_0000;
// Tab-separated Name<TAB>AppID per line (names/AppIDs don't contain tabs).
let script = "[Console]::OutputEncoding=[Text.Encoding]::UTF8; \
Get-StartApps | ForEach-Object { \"$($_.Name)`t$($_.AppID)\" }";
let encoded = {
let utf16: Vec<u8> = script.encode_utf16().flat_map(|u| u.to_le_bytes()).collect();
base64::engine::general_purpose::STANDARD.encode(utf16)
};
let output = std::process::Command::new("powershell")
.args(["-NoProfile", "-NonInteractive", "-EncodedCommand", &encoded])
.creation_flags(CREATE_NO_WINDOW)
.output();
let Ok(out) = output else {
return Vec::new();
};
let text = String::from_utf8_lossy(&out.stdout);
parse_start_apps(&text)
}
/// Parse `Get-StartApps` tab-separated output into `(name, app_id)` pairs (pure).
#[cfg(target_os = "windows")]
fn parse_start_apps(text: &str) -> Vec<(String, String)> {
text.lines()
.filter_map(|line| {
let mut it = line.splitn(2, '\t');
let name = it.next()?.trim();
let app_id = it.next()?.trim();
if name.is_empty() || app_id.is_empty() {
None
} else {
Some((name.to_string(), app_id.to_string()))
}
})
.collect()
}
/// Normalize for matching: lowercase, drop all whitespace.
#[cfg(target_os = "windows")]
fn norm(s: &str) -> String {
s.chars()
.filter(|c| !c.is_whitespace())
.flat_map(|c| c.to_lowercase())
.collect()
}
/// The exe basename of an AppID path (without `.exe`), for matching a query like
/// "qqmusic" against `…\QQMusic.exe`. Empty for AUMIDs (no `\`).
#[cfg(target_os = "windows")]
fn exe_basename(app_id: &str) -> String {
if !app_id.contains('\\') {
return String::new();
}
let base = app_id.rsplit('\\').next().unwrap_or(app_id);
base.strip_suffix(".exe")
.or_else(|| base.strip_suffix(".EXE"))
.unwrap_or(base)
.to_string()
}
/// Pick the best-matching AppID for `query` from the Start-Menu list (pure).
/// Priority: exact (normalized) match on the display name or the exe basename;
/// then a containment match (shortest name = most specific). `None` if nothing
/// reasonable matches.
#[cfg(target_os = "windows")]
fn match_app(apps: &[(String, String)], query: &str) -> Option<String> {
let q = norm(query);
if q.is_empty() {
return None;
}
let mut best_contains: Option<(&str, usize)> = None;
for (name, app_id) in apps {
let nn = norm(name);
let bn = norm(&exe_basename(app_id));
if nn == q || (!bn.is_empty() && bn == q) {
return Some(app_id.clone()); // exact wins immediately
}
if q.len() >= 2 && (nn.contains(&q) || (!bn.is_empty() && bn.contains(&q))) {
let score = name.chars().count();
if best_contains.map(|(_, s)| score < s).unwrap_or(true) {
best_contains = Some((app_id, score));
}
}
}
best_contains.map(|(a, _)| a.to_string())
}
/// Split a `{KNOWN-FOLDER-GUID}\rest` AppID into the GUID and the remainder
/// (pure). `None` when there is no leading `{…}` brace group.
#[cfg(target_os = "windows")]
fn split_guid_prefix(app_id: &str) -> Option<(&str, &str)> {
if !app_id.starts_with('{') {
return None;
}
let end = app_id.find('}')?;
let guid = &app_id[..=end];
let rest = app_id[end + 1..].trim_start_matches('\\');
Some((guid, rest))
}
/// Convert a `Get-StartApps` AppID into a ShellExecute-able launch target.
/// * `{GUID}\rest` → resolve the known-folder GUID and join `rest` (a real path).
/// * other `…\…` → a plain absolute path, used as-is.
/// * no backslash → an AUMID (packaged app) → `shell:AppsFolder\<AUMID>`.
/// An unresolvable GUID falls back to the raw AppID (best effort).
#[cfg(target_os = "windows")]
fn app_id_to_launch_target(app_id: &str) -> String {
if !app_id.contains('\\') {
return format!("shell:AppsFolder\\{app_id}");
}
if let Some((guid, rest)) = split_guid_prefix(app_id) {
if let Some(base) = known_folder_path(guid) {
return std::path::Path::new(&base)
.join(rest)
.to_string_lossy()
.into_owned();
}
}
app_id.to_string()
}
/// Resolve the common known-folder GUIDs that `Get-StartApps` prefixes desktop
/// app paths with, via their environment variables (no COM/FFI needed). Covers
/// ProgramFiles(x86/x64), Windows/System32, LocalAppData/AppData, UserProfile.
#[cfg(target_os = "windows")]
fn known_folder_path(guid: &str) -> Option<String> {
let g = guid.to_ascii_uppercase();
let env = |k: &str| std::env::var(k).ok().filter(|v| !v.is_empty());
let win = || env("SystemRoot").or_else(|| env("windir"));
match g.as_str() {
// FOLDERID_ProgramFilesX86
"{7C5A40EF-A0FB-4BFC-874A-C0F2E0B9FA8E}" => env("ProgramFiles(x86)").or_else(|| env("ProgramFiles")),
// FOLDERID_ProgramFilesX64 / FOLDERID_ProgramFiles
"{6D809377-6AF0-444B-8957-A3773F02200E}" => env("ProgramW6432").or_else(|| env("ProgramFiles")),
"{905E63B6-C1BF-494E-B29C-65B732D3D21A}" => env("ProgramFiles"),
// FOLDERID_Windows
"{F38BF404-1D43-42F2-9305-67DE0B28FC23}" => win(),
// FOLDERID_System (System32) / FOLDERID_SystemX86
"{1AC14E77-02E7-4E5D-B744-2EB1AE5198B7}" | "{D65231B0-B2F1-4857-A4CE-A8E7C6EA7D27}" => {
win().map(|w| format!("{w}\\System32"))
}
// FOLDERID_LocalAppData
"{F1B32785-6FBA-4FCF-9D55-7B8E7F157091}" => env("LOCALAPPDATA"),
// FOLDERID_RoamingAppData
"{3EB685DB-65F9-4CF6-A03A-E3EF65729F3D}" => env("APPDATA"),
// FOLDERID_Profile
"{5E6C858F-0E22-4760-9AFE-EA3317B67173}" => env("USERPROFILE"),
// FOLDERID_Programs (user Start Menu\Programs)
"{A77F5D77-2E2B-44C3-A6A2-ABA601054A51}" => {
env("APPDATA").map(|a| format!("{a}\\Microsoft\\Windows\\Start Menu\\Programs"))
}
// FOLDERID_CommonPrograms (all-users Start Menu\Programs)
"{0139D44E-6AFE-49F2-8690-3DAFCAE6FFB8}" => {
env("ProgramData").map(|a| format!("{a}\\Microsoft\\Windows\\Start Menu\\Programs"))
}
_ => None,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn rejects_empty_whitespace_and_bare_separators() {
assert!(validate_launch_target("").is_err());
assert!(validate_launch_target(" ").is_err());
assert!(validate_launch_target("\\").is_err());
assert!(validate_launch_target("//").is_err());
assert!(validate_launch_target("\\\\").is_err());
assert!(validate_launch_target("/\\/").is_err());
}
#[test]
fn accepts_url_path_and_app_name() {
assert!(validate_launch_target("https://www.example.com").is_ok());
assert!(validate_launch_target("C:\\Windows\\notepad.exe").is_ok());
assert!(validate_launch_target("msedge").is_ok());
assert!(validate_launch_target("/home/user/file.txt").is_ok());
}
#[test]
fn is_url_classifies_schemes_not_drive_paths() {
assert!(is_url("https://example.com"));
assert!(is_url("http://x"));
assert!(is_url("mailto:a@b.com"));
assert!(is_url("microsoft-edge:https://x"));
assert!(!is_url("C:\\Program Files\\x.exe")); // drive letter, not a scheme
assert!(!is_url("QQ音乐"));
assert!(!is_url("notepad"));
}
// ---- Windows Start-Menu resolution (pure logic) ----
#[cfg(target_os = "windows")]
mod windows_resolution {
use super::super::*;
/// Real `Get-StartApps` rows from this machine (QQ Music is a
/// GUID-prefixed desktop path; QQ is an AUMID-like bare id).
fn sample() -> Vec<(String, String)> {
vec![
("QQ".to_string(), "QQ".to_string()),
(
"QQ音乐".to_string(),
"{7C5A40EF-A0FB-4BFC-874A-C0F2E0B9FA8E}\\Tencent\\QQMusic\\QQMusic.exe".to_string(),
),
(
"网易云音乐".to_string(),
"{6D809377-6AF0-444B-8957-A3773F02200E}\\NetEase\\CloudMusic\\cloudmusic.exe".to_string(),
),
("Notepad".to_string(), "Microsoft.Windows.Notepad_8wekyb3d8bbwe!App".to_string()),
]
}
#[test]
fn parse_start_apps_splits_tab_separated() {
let out = "QQ音乐\t{7C5A40EF}\\a\\b.exe\nNotepad\tNotepad.AUMID\n\n";
let v = parse_start_apps(out);
assert_eq!(v.len(), 2);
assert_eq!(v[0].0, "QQ音乐");
assert_eq!(v[0].1, "{7C5A40EF}\\a\\b.exe");
assert_eq!(v[1], ("Notepad".to_string(), "Notepad.AUMID".to_string()));
}
#[test]
fn match_app_exact_chinese_name() {
let m = match_app(&sample(), "QQ音乐").unwrap();
assert!(m.contains("QQMusic.exe"), "got {m}");
}
#[test]
fn match_app_matches_exe_basename_for_qqmusic() {
// "qqmusic" matches the exe basename of QQ音乐's AppID, not its name.
let m = match_app(&sample(), "qqmusic").unwrap();
assert!(m.contains("QQMusic.exe"), "got {m}");
}
#[test]
fn match_app_contains_for_partial_chinese() {
let m = match_app(&sample(), "网易云").unwrap();
assert!(m.contains("cloudmusic.exe"), "got {m}");
}
#[test]
fn match_app_aumid_exact() {
let m = match_app(&sample(), "notepad").unwrap();
assert!(m.contains("Microsoft.Windows.Notepad"), "got {m}");
}
#[test]
fn match_app_none_for_unknown() {
assert!(match_app(&sample(), "definitely-not-an-app-xyz").is_none());
}
#[test]
fn split_guid_prefix_extracts_guid_and_rest() {
let (g, r) = split_guid_prefix("{7C5A40EF-A0FB-4BFC-874A-C0F2E0B9FA8E}\\Tencent\\QQMusic\\QQMusic.exe").unwrap();
assert_eq!(g, "{7C5A40EF-A0FB-4BFC-874A-C0F2E0B9FA8E}");
assert_eq!(r, "Tencent\\QQMusic\\QQMusic.exe");
assert!(split_guid_prefix("C:\\plain\\path.exe").is_none());
assert!(split_guid_prefix("Some.AUMID!App").is_none());
}
#[test]
fn app_id_to_launch_target_aumid_and_plain_path() {
assert_eq!(
app_id_to_launch_target("Microsoft.Windows.Notepad_8wekyb3d8bbwe!App"),
"shell:AppsFolder\\Microsoft.Windows.Notepad_8wekyb3d8bbwe!App"
);
assert_eq!(app_id_to_launch_target("C:\\plain\\x.exe"), "C:\\plain\\x.exe");
}
#[test]
fn app_id_to_launch_target_resolves_program_files_x86_guid() {
// {7C5A40EF…} = ProgramFilesX86 → real path under %ProgramFiles(x86)%.
let t = app_id_to_launch_target(
"{7C5A40EF-A0FB-4BFC-874A-C0F2E0B9FA8E}\\Tencent\\QQMusic\\QQMusic.exe",
);
assert!(t.ends_with("\\Tencent\\QQMusic\\QQMusic.exe"), "got {t}");
assert!(!t.starts_with('{'), "GUID must be resolved away: {t}");
}
}
}
nomifun-tauri/crates/agent/nomi-computer/src/lib.rs
+15
View File
@@ -0,0 +1,15 @@
pub mod fallback_backend;
pub mod input;
pub mod keys;
pub mod launch;
pub mod permissions;
pub mod scale;
pub mod screen;
pub mod tool;
pub use tool::ComputerTool;
// Re-exported so a host that pulls computer-use (but not nomi-a11y directly)
// can brand the permission-error guidance with its own app name — see
// `nomi_a11y::set_host_app_label`.
pub use nomi_a11y::{host_app_label, set_host_app_label};
nomifun-tauri/crates/agent/nomi-computer/src/permissions.rs
+310
View File
@@ -0,0 +1,310 @@
//! Best-effort OS permission diagnostics for screen capture and input.
//!
//! macOS gates screen capture behind "Screen Recording" and input synthesis
//! behind "Accessibility". This module provides PROACTIVE status probing and
//! prompting (via TCC APIs) plus the legacy reactive hints/heuristics as
//! corroborating fallbacks. On non-macOS platforms the status calls report
//! "unknown" (`None`) and requests are no-ops.
/// A point-in-time snapshot of the two TCC permissions computer-use needs.
/// `None` for a field means the platform cannot report it (non-macOS).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct PermissionStatus {
pub accessibility: Option<bool>,
pub screen_recording: Option<bool>,
}
/// Live Accessibility (input synthesis / future a11y-tree) grant state.
/// `None` where the platform has no such gate.
pub fn accessibility_granted() -> Option<bool> {
#[cfg(target_os = "macos")]
{
Some(macos_tcc::accessibility_trusted())
}
#[cfg(not(target_os = "macos"))]
{
None
}
}
/// Live Screen Recording (screenshot) grant state. `None` off macOS.
pub fn screen_recording_granted() -> Option<bool> {
#[cfg(target_os = "macos")]
{
Some(macos_tcc::screen_capture_granted())
}
#[cfg(not(target_os = "macos"))]
{
None
}
}
/// Snapshot both permissions at once (for a host/UI permission panel).
pub fn permission_status() -> PermissionStatus {
PermissionStatus {
accessibility: accessibility_granted(),
screen_recording: screen_recording_granted(),
}
}
/// Trigger the OS Accessibility prompt (macOS shows the system dialog and the
/// Settings deep-link). Returns the post-call grant state. No-op → `true`
/// where the platform has no such gate.
pub fn request_accessibility() -> bool {
#[cfg(target_os = "macos")]
{
macos_tcc::request_accessibility()
}
#[cfg(not(target_os = "macos"))]
{
true
}
}
/// Trigger the OS Screen Recording prompt (macOS). Returns the post-call grant
/// state. Note: macOS caches the grant per process, so a freshly-granted
/// permission typically requires an app relaunch to take effect.
pub fn request_screen_recording() -> bool {
#[cfg(target_os = "macos")]
{
macos_tcc::request_screen_capture()
}
#[cfg(not(target_os = "macos"))]
{
true
}
}
fn live_status_note(granted: Option<bool>, name: &str) -> String {
match granted {
Some(false) => format!(" (checked just now: {name} is NOT granted)"),
Some(true) => format!(" (checked just now: {name} is granted)"),
None => String::new(),
}
}
/// `screen_capture_hint()` plus the live grant state when known.
pub fn screen_capture_hint_detailed() -> String {
format!(
"{}{}",
screen_capture_hint(),
live_status_note(screen_recording_granted(), "Screen Recording")
)
}
/// `accessibility_hint()` plus the live grant state when known.
pub fn accessibility_hint_detailed() -> String {
format!(
"{}{}",
accessibility_hint(),
live_status_note(accessibility_granted(), "Accessibility")
)
}
/// Guidance appended to screen-capture failures. Names the host app (see
/// [`nomi_a11y::host_app_label`]) and stresses the relaunch, because macOS never
/// hot-loads Screen Recording into an already-running process.
pub fn screen_capture_hint() -> String {
if cfg!(target_os = "macos") {
let app = nomi_a11y::host_app_label();
format!(
"If this keeps failing, grant Screen Recording permission to {app} in \
System Settings → Privacy & Security → Screen Recording, then COMPLETELY quit and \
reopen {app} (a freshly-granted Screen Recording permission only takes effect after \
a relaunch)."
)
} else {
"Check that a display is connected and the app is allowed to capture the screen.".to_string()
}
}
/// Guidance appended to input-synthesis failures. Names the host app and the
/// relaunch — the generic "this app" otherwise leads a model to send the user
/// to grant a terminal/editor rather than the desktop host itself.
pub fn accessibility_hint() -> String {
if cfg!(target_os = "macos") {
let app = nomi_a11y::host_app_label();
format!(
"If this keeps failing, grant Accessibility permission to {app} in \
System Settings → Privacy & Security → Accessibility, then COMPLETELY quit and reopen \
{app}. Computer-use runs inside {app} itself, so grant {app} — not a terminal or editor."
)
} else {
"Check that the app is allowed to control the mouse and keyboard.".to_string()
}
}
/// Verify a captured frame is usable. macOS without Screen Recording can
/// "succeed" but return an all-black frame; flag that. The authoritative TCC
/// preflight lives at the capture call site (`screen::capture_screen`); this
/// stays a pure, environment-independent heuristic so it remains unit-testable
/// and also catches edge cases where the preflight and the real capture
/// disagree (e.g. the macOS 26 "responsible process" drift). Always Ok off
/// macOS.
pub fn screenshot_permission_check(img: &image::RgbaImage) -> Result<(), String> {
if cfg!(target_os = "macos") && looks_all_black(img) {
return Err(format!(
"Screenshot came back entirely black, which usually means the \
Screen Recording permission is missing or stale. {}",
screen_capture_hint_detailed()
));
}
Ok(())
}
/// macOS TCC FFI: proactive Accessibility / Screen Recording status + prompt.
#[cfg(target_os = "macos")]
mod macos_tcc {
use core_foundation::base::TCFType;
use core_foundation::boolean::CFBoolean;
use core_foundation::dictionary::{CFDictionary, CFDictionaryRef};
use core_foundation::string::{CFString, CFStringRef};
// HIServices (umbrella: ApplicationServices). `Boolean` is `unsigned char`.
#[link(name = "ApplicationServices", kind = "framework")]
unsafe extern "C" {
fn AXIsProcessTrusted() -> u8;
fn AXIsProcessTrustedWithOptions(options: CFDictionaryRef) -> u8;
static kAXTrustedCheckOptionPrompt: CFStringRef;
}
// CoreGraphics screen-capture access (C `bool`).
#[link(name = "CoreGraphics", kind = "framework")]
unsafe extern "C" {
fn CGPreflightScreenCaptureAccess() -> bool;
fn CGRequestScreenCaptureAccess() -> bool;
}
pub fn accessibility_trusted() -> bool {
unsafe { AXIsProcessTrusted() != 0 }
}
/// Probe Accessibility AND show the system prompt + Settings deep-link when
/// not yet granted.
pub fn request_accessibility() -> bool {
unsafe {
let key = CFString::wrap_under_get_rule(kAXTrustedCheckOptionPrompt);
let opts = CFDictionary::from_CFType_pairs(&[(
key.as_CFType(),
CFBoolean::true_value().as_CFType(),
)]);
AXIsProcessTrustedWithOptions(opts.as_concrete_TypeRef()) != 0
}
}
pub fn screen_capture_granted() -> bool {
unsafe { CGPreflightScreenCaptureAccess() }
}
pub fn request_screen_capture() -> bool {
unsafe { CGRequestScreenCaptureAccess() }
}
}
/// True if every sampled pixel is (near-)black. Samples a grid rather than
/// every pixel to keep this cheap on Retina-sized captures.
pub fn looks_all_black(img: &image::RgbaImage) -> bool {
let (w, h) = img.dimensions();
if w == 0 || h == 0 {
return true;
}
let step_x = (w / 64).max(1);
let step_y = (h / 64).max(1);
let mut y = 0;
while y < h {
let mut x = 0;
while x < w {
let p = img.get_pixel(x, y);
if p[0] > 2 || p[1] > 2 || p[2] > 2 {
return false;
}
x += step_x;
}
y += step_y;
}
true
}
#[cfg(test)]
mod tests {
use super::*;
use image::{Rgba, RgbaImage};
#[test]
fn all_black_image_detected() {
let img = RgbaImage::from_pixel(64, 64, Rgba([0, 0, 0, 255]));
assert!(looks_all_black(&img));
}
#[test]
fn near_black_noise_still_counts_as_black() {
let img = RgbaImage::from_pixel(64, 64, Rgba([1, 2, 1, 255]));
assert!(looks_all_black(&img));
}
#[test]
fn single_bright_pixel_is_not_black() {
let mut img = RgbaImage::from_pixel(64, 64, Rgba([0, 0, 0, 255]));
// Place it on the sampling grid origin so the sparse scan sees it.
img.put_pixel(0, 0, Rgba([255, 255, 255, 255]));
assert!(!looks_all_black(&img));
}
#[test]
fn empty_image_counts_as_black() {
let img = RgbaImage::new(0, 0);
assert!(looks_all_black(&img));
}
#[test]
fn permission_check_passes_on_normal_image() {
let img = RgbaImage::from_pixel(8, 8, Rgba([120, 40, 200, 255]));
assert!(screenshot_permission_check(&img).is_ok());
}
#[test]
fn hints_are_nonempty() {
assert!(!screen_capture_hint().is_empty());
assert!(!accessibility_hint().is_empty());
// Detailed variants embed live status and must still be non-empty.
assert!(!screen_capture_hint_detailed().is_empty());
assert!(!accessibility_hint_detailed().is_empty());
}
// The hints must name the host app on macOS so a model stops sending the
// user to grant a terminal/editor. Off macOS the message is generic.
#[test]
fn macos_hints_name_the_host_app() {
nomi_a11y::set_host_app_label("NomiFun");
if cfg!(target_os = "macos") {
assert!(
accessibility_hint().contains("NomiFun"),
"accessibility hint should name the host app: {}",
accessibility_hint()
);
assert!(
screen_capture_hint().contains("NomiFun"),
"screen-capture hint should name the host app: {}",
screen_capture_hint()
);
}
}
#[test]
fn permission_status_does_not_panic_and_is_consistent() {
// Calls the real TCC APIs on macOS (read-only, safe); no-op elsewhere.
let s = permission_status();
assert_eq!(s.accessibility, accessibility_granted());
assert_eq!(s.screen_recording, screen_recording_granted());
#[cfg(target_os = "macos")]
{
assert!(s.accessibility.is_some());
assert!(s.screen_recording.is_some());
}
#[cfg(not(target_os = "macos"))]
{
assert_eq!(s.accessibility, None);
assert_eq!(s.screen_recording, None);
}
}
}
nomifun-tauri/crates/agent/nomi-computer/src/scale.rs
+191
View File
@@ -0,0 +1,191 @@
//! Pure geometry helpers for screenshot downscaling and coordinate mapping.
//!
//! The LLM sees a (possibly downscaled) screenshot and replies with pixel
//! coordinates in that image. Input synthesis needs logical screen
//! coordinates (on macOS Retina xcap captures physical pixels while enigo
//! expects logical points), so every pointer action goes through
//! `map_llm_coord` and cursor reporting through `map_screen_coord`.
/// Compute downscaled dimensions so the longest edge fits `max_edge`.
/// Returns the original size when it already fits, or when `max_edge` is 0
/// (treated as "no limit"). Never returns a zero dimension.
pub fn fit_within(width: u32, height: u32, max_edge: u32) -> (u32, u32) {
if max_edge == 0 || (width <= max_edge && height <= max_edge) {
return (width, height);
}
let longest = width.max(height) as f64;
let scale = max_edge as f64 / longest;
let w = ((width as f64 * scale).round() as u32).max(1);
let h = ((height as f64 * scale).round() as u32).max(1);
(w, h)
}
/// Map a coordinate in screenshot pixel space to logical screen space.
///
/// Uses pixel-center mapping so the round trip with `map_screen_coord` is
/// exact whenever the logical size is >= the image size. The result is
/// clamped into the screen bounds so out-of-range model output cannot click
/// outside the display.
pub fn map_llm_coord(
llm_x: i32,
llm_y: i32,
img_w: u32,
img_h: u32,
logical_w: u32,
logical_h: u32,
) -> (i32, i32) {
if img_w == 0 || img_h == 0 || logical_w == 0 || logical_h == 0 {
return (llm_x, llm_y);
}
let x = (f64::from(llm_x) + 0.5) * f64::from(logical_w) / f64::from(img_w);
let y = (f64::from(llm_y) + 0.5) * f64::from(logical_h) / f64::from(img_h);
(
(x.floor() as i32).clamp(0, logical_w as i32 - 1),
(y.floor() as i32).clamp(0, logical_h as i32 - 1),
)
}
/// Inverse of `map_llm_coord`: map a logical screen coordinate back to
/// screenshot pixel space (e.g. to report the cursor position in the
/// coordinate system the model is working in).
pub fn map_screen_coord(
screen_x: i32,
screen_y: i32,
img_w: u32,
img_h: u32,
logical_w: u32,
logical_h: u32,
) -> (i32, i32) {
if img_w == 0 || img_h == 0 || logical_w == 0 || logical_h == 0 {
return (screen_x, screen_y);
}
let x = (f64::from(screen_x) + 0.5) * f64::from(img_w) / f64::from(logical_w);
let y = (f64::from(screen_y) + 0.5) * f64::from(img_h) / f64::from(logical_h);
(
(x.floor() as i32).clamp(0, img_w as i32 - 1),
(y.floor() as i32).clamp(0, img_h as i32 - 1),
)
}
#[cfg(test)]
mod tests {
use super::*;
// --- fit_within ---
#[test]
fn fit_within_already_fits() {
assert_eq!(fit_within(800, 600, 1568), (800, 600));
}
#[test]
fn fit_within_exactly_at_limit() {
assert_eq!(fit_within(1568, 980, 1568), (1568, 980));
}
#[test]
fn fit_within_landscape_downscale() {
// Retina MacBook: 2880x1800 physical -> longest edge 1568
assert_eq!(fit_within(2880, 1800, 1568), (1568, 980));
}
#[test]
fn fit_within_portrait_downscale() {
assert_eq!(fit_within(1800, 2880, 1568), (980, 1568));
}
#[test]
fn fit_within_square() {
assert_eq!(fit_within(2000, 2000, 1000), (1000, 1000));
}
#[test]
fn fit_within_extreme_ratio_never_zero() {
// 10000:1 aspect ratio must not collapse the short edge to 0
assert_eq!(fit_within(10000, 1, 100), (100, 1));
assert_eq!(fit_within(1, 10000, 100), (1, 100));
}
#[test]
fn fit_within_zero_max_edge_means_no_limit() {
assert_eq!(fit_within(2880, 1800, 0), (2880, 1800));
}
#[test]
fn fit_within_longest_edge_is_exact() {
let (w, h) = fit_within(2879, 1799, 1568);
assert_eq!(w.max(h), 1568);
assert!(w >= 1 && h >= 1);
}
// --- map_llm_coord ---
#[test]
fn map_llm_identity_when_same_size() {
assert_eq!(map_llm_coord(10, 20, 1440, 900, 1440, 900), (10, 20));
assert_eq!(map_llm_coord(0, 0, 1440, 900, 1440, 900), (0, 0));
assert_eq!(map_llm_coord(1439, 899, 1440, 900, 1440, 900), (1439, 899));
}
#[test]
fn map_llm_corners_map_to_screen_corners() {
// Screenshot 1568x980 of a 1440x900 logical screen (Retina capture
// downscaled, but still larger than logical).
assert_eq!(map_llm_coord(0, 0, 1568, 980, 1440, 900), (0, 0));
assert_eq!(map_llm_coord(1567, 979, 1568, 980, 1440, 900), (1439, 899));
}
#[test]
fn map_llm_center_maps_to_center() {
let (x, y) = map_llm_coord(784, 490, 1568, 980, 1440, 900);
assert!((x - 720).abs() <= 1, "x = {x}");
assert!((y - 450).abs() <= 1, "y = {y}");
}
#[test]
fn map_llm_clamps_out_of_range() {
assert_eq!(map_llm_coord(-50, -50, 1000, 800, 1440, 900), (0, 0));
assert_eq!(
map_llm_coord(99999, 99999, 1000, 800, 1440, 900),
(1439, 899)
);
}
#[test]
fn map_llm_zero_dims_passthrough() {
assert_eq!(map_llm_coord(5, 7, 0, 0, 1440, 900), (5, 7));
assert_eq!(map_llm_coord(5, 7, 1000, 800, 0, 0), (5, 7));
}
// --- round trips ---
#[test]
fn round_trip_exact_when_screen_larger_than_image() {
// image 1000 wide, logical 1440 wide: llm -> screen -> llm is identity
for k in 0..1000 {
let (sx, _) = map_llm_coord(k, 0, 1000, 800, 1440, 900);
let (back, _) = map_screen_coord(sx, 0, 1000, 800, 1440, 900);
assert_eq!(back, k, "round trip failed for {k}");
}
}
#[test]
fn round_trip_within_one_pixel_when_image_larger_than_screen() {
// image 1568 wide, logical 1440 wide: contraction loses at most 1px
for k in (0..1568).step_by(7) {
let (sx, _) = map_llm_coord(k, 0, 1568, 980, 1440, 900);
let (back, _) = map_screen_coord(sx, 0, 1568, 980, 1440, 900);
assert!((back - k).abs() <= 1, "round trip drifted for {k}: {back}");
}
}
#[test]
fn screen_round_trip_exact_when_image_larger_than_screen() {
// screen -> llm -> screen is identity in the contractive direction
for k in 0..1440 {
let (ix, _) = map_screen_coord(k, 0, 1568, 980, 1440, 900);
let (back, _) = map_llm_coord(ix, 0, 1568, 980, 1440, 900);
assert_eq!(back, k, "screen round trip failed for {k}");
}
}
}
nomifun-tauri/crates/agent/nomi-computer/src/screen.rs
+173
View File
@@ -0,0 +1,173 @@
//! Screen capture via xcap, downscaled and PNG/base64-encoded for the LLM.
//!
//! Coordinate systems: `Monitor::width()/height()` report the size enigo's
//! absolute mouse coordinates use (logical points on macOS, device pixels on
//! Windows/Linux), while `capture_image()` returns physical pixels (2x on
//! Retina). `CaptureGeometry` records both so clicks can be mapped back.
use base64::Engine as _;
use xcap::Monitor;
use nomi_types::tool::ToolImage;
use crate::permissions;
use crate::scale::fit_within;
/// Geometry of the most recent capture, used to map LLM (screenshot-pixel)
/// coordinates to absolute screen coordinates.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct CaptureGeometry {
/// Dimensions of the image the LLM sees (after downscaling).
pub img_w: u32,
pub img_h: u32,
/// Monitor size in the coordinate system input synthesis uses.
pub logical_w: u32,
pub logical_h: u32,
/// Monitor origin in the global (multi-display) coordinate space.
pub origin_x: i32,
pub origin_y: i32,
}
/// A completed screen capture. `image` is the downscaled RGBA frame (not yet
/// encoded) so callers can draw a Set-of-Marks overlay before encoding; use
/// `encode_png` to produce the `ToolImage`.
#[derive(Debug)]
pub struct CapturedScreen {
pub image: image::RgbaImage,
pub geometry: CaptureGeometry,
/// Raw captured frame size in physical pixels (before downscaling).
pub physical_w: u32,
pub physical_h: u32,
/// Index of the captured monitor within `Monitor::all()`.
pub display_index: usize,
}
/// Encode a (possibly overlay-annotated) RGBA frame as a base64 PNG `ToolImage`.
pub fn encode_png(img: &image::RgbaImage) -> Result<ToolImage, String> {
let mut png = Vec::new();
image::DynamicImage::ImageRgba8(img.clone())
.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png)
.map_err(|e| format!("Failed to encode screenshot as PNG: {e}"))?;
Ok(ToolImage {
media_type: "image/png".to_string(),
data: base64::engine::general_purpose::STANDARD.encode(&png),
})
}
/// Pick the monitor to capture: explicit index, else the primary, else the
/// first one listed.
fn select_monitor(monitors: &[Monitor], display: Option<usize>) -> Result<usize, String> {
if monitors.is_empty() {
return Err(format!(
"No displays found. {}",
permissions::screen_capture_hint_detailed()
));
}
match display {
Some(idx) => {
if idx < monitors.len() {
Ok(idx)
} else {
Err(format!(
"Display {idx} does not exist; {} display(s) available (0-{}).",
monitors.len(),
monitors.len() - 1
))
}
}
None => Ok(monitors
.iter()
.position(|m| m.is_primary().unwrap_or(false))
.unwrap_or(0)),
}
}
/// Capture a monitor, downscale to `max_edge`, and encode as base64 PNG.
/// Blocking: call from `spawn_blocking`.
pub fn capture_screen(display: Option<usize>, max_edge: u32) -> Result<CapturedScreen, String> {
// Proactive + authoritative on macOS: a denied Screen Recording grant lets
// capture "succeed" with a black frame, so fail fast with a clear message
// instead of relying solely on the downstream black-frame heuristic.
if permissions::screen_recording_granted() == Some(false) {
return Err(format!(
"Screen Recording permission is not granted, so the screen cannot be captured. {}",
permissions::screen_capture_hint_detailed()
));
}
let monitors = Monitor::all().map_err(|e| {
format!(
"Failed to enumerate displays: {e}. {}",
permissions::screen_capture_hint_detailed()
)
})?;
let display_index = select_monitor(&monitors, display)?;
let monitor = &monitors[display_index];
let frame = monitor.capture_image().map_err(|e| {
format!(
"Failed to capture the screen: {e}. {}",
permissions::screen_capture_hint_detailed()
)
})?;
permissions::screenshot_permission_check(&frame)?;
let (physical_w, physical_h) = frame.dimensions();
if physical_w == 0 || physical_h == 0 {
return Err(format!(
"Capture returned an empty frame. {}",
permissions::screen_capture_hint_detailed()
));
}
let (img_w, img_h) = fit_within(physical_w, physical_h, max_edge);
let scaled = if (img_w, img_h) == (physical_w, physical_h) {
frame
} else {
image::imageops::resize(&frame, img_w, img_h, image::imageops::FilterType::Triangle)
};
let logical_w = monitor.width().unwrap_or(physical_w);
let logical_h = monitor.height().unwrap_or(physical_h);
let origin_x = monitor.x().unwrap_or(0);
let origin_y = monitor.y().unwrap_or(0);
Ok(CapturedScreen {
image: scaled,
geometry: CaptureGeometry {
img_w,
img_h,
logical_w,
logical_h,
origin_x,
origin_y,
},
physical_w,
physical_h,
display_index,
})
}
#[cfg(test)]
mod tests {
use super::*;
// Requires a real display and (on macOS) Screen Recording permission.
#[test]
#[ignore]
fn capture_primary_screen_real() {
let captured = capture_screen(None, 1568).expect("capture should succeed");
assert!(captured.image.width() > 0 && captured.image.height() > 0);
let encoded = encode_png(&captured.image).expect("encode should succeed");
assert_eq!(encoded.media_type, "image/png");
assert!(!encoded.data.is_empty());
assert!(captured.geometry.img_w.max(captured.geometry.img_h) <= 1568);
assert!(captured.physical_w >= captured.geometry.img_w);
}
#[test]
#[ignore]
fn capture_invalid_display_errors_real() {
let err = capture_screen(Some(99), 1568).unwrap_err();
assert!(err.contains("99"), "error should name the display: {err}");
}
}
nomifun-tauri/crates/agent/nomi-computer/src/tool.rs
+1461
View File
File diff suppressed because it is too large Load Diff