Update: 将子项目从 submodule 转为完整内容
- 移除 GovAI, nomifun-tauri, 算力盒子 的 submodule 引用 - 添加所有子项目的完整源代码 - 保留原始 .git 为 .git.bak 备份
This commit is contained in:
freedak
@@ -0,0 +1,76 @@
|
||||
[package]
|
||||
name = "nomi-a11y"
|
||||
description = "Cross-platform accessibility-tree + Set-of-Marks engine for Nomi computer-use (macOS AX / Windows UIA / Linux AT-SPI)"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
nomi-types.workspace = true
|
||||
|
||||
tracing.workspace = true
|
||||
thiserror.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
image.workspace = true
|
||||
base64.workspace = true
|
||||
|
||||
# --- macOS backend: Accessibility (AXUIElement) + CoreGraphics actuation +
|
||||
# AppKit (NSRunningApplication focus) + Vision (OCR/text fusion). macOS only. ---
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
core-foundation = "0.10"
|
||||
core-graphics = "0.25"
|
||||
objc2 = "0.6"
|
||||
objc2-foundation = { version = "0.3", features = [
|
||||
"NSData",
|
||||
"NSArray",
|
||||
"NSString",
|
||||
"NSError",
|
||||
"NSDictionary",
|
||||
"NSValue",
|
||||
] }
|
||||
objc2-core-foundation = { version = "0.3", features = ["CFCGTypes"] }
|
||||
objc2-vision = { version = "0.3", features = [
|
||||
"VNRequest",
|
||||
"VNRecognizeTextRequest",
|
||||
"VNRequestHandler",
|
||||
"VNObservation",
|
||||
"VNTypes",
|
||||
"VNGeometry",
|
||||
"objc2-core-foundation",
|
||||
] }
|
||||
|
||||
# --- Windows backend: UI Automation (via the `uiautomation` high-level wrapper)
|
||||
# for the accessibility tree + actuation, and `windows` (windows-rs) for
|
||||
# Win32 foreground/window management + `Windows.Media.Ocr` text fusion.
|
||||
# Windows only. ---
|
||||
[target.'cfg(target_os = "windows")'.dependencies]
|
||||
# High-level IUIAutomation wrapper (COM init via new_direct + MTA, tree walkers,
|
||||
# patterns, control types). leexgone/uiautomation-rs.
|
||||
uiautomation = "0.25"
|
||||
# windows-rs: foreground window (Win32) + on-device OCR (Windows.Media.Ocr).
|
||||
# 0.61 aligns with the version xcap already pulls into Cargo.lock.
|
||||
windows = { version = "0.61", features = [
|
||||
"Win32_Foundation",
|
||||
"Win32_System_Com",
|
||||
"Win32_System_Threading",
|
||||
"Win32_UI_WindowsAndMessaging",
|
||||
"Media_Ocr",
|
||||
"Graphics_Imaging",
|
||||
"Storage_Streams",
|
||||
"Foundation",
|
||||
"Foundation_Collections",
|
||||
"Globalization",
|
||||
] }
|
||||
|
||||
# --- Linux backend: AT-SPI2 (pure-Rust, async via zbus over D-Bus). Compiled
|
||||
# only on Linux. No C deps — the actor thread runs a current-thread tokio
|
||||
# runtime and block_on's the async atspi calls. ---
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
atspi = "0.30"
|
||||
zbus = { version = "5", features = ["tokio"] }
|
||||
tokio = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
image.workspace = true
|
||||
@@ -0,0 +1,25 @@
|
||||
# Linux a11y dev/CI image for nomi-a11y.
|
||||
#
|
||||
# Purpose: develop + verify the Linux (AT-SPI2) backend from a non-Linux host.
|
||||
# - Native Rust build inside the container (no cross-linker needed).
|
||||
# - Headless AT-SPI test harness: Xvfb (virtual X11 display) + a private D-Bus
|
||||
# session + at-spi2-core (the a11y bus) + a real accessible GTK app
|
||||
# (gtk3-widget-factory) to `observe` and `do_action` against.
|
||||
#
|
||||
# Build: docker build -t nomi-a11y-linux:dev -f crates/agent/nomi-a11y/dev/Dockerfile.linux-a11y .
|
||||
# Use: see crates/agent/nomi-a11y/dev/run-linux-a11y.sh
|
||||
#
|
||||
# The `atspi`/`zbus` crates are pure Rust (no C deps), so nomi-a11y itself needs
|
||||
# no system libs to compile; the X11/dbus dev libs below are only so the wider
|
||||
# computer-use chain (xcap/enigo) can also be built for Linux if desired.
|
||||
FROM rust:1-bookworm
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
at-spi2-core \
|
||||
xvfb dbus dbus-x11 x11-utils \
|
||||
gtk-3-examples \
|
||||
libx11-dev libxtst-dev libxi-dev libxcb1-dev libxkbcommon-dev \
|
||||
libdbus-1-dev pkg-config ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /work
|
||||
@@ -0,0 +1,85 @@
|
||||
# Linux Accessibility Backend Validation
|
||||
|
||||
This directory contains helper assets for validating the Linux AT-SPI backend
|
||||
from `nomi-a11y`.
|
||||
|
||||
Use the lightest path that proves the behavior you are changing.
|
||||
|
||||
## Path A: Type And API Check
|
||||
|
||||
This catches most portability issues without linking or running Linux binaries.
|
||||
|
||||
```bash
|
||||
rustup target add x86_64-unknown-linux-gnu
|
||||
cargo check --target x86_64-unknown-linux-gnu \
|
||||
-p nomi-a11y --examples --tests
|
||||
```
|
||||
|
||||
## Path B: Docker
|
||||
|
||||
The Dockerfile installs Rust, AT-SPI, Xvfb, D-Bus, and GTK example widgets so
|
||||
the smoke test can run headlessly.
|
||||
|
||||
```bash
|
||||
docker build -t nomi-a11y-linux:dev \
|
||||
-f crates/agent/nomi-a11y/dev/Dockerfile.linux-a11y .
|
||||
|
||||
crates/agent/nomi-a11y/dev/run-linux-a11y.sh test
|
||||
crates/agent/nomi-a11y/dev/run-linux-a11y.sh smoke
|
||||
```
|
||||
|
||||
If the environment cannot pull base images, run the same commands inside any
|
||||
Linux VM with the dependencies below installed.
|
||||
|
||||
## Path C: Native Linux VM
|
||||
|
||||
Install the runtime dependencies:
|
||||
|
||||
```bash
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y \
|
||||
at-spi2-core gtk-3-examples xvfb dbus-x11 \
|
||||
build-essential pkg-config
|
||||
```
|
||||
|
||||
Build the smoke example with a VM-local target directory so host builds do not
|
||||
share artifacts:
|
||||
|
||||
```bash
|
||||
CARGO_TARGET_DIR="$HOME/nomi-a11y-target" \
|
||||
CARGO_BUILD_BUILD_DIR="$HOME/nomi-a11y-build" \
|
||||
cargo build -p nomi-a11y --example linux_smoke
|
||||
```
|
||||
|
||||
Run the smoke test under Xvfb and a private D-Bus session:
|
||||
|
||||
```bash
|
||||
export DISPLAY=:99
|
||||
Xvfb :99 -screen 0 1280x900x24 -nolisten tcp >/tmp/xvfb.log 2>&1 &
|
||||
|
||||
dbus-run-session -- bash -uc '
|
||||
export QT_LINUX_ACCESSIBILITY_ALWAYS_ON=1
|
||||
export GTK_MODULES=atk-bridge
|
||||
export NO_AT_BRIDGE=0
|
||||
export DISPLAY=:99
|
||||
gtk3-widget-factory >/tmp/app.log 2>&1 &
|
||||
sleep 4
|
||||
./target/debug/examples/linux_smoke
|
||||
'
|
||||
```
|
||||
|
||||
Set `NOMI_A11Y_CLICK=<substring>` to ask `linux_smoke` to invoke the first
|
||||
matching element action.
|
||||
|
||||
## Coverage Notes
|
||||
|
||||
- The smoke test validates an X11 session. Wayland support can legitimately
|
||||
degrade for synthetic pixel input while semantic actions remain available;
|
||||
check the reported `capabilities`.
|
||||
- KDE/Qt apps may require `QT_LINUX_ACCESSIBILITY_ALWAYS_ON=1`.
|
||||
- Electron apps often require `--force-renderer-accessibility`.
|
||||
- Sandboxed apps such as Flatpak may not expose a complete accessibility tree.
|
||||
|
||||
`atspi` / `zbus` are pure Rust dependencies, so cross-checking the Linux backend
|
||||
from a non-Linux host is practical. Behavior validation still needs a Linux
|
||||
runtime.
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build / test / smoke the nomi-a11y Linux backend inside the dev container.
|
||||
# Native Linux build (no cross-linker). Named volumes cache target + registry
|
||||
# so incremental builds are fast across runs.
|
||||
#
|
||||
# ./run-linux-a11y.sh build # cargo build -p nomi-a11y
|
||||
# ./run-linux-a11y.sh test # cargo test -p nomi-a11y
|
||||
# ./run-linux-a11y.sh check-chain # cargo check -p nomi-agent --features computer-use
|
||||
# ./run-linux-a11y.sh smoke # headless AT-SPI behavioral run (Xvfb+dbus+gtk app)
|
||||
# ./run-linux-a11y.sh shell # interactive shell in the container
|
||||
set -euo pipefail
|
||||
REPO="$(cd "$(dirname "$0")/../../../.." && pwd)"
|
||||
IMG=nomi-a11y-linux:dev
|
||||
COMMON=(--rm -v "$REPO":/work -w /work
|
||||
-v nomi-a11y-target:/target -e CARGO_TARGET_DIR=/target
|
||||
-v nomi-a11y-cargo-registry:/usr/local/cargo/registry)
|
||||
|
||||
case "${1:-test}" in
|
||||
build) docker run "${COMMON[@]}" "$IMG" cargo build -p nomi-a11y ;;
|
||||
test) docker run "${COMMON[@]}" "$IMG" cargo test -p nomi-a11y ;;
|
||||
check-chain) docker run "${COMMON[@]}" "$IMG" cargo check -p nomi-agent --features computer-use ;;
|
||||
smoke) docker run "${COMMON[@]}" "$IMG" bash crates/agent/nomi-a11y/dev/smoke.sh ;;
|
||||
shell) docker run -it "${COMMON[@]}" "$IMG" bash ;;
|
||||
*) echo "usage: $0 {build|test|check-chain|smoke|shell}" >&2; exit 1 ;;
|
||||
esac
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env bash
|
||||
# Headless AT-SPI behavioral harness (runs INSIDE the dev container):
|
||||
# Xvfb virtual display + private D-Bus session + at-spi2 a11y bus + a real
|
||||
# accessible GTK app, then run the nomi-a11y `linux_smoke` example against it.
|
||||
set -uo pipefail
|
||||
|
||||
export DISPLAY=:99
|
||||
Xvfb :99 -screen 0 1280x900x24 -nolisten tcp >/tmp/xvfb.log 2>&1 &
|
||||
XVFB_PID=$!
|
||||
sleep 1
|
||||
|
||||
dbus-run-session -- bash -u -c '
|
||||
export QT_LINUX_ACCESSIBILITY_ALWAYS_ON=1
|
||||
export GTK_MODULES="${GTK_MODULES:-}:atk-bridge"
|
||||
export NO_AT_BRIDGE=0
|
||||
# Launch the AT-SPI registry/bus (path differs across Debian versions; try both).
|
||||
for d in /usr/libexec /usr/lib/at-spi2-core /usr/lib/at-spi2; do
|
||||
[ -x "$d/at-spi-bus-launcher" ] && ( "$d/at-spi-bus-launcher" --launch-immediately >/tmp/atspi-bus.log 2>&1 & )
|
||||
[ -x "$d/at-spi2-registryd" ] && ( "$d/at-spi2-registryd" >/tmp/atspi-reg.log 2>&1 & )
|
||||
done
|
||||
sleep 1
|
||||
gtk3-widget-factory >/tmp/app.log 2>&1 &
|
||||
APP_PID=$!
|
||||
sleep 3
|
||||
echo "=== AT-SPI bus address: ${AT_SPI_BUS_ADDRESS:-<unset>} ==="
|
||||
echo "=== running nomi-a11y linux_smoke example ==="
|
||||
CARGO_TARGET_DIR=/target cargo run -p nomi-a11y --example linux_smoke
|
||||
rc=$?
|
||||
kill "$APP_PID" 2>/dev/null || true
|
||||
exit $rc
|
||||
'
|
||||
rc=$?
|
||||
kill "$XVFB_PID" 2>/dev/null || true
|
||||
exit $rc
|
||||
@@ -0,0 +1,56 @@
|
||||
//! Headless smoke for the Linux AT-SPI backend (run inside the dev container's
|
||||
//! `smoke.sh`: Xvfb + dbus + at-spi2 + a GTK app). Connects, observes the
|
||||
//! focused window, prints the element list, and optionally activates the
|
||||
//! element whose name matches $NOMI_A11Y_CLICK (to exercise `do_action`).
|
||||
//!
|
||||
//! Usage: `cargo run -p nomi-a11y --example linux_smoke`
|
||||
|
||||
fn main() {
|
||||
let engine = match nomi_a11y::create_engine() {
|
||||
Ok(e) => e,
|
||||
Err(e) => {
|
||||
eprintln!("create_engine failed: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
|
||||
println!("capabilities: {:?}", engine.capabilities());
|
||||
|
||||
let snap = match engine.observe(&nomi_a11y::ObserveOpts::default()) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
eprintln!("observe failed: {e}");
|
||||
std::process::exit(2);
|
||||
}
|
||||
};
|
||||
|
||||
println!(
|
||||
"observed {} interactable element(s) app={:?} window={:?} truncated={}",
|
||||
snap.entries.len(),
|
||||
snap.app_name,
|
||||
snap.window_title,
|
||||
snap.truncated,
|
||||
);
|
||||
println!("--- element list ---\n{}", snap.text);
|
||||
|
||||
// Optional: activate the first element whose name contains $NOMI_A11Y_CLICK.
|
||||
if let Ok(needle) = std::env::var("NOMI_A11Y_CLICK") {
|
||||
if let Some(e) = snap
|
||||
.entries
|
||||
.iter()
|
||||
.find(|e| e.name.as_deref().is_some_and(|n| n.contains(&needle)))
|
||||
{
|
||||
println!("activating element [{}] {:?}…", e.r#ref, e.name);
|
||||
match engine.invoke(
|
||||
&nomi_a11y::Target::Ref(e.r#ref),
|
||||
snap.generation,
|
||||
nomi_a11y::ElementAction::LeftClick,
|
||||
) {
|
||||
Ok(eff) => println!("invoke ok: {}", eff.message),
|
||||
Err(err) => eprintln!("invoke failed: {err}"),
|
||||
}
|
||||
} else {
|
||||
eprintln!("no element matching {needle:?} to click");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
//! Real-machine smoke test for the Windows UIA backend.
|
||||
//!
|
||||
//! Run with: cargo run -p nomi-a11y --example winsmoke
|
||||
//! Optional: cargo run -p nomi-a11y --example winsmoke -- <pid>
|
||||
//!
|
||||
//! Opens Notepad on a temp file with known (Chinese) content, then exercises
|
||||
//! the engine: observe → numbered element list + bounds + app_name → read the
|
||||
//! document's text back via the Text pattern (validates the TextPattern value
|
||||
//! path) → attempt SetValue → stale-generation guard. Prints a numbered
|
||||
//! Set-of-Marks-style listing so coordinates can be eyeballed against the window.
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
fn main() {
|
||||
use std::{thread::sleep, time::Duration};
|
||||
|
||||
use nomi_a11y::{ElementAction, ObserveOpts, Snapshot, Target};
|
||||
|
||||
fn dump(snap: &Snapshot) {
|
||||
println!(
|
||||
" app={:?} window={:?} pid={:?} entries={} truncated={}",
|
||||
snap.app_name,
|
||||
snap.window_title,
|
||||
snap.pid,
|
||||
snap.entries.len(),
|
||||
snap.truncated
|
||||
);
|
||||
for e in &snap.entries {
|
||||
let b = e.bounds;
|
||||
println!(
|
||||
" [{:>2}] {:<11} name={:?} value={:?} states={:?} @ ({:.0},{:.0}) {:.0}x{:.0}",
|
||||
e.r#ref, e.role, e.name, e.value, e.states, b.x, b.y, b.w, b.h
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let arg_pid: Option<i32> = std::env::args().nth(1).and_then(|s| s.parse().ok());
|
||||
|
||||
// A temp file with known content lets us verify the TextPattern read path
|
||||
// (the Win11 RichEdit Notepad exposes text via TextPattern, not ValuePattern).
|
||||
const MARKER: &str = "你好世界";
|
||||
let tmp = std::env::temp_dir().join("nomi_winsmoke.txt");
|
||||
let _ = std::fs::write(&tmp, format!("NomiFun TextPattern 验证 Hello {MARKER}\n第二行 line two\n"));
|
||||
|
||||
let mut child = None;
|
||||
let target_pid = match arg_pid {
|
||||
Some(p) => {
|
||||
println!("== using provided pid {p} ==");
|
||||
Some(p)
|
||||
}
|
||||
None => {
|
||||
println!("== launching notepad.exe on {} ==", tmp.display());
|
||||
match std::process::Command::new("notepad.exe").arg(&tmp).spawn() {
|
||||
Ok(c) => {
|
||||
let p = c.id() as i32;
|
||||
println!(" spawned notepad, launcher pid = {p}");
|
||||
child = Some(c);
|
||||
sleep(Duration::from_millis(2500)); // allow the file to load
|
||||
Some(p)
|
||||
}
|
||||
Err(e) => {
|
||||
println!(" failed to launch notepad: {e}");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let engine = match nomi_a11y::create_engine() {
|
||||
Ok(e) => e,
|
||||
Err(e) => {
|
||||
println!("FATAL: create_engine failed: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
println!("capabilities: {:?}", engine.capabilities());
|
||||
|
||||
println!("\n== observe(foreground) ==");
|
||||
let t0 = std::time::Instant::now();
|
||||
let fg = engine.observe(&ObserveOpts::default());
|
||||
let elapsed = t0.elapsed();
|
||||
match &fg {
|
||||
Ok(s) => {
|
||||
println!(
|
||||
" observe latency: {:?} ({} entries, truncated={})",
|
||||
elapsed,
|
||||
s.entries.len(),
|
||||
s.truncated
|
||||
);
|
||||
dump(s);
|
||||
println!("\n -- semantic tree (snap.text) --");
|
||||
for line in s.text.lines() {
|
||||
println!(" {line}");
|
||||
}
|
||||
}
|
||||
Err(e) => println!(" observe(foreground) error: {e} (after {elapsed:?})"),
|
||||
}
|
||||
|
||||
let mut pid_snap = None;
|
||||
if let Some(pid) = target_pid {
|
||||
println!("\n== observe(pid={pid}) ==");
|
||||
match engine.observe(&ObserveOpts {
|
||||
pid: Some(pid),
|
||||
..Default::default()
|
||||
}) {
|
||||
Ok(s) => {
|
||||
dump(&s);
|
||||
pid_snap = Some(s);
|
||||
}
|
||||
Err(e) => {
|
||||
println!(" observe(pid) error: {e} (Store Notepad reparents; using foreground)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let snap = pid_snap
|
||||
.filter(|s| !s.entries.is_empty())
|
||||
.or_else(|| fg.ok().filter(|s| !s.entries.is_empty()));
|
||||
|
||||
let Some(snap) = snap else {
|
||||
println!("\nNo usable snapshot with elements; aborting actuation phase.");
|
||||
if let Some(mut c) = child {
|
||||
let _ = c.kill();
|
||||
}
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
return;
|
||||
};
|
||||
|
||||
// --- TextPattern value read: the document should show the file's text ---
|
||||
let doc = snap
|
||||
.entries
|
||||
.iter()
|
||||
.find(|e| matches!(e.role.as_str(), "edit" | "document"));
|
||||
println!("\n== TextPattern value read ==");
|
||||
match doc.and_then(|e| e.value.clone()) {
|
||||
Some(v) => println!(
|
||||
" TEXTPATTERN READ: {} (value={:?})",
|
||||
if v.contains(MARKER) { "PASS ✔" } else { "got text but no marker" },
|
||||
v
|
||||
),
|
||||
None => println!(" document value = None (no text read)"),
|
||||
}
|
||||
|
||||
// --- SetValue actuation (RichEdit Notepad often no-ops ValuePattern.SetValue;
|
||||
// the tool layer types instead — we just confirm the call is honest) ---
|
||||
if let Some(e) = doc {
|
||||
let text = "NomiFun SetValue 测试".to_string();
|
||||
println!("\n== invoke SetValue on [{}] {} ==", e.r#ref, e.role);
|
||||
match engine.invoke(
|
||||
&Target::Ref(e.r#ref),
|
||||
snap.generation,
|
||||
ElementAction::SetValue(text),
|
||||
) {
|
||||
Ok(eff) => println!(" ok: {}", eff.message),
|
||||
Err(err) => println!(" SetValue error (data, not a panic): {err}"),
|
||||
}
|
||||
}
|
||||
|
||||
// --- press_chain demonstration: focus then activate a button via the chain ---
|
||||
if let Some(btn) = snap.entries.iter().find(|e| e.role == "button") {
|
||||
println!("\n== invoke Focus on [{}] button {:?} ==", btn.r#ref, btn.name);
|
||||
match engine.invoke(&Target::Ref(btn.r#ref), snap.generation, ElementAction::Focus) {
|
||||
Ok(eff) => println!(" ok: {}", eff.message),
|
||||
Err(err) => println!(" error: {err}"),
|
||||
}
|
||||
}
|
||||
|
||||
// --- stale-ref guard ---
|
||||
println!("\n== stale generation guard ==");
|
||||
let stale = engine.invoke(
|
||||
&Target::Ref(1),
|
||||
nomi_a11y::SnapshotGen(snap.generation.0.wrapping_sub(1)),
|
||||
ElementAction::Focus,
|
||||
);
|
||||
println!(" invoke with old generation → {stale:?}");
|
||||
|
||||
if let Some(mut c) = child {
|
||||
sleep(Duration::from_millis(300));
|
||||
let _ = c.kill();
|
||||
println!("\n(killed spawned notepad)");
|
||||
}
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
println!("\n== done ==");
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
fn main() {
|
||||
eprintln!("winsmoke is a Windows-only example.");
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
//! Platform-neutral engine types + the `A11yEngine` trait every OS backend
|
||||
//! implements.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use nomi_types::tool::ToolImage;
|
||||
|
||||
use crate::selector::Selector;
|
||||
|
||||
/// Monotonic snapshot generation. A `ref` (index into a snapshot's element
|
||||
/// list) is only valid against the generation it was produced in; backends use
|
||||
/// this to reject stale references instead of acting on a moved element.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
pub struct SnapshotGen(pub u64);
|
||||
|
||||
/// An opaque, generation-tagged handle to an element in a backend's registry.
|
||||
/// The raw OS handle (AXUIElement / IUIAutomationElement / AT-SPI Accessible)
|
||||
/// never crosses the engine boundary — only this token and serializable data.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ElementId {
|
||||
pub generation: SnapshotGen,
|
||||
pub index: u32,
|
||||
}
|
||||
|
||||
/// A rectangle. Backends return element bounds in **OS accessibility
|
||||
/// coordinates** (e.g. macOS global screen points, top-left origin); mapping to
|
||||
/// screenshot-pixel space for overlays/pixel-fallback is the caller's job (see
|
||||
/// the design's AX-points→pixel conversion).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Rect {
|
||||
pub x: f64,
|
||||
pub y: f64,
|
||||
pub w: f64,
|
||||
pub h: f64,
|
||||
}
|
||||
|
||||
impl Rect {
|
||||
pub fn center(&self) -> (f64, f64) {
|
||||
(self.x + self.w / 2.0, self.y + self.h / 2.0)
|
||||
}
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.w <= 0.0 || self.h <= 0.0
|
||||
}
|
||||
}
|
||||
|
||||
/// Where an element entry came from. Set-of-Marks fuses these.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Source {
|
||||
/// Native accessibility tree (most reliable).
|
||||
A11y,
|
||||
/// OCR text recognition (fallback where a11y is thin).
|
||||
Ocr,
|
||||
/// Vision/icon classification (fallback).
|
||||
Vision,
|
||||
}
|
||||
|
||||
/// One interactable element exposed to the model as `[ref]`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ElementEntry {
|
||||
/// The number the model targets: "click element [ref]".
|
||||
pub r#ref: u32,
|
||||
pub role: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub name: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub value: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub states: Vec<String>,
|
||||
pub bounds: Rect,
|
||||
pub source: Source,
|
||||
}
|
||||
|
||||
/// A line of text recognized by OCR, with bounds in screenshot-pixel space
|
||||
/// (top-left origin). Fused into the Set-of-Marks list where the accessibility
|
||||
/// tree is thin (Electron/canvas/games).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct OcrLine {
|
||||
pub text: String,
|
||||
pub bounds: Rect,
|
||||
}
|
||||
|
||||
/// How synthetic input is delivered on this platform/session.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum InputKind {
|
||||
/// Native event posting (macOS CGEvent / Windows SendInput / AT-SPI action).
|
||||
Native,
|
||||
/// X11 XTest.
|
||||
X11,
|
||||
/// Wayland xdg-desktop-portal RemoteDesktop (per-session consent).
|
||||
WaylandPortal,
|
||||
/// No reliable synthetic-input path in this session.
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
/// What the engine can actually do this session — injected into the system
|
||||
/// prompt so the model knows its real abilities up front.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Capabilities {
|
||||
pub os: String,
|
||||
/// Can read the accessibility tree (`observe`).
|
||||
pub tree_read: bool,
|
||||
/// Can capture a screenshot for the Set-of-Marks overlay.
|
||||
pub screenshot: bool,
|
||||
/// Can perform semantic actions (AXPress / Invoke / do_action) on elements.
|
||||
pub semantic_action: bool,
|
||||
pub synthetic_input: InputKind,
|
||||
/// Can move/resize/raise windows.
|
||||
pub window_management: bool,
|
||||
}
|
||||
|
||||
/// A completed `observe`: the filtered interactable elements + an optional
|
||||
/// Set-of-Marks overlay image, plus the indented text rendering for the model.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Snapshot {
|
||||
pub generation: SnapshotGen,
|
||||
pub entries: Vec<ElementEntry>,
|
||||
/// Set-of-Marks overlay (numbered boxes on the screenshot), when produced.
|
||||
pub overlay: Option<ToolImage>,
|
||||
/// Indented text rendering: `[14] button "Submit" enabled`.
|
||||
pub text: String,
|
||||
/// True if the tree exceeded the node budget and was truncated.
|
||||
pub truncated: bool,
|
||||
/// Process id of the observed application (for `focus_window`).
|
||||
pub pid: Option<i32>,
|
||||
pub app_name: Option<String>,
|
||||
pub window_title: Option<String>,
|
||||
}
|
||||
|
||||
/// How the model addresses an element. Three mutually-exclusive modes, shared
|
||||
/// with the browser tool's contract.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum Target {
|
||||
/// A `[ref]` from the most recent snapshot.
|
||||
Ref(u32),
|
||||
/// A deterministic selector (`role:Button && name:Save`).
|
||||
Selector(Selector),
|
||||
/// Last-resort absolute screen coordinates (pixel fallback).
|
||||
Pixel { x: i32, y: i32 },
|
||||
}
|
||||
|
||||
/// A semantic action to perform on a resolved element.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum ElementAction {
|
||||
/// The element's default action (AXPress / Invoke / do_action).
|
||||
Press,
|
||||
LeftClick,
|
||||
RightClick,
|
||||
DoubleClick,
|
||||
Focus,
|
||||
SetValue(String),
|
||||
}
|
||||
|
||||
/// The observed effect of an action, for closed-loop verification.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Effect {
|
||||
pub changed: bool,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
/// Options controlling an `observe` tree walk.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ObserveOpts {
|
||||
/// Maximum tree depth to traverse.
|
||||
pub max_depth: usize,
|
||||
/// Stop after this many interactable elements (then set `truncated`).
|
||||
pub node_budget: usize,
|
||||
/// Restrict to a specific process; `None` = the frontmost app.
|
||||
pub pid: Option<i32>,
|
||||
}
|
||||
|
||||
impl Default for ObserveOpts {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_depth: 12,
|
||||
node_budget: 120,
|
||||
pid: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Errors are data the model reads and routes around — never a panic, never a
|
||||
/// silent no-op.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum A11yError {
|
||||
#[error("not supported ({capability}): {hint}")]
|
||||
Unsupported { capability: String, hint: String },
|
||||
#[error("element not found: {0}")]
|
||||
NotFound(String),
|
||||
#[error("stale reference: {0}")]
|
||||
Stale(String),
|
||||
#[error("permission required: {0}")]
|
||||
Permission(String),
|
||||
#[error("accessibility backend error: {0}")]
|
||||
Backend(String),
|
||||
}
|
||||
|
||||
/// The contract every OS backend implements. Methods are synchronous; callers
|
||||
/// invoke them from `spawn_blocking`. macOS marshals each call to a single
|
||||
/// CFRunLoop actor thread internally, so the engine is `Send + Sync`.
|
||||
pub trait A11yEngine: Send + Sync {
|
||||
/// Honest report of what this session can do.
|
||||
fn capabilities(&self) -> Capabilities;
|
||||
|
||||
/// Read the frontmost (or `opts.pid`) window's accessibility tree, filter to
|
||||
/// interactable elements, and return them numbered as a Set-of-Marks
|
||||
/// snapshot. Element `bounds` are in OS accessibility coordinates.
|
||||
fn observe(&self, opts: &ObserveOpts) -> Result<Snapshot, A11yError>;
|
||||
|
||||
/// Perform `action` on the element addressed by `target`. `Ref` targets are
|
||||
/// validated against `generation` and rejected if stale.
|
||||
fn invoke(
|
||||
&self,
|
||||
target: &Target,
|
||||
generation: SnapshotGen,
|
||||
action: ElementAction,
|
||||
) -> Result<Effect, A11yError>;
|
||||
|
||||
/// Raise/activate a window by its owning process id.
|
||||
fn focus_window(&self, pid: i32) -> Result<Effect, A11yError>;
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
//! Cross-platform accessibility-tree + Set-of-Marks engine for Nomi computer-use.
|
||||
//!
|
||||
//! The platform-neutral layer (engine trait/types, selector grammar, tree
|
||||
//! model + filtering, Set-of-Marks overlay) compiles on every target. Per-OS
|
||||
//! backends live behind `#[cfg(target_os = …)]`:
|
||||
//! - macOS: AXUIElement via a dedicated CFRunLoop actor thread (implemented).
|
||||
//! - Windows: UI Automation via a dedicated MTA actor thread (implemented).
|
||||
//! - Linux: AT-SPI2 over D-Bus (implemented).
|
||||
//!
|
||||
//! Backends report honest `Capabilities`; unimplemented operations return
|
||||
//! `A11yError::Unsupported { capability, hint }` (never panic, never a silent
|
||||
//! no-op) so the agent can route around them.
|
||||
|
||||
pub mod engine;
|
||||
pub mod overlay;
|
||||
pub mod selector;
|
||||
pub mod tree;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos;
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
mod windows;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
|
||||
pub use engine::{
|
||||
A11yEngine, A11yError, Capabilities, Effect, ElementAction, ElementEntry, ElementId,
|
||||
InputKind, ObserveOpts, OcrLine, Rect, Snapshot, SnapshotGen, Source, Target,
|
||||
};
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
/// Process-wide label for the host application, woven into permission-error
|
||||
/// guidance so the message names the *actual* app the user must grant (and
|
||||
/// restart) instead of a generic "this app". On a desktop host that ambiguity
|
||||
/// is actively harmful: computer-use runs IN-PROCESS inside the host app, but a
|
||||
/// model reading "this app" reliably misattributes it to the terminal/editor it
|
||||
/// imagines is hosting the session and sends the user to grant the wrong
|
||||
/// process. The host sets this once at startup (the desktop shell sets
|
||||
/// "NomiFun"); library/headless embeddings leave it unset and get "this app".
|
||||
static HOST_APP_LABEL: RwLock<Option<String>> = RwLock::new(None);
|
||||
|
||||
/// Set the host-application label used in permission-error guidance (e.g.
|
||||
/// "NomiFun"). Last writer wins; call once early in host startup. A poisoned
|
||||
/// lock is ignored — the default ("this app") is a safe fallback, never a panic.
|
||||
pub fn set_host_app_label(label: impl Into<String>) {
|
||||
if let Ok(mut guard) = HOST_APP_LABEL.write() {
|
||||
*guard = Some(label.into());
|
||||
}
|
||||
}
|
||||
|
||||
/// The host-application label for permission guidance, or `"this app"` when the
|
||||
/// host has not set one. Always returns an owned, non-empty string.
|
||||
pub fn host_app_label() -> String {
|
||||
HOST_APP_LABEL
|
||||
.read()
|
||||
.ok()
|
||||
.and_then(|g| g.clone())
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or_else(|| "this app".to_string())
|
||||
}
|
||||
|
||||
/// Construct the platform's accessibility engine, or report why it is
|
||||
/// unavailable. The returned engine is `Send + Sync` and its methods are
|
||||
/// synchronous (call them from `spawn_blocking`); macOS marshals every call to
|
||||
/// a single CFRunLoop actor thread internally.
|
||||
pub fn create_engine() -> Result<Arc<dyn A11yEngine>, A11yError> {
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
let engine = macos::MacEngine::start()?;
|
||||
Ok(Arc::new(engine))
|
||||
}
|
||||
#[cfg(target_os = "windows")]
|
||||
{
|
||||
let engine = windows::WinEngine::start()?;
|
||||
Ok(Arc::new(engine))
|
||||
}
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let engine = linux::LinuxEngine::start()?;
|
||||
Ok(Arc::new(engine))
|
||||
}
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))]
|
||||
{
|
||||
Err(A11yError::Unsupported {
|
||||
capability: "accessibility engine".to_string(),
|
||||
hint: "The accessibility-tree backend is implemented on macOS, Windows, and Linux. \
|
||||
Pixel-based computer-use still works."
|
||||
.to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod host_label_tests {
|
||||
use super::{host_app_label, set_host_app_label};
|
||||
|
||||
// The only test in this crate that touches the process-global label, so its
|
||||
// steps observe each other deterministically under the parallel runner.
|
||||
#[test]
|
||||
fn label_defaults_then_reflects_set_and_ignores_empty() {
|
||||
assert_eq!(host_app_label(), "this app", "default before any host sets it");
|
||||
set_host_app_label("NomiFun");
|
||||
assert_eq!(host_app_label(), "NomiFun");
|
||||
// An empty label is ignored so a mis-set never blanks the guidance.
|
||||
set_host_app_label("");
|
||||
assert_eq!(host_app_label(), "this app");
|
||||
}
|
||||
}
|
||||
|
||||
/// Recognize on-screen text in a screenshot via the OS OCR engine (macOS:
|
||||
/// Vision.framework `VNRecognizeTextRequest`, on-device, with CJK support).
|
||||
/// `langs` are BCP-47 hints (e.g. `["zh-Hans", "en-US"]`). Bounds are in the
|
||||
/// image's pixel space (top-left origin). Used to fuse text into Set-of-Marks
|
||||
/// where the accessibility tree is thin. Returns `Unsupported` off macOS.
|
||||
pub fn ocr_screenshot(img: &image::RgbaImage, langs: &[String]) -> Result<Vec<OcrLine>, A11yError> {
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
macos::ocr_screenshot(img, langs)
|
||||
}
|
||||
#[cfg(target_os = "windows")]
|
||||
{
|
||||
windows::ocr_screenshot(img, langs)
|
||||
}
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
linux::ocr_screenshot(img, langs)
|
||||
}
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))]
|
||||
{
|
||||
let _ = (img, langs);
|
||||
Err(A11yError::Unsupported {
|
||||
capability: "OCR".to_string(),
|
||||
hint: "OCR fusion is implemented on macOS (Vision.framework) and Windows \
|
||||
(Windows.Media.Ocr)."
|
||||
.to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,519 @@
|
||||
//! Linux AT-SPI actor: a dedicated thread owns a current-thread tokio runtime
|
||||
//! and the `AccessibilityConnection`; the synchronous `A11yEngine` methods
|
||||
//! marshal here over a command channel and `block_on` the async AT-SPI calls.
|
||||
//!
|
||||
//! `invoke` prefers AT-SPI semantic actions (Action.do_action / EditableText /
|
||||
//! grab_focus) — coordinate-free and reliable on both X11 and Wayland. Element
|
||||
//! bounds come from Component.get_extents(Screen) (valid pixels on X11; often
|
||||
//! unavailable on Wayland, where the tool's pixel fallback is degraded anyway).
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::mpsc::{Sender, channel};
|
||||
|
||||
use atspi::connection::AccessibilityConnection;
|
||||
use atspi::proxy::accessible::ObjectRefExt;
|
||||
use atspi::proxy::action::ActionProxy;
|
||||
use atspi::proxy::component::ComponentProxy;
|
||||
use atspi::proxy::editable_text::EditableTextProxy;
|
||||
use atspi::{CoordType, ObjectRefOwned, State as AtspiState};
|
||||
|
||||
use crate::engine::{
|
||||
A11yError, Capabilities, Effect, ElementAction, ElementEntry, InputKind, ObserveOpts, Rect,
|
||||
Snapshot, SnapshotGen, Source, Target,
|
||||
};
|
||||
use crate::tree::format_entries;
|
||||
|
||||
enum Cmd {
|
||||
Capabilities(Sender<Capabilities>),
|
||||
Observe(ObserveOpts, Sender<Result<Snapshot, A11yError>>),
|
||||
Invoke(
|
||||
Target,
|
||||
SnapshotGen,
|
||||
ElementAction,
|
||||
Sender<Result<Effect, A11yError>>,
|
||||
),
|
||||
Focus(i32, Sender<Result<Effect, A11yError>>),
|
||||
}
|
||||
|
||||
pub struct ActorHandle {
|
||||
tx: Mutex<Sender<Cmd>>,
|
||||
}
|
||||
|
||||
struct State {
|
||||
gen_counter: u64,
|
||||
current_gen: SnapshotGen,
|
||||
registry: HashMap<u32, ObjectRefOwned>,
|
||||
}
|
||||
|
||||
/// Session probe → honest `Capabilities`. AT-SPI tree-read works on X11 +
|
||||
/// Wayland; input/coordinates/window-mgmt degrade on Wayland.
|
||||
fn detect_caps() -> Capabilities {
|
||||
let session = std::env::var("XDG_SESSION_TYPE").unwrap_or_default();
|
||||
let wayland =
|
||||
session.eq_ignore_ascii_case("wayland") || std::env::var_os("WAYLAND_DISPLAY").is_some();
|
||||
let x11 = !wayland
|
||||
&& (session.eq_ignore_ascii_case("x11") || std::env::var_os("DISPLAY").is_some());
|
||||
Capabilities {
|
||||
os: "linux".to_string(),
|
||||
tree_read: true,
|
||||
screenshot: true,
|
||||
semantic_action: true,
|
||||
synthetic_input: if x11 {
|
||||
InputKind::X11
|
||||
} else {
|
||||
// No reliable persistent unattended Wayland input without a portal grant.
|
||||
InputKind::Unsupported
|
||||
},
|
||||
window_management: x11,
|
||||
}
|
||||
}
|
||||
|
||||
// ---- AT-SPI helpers (run on the actor thread's runtime) ------------------
|
||||
|
||||
/// Map an AT-SPI role to a stable lowercase name (Debug form, e.g. `pushbutton`,
|
||||
/// `entry`, `text`). The model just needs readable, stable role names.
|
||||
fn role_name(acc_role: Option<atspi::Role>) -> String {
|
||||
match acc_role {
|
||||
Some(r) => format!("{r:?}").to_lowercase(),
|
||||
None => "element".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_click_action(name: &str) -> bool {
|
||||
matches!(
|
||||
name.to_lowercase().as_str(),
|
||||
"click" | "activate" | "press" | "jump" | "open" | "do default" | "default"
|
||||
)
|
||||
}
|
||||
|
||||
/// Find the focused (Active) toplevel window by scanning each application's
|
||||
/// children. Returns its object reference.
|
||||
async fn find_active_window(conn: &AccessibilityConnection) -> Result<ObjectRefOwned, A11yError> {
|
||||
let zconn = conn.connection();
|
||||
let root = conn
|
||||
.root_accessible_on_registry()
|
||||
.await
|
||||
.map_err(|e| A11yError::Backend(format!("cannot read the AT-SPI registry root: {e}")))?;
|
||||
let apps = root.get_children().await.map_err(|e| {
|
||||
A11yError::Backend(format!("cannot list accessible applications: {e}"))
|
||||
})?;
|
||||
|
||||
let mut first_window: Option<ObjectRefOwned> = None;
|
||||
for app in apps {
|
||||
let Ok(app_acc) = app.as_accessible_proxy(zconn).await else {
|
||||
continue;
|
||||
};
|
||||
let Ok(windows) = app_acc.get_children().await else {
|
||||
continue;
|
||||
};
|
||||
for win in windows {
|
||||
let Ok(win_acc) = win.as_accessible_proxy(zconn).await else {
|
||||
continue;
|
||||
};
|
||||
let states = win_acc.get_state().await.unwrap_or_default();
|
||||
if states.contains(AtspiState::Active) {
|
||||
return Ok(win);
|
||||
}
|
||||
if first_window.is_none() {
|
||||
first_window = Some(win);
|
||||
}
|
||||
}
|
||||
}
|
||||
// No window reported Active (common headless) — fall back to the first one.
|
||||
first_window.ok_or_else(|| {
|
||||
A11yError::NotFound(
|
||||
"no accessible window found. Ensure the app exposes accessibility \
|
||||
(KDE: QT_LINUX_ACCESSIBILITY_ALWAYS_ON=1; Electron: --force-renderer-accessibility)."
|
||||
.to_string(),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
struct Collected {
|
||||
obj: ObjectRefOwned,
|
||||
role: String,
|
||||
name: Option<String>,
|
||||
value: Option<String>,
|
||||
states: Vec<String>,
|
||||
bounds: Rect,
|
||||
}
|
||||
|
||||
/// Walk the window subtree (iteratively, to avoid async recursion), collecting
|
||||
/// interactable elements with screen-pixel bounds.
|
||||
async fn walk_window(
|
||||
conn: &AccessibilityConnection,
|
||||
window: ObjectRefOwned,
|
||||
opts: &ObserveOpts,
|
||||
) -> (Vec<Collected>, bool) {
|
||||
let zconn = conn.connection();
|
||||
let mut out: Vec<Collected> = Vec::new();
|
||||
let mut truncated = false;
|
||||
let mut stack: Vec<(ObjectRefOwned, usize)> = vec![(window, 0)];
|
||||
|
||||
while let Some((obj, depth)) = stack.pop() {
|
||||
if out.len() >= opts.node_budget {
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
let Ok(acc) = obj.as_accessible_proxy(zconn).await else {
|
||||
continue;
|
||||
};
|
||||
let dest = acc.inner().destination().to_owned();
|
||||
let path = acc.inner().path().to_owned();
|
||||
|
||||
let role = acc.get_role().await.ok();
|
||||
let name = acc.name().await.ok().filter(|s| !s.trim().is_empty());
|
||||
let states = acc.get_state().await.unwrap_or_default();
|
||||
|
||||
let bounds = match ComponentProxy::builder(zconn)
|
||||
.destination(dest.clone())
|
||||
.and_then(|b| b.path(path.clone()))
|
||||
{
|
||||
Ok(builder) => match builder.build().await {
|
||||
Ok(comp) => comp.get_extents(CoordType::Screen).await.ok().map(|(x, y, w, h)| {
|
||||
Rect {
|
||||
x: x as f64,
|
||||
y: y as f64,
|
||||
w: w as f64,
|
||||
h: h as f64,
|
||||
}
|
||||
}),
|
||||
Err(_) => None,
|
||||
},
|
||||
Err(_) => None,
|
||||
};
|
||||
|
||||
let has_action = match ActionProxy::builder(zconn)
|
||||
.destination(dest.clone())
|
||||
.and_then(|b| b.path(path.clone()))
|
||||
{
|
||||
Ok(builder) => match builder.build().await {
|
||||
Ok(act) => act.n_actions().await.map(|n| n > 0).unwrap_or(false),
|
||||
Err(_) => false,
|
||||
},
|
||||
Err(_) => false,
|
||||
};
|
||||
|
||||
let focusable = states.contains(AtspiState::Focusable);
|
||||
let editable = states.contains(AtspiState::Editable);
|
||||
let enabled = states.contains(AtspiState::Enabled) || states.contains(AtspiState::Sensitive);
|
||||
|
||||
if let Some(b) = bounds {
|
||||
if b.w > 0.0 && b.h > 0.0 && (has_action || focusable || editable) {
|
||||
let mut st = Vec::new();
|
||||
if !enabled {
|
||||
st.push("disabled".to_string());
|
||||
}
|
||||
if states.contains(AtspiState::Focused) {
|
||||
st.push("focused".to_string());
|
||||
}
|
||||
out.push(Collected {
|
||||
obj: obj.clone(),
|
||||
role: role_name(role),
|
||||
name,
|
||||
value: None,
|
||||
states: st,
|
||||
bounds: b,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if depth < opts.max_depth {
|
||||
if let Ok(children) = acc.get_children().await {
|
||||
for c in children {
|
||||
stack.push((c, depth + 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
(out, truncated)
|
||||
}
|
||||
|
||||
async fn do_observe(
|
||||
conn: &AccessibilityConnection,
|
||||
opts: &ObserveOpts,
|
||||
state: &mut State,
|
||||
) -> Result<Snapshot, A11yError> {
|
||||
let window = find_active_window(conn).await?;
|
||||
let window_title = match window.as_accessible_proxy(conn.connection()).await {
|
||||
Ok(w) => w.name().await.ok().filter(|s| !s.trim().is_empty()),
|
||||
Err(_) => None,
|
||||
};
|
||||
|
||||
let (mut collected, truncated) = walk_window(conn, window, opts).await;
|
||||
|
||||
// Reading order: top-to-bottom, left-to-right.
|
||||
collected.sort_by(|a, b| {
|
||||
(a.bounds.y.round() as i64, a.bounds.x.round() as i64)
|
||||
.cmp(&(b.bounds.y.round() as i64, b.bounds.x.round() as i64))
|
||||
});
|
||||
|
||||
state.gen_counter += 1;
|
||||
let generation = SnapshotGen(state.gen_counter);
|
||||
state.current_gen = generation;
|
||||
state.registry.clear();
|
||||
|
||||
let mut entries = Vec::with_capacity(collected.len());
|
||||
for (i, c) in collected.into_iter().enumerate() {
|
||||
let r = i as u32 + 1;
|
||||
state.registry.insert(r, c.obj);
|
||||
entries.push(ElementEntry {
|
||||
r#ref: r,
|
||||
role: c.role,
|
||||
name: c.name,
|
||||
value: c.value,
|
||||
states: c.states,
|
||||
bounds: c.bounds,
|
||||
source: Source::A11y,
|
||||
});
|
||||
}
|
||||
|
||||
let text = format_entries(&entries);
|
||||
Ok(Snapshot {
|
||||
generation,
|
||||
entries,
|
||||
overlay: None,
|
||||
text,
|
||||
truncated,
|
||||
pid: None,
|
||||
app_name: None,
|
||||
window_title,
|
||||
})
|
||||
}
|
||||
|
||||
async fn do_invoke(
|
||||
conn: &AccessibilityConnection,
|
||||
target: &Target,
|
||||
generation: SnapshotGen,
|
||||
action: &ElementAction,
|
||||
state: &State,
|
||||
) -> Result<Effect, A11yError> {
|
||||
let r = match target {
|
||||
Target::Ref(r) => *r,
|
||||
Target::Selector(_) => {
|
||||
return Err(A11yError::Unsupported {
|
||||
capability: "selector targeting".to_string(),
|
||||
hint: "Resolve a selector against the latest observe() result and act by [ref]."
|
||||
.to_string(),
|
||||
});
|
||||
}
|
||||
Target::Pixel { .. } => {
|
||||
return Err(A11yError::Unsupported {
|
||||
capability: "pixel targeting".to_string(),
|
||||
hint: "Pixel fallback is handled by the computer tool's input layer.".to_string(),
|
||||
});
|
||||
}
|
||||
};
|
||||
if generation != state.current_gen {
|
||||
return Err(A11yError::Stale(format!(
|
||||
"ref [{r}] is from an older snapshot; re-run observe and use a fresh [ref]"
|
||||
)));
|
||||
}
|
||||
let obj = state
|
||||
.registry
|
||||
.get(&r)
|
||||
.cloned()
|
||||
.ok_or_else(|| A11yError::NotFound(format!("no element [{r}] in the latest snapshot")))?;
|
||||
|
||||
let zconn = conn.connection();
|
||||
let acc = obj
|
||||
.as_accessible_proxy(zconn)
|
||||
.await
|
||||
.map_err(|e| A11yError::Backend(format!("cannot resolve [{r}]: {e}")))?;
|
||||
let dest = acc.inner().destination().to_owned();
|
||||
let path = acc.inner().path().to_owned();
|
||||
|
||||
match action {
|
||||
ElementAction::Press | ElementAction::LeftClick | ElementAction::DoubleClick => {
|
||||
let act = ActionProxy::builder(zconn)
|
||||
.destination(dest)
|
||||
.and_then(|b| b.path(path))
|
||||
.map_err(|e| A11yError::Backend(format!("action proxy: {e}")))?
|
||||
.build()
|
||||
.await
|
||||
.map_err(|e| A11yError::Backend(format!("action proxy: {e}")))?;
|
||||
let actions = act.get_actions().await.unwrap_or_default();
|
||||
let idx = actions
|
||||
.iter()
|
||||
.position(|a| is_click_action(&a.name))
|
||||
.unwrap_or(0);
|
||||
if act.n_actions().await.unwrap_or(0) <= 0 {
|
||||
return Err(A11yError::Backend(format!(
|
||||
"element [{r}] exposes no AT-SPI action; fall back to a pixel click"
|
||||
)));
|
||||
}
|
||||
match act.do_action(idx as i32).await {
|
||||
Ok(true) => Ok(Effect {
|
||||
changed: true,
|
||||
message: format!("performed action {idx} on element [{r}]"),
|
||||
}),
|
||||
Ok(false) => Err(A11yError::Backend(format!(
|
||||
"do_action on [{r}] returned false; try a pixel click"
|
||||
))),
|
||||
Err(e) => Err(A11yError::Backend(format!("do_action on [{r}] failed: {e}"))),
|
||||
}
|
||||
}
|
||||
ElementAction::RightClick => Err(A11yError::Unsupported {
|
||||
capability: "right click".to_string(),
|
||||
hint: "AT-SPI has no standard right-click action; use a pixel right-click.".to_string(),
|
||||
}),
|
||||
ElementAction::Focus => {
|
||||
let comp = ComponentProxy::builder(zconn)
|
||||
.destination(dest)
|
||||
.and_then(|b| b.path(path))
|
||||
.map_err(|e| A11yError::Backend(format!("component proxy: {e}")))?
|
||||
.build()
|
||||
.await
|
||||
.map_err(|e| A11yError::Backend(format!("component proxy: {e}")))?;
|
||||
match comp.grab_focus().await {
|
||||
Ok(_) => Ok(Effect {
|
||||
changed: true,
|
||||
message: format!("focused element [{r}]"),
|
||||
}),
|
||||
Err(e) => Err(A11yError::Backend(format!("grab_focus on [{r}] failed: {e}"))),
|
||||
}
|
||||
}
|
||||
ElementAction::SetValue(v) => {
|
||||
let et = EditableTextProxy::builder(zconn)
|
||||
.destination(dest)
|
||||
.and_then(|b| b.path(path))
|
||||
.map_err(|e| A11yError::Backend(format!("editable-text proxy: {e}")))?
|
||||
.build()
|
||||
.await
|
||||
.map_err(|e| A11yError::Backend(format!("editable-text proxy: {e}")))?;
|
||||
match et.set_text_contents(v).await {
|
||||
Ok(_) => Ok(Effect {
|
||||
changed: true,
|
||||
message: format!("set value of element [{r}]"),
|
||||
}),
|
||||
Err(e) => Err(A11yError::Backend(format!(
|
||||
"set_text_contents on [{r}] failed ({e}); fall back to focus + type"
|
||||
))),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn do_focus(_pid: i32) -> Result<Effect, A11yError> {
|
||||
Err(A11yError::Unsupported {
|
||||
capability: "window activation".to_string(),
|
||||
hint: "Cross-application window activation is not wired on Linux yet (X11 EWMH / Wayland \
|
||||
has no portable protocol); the focused window is used by observe."
|
||||
.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
// ---- thread + channel plumbing ------------------------------------------
|
||||
|
||||
impl ActorHandle {
|
||||
pub fn spawn() -> Result<Self, A11yError> {
|
||||
let (tx, rx) = channel::<Cmd>();
|
||||
let (ready_tx, ready_rx) = channel::<Result<(), A11yError>>();
|
||||
|
||||
std::thread::Builder::new()
|
||||
.name("nomi-a11y-linux".to_string())
|
||||
.spawn(move || {
|
||||
let rt = match tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
{
|
||||
Ok(rt) => rt,
|
||||
Err(e) => {
|
||||
let _ =
|
||||
ready_tx.send(Err(A11yError::Backend(format!("tokio runtime: {e}"))));
|
||||
return;
|
||||
}
|
||||
};
|
||||
let conn = match rt.block_on(AccessibilityConnection::new()) {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
let _ = ready_tx.send(Err(A11yError::Permission(format!(
|
||||
"cannot connect to the AT-SPI accessibility bus: {e}. Ensure \
|
||||
at-spi2-core is running; KDE needs QT_LINUX_ACCESSIBILITY_ALWAYS_ON=1, \
|
||||
Electron apps need --force-renderer-accessibility."
|
||||
))));
|
||||
return;
|
||||
}
|
||||
};
|
||||
let _ = ready_tx.send(Ok(()));
|
||||
|
||||
let mut state = State {
|
||||
gen_counter: 0,
|
||||
current_gen: SnapshotGen(0),
|
||||
registry: HashMap::new(),
|
||||
};
|
||||
|
||||
while let Ok(cmd) = rx.recv() {
|
||||
match cmd {
|
||||
Cmd::Capabilities(reply) => {
|
||||
let _ = reply.send(detect_caps());
|
||||
}
|
||||
Cmd::Observe(opts, reply) => {
|
||||
let _ = reply.send(rt.block_on(do_observe(&conn, &opts, &mut state)));
|
||||
}
|
||||
Cmd::Invoke(target, generation, action, reply) => {
|
||||
let _ = reply.send(rt.block_on(do_invoke(
|
||||
&conn,
|
||||
&target,
|
||||
generation,
|
||||
&action,
|
||||
&state,
|
||||
)));
|
||||
}
|
||||
Cmd::Focus(pid, reply) => {
|
||||
let _ = reply.send(do_focus(pid));
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.map_err(|e| A11yError::Backend(format!("failed to start AT-SPI actor thread: {e}")))?;
|
||||
|
||||
ready_rx
|
||||
.recv()
|
||||
.map_err(|_| A11yError::Backend("AT-SPI actor died at startup".to_string()))??;
|
||||
Ok(Self { tx: Mutex::new(tx) })
|
||||
}
|
||||
|
||||
fn send(&self, cmd: Cmd) -> Result<(), A11yError> {
|
||||
self.tx
|
||||
.lock()
|
||||
.map_err(|_| A11yError::Backend("AT-SPI actor lock poisoned".to_string()))?
|
||||
.send(cmd)
|
||||
.map_err(|_| A11yError::Backend("AT-SPI actor thread is gone".to_string()))
|
||||
}
|
||||
|
||||
pub fn capabilities(&self) -> Capabilities {
|
||||
let (tx, rx) = channel();
|
||||
if self.send(Cmd::Capabilities(tx)).is_err() {
|
||||
return detect_caps();
|
||||
}
|
||||
rx.recv().unwrap_or_else(|_| detect_caps())
|
||||
}
|
||||
|
||||
pub fn observe(&self, opts: ObserveOpts) -> Result<Snapshot, A11yError> {
|
||||
let (tx, rx) = channel();
|
||||
self.send(Cmd::Observe(opts, tx))?;
|
||||
rx.recv()
|
||||
.map_err(|_| A11yError::Backend("AT-SPI actor dropped the reply".to_string()))?
|
||||
}
|
||||
|
||||
pub fn invoke(
|
||||
&self,
|
||||
target: Target,
|
||||
generation: SnapshotGen,
|
||||
action: ElementAction,
|
||||
) -> Result<Effect, A11yError> {
|
||||
let (tx, rx) = channel();
|
||||
self.send(Cmd::Invoke(target, generation, action, tx))?;
|
||||
rx.recv()
|
||||
.map_err(|_| A11yError::Backend("AT-SPI actor dropped the reply".to_string()))?
|
||||
}
|
||||
|
||||
pub fn focus_window(&self, pid: i32) -> Result<Effect, A11yError> {
|
||||
let (tx, rx) = channel();
|
||||
self.send(Cmd::Focus(pid, tx))?;
|
||||
rx.recv()
|
||||
.map_err(|_| A11yError::Backend("AT-SPI actor dropped the reply".to_string()))?
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
//! Linux AT-SPI2 backend.
|
||||
//!
|
||||
//! AT-SPI2 is a D-Bus protocol; the `atspi` crate is a pure-Rust (zbus) async
|
||||
//! client. We mirror the macOS actor: a dedicated thread owns a current-thread
|
||||
//! tokio runtime + the `AccessibilityConnection`, and the synchronous
|
||||
//! `A11yEngine` methods `block_on` async AT-SPI calls via a command channel.
|
||||
//!
|
||||
//! Status: skeleton (reports honest capabilities; observe/invoke wired in
|
||||
//! `actor.rs`). Compiled only on Linux.
|
||||
|
||||
use crate::engine::{
|
||||
A11yEngine, A11yError, Capabilities, Effect, ElementAction, ObserveOpts, OcrLine, Snapshot,
|
||||
SnapshotGen, Target,
|
||||
};
|
||||
|
||||
mod actor;
|
||||
|
||||
pub struct LinuxEngine {
|
||||
inner: actor::ActorHandle,
|
||||
}
|
||||
|
||||
impl LinuxEngine {
|
||||
pub fn start() -> Result<Self, A11yError> {
|
||||
let inner = actor::ActorHandle::spawn()?;
|
||||
Ok(Self { inner })
|
||||
}
|
||||
}
|
||||
|
||||
impl A11yEngine for LinuxEngine {
|
||||
fn capabilities(&self) -> Capabilities {
|
||||
self.inner.capabilities()
|
||||
}
|
||||
fn observe(&self, opts: &ObserveOpts) -> Result<Snapshot, A11yError> {
|
||||
self.inner.observe(opts.clone())
|
||||
}
|
||||
fn invoke(
|
||||
&self,
|
||||
target: &Target,
|
||||
generation: SnapshotGen,
|
||||
action: ElementAction,
|
||||
) -> Result<Effect, A11yError> {
|
||||
self.inner.invoke(target.clone(), generation, action)
|
||||
}
|
||||
fn focus_window(&self, pid: i32) -> Result<Effect, A11yError> {
|
||||
self.inner.focus_window(pid)
|
||||
}
|
||||
}
|
||||
|
||||
/// Linux has no OS-native OCR (unlike macOS Vision / Windows.Media.Ocr). The
|
||||
/// tool layer handles this `Unsupported` gracefully (it just skips OCR fusion).
|
||||
/// A `tesseract`-backed path could be added behind a cargo feature later.
|
||||
pub fn ocr_screenshot(_img: &image::RgbaImage, _langs: &[String]) -> Result<Vec<OcrLine>, A11yError> {
|
||||
Err(A11yError::Unsupported {
|
||||
capability: "OCR".to_string(),
|
||||
hint: "Linux has no built-in OCR engine; accessibility-tree targeting still works, and \
|
||||
a11y-thin content falls back to pixel actions."
|
||||
.to_string(),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,796 @@
|
||||
//! The macOS AX actor: a single dedicated thread that owns every AXUIElement
|
||||
//! and is the sole caller of the Accessibility C API. The public engine sends
|
||||
//! commands over a channel and blocks on a per-command reply, so AX handles
|
||||
//! (which are not `Send` and have thread affinity) never leave this thread.
|
||||
//!
|
||||
//! Raw FFI is used (rather than a higher-level AX crate) so the whole backend
|
||||
//! pins to one CoreFoundation version (0.10, shared with core-graphics 0.25)
|
||||
//! and we control retain/release precisely. Attribute names are plain CFStrings
|
||||
//! ("AXRole", "AXTitle", …) so no framework string constants need linking.
|
||||
//!
|
||||
//! The actor thread runs a CFRunLoop (polled via `recv_timeout` + a
|
||||
//! non-blocking `CFRunLoopRunInMode`, so it never hot-spins) and owns an
|
||||
//! AXObserver on the frontmost app. Change notifications flip a `dirty` flag so
|
||||
//! `observe` re-serves the cached snapshot when nothing has changed and
|
||||
//! re-walks the tree otherwise. Every mutating command also marks `dirty`, so
|
||||
//! the cache is never stale after one of our own actions. OCR/vision fusion
|
||||
//! lives one layer up (the computer tool fuses `nomi_a11y::ocr_screenshot`).
|
||||
|
||||
// This whole module is FFI against the Accessibility C API; every helper is an
|
||||
// `unsafe fn` that is only valid on the actor thread. We keep the pre-2024
|
||||
// "unsafe fn body is unsafe" ergonomics rather than wrapping each FFI call.
|
||||
#![allow(unsafe_op_in_unsafe_fn)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::ffi::c_void;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::mpsc::{Sender, channel};
|
||||
use std::time::Duration;
|
||||
|
||||
use core_foundation::base::TCFType;
|
||||
use core_foundation::string::{CFString, CFStringRef};
|
||||
use core_graphics::geometry::{CGPoint, CGSize};
|
||||
|
||||
use crate::engine::{
|
||||
A11yError, Effect, ElementAction, ElementEntry, ObserveOpts, Rect, Snapshot, SnapshotGen,
|
||||
Source, Target,
|
||||
};
|
||||
use crate::tree::{format_entries, normalize_role};
|
||||
|
||||
// ---- FFI ---------------------------------------------------------------
|
||||
|
||||
#[link(name = "CoreFoundation", kind = "framework")]
|
||||
unsafe extern "C" {
|
||||
fn CFRetain(cf: *const c_void) -> *const c_void;
|
||||
fn CFRelease(cf: *const c_void);
|
||||
fn CFGetTypeID(cf: *const c_void) -> usize;
|
||||
fn CFStringGetTypeID() -> usize;
|
||||
fn CFBooleanGetTypeID() -> usize;
|
||||
fn CFBooleanGetValue(b: *const c_void) -> u8;
|
||||
fn CFArrayGetCount(arr: *const c_void) -> isize;
|
||||
fn CFArrayGetValueAtIndex(arr: *const c_void, idx: isize) -> *const c_void;
|
||||
fn CFRunLoopGetCurrent() -> *mut c_void;
|
||||
fn CFRunLoopRunInMode(mode: CFStringRef, seconds: f64, return_after_source_handled: u8) -> i32;
|
||||
fn CFRunLoopAddSource(rl: *mut c_void, source: *const c_void, mode: CFStringRef);
|
||||
fn CFRunLoopRemoveSource(rl: *mut c_void, source: *const c_void, mode: CFStringRef);
|
||||
}
|
||||
|
||||
/// AXObserver notification callback: flips the `dirty` flag (passed as `refcon`)
|
||||
/// so the next `observe` re-walks instead of re-serving a stale snapshot. Runs
|
||||
/// on the actor thread (the run loop that owns the observer source).
|
||||
unsafe extern "C" fn observer_callback(
|
||||
_observer: *mut c_void,
|
||||
_element: *const c_void,
|
||||
_notification: CFStringRef,
|
||||
refcon: *mut c_void,
|
||||
) {
|
||||
if !refcon.is_null() {
|
||||
(*(refcon as *const AtomicBool)).store(true, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
type AXObserverCallback = unsafe extern "C" fn(*mut c_void, *const c_void, CFStringRef, *mut c_void);
|
||||
|
||||
#[link(name = "ApplicationServices", kind = "framework")]
|
||||
unsafe extern "C" {
|
||||
fn AXUIElementCreateSystemWide() -> *const c_void;
|
||||
fn AXUIElementCreateApplication(pid: i32) -> *const c_void;
|
||||
fn AXUIElementCopyAttributeValue(
|
||||
el: *const c_void,
|
||||
attr: CFStringRef,
|
||||
out: *mut *const c_void,
|
||||
) -> i32;
|
||||
fn AXUIElementSetAttributeValue(el: *const c_void, attr: CFStringRef, val: *const c_void)
|
||||
-> i32;
|
||||
fn AXUIElementCopyActionNames(el: *const c_void, out: *mut *const c_void) -> i32;
|
||||
fn AXUIElementPerformAction(el: *const c_void, action: CFStringRef) -> i32;
|
||||
fn AXUIElementGetPid(el: *const c_void, out: *mut i32) -> i32;
|
||||
fn AXValueGetValue(value: *const c_void, the_type: u32, out: *mut c_void) -> u8;
|
||||
fn AXObserverCreate(
|
||||
application: i32,
|
||||
callback: AXObserverCallback,
|
||||
out: *mut *mut c_void,
|
||||
) -> i32;
|
||||
fn AXObserverAddNotification(
|
||||
observer: *mut c_void,
|
||||
element: *const c_void,
|
||||
notification: CFStringRef,
|
||||
refcon: *mut c_void,
|
||||
) -> i32;
|
||||
fn AXObserverRemoveNotification(
|
||||
observer: *mut c_void,
|
||||
element: *const c_void,
|
||||
notification: CFStringRef,
|
||||
) -> i32;
|
||||
fn AXObserverGetRunLoopSource(observer: *mut c_void) -> *const c_void;
|
||||
}
|
||||
|
||||
const AX_VALUE_CGPOINT: u32 = 1;
|
||||
const AX_VALUE_CGSIZE: u32 = 2;
|
||||
|
||||
// ---- AxElem: RAII owner of one AXUIElement (thread-confined, !Send) -----
|
||||
|
||||
struct AxElem(*const c_void);
|
||||
|
||||
impl AxElem {
|
||||
/// Take ownership of a +1 reference (from a Create/Copy call).
|
||||
unsafe fn from_create(p: *const c_void) -> Option<Self> {
|
||||
if p.is_null() {
|
||||
None
|
||||
} else {
|
||||
Some(AxElem(p))
|
||||
}
|
||||
}
|
||||
/// Retain a borrowed (+0) reference and own the new count.
|
||||
unsafe fn from_borrowed(p: *const c_void) -> Option<Self> {
|
||||
if p.is_null() {
|
||||
None
|
||||
} else {
|
||||
Some(AxElem(CFRetain(p)))
|
||||
}
|
||||
}
|
||||
fn ptr(&self) -> *const c_void {
|
||||
self.0
|
||||
}
|
||||
fn retain(&self) -> AxElem {
|
||||
unsafe { AxElem(CFRetain(self.0)) }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for AxElem {
|
||||
fn drop(&mut self) {
|
||||
unsafe { CFRelease(self.0) }
|
||||
}
|
||||
}
|
||||
|
||||
// ---- low-level attribute helpers (call only on the actor thread) --------
|
||||
|
||||
unsafe fn copy_attr_raw(el: *const c_void, name: &str) -> *const c_void {
|
||||
let attr = CFString::new(name);
|
||||
let mut out: *const c_void = std::ptr::null();
|
||||
let err = AXUIElementCopyAttributeValue(el, attr.as_concrete_TypeRef(), &mut out);
|
||||
if err != 0 {
|
||||
std::ptr::null()
|
||||
} else {
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn copy_str_attr(el: *const c_void, name: &str) -> Option<String> {
|
||||
let out = copy_attr_raw(el, name);
|
||||
if out.is_null() {
|
||||
return None;
|
||||
}
|
||||
if CFGetTypeID(out) == CFStringGetTypeID() {
|
||||
// Take the +1 directly as a CFString and let it release on drop.
|
||||
Some(CFString::wrap_under_create_rule(out as CFStringRef).to_string())
|
||||
} else {
|
||||
CFRelease(out);
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn copy_bool_attr(el: *const c_void, name: &str) -> Option<bool> {
|
||||
let out = copy_attr_raw(el, name);
|
||||
if out.is_null() {
|
||||
return None;
|
||||
}
|
||||
let r = if CFGetTypeID(out) == CFBooleanGetTypeID() {
|
||||
Some(CFBooleanGetValue(out) != 0)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
CFRelease(out);
|
||||
r
|
||||
}
|
||||
|
||||
unsafe fn copy_elem_attr(el: *const c_void, name: &str) -> Option<AxElem> {
|
||||
AxElem::from_create(copy_attr_raw(el, name))
|
||||
}
|
||||
|
||||
unsafe fn copy_children(el: *const c_void) -> Vec<AxElem> {
|
||||
let out = copy_attr_raw(el, "AXChildren");
|
||||
if out.is_null() {
|
||||
return Vec::new();
|
||||
}
|
||||
let n = CFArrayGetCount(out);
|
||||
let mut v = Vec::with_capacity(n.max(0) as usize);
|
||||
for i in 0..n {
|
||||
let item = CFArrayGetValueAtIndex(out, i);
|
||||
if let Some(e) = AxElem::from_borrowed(item) {
|
||||
v.push(e);
|
||||
}
|
||||
}
|
||||
CFRelease(out);
|
||||
v
|
||||
}
|
||||
|
||||
unsafe fn copy_point(el: *const c_void, name: &str) -> Option<(f64, f64)> {
|
||||
let out = copy_attr_raw(el, name);
|
||||
if out.is_null() {
|
||||
return None;
|
||||
}
|
||||
let mut p = CGPoint { x: 0.0, y: 0.0 };
|
||||
let ok = AXValueGetValue(out, AX_VALUE_CGPOINT, &mut p as *mut _ as *mut c_void);
|
||||
CFRelease(out);
|
||||
if ok != 0 {
|
||||
Some((p.x, p.y))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn copy_size(el: *const c_void, name: &str) -> Option<(f64, f64)> {
|
||||
let out = copy_attr_raw(el, name);
|
||||
if out.is_null() {
|
||||
return None;
|
||||
}
|
||||
let mut s = CGSize {
|
||||
width: 0.0,
|
||||
height: 0.0,
|
||||
};
|
||||
let ok = AXValueGetValue(out, AX_VALUE_CGSIZE, &mut s as *mut _ as *mut c_void);
|
||||
CFRelease(out);
|
||||
if ok != 0 {
|
||||
Some((s.width, s.height))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn copy_actions(el: *const c_void) -> Vec<String> {
|
||||
let mut out: *const c_void = std::ptr::null();
|
||||
let err = AXUIElementCopyActionNames(el, &mut out);
|
||||
if err != 0 || out.is_null() {
|
||||
return Vec::new();
|
||||
}
|
||||
let n = CFArrayGetCount(out);
|
||||
let mut v = Vec::new();
|
||||
for i in 0..n {
|
||||
let item = CFArrayGetValueAtIndex(out, i);
|
||||
if !item.is_null() && CFGetTypeID(item) == CFStringGetTypeID() {
|
||||
v.push(CFString::wrap_under_get_rule(item as CFStringRef).to_string());
|
||||
}
|
||||
}
|
||||
CFRelease(out);
|
||||
v
|
||||
}
|
||||
|
||||
unsafe fn pid_of(el: *const c_void) -> Option<i32> {
|
||||
let mut p = 0i32;
|
||||
if AXUIElementGetPid(el, &mut p) == 0 {
|
||||
Some(p)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn perform(el: *const c_void, action: &str) -> i32 {
|
||||
let a = CFString::new(action);
|
||||
AXUIElementPerformAction(el, a.as_concrete_TypeRef())
|
||||
}
|
||||
|
||||
unsafe fn set_string_value(el: *const c_void, val: &str) -> i32 {
|
||||
let attr = CFString::new("AXValue");
|
||||
let v = CFString::new(val);
|
||||
AXUIElementSetAttributeValue(
|
||||
el,
|
||||
attr.as_concrete_TypeRef(),
|
||||
v.as_concrete_TypeRef() as *const c_void,
|
||||
)
|
||||
}
|
||||
|
||||
fn is_action_actionable(actions: &[String]) -> bool {
|
||||
actions.iter().any(|a| {
|
||||
matches!(
|
||||
a.as_str(),
|
||||
"AXPress" | "AXConfirm" | "AXOpen" | "AXShowMenu" | "AXPick" | "AXIncrement"
|
||||
| "AXDecrement"
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
// ---- collection: walk the focused window into numbered entries ----------
|
||||
|
||||
struct Collected {
|
||||
elem: AxElem,
|
||||
role: String,
|
||||
name: Option<String>,
|
||||
value: Option<String>,
|
||||
states: Vec<String>,
|
||||
bounds: Rect,
|
||||
}
|
||||
|
||||
unsafe fn walk(
|
||||
el: &AxElem,
|
||||
depth: usize,
|
||||
opts: &ObserveOpts,
|
||||
out: &mut Vec<Collected>,
|
||||
truncated: &mut bool,
|
||||
) {
|
||||
if out.len() >= opts.node_budget {
|
||||
*truncated = true;
|
||||
return;
|
||||
}
|
||||
let role = copy_str_attr(el.ptr(), "AXRole");
|
||||
let name = copy_str_attr(el.ptr(), "AXTitle")
|
||||
.or_else(|| copy_str_attr(el.ptr(), "AXDescription"))
|
||||
.filter(|s| !s.trim().is_empty());
|
||||
let value = copy_str_attr(el.ptr(), "AXValue").filter(|s| !s.trim().is_empty());
|
||||
let pos = copy_point(el.ptr(), "AXPosition");
|
||||
let size = copy_size(el.ptr(), "AXSize");
|
||||
let actions = copy_actions(el.ptr());
|
||||
let actionable = is_action_actionable(&actions);
|
||||
let enabled = copy_bool_attr(el.ptr(), "AXEnabled").unwrap_or(true);
|
||||
|
||||
if let (Some((x, y)), Some((w, h))) = (pos, size) {
|
||||
let bounds = Rect { x, y, w, h };
|
||||
if !bounds.is_empty() && (actionable || name.is_some()) {
|
||||
let mut states = Vec::new();
|
||||
if !enabled {
|
||||
states.push("disabled".to_string());
|
||||
}
|
||||
if copy_bool_attr(el.ptr(), "AXFocused").unwrap_or(false) {
|
||||
states.push("focused".to_string());
|
||||
}
|
||||
out.push(Collected {
|
||||
elem: el.retain(),
|
||||
role: role.clone().unwrap_or_else(|| "element".to_string()),
|
||||
name,
|
||||
value,
|
||||
states,
|
||||
bounds,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if depth >= opts.max_depth {
|
||||
return;
|
||||
}
|
||||
for child in copy_children(el.ptr()) {
|
||||
if out.len() >= opts.node_budget {
|
||||
*truncated = true;
|
||||
return;
|
||||
}
|
||||
walk(&child, depth + 1, opts, out, truncated);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- actor thread state + command handling ------------------------------
|
||||
|
||||
/// A registered AXObserver watching one application for change notifications.
|
||||
/// Dropping it removes the run-loop source and notifications before the `dirty`
|
||||
/// flag it points at can be freed (see `State` field order).
|
||||
struct AxObserver {
|
||||
observer: *mut c_void,
|
||||
app: AxElem,
|
||||
runloop: *mut c_void,
|
||||
notifications: Vec<CFString>,
|
||||
}
|
||||
|
||||
impl Drop for AxObserver {
|
||||
fn drop(&mut self) {
|
||||
unsafe {
|
||||
let src = AXObserverGetRunLoopSource(self.observer);
|
||||
if !src.is_null() {
|
||||
let mode = CFString::new("kCFRunLoopDefaultMode");
|
||||
CFRunLoopRemoveSource(self.runloop, src, mode.as_concrete_TypeRef());
|
||||
}
|
||||
for n in &self.notifications {
|
||||
AXObserverRemoveNotification(self.observer, self.app.ptr(), n.as_concrete_TypeRef());
|
||||
}
|
||||
CFRelease(self.observer as *const c_void);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Register a change observer for `pid` on the run loop, with `refcon` pointing
|
||||
/// at the `dirty` flag. Returns `None` (caller then never caches) on failure.
|
||||
unsafe fn register_observer(
|
||||
pid: i32,
|
||||
app: &AxElem,
|
||||
runloop: *mut c_void,
|
||||
refcon: *mut c_void,
|
||||
) -> Option<AxObserver> {
|
||||
let mut obs: *mut c_void = std::ptr::null_mut();
|
||||
if AXObserverCreate(pid, observer_callback, &mut obs) != 0 || obs.is_null() {
|
||||
return None;
|
||||
}
|
||||
const NOTIFS: &[&str] = &[
|
||||
"AXValueChanged",
|
||||
"AXUIElementDestroyed",
|
||||
"AXFocusedUIElementChanged",
|
||||
"AXMainWindowChanged",
|
||||
"AXFocusedWindowChanged",
|
||||
"AXWindowResized",
|
||||
"AXWindowMoved",
|
||||
"AXCreated",
|
||||
"AXLayoutChanged",
|
||||
"AXSelectedChildrenChanged",
|
||||
"AXRowCountChanged",
|
||||
"AXTitleChanged",
|
||||
"AXMenuOpened",
|
||||
"AXMenuClosed",
|
||||
];
|
||||
let mut notifications = Vec::new();
|
||||
for name in NOTIFS {
|
||||
let cf = CFString::new(name);
|
||||
// Not every notification applies to every app element; ignore failures.
|
||||
if AXObserverAddNotification(obs, app.ptr(), cf.as_concrete_TypeRef(), refcon) == 0 {
|
||||
notifications.push(cf);
|
||||
}
|
||||
}
|
||||
let src = AXObserverGetRunLoopSource(obs);
|
||||
if src.is_null() {
|
||||
CFRelease(obs as *const c_void);
|
||||
return None;
|
||||
}
|
||||
let mode = CFString::new("kCFRunLoopDefaultMode");
|
||||
CFRunLoopAddSource(runloop, src, mode.as_concrete_TypeRef());
|
||||
Some(AxObserver {
|
||||
observer: obs,
|
||||
app: app.retain(),
|
||||
runloop,
|
||||
notifications,
|
||||
})
|
||||
}
|
||||
|
||||
/// The last walk, kept so repeated `observe`s on an unchanged window re-serve
|
||||
/// instead of re-walking the tree.
|
||||
struct CachedWalk {
|
||||
entries: Vec<ElementEntry>,
|
||||
app_name: Option<String>,
|
||||
window_title: Option<String>,
|
||||
pid: Option<i32>,
|
||||
truncated: bool,
|
||||
}
|
||||
|
||||
struct State {
|
||||
gen_counter: u64,
|
||||
current_gen: SnapshotGen,
|
||||
registry: HashMap<u32, AxElem>,
|
||||
/// This thread's run loop (observer sources are attached to it).
|
||||
runloop: *mut c_void,
|
||||
/// MUST be declared before `dirty`: dropping the observer removes its
|
||||
/// callback source before the `dirty` flag it references is freed.
|
||||
observer: Option<AxObserver>,
|
||||
/// Boxed for a stable address (the observer's `refcon`). Set true by the
|
||||
/// observer callback and by every mutating command; cleared on a fresh walk.
|
||||
dirty: Box<AtomicBool>,
|
||||
observed_pid: Option<i32>,
|
||||
cached: Option<CachedWalk>,
|
||||
}
|
||||
|
||||
unsafe fn focused_app() -> Result<AxElem, A11yError> {
|
||||
let sw = AxElem::from_create(AXUIElementCreateSystemWide()).ok_or_else(|| {
|
||||
A11yError::Backend("AXUIElementCreateSystemWide returned null".to_string())
|
||||
})?;
|
||||
copy_elem_attr(sw.ptr(), "AXFocusedApplication").ok_or_else(|| {
|
||||
let app = crate::host_app_label();
|
||||
A11yError::Permission(format!(
|
||||
"No focused application is readable — Accessibility permission is not in effect for \
|
||||
{app}. Grant it in System Settings → Privacy & Security → Accessibility (the entry is \
|
||||
named \"{app}\"), then COMPLETELY quit and reopen {app} — macOS does not apply this \
|
||||
permission to an already-running process. Computer-use runs inside {app} itself, so \
|
||||
do not grant a terminal or editor."
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
unsafe fn focused_window(app: &AxElem) -> Option<AxElem> {
|
||||
copy_elem_attr(app.ptr(), "AXFocusedWindow")
|
||||
.or_else(|| copy_elem_attr(app.ptr(), "AXMainWindow"))
|
||||
.or_else(|| copy_children(app.ptr()).into_iter().next())
|
||||
}
|
||||
|
||||
fn do_observe(opts: &ObserveOpts, state: &mut State) -> Result<Snapshot, A11yError> {
|
||||
unsafe {
|
||||
let app = match opts.pid {
|
||||
Some(pid) => AxElem::from_create(AXUIElementCreateApplication(pid))
|
||||
.ok_or_else(|| A11yError::NotFound(format!("no app for pid {pid}")))?,
|
||||
None => focused_app()?,
|
||||
};
|
||||
let app_pid = pid_of(app.ptr());
|
||||
|
||||
// Cache re-serve: frontmost app unchanged, an observer is watching it,
|
||||
// and nothing has dirtied the snapshot since the last walk. (Explicit-pid
|
||||
// observes always re-walk.)
|
||||
if opts.pid.is_none()
|
||||
&& app_pid.is_some()
|
||||
&& state.observed_pid == app_pid
|
||||
&& state.observer.is_some()
|
||||
&& !state.dirty.load(Ordering::Relaxed)
|
||||
{
|
||||
if let Some(c) = &state.cached {
|
||||
return Ok(Snapshot {
|
||||
generation: state.current_gen,
|
||||
entries: c.entries.clone(),
|
||||
overlay: None,
|
||||
text: format_entries(&c.entries),
|
||||
truncated: c.truncated,
|
||||
pid: c.pid,
|
||||
app_name: c.app_name.clone(),
|
||||
window_title: c.window_title.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let app_name = copy_str_attr(app.ptr(), "AXTitle");
|
||||
let window = focused_window(&app).ok_or_else(|| {
|
||||
A11yError::NotFound("the focused application has no readable window".to_string())
|
||||
})?;
|
||||
let window_title = copy_str_attr(window.ptr(), "AXTitle");
|
||||
|
||||
let mut collected = Vec::new();
|
||||
let mut truncated = false;
|
||||
walk(&window, 0, opts, &mut collected, &mut truncated);
|
||||
|
||||
// Reading order: top-to-bottom, left-to-right.
|
||||
collected.sort_by(|a, b| {
|
||||
(a.bounds.y.round() as i64, a.bounds.x.round() as i64)
|
||||
.cmp(&(b.bounds.y.round() as i64, b.bounds.x.round() as i64))
|
||||
});
|
||||
|
||||
state.gen_counter += 1;
|
||||
let generation = SnapshotGen(state.gen_counter);
|
||||
state.current_gen = generation;
|
||||
state.registry.clear();
|
||||
|
||||
let mut entries = Vec::with_capacity(collected.len());
|
||||
for (i, c) in collected.into_iter().enumerate() {
|
||||
let r = i as u32 + 1;
|
||||
state.registry.insert(r, c.elem);
|
||||
entries.push(ElementEntry {
|
||||
r#ref: r,
|
||||
role: normalize_role(&c.role),
|
||||
name: c.name,
|
||||
value: c.value,
|
||||
states: c.states,
|
||||
bounds: c.bounds,
|
||||
source: Source::A11y,
|
||||
});
|
||||
}
|
||||
|
||||
// (Re)register the change observer if the frontmost app changed.
|
||||
if state.observed_pid != app_pid {
|
||||
state.observer = None; // drop the old observer first (removes its source)
|
||||
if let Some(p) = app_pid {
|
||||
let refcon = (&*state.dirty as *const AtomicBool) as *mut c_void;
|
||||
state.observer = register_observer(p, &app, state.runloop, refcon);
|
||||
}
|
||||
state.observed_pid = app_pid;
|
||||
}
|
||||
state.dirty.store(false, Ordering::Relaxed);
|
||||
|
||||
let text = format_entries(&entries);
|
||||
state.cached = Some(CachedWalk {
|
||||
entries: entries.clone(),
|
||||
app_name: app_name.clone(),
|
||||
window_title: window_title.clone(),
|
||||
pid: app_pid,
|
||||
truncated,
|
||||
});
|
||||
Ok(Snapshot {
|
||||
generation,
|
||||
entries,
|
||||
overlay: None, // the tool captures the screenshot + draws the overlay
|
||||
text,
|
||||
truncated,
|
||||
pid: app_pid,
|
||||
app_name,
|
||||
window_title,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn do_invoke(
|
||||
target: &Target,
|
||||
generation: SnapshotGen,
|
||||
action: &ElementAction,
|
||||
state: &State,
|
||||
) -> Result<Effect, A11yError> {
|
||||
let r = match target {
|
||||
Target::Ref(r) => *r,
|
||||
Target::Selector(_) => {
|
||||
return Err(A11yError::Unsupported {
|
||||
capability: "selector targeting".to_string(),
|
||||
hint: "Resolve a selector against the latest observe() result and act by [ref]; \
|
||||
direct selector actuation is not yet implemented."
|
||||
.to_string(),
|
||||
})
|
||||
}
|
||||
Target::Pixel { .. } => {
|
||||
return Err(A11yError::Unsupported {
|
||||
capability: "pixel targeting".to_string(),
|
||||
hint: "Pixel fallback is handled by the computer tool's input layer, not the \
|
||||
accessibility engine."
|
||||
.to_string(),
|
||||
})
|
||||
}
|
||||
};
|
||||
if generation != state.current_gen {
|
||||
return Err(A11yError::Stale(format!(
|
||||
"ref [{r}] is from an older snapshot (the UI may have changed); re-run observe and \
|
||||
use a fresh [ref]"
|
||||
)));
|
||||
}
|
||||
let elem = state
|
||||
.registry
|
||||
.get(&r)
|
||||
.ok_or_else(|| A11yError::NotFound(format!("no element [{r}] in the latest snapshot")))?;
|
||||
|
||||
unsafe {
|
||||
let err = match action {
|
||||
ElementAction::Press | ElementAction::LeftClick | ElementAction::DoubleClick => {
|
||||
perform(elem.ptr(), "AXPress")
|
||||
}
|
||||
ElementAction::RightClick => perform(elem.ptr(), "AXShowMenu"),
|
||||
ElementAction::Focus => {
|
||||
let attr = CFString::new("AXFocused");
|
||||
let t = core_foundation::boolean::CFBoolean::true_value();
|
||||
AXUIElementSetAttributeValue(
|
||||
elem.ptr(),
|
||||
attr.as_concrete_TypeRef(),
|
||||
t.as_concrete_TypeRef() as *const c_void,
|
||||
)
|
||||
}
|
||||
ElementAction::SetValue(v) => set_string_value(elem.ptr(), v),
|
||||
};
|
||||
if err == 0 {
|
||||
Ok(Effect {
|
||||
changed: true,
|
||||
message: format!("performed {action:?} on element [{r}]"),
|
||||
})
|
||||
} else {
|
||||
Err(A11yError::Backend(format!(
|
||||
"AX action on [{r}] failed (AXError {err}); the element may be a web view \
|
||||
(AXWebArea) that ignores AXPress — fall back to a pixel click"
|
||||
)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn do_focus(pid: i32) -> Result<Effect, A11yError> {
|
||||
unsafe {
|
||||
let app = AxElem::from_create(AXUIElementCreateApplication(pid))
|
||||
.ok_or_else(|| A11yError::NotFound(format!("no app for pid {pid}")))?;
|
||||
let attr = CFString::new("AXFrontmost");
|
||||
let t = core_foundation::boolean::CFBoolean::true_value();
|
||||
let err = AXUIElementSetAttributeValue(
|
||||
app.ptr(),
|
||||
attr.as_concrete_TypeRef(),
|
||||
t.as_concrete_TypeRef() as *const c_void,
|
||||
);
|
||||
if let Some(win) = focused_window(&app) {
|
||||
let _ = perform(win.ptr(), "AXRaise");
|
||||
}
|
||||
if err == 0 {
|
||||
Ok(Effect {
|
||||
changed: true,
|
||||
message: format!("brought pid {pid} to the front"),
|
||||
})
|
||||
} else {
|
||||
Err(A11yError::Backend(format!(
|
||||
"could not activate pid {pid} (AXError {err})"
|
||||
)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- command plumbing ---------------------------------------------------
|
||||
|
||||
enum Cmd {
|
||||
Observe(ObserveOpts, Sender<Result<Snapshot, A11yError>>),
|
||||
Invoke(
|
||||
Target,
|
||||
SnapshotGen,
|
||||
ElementAction,
|
||||
Sender<Result<Effect, A11yError>>,
|
||||
),
|
||||
Focus(i32, Sender<Result<Effect, A11yError>>),
|
||||
}
|
||||
|
||||
pub struct ActorHandle {
|
||||
// mpsc::Sender is not Sync; the Mutex makes the handle Sync (calls are
|
||||
// serialized anyway — the tool marks the tool non-concurrency-safe).
|
||||
tx: Mutex<Sender<Cmd>>,
|
||||
}
|
||||
|
||||
impl ActorHandle {
|
||||
pub fn spawn() -> Result<Self, A11yError> {
|
||||
let (tx, rx) = channel::<Cmd>();
|
||||
std::thread::Builder::new()
|
||||
.name("nomi-a11y-macos".to_string())
|
||||
.spawn(move || {
|
||||
let runloop = unsafe { CFRunLoopGetCurrent() };
|
||||
let mut state = State {
|
||||
gen_counter: 0,
|
||||
current_gen: SnapshotGen(0),
|
||||
registry: HashMap::new(),
|
||||
runloop,
|
||||
observer: None,
|
||||
dirty: Box::new(AtomicBool::new(true)),
|
||||
observed_pid: None,
|
||||
cached: None,
|
||||
};
|
||||
let mode = CFString::new("kCFRunLoopDefaultMode");
|
||||
loop {
|
||||
// Block for a command. Before there are any observer sources
|
||||
// (first observe not run yet), the run loop has nothing to
|
||||
// wait on, so we must NOT spin on CFRunLoopRunInMode — block
|
||||
// on the channel instead and pump callbacks non-blocking.
|
||||
match rx.recv_timeout(Duration::from_millis(100)) {
|
||||
Ok(cmd) => {
|
||||
// Flush any pending observer callbacks (→ `dirty`)
|
||||
// before handling, so `observe` sees the freshest state.
|
||||
unsafe {
|
||||
CFRunLoopRunInMode(mode.as_concrete_TypeRef(), 0.0, 0);
|
||||
}
|
||||
match cmd {
|
||||
Cmd::Observe(opts, reply) => {
|
||||
let _ = reply.send(do_observe(&opts, &mut state));
|
||||
}
|
||||
Cmd::Invoke(target, generation, action, reply) => {
|
||||
let r = do_invoke(&target, generation, &action, &state);
|
||||
// A mutating action invalidates the cache even
|
||||
// before the observer notification arrives.
|
||||
state.dirty.store(true, Ordering::Relaxed);
|
||||
let _ = reply.send(r);
|
||||
}
|
||||
Cmd::Focus(pid, reply) => {
|
||||
let r = do_focus(pid);
|
||||
state.dirty.store(true, Ordering::Relaxed);
|
||||
let _ = reply.send(r);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
|
||||
// Periodically service the observer source so pending
|
||||
// notifications don't pile up while idle.
|
||||
unsafe {
|
||||
CFRunLoopRunInMode(mode.as_concrete_TypeRef(), 0.0, 0);
|
||||
}
|
||||
}
|
||||
Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => return,
|
||||
}
|
||||
}
|
||||
})
|
||||
.map_err(|e| A11yError::Backend(format!("failed to start AX actor thread: {e}")))?;
|
||||
Ok(Self { tx: Mutex::new(tx) })
|
||||
}
|
||||
|
||||
fn send(&self, cmd: Cmd) -> Result<(), A11yError> {
|
||||
self.tx
|
||||
.lock()
|
||||
.map_err(|_| A11yError::Backend("AX actor lock poisoned".to_string()))?
|
||||
.send(cmd)
|
||||
.map_err(|_| A11yError::Backend("AX actor thread is gone".to_string()))
|
||||
}
|
||||
|
||||
pub fn observe(&self, opts: ObserveOpts) -> Result<Snapshot, A11yError> {
|
||||
let (tx, rx) = channel();
|
||||
self.send(Cmd::Observe(opts, tx))?;
|
||||
rx.recv()
|
||||
.map_err(|_| A11yError::Backend("AX actor dropped the reply".to_string()))?
|
||||
}
|
||||
|
||||
pub fn invoke(
|
||||
&self,
|
||||
target: Target,
|
||||
generation: SnapshotGen,
|
||||
action: ElementAction,
|
||||
) -> Result<Effect, A11yError> {
|
||||
let (tx, rx) = channel();
|
||||
self.send(Cmd::Invoke(target, generation, action, tx))?;
|
||||
rx.recv()
|
||||
.map_err(|_| A11yError::Backend("AX actor dropped the reply".to_string()))?
|
||||
}
|
||||
|
||||
pub fn focus_window(&self, pid: i32) -> Result<Effect, A11yError> {
|
||||
let (tx, rx) = channel();
|
||||
self.send(Cmd::Focus(pid, tx))?;
|
||||
rx.recv()
|
||||
.map_err(|_| A11yError::Backend("AX actor dropped the reply".to_string()))?
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
//! macOS accessibility backend (AXUIElement).
|
||||
//!
|
||||
//! Threading model: AXUIElement / AXObserver have CFRunLoop / main-thread
|
||||
//! affinity, so all AX calls are marshaled to a single dedicated actor thread
|
||||
//! that owns a CFRunLoop and is the sole caller of the AX APIs. The public
|
||||
//! `MacEngine` is a `Send + Sync` handle that sends commands to that actor and
|
||||
//! blocks on a reply channel. Raw `AXUIElement` handles never cross the actor
|
||||
//! boundary — only serializable `Snapshot` / `Effect` data does.
|
||||
//!
|
||||
//! Status: the actor scaffolding + capabilities are in place; the AX tree walk
|
||||
//! and actuation are wired in `actor.rs` (see below). This module is compiled
|
||||
//! only on macOS.
|
||||
|
||||
use crate::engine::{
|
||||
A11yEngine, A11yError, Capabilities, Effect, ElementAction, InputKind, ObserveOpts, Snapshot,
|
||||
SnapshotGen, Target,
|
||||
};
|
||||
|
||||
pub struct MacEngine {
|
||||
inner: actor::ActorHandle,
|
||||
}
|
||||
|
||||
impl MacEngine {
|
||||
pub fn start() -> Result<Self, A11yError> {
|
||||
let inner = actor::ActorHandle::spawn()?;
|
||||
Ok(Self { inner })
|
||||
}
|
||||
}
|
||||
|
||||
impl A11yEngine for MacEngine {
|
||||
fn capabilities(&self) -> Capabilities {
|
||||
Capabilities {
|
||||
os: "macos".to_string(),
|
||||
tree_read: true,
|
||||
screenshot: true,
|
||||
semantic_action: true,
|
||||
synthetic_input: InputKind::Native,
|
||||
window_management: true,
|
||||
}
|
||||
}
|
||||
|
||||
fn observe(&self, opts: &ObserveOpts) -> Result<Snapshot, A11yError> {
|
||||
self.inner.observe(opts.clone())
|
||||
}
|
||||
|
||||
fn invoke(
|
||||
&self,
|
||||
target: &Target,
|
||||
generation: SnapshotGen,
|
||||
action: ElementAction,
|
||||
) -> Result<Effect, A11yError> {
|
||||
self.inner.invoke(target.clone(), generation, action)
|
||||
}
|
||||
|
||||
fn focus_window(&self, pid: i32) -> Result<Effect, A11yError> {
|
||||
self.inner.focus_window(pid)
|
||||
}
|
||||
}
|
||||
|
||||
mod actor;
|
||||
mod ocr;
|
||||
|
||||
pub use ocr::ocr_screenshot;
|
||||
@@ -0,0 +1,112 @@
|
||||
//! macOS OCR via Vision.framework (`VNRecognizeTextRequest`) — on-device, with
|
||||
//! CJK support (essential for a Chinese-first product). The screenshot is fed
|
||||
//! in as PNG data (`VNImageRequestHandler initWithData:`) so we avoid
|
||||
//! hand-building a CGImage. Vision returns normalized bounding boxes with a
|
||||
//! bottom-left origin; we convert them to pixel rectangles with a top-left
|
||||
//! origin to match the screenshot space the overlay/tool use.
|
||||
//!
|
||||
//! OCR has no main-thread/run-loop affinity, so this runs on whatever thread
|
||||
//! the caller uses (the tool calls it from `spawn_blocking`).
|
||||
|
||||
use objc2::AnyThread;
|
||||
use objc2::rc::Retained;
|
||||
use objc2::runtime::AnyObject;
|
||||
use objc2_foundation::{NSArray, NSData, NSDictionary, NSString};
|
||||
use objc2_vision::{
|
||||
VNImageRequestHandler, VNRecognizeTextRequest, VNRequest, VNRequestTextRecognitionLevel,
|
||||
};
|
||||
|
||||
use crate::engine::{A11yError, OcrLine, Rect};
|
||||
|
||||
pub fn ocr_screenshot(img: &image::RgbaImage, langs: &[String]) -> Result<Vec<OcrLine>, A11yError> {
|
||||
let (w, h) = img.dimensions();
|
||||
if w == 0 || h == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
// Encode to PNG so VNImageRequestHandler can decode it directly.
|
||||
let mut png = Vec::new();
|
||||
image::DynamicImage::ImageRgba8(img.clone())
|
||||
.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png)
|
||||
.map_err(|e| A11yError::Backend(format!("OCR: PNG encode failed: {e}")))?;
|
||||
|
||||
unsafe {
|
||||
let data = NSData::with_bytes(&png);
|
||||
|
||||
let request = VNRecognizeTextRequest::new();
|
||||
request.setRecognitionLevel(VNRequestTextRecognitionLevel::Accurate);
|
||||
request.setUsesLanguageCorrection(true);
|
||||
if !langs.is_empty() {
|
||||
let ns: Vec<Retained<NSString>> =
|
||||
langs.iter().map(|l| NSString::from_str(l)).collect();
|
||||
let arr = NSArray::from_retained_slice(&ns);
|
||||
request.setRecognitionLanguages(&arr);
|
||||
}
|
||||
|
||||
let options: Retained<NSDictionary<NSString, AnyObject>> = NSDictionary::new();
|
||||
let handler = VNImageRequestHandler::initWithData_options(
|
||||
VNImageRequestHandler::alloc(),
|
||||
&data,
|
||||
&options,
|
||||
);
|
||||
|
||||
let req_ref: &VNRequest = &request;
|
||||
let requests = NSArray::from_slice(&[req_ref]);
|
||||
handler
|
||||
.performRequests_error(&requests)
|
||||
.map_err(|e| A11yError::Backend(format!("OCR perform failed: {e:?}")))?;
|
||||
|
||||
let Some(results) = request.results() else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
|
||||
let mut lines = Vec::new();
|
||||
for obs in results.iter() {
|
||||
let top = obs.topCandidates(1);
|
||||
let Some(text) = top.firstObject() else {
|
||||
continue;
|
||||
};
|
||||
let s = text.string().to_string();
|
||||
if s.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
// Normalized (0..1), bottom-left origin → pixel, top-left origin.
|
||||
let bb = obs.boundingBox();
|
||||
let px = bb.origin.x * w as f64;
|
||||
let pw = bb.size.width * w as f64;
|
||||
let ph = bb.size.height * h as f64;
|
||||
let py = (1.0 - bb.origin.y - bb.size.height) * h as f64;
|
||||
lines.push(OcrLine {
|
||||
text: s,
|
||||
bounds: Rect {
|
||||
x: px,
|
||||
y: py,
|
||||
w: pw,
|
||||
h: ph,
|
||||
},
|
||||
});
|
||||
}
|
||||
Ok(lines)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn ocr_blank_image_runs_without_error() {
|
||||
// Exercises the full Vision FFI path (compile + link + run). A blank
|
||||
// image yields no recognized text; OCR needs no TCC permission.
|
||||
let img = image::RgbaImage::from_pixel(80, 40, image::Rgba([255, 255, 255, 255]));
|
||||
let lines = ocr_screenshot(&img, &["en-US".to_string()]).expect("ocr should not error");
|
||||
assert!(lines.iter().all(|l| !l.text.trim().is_empty()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ocr_empty_image_is_empty() {
|
||||
let img = image::RgbaImage::new(0, 0);
|
||||
assert!(ocr_screenshot(&img, &[]).unwrap().is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
//! Set-of-Marks overlay: draw numbered boxes for each interactable element onto
|
||||
//! the screenshot the model sees. Self-contained (only the `image` crate) — a
|
||||
//! tiny embedded 3×5 bitmap digit font renders the `[ref]` labels, so no font
|
||||
//! asset or extra dependency is needed.
|
||||
//!
|
||||
//! Element `bounds` MUST already be in the image's pixel space (the caller
|
||||
//! converts OS accessibility coordinates → screenshot pixels before calling).
|
||||
|
||||
use image::{Rgba, RgbaImage};
|
||||
|
||||
use crate::engine::ElementEntry;
|
||||
|
||||
/// Distinct, high-contrast mark colors cycled by ref so neighbors differ.
|
||||
const PALETTE: [[u8; 3]; 6] = [
|
||||
[255, 59, 48], // red
|
||||
[0, 122, 255], // blue
|
||||
[52, 199, 89], // green
|
||||
[255, 149, 0], // orange
|
||||
[175, 82, 222], // purple
|
||||
[255, 45, 85], // pink
|
||||
];
|
||||
|
||||
/// 3×5 bitmap font, digits 0-9. Each row's low 3 bits are pixels (MSB = left).
|
||||
const DIGITS: [[u8; 5]; 10] = [
|
||||
[0b111, 0b101, 0b101, 0b101, 0b111], // 0
|
||||
[0b010, 0b110, 0b010, 0b010, 0b111], // 1
|
||||
[0b111, 0b001, 0b111, 0b100, 0b111], // 2
|
||||
[0b111, 0b001, 0b111, 0b001, 0b111], // 3
|
||||
[0b101, 0b101, 0b111, 0b001, 0b001], // 4
|
||||
[0b111, 0b100, 0b111, 0b001, 0b111], // 5
|
||||
[0b111, 0b100, 0b111, 0b101, 0b111], // 6
|
||||
[0b111, 0b001, 0b010, 0b010, 0b010], // 7
|
||||
[0b111, 0b101, 0b111, 0b101, 0b111], // 8
|
||||
[0b111, 0b101, 0b111, 0b001, 0b111], // 9
|
||||
];
|
||||
|
||||
const SCALE: i64 = 3; // pixels per font cell
|
||||
const DIGIT_W: i64 = 3 * SCALE;
|
||||
const DIGIT_H: i64 = 5 * SCALE;
|
||||
const GAP: i64 = SCALE;
|
||||
const PAD: i64 = SCALE;
|
||||
|
||||
/// Draw a numbered box for each entry.
|
||||
pub fn draw_set_of_marks(img: &mut RgbaImage, entries: &[ElementEntry]) {
|
||||
let (iw, ih) = img.dimensions();
|
||||
for e in entries {
|
||||
if e.bounds.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let color = PALETTE[(e.r#ref as usize) % PALETTE.len()];
|
||||
let x = e.bounds.x.round() as i64;
|
||||
let y = e.bounds.y.round() as i64;
|
||||
let w = e.bounds.w.round() as i64;
|
||||
let h = e.bounds.h.round() as i64;
|
||||
draw_rect_border(img, x, y, w, h, color, 2, iw, ih);
|
||||
draw_label(img, x, y, e.r#ref, color, iw, ih);
|
||||
}
|
||||
}
|
||||
|
||||
fn put(img: &mut RgbaImage, x: i64, y: i64, c: [u8; 3], iw: u32, ih: u32) {
|
||||
if x < 0 || y < 0 || x >= iw as i64 || y >= ih as i64 {
|
||||
return;
|
||||
}
|
||||
img.put_pixel(x as u32, y as u32, Rgba([c[0], c[1], c[2], 255]));
|
||||
}
|
||||
|
||||
fn fill_rect(img: &mut RgbaImage, x: i64, y: i64, w: i64, h: i64, c: [u8; 3], iw: u32, ih: u32) {
|
||||
for dy in 0..h {
|
||||
for dx in 0..w {
|
||||
put(img, x + dx, y + dy, c, iw, ih);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn draw_rect_border(
|
||||
img: &mut RgbaImage,
|
||||
x: i64,
|
||||
y: i64,
|
||||
w: i64,
|
||||
h: i64,
|
||||
c: [u8; 3],
|
||||
t: i64,
|
||||
iw: u32,
|
||||
ih: u32,
|
||||
) {
|
||||
for k in 0..t {
|
||||
// top / bottom
|
||||
for dx in 0..w {
|
||||
put(img, x + dx, y + k, c, iw, ih);
|
||||
put(img, x + dx, y + h - 1 - k, c, iw, ih);
|
||||
}
|
||||
// left / right
|
||||
for dy in 0..h {
|
||||
put(img, x + k, y + dy, c, iw, ih);
|
||||
put(img, x + w - 1 - k, y + dy, c, iw, ih);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn label_size(n: u32) -> (i64, i64) {
|
||||
let digits = n.max(1).to_string().len() as i64;
|
||||
let w = PAD * 2 + digits * DIGIT_W + (digits - 1) * GAP;
|
||||
let h = PAD * 2 + DIGIT_H;
|
||||
(w, h)
|
||||
}
|
||||
|
||||
fn draw_label(img: &mut RgbaImage, ex: i64, ey: i64, n: u32, bg: [u8; 3], iw: u32, ih: u32) {
|
||||
let (lw, lh) = label_size(n);
|
||||
// Prefer just above the element's top-left; if no room, place inside.
|
||||
let lx = ex.max(0);
|
||||
let ly = if ey - lh >= 0 { ey - lh } else { ey };
|
||||
fill_rect(img, lx, ly, lw, lh, bg, iw, ih);
|
||||
|
||||
let fg = [255u8, 255, 255]; // white digits on the colored chip
|
||||
let mut cx = lx + PAD;
|
||||
let cy = ly + PAD;
|
||||
for ch in n.to_string().chars() {
|
||||
let d = ch.to_digit(10).unwrap_or(0) as usize;
|
||||
draw_digit(img, cx, cy, DIGITS[d], fg, iw, ih);
|
||||
cx += DIGIT_W + GAP;
|
||||
}
|
||||
}
|
||||
|
||||
fn draw_digit(img: &mut RgbaImage, x: i64, y: i64, glyph: [u8; 5], c: [u8; 3], iw: u32, ih: u32) {
|
||||
for (row, bits) in glyph.iter().enumerate() {
|
||||
for col in 0..3i64 {
|
||||
// MSB is the leftmost column.
|
||||
if bits & (1 << (2 - col)) != 0 {
|
||||
fill_rect(
|
||||
img,
|
||||
x + col * SCALE,
|
||||
y + row as i64 * SCALE,
|
||||
SCALE,
|
||||
SCALE,
|
||||
c,
|
||||
iw,
|
||||
ih,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::engine::{Rect, Source};
|
||||
|
||||
fn entry(r: u32, x: f64, y: f64) -> ElementEntry {
|
||||
ElementEntry {
|
||||
r#ref: r,
|
||||
role: "button".into(),
|
||||
name: Some("x".into()),
|
||||
value: None,
|
||||
states: vec![],
|
||||
bounds: Rect { x, y, w: 40.0, h: 20.0 },
|
||||
source: Source::A11y,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn draws_marks_without_panicking_at_edges() {
|
||||
let mut img = RgbaImage::from_pixel(100, 100, Rgba([0, 0, 0, 255]));
|
||||
// One in-bounds, one clipped at the top edge (label would go off-screen).
|
||||
draw_set_of_marks(&mut img, &[entry(1, 30.0, 40.0), entry(12, 0.0, 0.0)]);
|
||||
// Some pixels must now be non-black (a border or label was drawn).
|
||||
let changed = img.pixels().any(|p| p[0] > 2 || p[1] > 2 || p[2] > 2);
|
||||
assert!(changed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn label_size_grows_with_digits() {
|
||||
assert!(label_size(7).0 < label_size(42).0);
|
||||
assert!(label_size(42).0 < label_size(123).0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
//! A focused, platform-independent selector grammar for addressing
|
||||
//! accessibility elements deterministically — the durable way to re-locate an
|
||||
//! element across snapshots (vs a `[ref]`, which is snapshot-scoped).
|
||||
//!
|
||||
//! Grammar (v1 subset; positional/relative combinators are a planned
|
||||
//! extension): `prefix:value` terms joined by `&&` / `||`, each optionally
|
||||
//! negated with a leading `!`.
|
||||
//!
|
||||
//! ```text
|
||||
//! role:Button && name:Save
|
||||
//! name:Save || name:Submit
|
||||
//! role:Button && !name:Cancel
|
||||
//! role:Button && name:Item && nth:2
|
||||
//! ```
|
||||
//!
|
||||
//! Prefixes: `role:` `name:` `text:` `nth:`. A bare term with no prefix is
|
||||
//! treated as `name:`. `name`/`role` match case-insensitively as substrings;
|
||||
//! `text` matches case-sensitively (visible-text semantics).
|
||||
|
||||
use crate::engine::ElementEntry;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Selector {
|
||||
/// Accessibility role (case-insensitive substring).
|
||||
Role(String),
|
||||
/// Accessible name/label (case-insensitive substring).
|
||||
Name(String),
|
||||
/// Visible text / value (case-sensitive substring).
|
||||
Text(String),
|
||||
/// Pick the Nth (0-based) of the otherwise-matching set.
|
||||
Nth(usize),
|
||||
/// All must match.
|
||||
And(Vec<Selector>),
|
||||
/// Any may match.
|
||||
Or(Vec<Selector>),
|
||||
/// Must not match.
|
||||
Not(Box<Selector>),
|
||||
}
|
||||
|
||||
impl Selector {
|
||||
/// Parse a selector expression. Returns a human-readable error on malformed
|
||||
/// input (which the caller surfaces to the model, not a panic).
|
||||
pub fn parse(input: &str) -> Result<Selector, String> {
|
||||
let s = input.trim();
|
||||
if s.is_empty() {
|
||||
return Err("empty selector".to_string());
|
||||
}
|
||||
parse_or(s)
|
||||
}
|
||||
|
||||
/// True if this selector (ignoring any positional `Nth`) matches `e`.
|
||||
pub fn matches(&self, e: &ElementEntry) -> bool {
|
||||
match self {
|
||||
Selector::Role(r) => contains_ci(&e.role, r),
|
||||
Selector::Name(n) => e.name.as_deref().is_some_and(|v| contains_ci(v, n)),
|
||||
Selector::Text(t) => {
|
||||
e.name.as_deref().is_some_and(|v| v.contains(t.as_str()))
|
||||
|| e.value.as_deref().is_some_and(|v| v.contains(t.as_str()))
|
||||
}
|
||||
Selector::Nth(_) => true, // positional; applied in `select`
|
||||
Selector::And(parts) => parts.iter().all(|p| p.matches(e)),
|
||||
Selector::Or(parts) => parts.iter().any(|p| p.matches(e)),
|
||||
Selector::Not(inner) => !inner.matches(e),
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve against a snapshot's entries: filter by the match predicate, then
|
||||
/// apply any top-level `Nth` positional pick. Returns matching refs in order.
|
||||
pub fn select<'a>(&self, entries: &'a [ElementEntry]) -> Vec<&'a ElementEntry> {
|
||||
let matched: Vec<&ElementEntry> = entries.iter().filter(|e| self.matches(e)).collect();
|
||||
match self.find_nth() {
|
||||
Some(n) => matched.into_iter().skip(n).take(1).collect(),
|
||||
None => matched,
|
||||
}
|
||||
}
|
||||
|
||||
/// Find a top-level `Nth` index if present (directly or inside a top `And`).
|
||||
fn find_nth(&self) -> Option<usize> {
|
||||
match self {
|
||||
Selector::Nth(n) => Some(*n),
|
||||
Selector::And(parts) => parts.iter().find_map(|p| match p {
|
||||
Selector::Nth(n) => Some(*n),
|
||||
_ => None,
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn contains_ci(haystack: &str, needle: &str) -> bool {
|
||||
haystack.to_lowercase().contains(&needle.to_lowercase())
|
||||
}
|
||||
|
||||
fn parse_or(s: &str) -> Result<Selector, String> {
|
||||
let parts = split_top(s, "||");
|
||||
if parts.len() == 1 {
|
||||
return parse_and(parts[0]);
|
||||
}
|
||||
let parsed: Result<Vec<_>, _> = parts.iter().map(|p| parse_and(p)).collect();
|
||||
Ok(Selector::Or(parsed?))
|
||||
}
|
||||
|
||||
fn parse_and(s: &str) -> Result<Selector, String> {
|
||||
let parts = split_top(s, "&&");
|
||||
if parts.len() == 1 {
|
||||
return parse_term(parts[0]);
|
||||
}
|
||||
let parsed: Result<Vec<_>, _> = parts.iter().map(|p| parse_term(p)).collect();
|
||||
Ok(Selector::And(parsed?))
|
||||
}
|
||||
|
||||
fn parse_term(s: &str) -> Result<Selector, String> {
|
||||
let t = s.trim();
|
||||
if let Some(rest) = t.strip_prefix('!') {
|
||||
return Ok(Selector::Not(Box::new(parse_simple(rest.trim())?)));
|
||||
}
|
||||
parse_simple(t)
|
||||
}
|
||||
|
||||
fn parse_simple(s: &str) -> Result<Selector, String> {
|
||||
let t = s.trim();
|
||||
if t.is_empty() {
|
||||
return Err("empty selector term".to_string());
|
||||
}
|
||||
let (prefix, value) = match t.split_once(':') {
|
||||
Some((p, v)) => (p.trim().to_lowercase(), v.trim().to_string()),
|
||||
None => ("name".to_string(), t.to_string()),
|
||||
};
|
||||
if value.is_empty() && prefix != "nth" {
|
||||
return Err(format!("selector term `{t}` has an empty value"));
|
||||
}
|
||||
match prefix.as_str() {
|
||||
"role" => Ok(Selector::Role(value)),
|
||||
"name" => Ok(Selector::Name(value)),
|
||||
"text" => Ok(Selector::Text(value)),
|
||||
"nth" => value
|
||||
.parse::<usize>()
|
||||
.map(Selector::Nth)
|
||||
.map_err(|_| format!("`nth:` expects a non-negative integer, got `{value}`")),
|
||||
other => Err(format!(
|
||||
"unknown selector prefix `{other}:` (supported: role, name, text, nth)"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Split on a two-char operator at the top level. (v1 has no parentheses, so
|
||||
/// this is a plain delimiter split; selector values do not contain `&&`/`||`.)
|
||||
fn split_top<'a>(s: &'a str, op: &str) -> Vec<&'a str> {
|
||||
s.split(op).map(|p| p.trim()).filter(|p| !p.is_empty()).collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::engine::{Rect, Source};
|
||||
|
||||
fn entry(r: u32, role: &str, name: Option<&str>) -> ElementEntry {
|
||||
ElementEntry {
|
||||
r#ref: r,
|
||||
role: role.to_string(),
|
||||
name: name.map(|s| s.to_string()),
|
||||
value: None,
|
||||
states: vec![],
|
||||
bounds: Rect { x: 0.0, y: 0.0, w: 1.0, h: 1.0 },
|
||||
source: Source::A11y,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_role_and_name() {
|
||||
let s = Selector::parse("role:Button && name:Save").unwrap();
|
||||
assert_eq!(
|
||||
s,
|
||||
Selector::And(vec![
|
||||
Selector::Role("Button".into()),
|
||||
Selector::Name("Save".into())
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bare_term_is_name() {
|
||||
assert_eq!(Selector::parse("Submit").unwrap(), Selector::Name("Submit".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_or_and_not() {
|
||||
let s = Selector::parse("name:Save || name:Submit").unwrap();
|
||||
assert!(matches!(s, Selector::Or(_)));
|
||||
let n = Selector::parse("!name:Cancel").unwrap();
|
||||
assert!(matches!(n, Selector::Not(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_case_insensitive_substring() {
|
||||
let e = entry(1, "AXButton", Some("Save Document"));
|
||||
assert!(Selector::parse("role:button && name:save").unwrap().matches(&e));
|
||||
assert!(!Selector::parse("name:delete").unwrap().matches(&e));
|
||||
assert!(Selector::parse("role:Button && !name:Cancel").unwrap().matches(&e));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nth_picks_positionally() {
|
||||
let entries = vec![
|
||||
entry(1, "AXButton", Some("Item")),
|
||||
entry(2, "AXButton", Some("Item")),
|
||||
entry(3, "AXButton", Some("Item")),
|
||||
];
|
||||
let s = Selector::parse("role:Button && name:Item && nth:1").unwrap();
|
||||
let got = s.select(&entries);
|
||||
assert_eq!(got.len(), 1);
|
||||
assert_eq!(got[0].r#ref, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_is_error() {
|
||||
assert!(Selector::parse(" ").is_err());
|
||||
assert!(Selector::parse("bogus:x").is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
//! Neutral accessibility-tree model + interactable filtering and text
|
||||
//! formatting, shared by every OS backend. Backends build a `UiNode` tree from
|
||||
//! their native API; this module turns it into the numbered `ElementEntry`
|
||||
//! list the model consumes (and the overlay draws).
|
||||
|
||||
use crate::engine::{ElementEntry, Rect, Source};
|
||||
|
||||
/// A raw accessibility node as captured by a backend, before filtering.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct UiNode {
|
||||
pub role: String,
|
||||
pub name: Option<String>,
|
||||
pub value: Option<String>,
|
||||
pub states: Vec<String>,
|
||||
pub bounds: Option<Rect>,
|
||||
/// Backend's verdict that this node is actionable (has a default action /
|
||||
/// is a control role) — the primary interactability signal.
|
||||
pub actionable: bool,
|
||||
pub children: Vec<UiNode>,
|
||||
}
|
||||
|
||||
impl UiNode {
|
||||
pub fn leaf(role: impl Into<String>) -> Self {
|
||||
Self {
|
||||
role: role.into(),
|
||||
name: None,
|
||||
value: None,
|
||||
states: vec![],
|
||||
bounds: None,
|
||||
actionable: false,
|
||||
children: vec![],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_interactable(n: &UiNode) -> bool {
|
||||
let Some(b) = n.bounds else { return false };
|
||||
if b.is_empty() {
|
||||
return false;
|
||||
}
|
||||
// Actionable per the backend, or a control that carries a label/value worth
|
||||
// targeting even if no explicit action was reported.
|
||||
n.actionable || n.name.is_some() || n.value.is_some()
|
||||
}
|
||||
|
||||
/// Depth-first collect interactable nodes (honoring depth + budget), then number
|
||||
/// them in reading order (top-to-bottom, left-to-right). Returns
|
||||
/// `(entries, truncated)`.
|
||||
pub fn flatten_interactable(
|
||||
root: &UiNode,
|
||||
max_depth: usize,
|
||||
node_budget: usize,
|
||||
) -> (Vec<ElementEntry>, bool) {
|
||||
let mut collected: Vec<&UiNode> = Vec::new();
|
||||
let mut truncated = false;
|
||||
collect(root, 0, max_depth, node_budget, &mut collected, &mut truncated);
|
||||
|
||||
// Reading order: sort by rounded (y, x) so the model's numbering tracks the
|
||||
// visual layout. Stable so equal positions keep DFS order.
|
||||
collected.sort_by(|a, b| {
|
||||
let (ax, ay) = a.bounds.map(|r| (r.x, r.y)).unwrap_or((0.0, 0.0));
|
||||
let (bx, by) = b.bounds.map(|r| (r.x, r.y)).unwrap_or((0.0, 0.0));
|
||||
(ay.round() as i64, ax.round() as i64).cmp(&(by.round() as i64, bx.round() as i64))
|
||||
});
|
||||
|
||||
let entries = collected
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(i, n)| ElementEntry {
|
||||
r#ref: i as u32 + 1, // 1-based: matches the [ref] the model sees
|
||||
role: normalize_role(&n.role),
|
||||
name: n.name.clone().filter(|s| !s.trim().is_empty()),
|
||||
value: n.value.clone().filter(|s| !s.trim().is_empty()),
|
||||
states: n.states.clone(),
|
||||
bounds: n.bounds.unwrap_or(Rect { x: 0.0, y: 0.0, w: 0.0, h: 0.0 }),
|
||||
source: Source::A11y,
|
||||
})
|
||||
.collect();
|
||||
(entries, truncated)
|
||||
}
|
||||
|
||||
fn collect<'a>(
|
||||
node: &'a UiNode,
|
||||
depth: usize,
|
||||
max_depth: usize,
|
||||
budget: usize,
|
||||
out: &mut Vec<&'a UiNode>,
|
||||
truncated: &mut bool,
|
||||
) {
|
||||
if is_interactable(node) {
|
||||
if out.len() >= budget {
|
||||
*truncated = true;
|
||||
return;
|
||||
}
|
||||
out.push(node);
|
||||
}
|
||||
if depth >= max_depth {
|
||||
if !node.children.is_empty() {
|
||||
*truncated = true;
|
||||
}
|
||||
return;
|
||||
}
|
||||
for child in &node.children {
|
||||
if out.len() >= budget {
|
||||
*truncated = true;
|
||||
return;
|
||||
}
|
||||
collect(child, depth + 1, max_depth, budget, out, truncated);
|
||||
}
|
||||
}
|
||||
|
||||
/// Strip the platform `AX`/`UIA_` prefix and lowercase so the model sees
|
||||
/// stable cross-platform role names (`button`, `textfield`, …).
|
||||
pub fn normalize_role(role: &str) -> String {
|
||||
let r = role
|
||||
.strip_prefix("AX")
|
||||
.or_else(|| role.strip_prefix("UIA_"))
|
||||
.unwrap_or(role);
|
||||
r.to_lowercase()
|
||||
}
|
||||
|
||||
/// Render entries as a numbered text list for the model:
|
||||
/// `[14] button "Submit" enabled`.
|
||||
pub fn format_entries(entries: &[ElementEntry]) -> String {
|
||||
if entries.is_empty() {
|
||||
return "No interactable elements found in the accessibility tree.".to_string();
|
||||
}
|
||||
let mut out = String::new();
|
||||
for e in entries {
|
||||
out.push_str(&format!("[{}] {}", e.r#ref, e.role));
|
||||
if let Some(name) = &e.name {
|
||||
out.push_str(&format!(" {:?}", truncate(name, 80)));
|
||||
}
|
||||
if let Some(value) = &e.value {
|
||||
if Some(value) != e.name.as_ref() {
|
||||
out.push_str(&format!(" = {:?}", truncate(value, 60)));
|
||||
}
|
||||
}
|
||||
if !e.states.is_empty() {
|
||||
out.push_str(&format!(" [{}]", e.states.join(",")));
|
||||
}
|
||||
out.push('\n');
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn truncate(s: &str, max: usize) -> String {
|
||||
if s.chars().count() <= max {
|
||||
s.to_string()
|
||||
} else {
|
||||
let t: String = s.chars().take(max).collect();
|
||||
format!("{t}…")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn node(role: &str, name: Option<&str>, x: f64, y: f64, actionable: bool, children: Vec<UiNode>) -> UiNode {
|
||||
UiNode {
|
||||
role: role.to_string(),
|
||||
name: name.map(|s| s.to_string()),
|
||||
value: None,
|
||||
states: vec![],
|
||||
bounds: Some(Rect { x, y, w: 50.0, h: 20.0 }),
|
||||
actionable,
|
||||
children,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flattens_filters_and_numbers_in_reading_order() {
|
||||
// Root window (not actionable, no name) with two buttons out of order.
|
||||
let root = UiNode {
|
||||
role: "AXWindow".into(),
|
||||
name: None,
|
||||
value: None,
|
||||
states: vec![],
|
||||
bounds: Some(Rect { x: 0.0, y: 0.0, w: 800.0, h: 600.0 }),
|
||||
actionable: false,
|
||||
children: vec![
|
||||
node("AXButton", Some("Bottom"), 10.0, 200.0, true, vec![]),
|
||||
node("AXButton", Some("Top"), 10.0, 10.0, true, vec![]),
|
||||
],
|
||||
};
|
||||
let (entries, truncated) = flatten_interactable(&root, 12, 120);
|
||||
assert!(!truncated);
|
||||
assert_eq!(entries.len(), 2);
|
||||
// Sorted top-to-bottom: "Top" gets [1].
|
||||
assert_eq!(entries[0].name.as_deref(), Some("Top"));
|
||||
assert_eq!(entries[0].role, "button");
|
||||
assert_eq!(entries[0].r#ref, 1);
|
||||
assert_eq!(entries[1].name.as_deref(), Some("Bottom"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn budget_truncates() {
|
||||
let children: Vec<UiNode> = (0..10)
|
||||
.map(|i| node("AXButton", Some("b"), 0.0, i as f64, true, vec![]))
|
||||
.collect();
|
||||
let root = UiNode {
|
||||
role: "AXWindow".into(),
|
||||
name: None,
|
||||
value: None,
|
||||
states: vec![],
|
||||
bounds: Some(Rect { x: 0.0, y: 0.0, w: 100.0, h: 100.0 }),
|
||||
actionable: false,
|
||||
children,
|
||||
};
|
||||
let (entries, truncated) = flatten_interactable(&root, 12, 3);
|
||||
assert!(truncated);
|
||||
assert!(entries.len() <= 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn format_is_readable() {
|
||||
let entries = flatten_interactable(
|
||||
&node("AXButton", Some("Save"), 0.0, 0.0, true, vec![]),
|
||||
12,
|
||||
120,
|
||||
)
|
||||
.0;
|
||||
let text = format_entries(&entries);
|
||||
assert!(text.contains("[1] button"));
|
||||
assert!(text.contains("Save"));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,66 @@
|
||||
//! Windows accessibility backend (UI Automation).
|
||||
//!
|
||||
//! Threading model (mirrors the macOS backend): every `IUIAutomation` /
|
||||
//! `IUIAutomationElement` call has COM apartment affinity, so all UIA work is
|
||||
//! marshaled to a single dedicated actor thread that initializes COM as MTA
|
||||
//! (`CoInitializeEx(COINIT_MULTITHREADED)`) and is the sole owner of the
|
||||
//! `UIAutomation` instance and every element handle. The public `WinEngine` is
|
||||
//! a `Send + Sync` handle that sends commands over a channel and blocks on a
|
||||
//! per-command reply; raw UIA element handles never cross the actor boundary —
|
||||
//! only serializable `Snapshot` / `Effect` data does (so `WinEngine` is
|
||||
//! `Send + Sync` automatically, no `unsafe impl` needed).
|
||||
//!
|
||||
//! OCR (`Windows.Media.Ocr`) has no apartment affinity and runs on whatever
|
||||
//! thread the caller uses (the computer tool calls it from `spawn_blocking`).
|
||||
|
||||
use crate::engine::{
|
||||
A11yEngine, A11yError, Capabilities, Effect, ElementAction, InputKind, ObserveOpts, Snapshot,
|
||||
SnapshotGen, Target,
|
||||
};
|
||||
|
||||
mod actor;
|
||||
mod ocr;
|
||||
mod tree_map;
|
||||
|
||||
pub use ocr::ocr_screenshot;
|
||||
|
||||
pub struct WinEngine {
|
||||
inner: actor::ActorHandle,
|
||||
}
|
||||
|
||||
impl WinEngine {
|
||||
pub fn start() -> Result<Self, A11yError> {
|
||||
let inner = actor::ActorHandle::spawn()?;
|
||||
Ok(Self { inner })
|
||||
}
|
||||
}
|
||||
|
||||
impl A11yEngine for WinEngine {
|
||||
fn capabilities(&self) -> Capabilities {
|
||||
Capabilities {
|
||||
os: "windows".to_string(),
|
||||
tree_read: true,
|
||||
screenshot: true,
|
||||
semantic_action: true,
|
||||
synthetic_input: InputKind::Native,
|
||||
window_management: true,
|
||||
}
|
||||
}
|
||||
|
||||
fn observe(&self, opts: &ObserveOpts) -> Result<Snapshot, A11yError> {
|
||||
self.inner.observe(opts.clone())
|
||||
}
|
||||
|
||||
fn invoke(
|
||||
&self,
|
||||
target: &Target,
|
||||
generation: SnapshotGen,
|
||||
action: ElementAction,
|
||||
) -> Result<Effect, A11yError> {
|
||||
self.inner.invoke(target.clone(), generation, action)
|
||||
}
|
||||
|
||||
fn focus_window(&self, pid: i32) -> Result<Effect, A11yError> {
|
||||
self.inner.focus_window(pid)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
//! Windows OCR via `Windows.Media.Ocr` — on-device, with CJK support (essential
|
||||
//! for a Chinese-first product). Mirrors the macOS Vision backend: the
|
||||
//! screenshot is encoded to PNG, decoded into a `SoftwareBitmap` via
|
||||
//! `BitmapDecoder` (so we avoid hand-building a pixel buffer), then recognized.
|
||||
//! `OcrEngine`/`OcrResult` report word bounding boxes already in pixel space
|
||||
//! with a top-left origin — the same space the overlay/tool use — so no flip is
|
||||
//! needed.
|
||||
//!
|
||||
//! WinRT activation requires COM to be initialized on the calling thread. The
|
||||
//! tool calls this from `spawn_blocking`, whose pooled threads are not
|
||||
//! necessarily initialized, so we initialize COM (idempotently) up front.
|
||||
|
||||
use std::cell::Cell;
|
||||
use std::io::Cursor;
|
||||
|
||||
use windows::Globalization::Language;
|
||||
use windows::Graphics::Imaging::{
|
||||
BitmapAlphaMode, BitmapDecoder, BitmapPixelFormat, SoftwareBitmap,
|
||||
};
|
||||
use windows::Media::Ocr::OcrEngine;
|
||||
use windows::Storage::Streams::{DataWriter, InMemoryRandomAccessStream};
|
||||
use windows::Win32::System::Com::{COINIT_MULTITHREADED, CoInitializeEx};
|
||||
use windows::core::HSTRING;
|
||||
|
||||
use crate::engine::{A11yError, OcrLine, Rect};
|
||||
|
||||
fn win_err(ctx: &str, e: windows::core::Error) -> A11yError {
|
||||
A11yError::Backend(format!("OCR: {ctx}: {e}"))
|
||||
}
|
||||
|
||||
thread_local! {
|
||||
/// Whether this thread has already initialized COM. WinRT activation
|
||||
/// (OcrEngine, BitmapDecoder) requires COM on the calling thread; the tool
|
||||
/// calls us from pooled `spawn_blocking` threads. Initialize at most ONCE
|
||||
/// per thread and intentionally never `CoUninitialize` — these are
|
||||
/// process-lifetime pool threads, so a single MTA init is correct and a
|
||||
/// per-call init/leak is avoided.
|
||||
static COM_READY: Cell<bool> = const { Cell::new(false) };
|
||||
}
|
||||
|
||||
fn ensure_com() {
|
||||
COM_READY.with(|ready| {
|
||||
if !ready.get() {
|
||||
// Ignore S_FALSE / RPC_E_CHANGED_MODE — the thread is usable either way.
|
||||
unsafe {
|
||||
let _ = CoInitializeEx(None, COINIT_MULTITHREADED);
|
||||
}
|
||||
ready.set(true);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Build an OCR engine: prefer the caller's requested languages (e.g.
|
||||
/// `zh-Hans`) when an OCR pack for them is installed, else fall back to the
|
||||
/// user-profile languages.
|
||||
fn make_engine(langs: &[String]) -> Result<OcrEngine, A11yError> {
|
||||
for l in langs {
|
||||
if let Ok(lang) = Language::CreateLanguage(&HSTRING::from(l.as_str())) {
|
||||
if OcrEngine::IsLanguageSupported(&lang).unwrap_or(false) {
|
||||
if let Ok(engine) = OcrEngine::TryCreateFromLanguage(&lang) {
|
||||
return Ok(engine);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
OcrEngine::TryCreateFromUserProfileLanguages().map_err(|e| {
|
||||
A11yError::Backend(format!(
|
||||
"OCR engine unavailable: {e}. Install an OCR language pack (Settings → Time & \
|
||||
Language → Language → add a language and enable its Optical character recognition \
|
||||
feature)."
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn ocr_screenshot(img: &image::RgbaImage, langs: &[String]) -> Result<Vec<OcrLine>, A11yError> {
|
||||
let (w, h) = img.dimensions();
|
||||
if w == 0 || h == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
// WinRT activation needs COM on this thread (initialized at most once).
|
||||
ensure_com();
|
||||
|
||||
// Encode to PNG so BitmapDecoder can decode it into a SoftwareBitmap.
|
||||
let mut png = Vec::new();
|
||||
image::DynamicImage::ImageRgba8(img.clone())
|
||||
.write_to(&mut Cursor::new(&mut png), image::ImageFormat::Png)
|
||||
.map_err(|e| A11yError::Backend(format!("OCR: PNG encode failed: {e}")))?;
|
||||
|
||||
let engine = make_engine(langs)?;
|
||||
|
||||
// OcrEngine rejects images larger than MaxImageDimension on a side. The tool
|
||||
// already downscales screenshots well under this, but guard defensively with
|
||||
// a clear error rather than letting RecognizeAsync fail opaquely.
|
||||
if let Ok(max) = OcrEngine::MaxImageDimension() {
|
||||
if w > max || h > max {
|
||||
return Err(A11yError::Backend(format!(
|
||||
"OCR: image {w}x{h} exceeds the engine's max dimension {max} per side; downscale \
|
||||
before OCR"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
// PNG bytes → in-memory stream → SoftwareBitmap.
|
||||
let stream = InMemoryRandomAccessStream::new().map_err(|e| win_err("create stream", e))?;
|
||||
let writer = DataWriter::CreateDataWriter(&stream).map_err(|e| win_err("create writer", e))?;
|
||||
writer.WriteBytes(&png).map_err(|e| win_err("write bytes", e))?;
|
||||
writer
|
||||
.StoreAsync()
|
||||
.map_err(|e| win_err("store", e))?
|
||||
.get()
|
||||
.map_err(|e| win_err("store.get", e))?;
|
||||
writer
|
||||
.FlushAsync()
|
||||
.map_err(|e| win_err("flush", e))?
|
||||
.get()
|
||||
.map_err(|e| win_err("flush.get", e))?;
|
||||
writer
|
||||
.DetachStream()
|
||||
.map_err(|e| win_err("detach stream", e))?;
|
||||
stream.Seek(0).map_err(|e| win_err("seek", e))?;
|
||||
|
||||
let decoder = BitmapDecoder::CreateAsync(&stream)
|
||||
.map_err(|e| win_err("create decoder", e))?
|
||||
.get()
|
||||
.map_err(|e| win_err("decoder.get", e))?;
|
||||
let bitmap = decoder
|
||||
.GetSoftwareBitmapAsync()
|
||||
.map_err(|e| win_err("get bitmap", e))?
|
||||
.get()
|
||||
.map_err(|e| win_err("bitmap.get", e))?;
|
||||
// The PNG decoder auto-selects the pixel format (often Rgba8); OcrEngine
|
||||
// reliably accepts Bgra8/Premultiplied, so normalize before recognition
|
||||
// instead of relying on an undocumented accepted-format set.
|
||||
let bitmap = SoftwareBitmap::ConvertWithAlpha(
|
||||
&bitmap,
|
||||
BitmapPixelFormat::Bgra8,
|
||||
BitmapAlphaMode::Premultiplied,
|
||||
)
|
||||
.map_err(|e| win_err("convert to bgra8", e))?;
|
||||
|
||||
let result = engine
|
||||
.RecognizeAsync(&bitmap)
|
||||
.map_err(|e| win_err("recognize", e))?
|
||||
.get()
|
||||
.map_err(|e| win_err("recognize.get", e))?;
|
||||
|
||||
let mut out = Vec::new();
|
||||
let lines = result.Lines().map_err(|e| win_err("lines", e))?;
|
||||
for line in lines {
|
||||
let text = line.Text().map_err(|e| win_err("line text", e))?.to_string();
|
||||
if text.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
// Union of the line's word bounding rects (already pixel, top-left).
|
||||
let words = line.Words().map_err(|e| win_err("words", e))?;
|
||||
let (mut min_x, mut min_y) = (f64::MAX, f64::MAX);
|
||||
let (mut max_x, mut max_y) = (f64::MIN, f64::MIN);
|
||||
let mut any = false;
|
||||
for word in words {
|
||||
let r = word.BoundingRect().map_err(|e| win_err("word rect", e))?;
|
||||
min_x = min_x.min(r.X as f64);
|
||||
min_y = min_y.min(r.Y as f64);
|
||||
max_x = max_x.max((r.X + r.Width) as f64);
|
||||
max_y = max_y.max((r.Y + r.Height) as f64);
|
||||
any = true;
|
||||
}
|
||||
let bounds = if any && max_x >= min_x && max_y >= min_y {
|
||||
Rect {
|
||||
x: min_x,
|
||||
y: min_y,
|
||||
w: max_x - min_x,
|
||||
h: max_y - min_y,
|
||||
}
|
||||
} else {
|
||||
Rect {
|
||||
x: 0.0,
|
||||
y: 0.0,
|
||||
w: 0.0,
|
||||
h: 0.0,
|
||||
}
|
||||
};
|
||||
out.push(OcrLine { text, bounds });
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn ocr_blank_image_runs_without_error() {
|
||||
// Exercises the full Windows.Media.Ocr FFI path (compile + link + run).
|
||||
// A blank image yields no recognized text; OCR needs no permission, but
|
||||
// does require an OCR language pack (English ships by default on
|
||||
// Windows 10/11).
|
||||
let img = image::RgbaImage::from_pixel(80, 40, image::Rgba([255, 255, 255, 255]));
|
||||
match ocr_screenshot(&img, &["en-US".to_string()]) {
|
||||
Ok(lines) => assert!(lines.iter().all(|l| !l.text.trim().is_empty())),
|
||||
Err(A11yError::Backend(msg)) if msg.contains("language pack") => {
|
||||
eprintln!("skipping: no OCR language pack installed ({msg})");
|
||||
}
|
||||
Err(e) => panic!("OCR failed unexpectedly: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ocr_empty_image_is_empty() {
|
||||
assert!(ocr_screenshot(&image::RgbaImage::new(0, 0), &[]).unwrap().is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,715 @@
|
||||
//! Pure transform layer for the Windows UIA backend.
|
||||
//!
|
||||
//! The actor (which owns COM) captures the cached UIA tree of every target
|
||||
//! window into a `RawNode` forest of plain data — a synthetic `desktop` root
|
||||
//! whose children are per-window subtrees, each node carrying an index into a
|
||||
//! parallel handle table — and this module turns it into two products:
|
||||
//! * the numbered `ElementEntry` Set-of-Marks list (`build_entries`), grouped
|
||||
//! per window then in reading order, with a `ref → handle_idx` map so the
|
||||
//! actor can rebuild `ref → UIElement` for actuation; and
|
||||
//! * a hierarchical **semantic tree** text rendering (`render_tree`):
|
||||
//! `desktop → window → structural container → [ref] control`, pruning empty
|
||||
//! containers — the view the model reasons over.
|
||||
//!
|
||||
//! Keeping it free of any `uiautomation` / COM type makes the non-trivial logic
|
||||
//! (interactability filter, off-screen handling, per-window reading-order
|
||||
//! numbering, depth/budget truncation, the ref→handle correlation that must
|
||||
//! survive the sort, and the structural-tree pruning) unit-testable without a
|
||||
//! live UIA session; the COM reads in the actor are exercised by the `winsmoke`
|
||||
//! example instead.
|
||||
//!
|
||||
//! It is intentionally Windows-local rather than routed through the neutral
|
||||
//! `tree::flatten_interactable`: every emitted entry must map back to its live
|
||||
//! `UIElement` handle for `invoke`, which the neutral `UiNode` cannot carry, the
|
||||
//! off-screen-park filtering is UIA-specific, and the neutral layer has no
|
||||
//! multi-window / semantic-tree concept. It reuses the neutral `normalize_role`
|
||||
//! so role names stay consistent across platforms.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::engine::{ElementEntry, Rect, Source};
|
||||
use crate::tree::normalize_role;
|
||||
|
||||
/// Synthetic role for the forest root the actor builds above the per-window
|
||||
/// subtrees. Excluded from emission and rendered as the literal `desktop` line.
|
||||
pub(crate) const DESKTOP_ROLE: &str = "desktop";
|
||||
/// Role the actor stamps on each top-level window node. Excluded from emission
|
||||
/// (you target controls, not the frame) and rendered as `window "title"`.
|
||||
pub(crate) const WINDOW_ROLE: &str = "window";
|
||||
|
||||
/// One UIA element captured as plain, COM-free data. `handle_idx` indexes the
|
||||
/// actor's parallel `Vec<UIElement>` handle table, so a surviving entry can be
|
||||
/// mapped back to its element for actuation. `children` preserve document order.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct RawNode {
|
||||
pub handle_idx: usize,
|
||||
pub role: String,
|
||||
pub name: Option<String>,
|
||||
pub value: Option<String>,
|
||||
pub states: Vec<String>,
|
||||
pub bounds: Rect,
|
||||
/// Inherently actionable or keyboard-focusable (the primary interactability
|
||||
/// signal). A named/valued node is also emitted even when this is false.
|
||||
pub actionable: bool,
|
||||
/// Has a ScrollPattern with a scrollable axis — emitted as a target so the
|
||||
/// model can scroll the region even when it is not otherwise actionable.
|
||||
pub scrollable: bool,
|
||||
/// Scrolled/clipped/parked off-screen: never emitted as a target, but its
|
||||
/// children are still traversed (a visible control inside an off-screen
|
||||
/// container is still reachable).
|
||||
pub offscreen: bool,
|
||||
pub children: Vec<RawNode>,
|
||||
}
|
||||
|
||||
impl RawNode {
|
||||
/// A synthetic structural node (desktop root / window node) carrying no live
|
||||
/// handle. `handle_idx` is a sentinel that is never inserted into the handle
|
||||
/// table; such nodes are excluded from emission by role.
|
||||
pub(crate) fn structural(role: &str, name: Option<String>, bounds: Rect, children: Vec<RawNode>) -> Self {
|
||||
RawNode {
|
||||
handle_idx: usize::MAX,
|
||||
role: role.to_string(),
|
||||
name,
|
||||
value: None,
|
||||
states: Vec::new(),
|
||||
bounds,
|
||||
actionable: false,
|
||||
scrollable: false,
|
||||
offscreen: false,
|
||||
children,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Roles that are never themselves emitted as targets even when named: the
|
||||
/// synthetic forest scaffolding. (Real window-like panes inside an app still
|
||||
/// surface via their control type.)
|
||||
fn is_scaffold_role(role: &str) -> bool {
|
||||
role == DESKTOP_ROLE || role == WINDOW_ROLE
|
||||
}
|
||||
|
||||
/// True if this node should be emitted as a numbered target: on-screen, with
|
||||
/// non-empty bounds, not forest scaffolding, and either actionable, scrollable,
|
||||
/// or a named/valued leaf. A named *structural container* (toolbar/group/pane/…)
|
||||
/// is a grouping branch in the semantic tree, not a click target, so it is not
|
||||
/// emitted unless it is itself actionable or scrollable.
|
||||
fn is_emittable(n: &RawNode) -> bool {
|
||||
if n.offscreen || n.bounds.is_empty() || is_scaffold_role(&n.role) {
|
||||
return false;
|
||||
}
|
||||
if n.actionable || n.scrollable {
|
||||
return true;
|
||||
}
|
||||
(n.name.is_some() || n.value.is_some()) && !is_structural_role(&n.role)
|
||||
}
|
||||
|
||||
/// Container roles promoted to a labelled branch in the semantic tree when they
|
||||
/// carry a name and have emittable descendants (otherwise pruned). Gives the
|
||||
/// model the grouping context ("this button is inside the Formatting toolbar")
|
||||
/// without numbering the container itself.
|
||||
fn is_structural_role(role: &str) -> bool {
|
||||
matches!(
|
||||
role,
|
||||
"pane" | "group" | "toolbar" | "menubar" | "menu" | "tab" | "tree" | "list"
|
||||
| "table" | "datagrid" | "statusbar" | "titlebar" | "header" | "tabitem"
|
||||
)
|
||||
}
|
||||
|
||||
/// The verb the model performs on a control of this role — surfaced in the
|
||||
/// semantic tree as `[action: …]` so the affordance is explicit. Mirrors
|
||||
/// Windows-MCP's action map (edit→fill, checkbox→toggle, …); scrollable regions
|
||||
/// override to `scroll`.
|
||||
pub(crate) fn action_for(role: &str, scrollable: bool) -> &'static str {
|
||||
if scrollable && role != "slider" {
|
||||
return "scroll";
|
||||
}
|
||||
match role {
|
||||
"edit" => "fill",
|
||||
"checkbox" => "toggle",
|
||||
"combobox" => "select",
|
||||
"radiobutton" => "select",
|
||||
"slider" => "slide",
|
||||
"document" => "scroll",
|
||||
_ => "click",
|
||||
}
|
||||
}
|
||||
|
||||
/// Map a UIA `ToggleState` code (Off=0, On=1, Indeterminate=2) to a state label.
|
||||
/// Off yields none (the unremarkable default), keeping the list terse.
|
||||
pub(crate) fn toggle_label(code: i32) -> Option<&'static str> {
|
||||
match code {
|
||||
1 => Some("checked"),
|
||||
2 => Some("indeterminate"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Map a UIA `ExpandCollapseState` code (Collapsed=0, Expanded=1,
|
||||
/// PartiallyExpanded=2, LeafNode=3) to a state label. LeafNode (nothing to
|
||||
/// expand) yields none.
|
||||
pub(crate) fn expand_label(code: i32) -> Option<&'static str> {
|
||||
match code {
|
||||
0 => Some("collapsed"),
|
||||
1 => Some("expanded"),
|
||||
2 => Some("partially-expanded"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Filter the `RawNode` forest to emittable targets (honoring `max_depth` +
|
||||
/// `node_budget`), number them per-window then in reading order (top-to-bottom,
|
||||
/// left-to-right), and return:
|
||||
/// * the `ElementEntry` list (1-based `ref`s),
|
||||
/// * the parallel handle indices in the SAME order (to rebuild ref→`UIElement`),
|
||||
/// * a `handle_idx → ref` map (so the semantic renderer can annotate nodes),
|
||||
/// * whether the forest was truncated by depth or budget.
|
||||
///
|
||||
/// `root` is either the synthetic `desktop` forest root (children = windows) or
|
||||
/// a single window subtree (used directly).
|
||||
pub(crate) fn build_entries(
|
||||
root: &RawNode,
|
||||
max_depth: usize,
|
||||
node_budget: usize,
|
||||
) -> (Vec<ElementEntry>, Vec<usize>, HashMap<usize, u32>, bool) {
|
||||
let windows: Vec<&RawNode> = if root.role == DESKTOP_ROLE {
|
||||
root.children.iter().collect()
|
||||
} else {
|
||||
vec![root]
|
||||
};
|
||||
|
||||
// Collect (window_index, node) so refs group by window (foreground first),
|
||||
// then within a window by reading order — matching the semantic tree layout.
|
||||
let mut collected: Vec<(usize, &RawNode)> = Vec::new();
|
||||
let mut truncated = false;
|
||||
for (wi, win) in windows.iter().enumerate() {
|
||||
collect(win, wi, 0, max_depth, node_budget, &mut collected, &mut truncated);
|
||||
}
|
||||
|
||||
// Stable sort by (window, rounded y, rounded x): equal positions keep DFS
|
||||
// order. Window grouping dominates so refs never interleave across windows.
|
||||
collected.sort_by(|(awi, a), (bwi, b)| {
|
||||
(
|
||||
*awi,
|
||||
a.bounds.y.round() as i64,
|
||||
a.bounds.x.round() as i64,
|
||||
)
|
||||
.cmp(&(*bwi, b.bounds.y.round() as i64, b.bounds.x.round() as i64))
|
||||
});
|
||||
|
||||
let mut entries = Vec::with_capacity(collected.len());
|
||||
let mut handle_indices = Vec::with_capacity(collected.len());
|
||||
let mut ref_by_handle = HashMap::with_capacity(collected.len());
|
||||
for (i, (_wi, node)) in collected.iter().enumerate() {
|
||||
let r = i as u32 + 1; // 1-based: matches the [ref] the model sees
|
||||
entries.push(ElementEntry {
|
||||
r#ref: r,
|
||||
role: normalize_role(&node.role),
|
||||
name: node.name.clone().filter(|s| !s.trim().is_empty()),
|
||||
value: node.value.clone().filter(|s| !s.trim().is_empty()),
|
||||
states: node.states.clone(),
|
||||
bounds: node.bounds,
|
||||
source: Source::A11y,
|
||||
});
|
||||
handle_indices.push(node.handle_idx);
|
||||
ref_by_handle.insert(node.handle_idx, r);
|
||||
}
|
||||
(entries, handle_indices, ref_by_handle, truncated)
|
||||
}
|
||||
|
||||
/// Depth-first collect of emittable nodes within one window subtree. An
|
||||
/// off-screen / non-emittable node is not pushed, but its children are still
|
||||
/// traversed (until the depth cap), so a visible control inside an off-screen
|
||||
/// container is reached. `depth` is measured from the window root (0).
|
||||
fn collect<'a>(
|
||||
node: &'a RawNode,
|
||||
win_idx: usize,
|
||||
depth: usize,
|
||||
max_depth: usize,
|
||||
budget: usize,
|
||||
out: &mut Vec<(usize, &'a RawNode)>,
|
||||
truncated: &mut bool,
|
||||
) {
|
||||
if is_emittable(node) {
|
||||
if out.len() >= budget {
|
||||
*truncated = true;
|
||||
return;
|
||||
}
|
||||
out.push((win_idx, node));
|
||||
}
|
||||
if depth >= max_depth {
|
||||
if !node.children.is_empty() {
|
||||
*truncated = true;
|
||||
}
|
||||
return;
|
||||
}
|
||||
for child in &node.children {
|
||||
if out.len() >= budget {
|
||||
*truncated = true;
|
||||
return;
|
||||
}
|
||||
collect(child, win_idx, depth + 1, max_depth, budget, out, truncated);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- semantic tree rendering --------------------------------------------
|
||||
//
|
||||
// Two phases (mirrors Windows-MCP's SemanticNode build + prune + render):
|
||||
// 1. `build_sem` collapses the raw forest to only meaningful nodes — desktop,
|
||||
// windows, named structural containers, and emittable controls — making
|
||||
// transparent wrapper panes disappear so descendants attach to the nearest
|
||||
// meaningful ancestor.
|
||||
// 2. `render_sem` draws it with ├──/└── connectors.
|
||||
|
||||
#[derive(Debug)]
|
||||
enum SemKind {
|
||||
Desktop,
|
||||
Window,
|
||||
Structural,
|
||||
/// An emittable control, carrying its `[ref]`.
|
||||
Control(u32),
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct SemNode {
|
||||
kind: SemKind,
|
||||
role: String,
|
||||
name: String,
|
||||
/// Center coordinates (emittable controls only).
|
||||
coords: Option<(i64, i64)>,
|
||||
scrollable: bool,
|
||||
states: Vec<String>,
|
||||
children: Vec<SemNode>,
|
||||
}
|
||||
|
||||
/// Build the meaningful-node tree for `node`, appending the resulting node(s)
|
||||
/// to `parent_children`. Transparent nodes (unnamed containers, plain wrappers)
|
||||
/// contribute their children directly to the parent. Returns nothing; mutates
|
||||
/// `parent_children`.
|
||||
fn build_sem(node: &RawNode, ref_by_handle: &HashMap<usize, u32>, parent_children: &mut Vec<SemNode>) {
|
||||
let role = normalize_role(&node.role);
|
||||
|
||||
// Desktop / window scaffolding: always a branch.
|
||||
if node.role == DESKTOP_ROLE {
|
||||
let mut me = SemNode {
|
||||
kind: SemKind::Desktop,
|
||||
role,
|
||||
name: node.name.clone().unwrap_or_default(),
|
||||
coords: None,
|
||||
scrollable: false,
|
||||
states: Vec::new(),
|
||||
children: Vec::new(),
|
||||
};
|
||||
for c in &node.children {
|
||||
build_sem(c, ref_by_handle, &mut me.children);
|
||||
}
|
||||
parent_children.push(me);
|
||||
return;
|
||||
}
|
||||
if node.role == WINDOW_ROLE {
|
||||
let mut me = SemNode {
|
||||
kind: SemKind::Window,
|
||||
role,
|
||||
name: node.name.clone().unwrap_or_default(),
|
||||
coords: None,
|
||||
scrollable: false,
|
||||
states: Vec::new(),
|
||||
children: Vec::new(),
|
||||
};
|
||||
for c in &node.children {
|
||||
build_sem(c, ref_by_handle, &mut me.children);
|
||||
}
|
||||
parent_children.push(me);
|
||||
return;
|
||||
}
|
||||
|
||||
// An emittable control: a numbered leaf-or-branch.
|
||||
if let Some(&r) = ref_by_handle.get(&node.handle_idx) {
|
||||
let (cx, cy) = node.bounds.center();
|
||||
let mut me = SemNode {
|
||||
kind: SemKind::Control(r),
|
||||
role,
|
||||
name: node.name.clone().unwrap_or_default(),
|
||||
coords: Some((cx.round() as i64, cy.round() as i64)),
|
||||
scrollable: node.scrollable,
|
||||
states: node.states.clone(),
|
||||
children: Vec::new(),
|
||||
};
|
||||
for c in &node.children {
|
||||
build_sem(c, ref_by_handle, &mut me.children);
|
||||
}
|
||||
parent_children.push(me);
|
||||
return;
|
||||
}
|
||||
|
||||
// A named structural container: tentatively a branch, kept only if it ends
|
||||
// up with children (pruned below otherwise).
|
||||
let named = node.name.as_deref().map(|s| !s.trim().is_empty()).unwrap_or(false);
|
||||
if named && is_structural_role(&role) && !node.offscreen {
|
||||
let mut me = SemNode {
|
||||
kind: SemKind::Structural,
|
||||
role,
|
||||
name: node.name.clone().unwrap_or_default(),
|
||||
coords: None,
|
||||
scrollable: false,
|
||||
states: Vec::new(),
|
||||
children: Vec::new(),
|
||||
};
|
||||
for c in &node.children {
|
||||
build_sem(c, ref_by_handle, &mut me.children);
|
||||
}
|
||||
if !me.children.is_empty() {
|
||||
parent_children.push(me);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Transparent: attach children to the current parent.
|
||||
for c in &node.children {
|
||||
build_sem(c, ref_by_handle, parent_children);
|
||||
}
|
||||
}
|
||||
|
||||
fn format_sem_line(node: &SemNode) -> String {
|
||||
match &node.kind {
|
||||
SemKind::Desktop => "desktop".to_string(),
|
||||
SemKind::Window => format!("window {:?}", node.name),
|
||||
SemKind::Structural => {
|
||||
if node.name.is_empty() {
|
||||
node.role.clone()
|
||||
} else {
|
||||
format!("{} {:?}", node.role, node.name)
|
||||
}
|
||||
}
|
||||
SemKind::Control(r) => {
|
||||
let mut s = format!("[{r}] {}", node.role);
|
||||
if !node.name.is_empty() {
|
||||
s.push_str(&format!(" {:?}", truncate(&node.name, 80)));
|
||||
}
|
||||
if let Some((x, y)) = node.coords {
|
||||
s.push_str(&format!(" ({x},{y})"));
|
||||
}
|
||||
s.push_str(&format!(" [action: {}]", action_for(&node.role, node.scrollable)));
|
||||
if !node.states.is_empty() {
|
||||
s.push_str(&format!(" [{}]", node.states.join(",")));
|
||||
}
|
||||
s
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn render_sem(node: &SemNode, lines: &mut Vec<String>, prefix: &str, is_last: bool, is_root: bool) {
|
||||
if is_root {
|
||||
lines.push(format_sem_line(node));
|
||||
} else {
|
||||
let connector = if is_last { "└── " } else { "├── " };
|
||||
lines.push(format!("{prefix}{connector}{}", format_sem_line(node)));
|
||||
}
|
||||
let extension = if is_root {
|
||||
""
|
||||
} else if is_last {
|
||||
" "
|
||||
} else {
|
||||
"│ "
|
||||
};
|
||||
let child_prefix = format!("{prefix}{extension}");
|
||||
let n = node.children.len();
|
||||
for (i, child) in node.children.iter().enumerate() {
|
||||
render_sem(child, lines, &child_prefix, i == n - 1, false);
|
||||
}
|
||||
}
|
||||
|
||||
/// Render the raw forest as the hierarchical semantic tree the model reads.
|
||||
/// `root` is the synthetic `desktop` root (or a single window subtree).
|
||||
pub(crate) fn render_tree(root: &RawNode, ref_by_handle: &HashMap<usize, u32>) -> String {
|
||||
let mut tops: Vec<SemNode> = Vec::new();
|
||||
build_sem(root, ref_by_handle, &mut tops);
|
||||
let mut lines: Vec<String> = Vec::new();
|
||||
let n = tops.len();
|
||||
for (i, top) in tops.iter().enumerate() {
|
||||
render_sem(top, &mut lines, "", i == n - 1, true);
|
||||
}
|
||||
lines.join("\n")
|
||||
}
|
||||
|
||||
fn truncate(s: &str, max: usize) -> String {
|
||||
if s.chars().count() <= max {
|
||||
s.to_string()
|
||||
} else {
|
||||
let t: String = s.chars().take(max).collect();
|
||||
format!("{t}…")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn n(handle_idx: usize, role: &str, name: Option<&str>, x: f64, y: f64, actionable: bool) -> RawNode {
|
||||
RawNode {
|
||||
handle_idx,
|
||||
role: role.to_string(),
|
||||
name: name.map(|s| s.to_string()),
|
||||
value: None,
|
||||
states: vec![],
|
||||
bounds: Rect { x, y, w: 40.0, h: 16.0 },
|
||||
actionable,
|
||||
scrollable: false,
|
||||
offscreen: false,
|
||||
children: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
fn window(children: Vec<RawNode>) -> RawNode {
|
||||
RawNode {
|
||||
handle_idx: 0,
|
||||
role: "window".to_string(),
|
||||
name: Some("Test Window".to_string()),
|
||||
value: None,
|
||||
states: vec![],
|
||||
bounds: Rect { x: 0.0, y: 0.0, w: 800.0, h: 600.0 },
|
||||
actionable: false,
|
||||
scrollable: false,
|
||||
offscreen: false,
|
||||
children,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn emits_actionable_or_named_and_skips_plain_containers() {
|
||||
let root = window(vec![
|
||||
n(1, "button", None, 10.0, 50.0, true), // actionable, no name → emit
|
||||
n(2, "group", None, 10.0, 80.0, false), // not actionable, no name → skip
|
||||
n(3, "text", Some("Hello"), 10.0, 110.0, false), // named label → emit
|
||||
]);
|
||||
let (entries, handles, ref_by_handle, trunc) = build_entries(&root, 12, 120);
|
||||
assert!(!trunc);
|
||||
// window root (role "window") skipped; button + text emitted.
|
||||
assert_eq!(entries.len(), 2, "entries: {entries:?}");
|
||||
assert_eq!(handles.len(), 2);
|
||||
assert_eq!(ref_by_handle.len(), 2);
|
||||
let roles: Vec<_> = entries.iter().map(|e| e.role.as_str()).collect();
|
||||
assert!(roles.contains(&"button"));
|
||||
assert!(roles.contains(&"text"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skips_empty_bounds() {
|
||||
let mut ghost = n(1, "button", Some("Ghost"), 10.0, 10.0, true);
|
||||
ghost.bounds = Rect { x: 10.0, y: 10.0, w: 0.0, h: 0.0 };
|
||||
let (entries, _, _, _) = build_entries(&window(vec![ghost]), 12, 120);
|
||||
assert_eq!(entries.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn offscreen_node_not_emitted_but_children_traversed() {
|
||||
let mut container = n(1, "pane", Some("Panel"), 5.0, 5.0, false);
|
||||
container.offscreen = true;
|
||||
container.bounds = Rect { x: 5.0, y: 5.0, w: 300.0, h: 300.0 };
|
||||
container.children = vec![n(2, "button", Some("Deep"), 20.0, 20.0, true)];
|
||||
let (entries, handles, _, _) = build_entries(&window(vec![container]), 12, 120);
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].name.as_deref(), Some("Deep"));
|
||||
assert_eq!(handles, vec![2]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn numbers_in_reading_order_and_correlates_handles_across_sort() {
|
||||
let root = window(vec![
|
||||
n(7, "button", Some("Bottom"), 10.0, 200.0, true),
|
||||
n(9, "button", Some("Top"), 10.0, 10.0, true),
|
||||
]);
|
||||
let (entries, handles, ref_by_handle, _) = build_entries(&root, 12, 120);
|
||||
assert_eq!(entries.len(), 2);
|
||||
assert_eq!(entries[0].name.as_deref(), Some("Top"));
|
||||
assert_eq!(entries[0].r#ref, 1);
|
||||
assert_eq!(handles[0], 9);
|
||||
assert_eq!(ref_by_handle[&9], 1);
|
||||
assert_eq!(entries[1].name.as_deref(), Some("Bottom"));
|
||||
assert_eq!(entries[1].r#ref, 2);
|
||||
assert_eq!(handles[1], 7);
|
||||
assert_eq!(ref_by_handle[&7], 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn budget_truncates() {
|
||||
let kids: Vec<RawNode> = (1..=10).map(|i| n(i, "button", Some("b"), 0.0, i as f64, true)).collect();
|
||||
let (entries, handles, _, trunc) = build_entries(&window(kids), 12, 3);
|
||||
assert!(trunc);
|
||||
assert!(entries.len() <= 3);
|
||||
assert_eq!(entries.len(), handles.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn depth_cap_truncates_and_drops_deep_nodes() {
|
||||
let gc = n(2, "button", Some("Deep"), 10.0, 10.0, true);
|
||||
let mut child = n(1, "pane", None, 5.0, 5.0, false);
|
||||
child.bounds = Rect { x: 5.0, y: 5.0, w: 100.0, h: 100.0 };
|
||||
child.children = vec![gc];
|
||||
let (entries, _, _, trunc) = build_entries(&window(vec![child]), 1, 120);
|
||||
assert!(trunc);
|
||||
assert!(entries.iter().all(|e| e.name.as_deref() != Some("Deep")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn states_and_value_pass_through() {
|
||||
let mut cb = n(1, "checkbox", Some("Agree"), 10.0, 10.0, true);
|
||||
cb.states = vec!["checked".into(), "focused".into()];
|
||||
cb.value = Some("on".into());
|
||||
let (entries, _, _, _) = build_entries(&window(vec![cb]), 12, 120);
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].states, vec!["checked".to_string(), "focused".to_string()]);
|
||||
assert_eq!(entries[0].value.as_deref(), Some("on"));
|
||||
assert_eq!(entries[0].role, "checkbox");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn toggle_and_expand_labels_map_uia_codes() {
|
||||
assert_eq!(toggle_label(0), None);
|
||||
assert_eq!(toggle_label(1), Some("checked"));
|
||||
assert_eq!(toggle_label(2), Some("indeterminate"));
|
||||
assert_eq!(toggle_label(99), None);
|
||||
assert_eq!(expand_label(0), Some("collapsed"));
|
||||
assert_eq!(expand_label(1), Some("expanded"));
|
||||
assert_eq!(expand_label(2), Some("partially-expanded"));
|
||||
assert_eq!(expand_label(3), None);
|
||||
}
|
||||
|
||||
// ---- new behavior: scrollables, action map, multi-window, rendering ----
|
||||
|
||||
#[test]
|
||||
fn scrollable_container_is_emitted_even_without_name_or_action() {
|
||||
let mut scroll = n(1, "pane", None, 0.0, 0.0, false);
|
||||
scroll.scrollable = true;
|
||||
scroll.bounds = Rect { x: 0.0, y: 0.0, w: 300.0, h: 300.0 };
|
||||
let (entries, _, _, _) = build_entries(&window(vec![scroll]), 12, 120);
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].role, "pane");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn action_map_matches_role_and_scrollable() {
|
||||
assert_eq!(action_for("edit", false), "fill");
|
||||
assert_eq!(action_for("checkbox", false), "toggle");
|
||||
assert_eq!(action_for("combobox", false), "select");
|
||||
assert_eq!(action_for("radiobutton", false), "select");
|
||||
assert_eq!(action_for("slider", false), "slide");
|
||||
assert_eq!(action_for("document", false), "scroll");
|
||||
assert_eq!(action_for("button", false), "click");
|
||||
assert_eq!(action_for("pane", true), "scroll"); // scrollable overrides
|
||||
assert_eq!(action_for("slider", true), "slide"); // slider keeps slide
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_entries_groups_refs_by_window_then_reading_order() {
|
||||
// Window B sits visually ABOVE window A (smaller y), but refs must group
|
||||
// by window order (A first in the forest), not by global y.
|
||||
let win_a = RawNode::structural(
|
||||
WINDOW_ROLE,
|
||||
Some("App A".into()),
|
||||
Rect { x: 0.0, y: 100.0, w: 400.0, h: 400.0 },
|
||||
vec![n(10, "button", Some("A1"), 10.0, 300.0, true)],
|
||||
);
|
||||
let win_b = RawNode::structural(
|
||||
WINDOW_ROLE,
|
||||
Some("App B".into()),
|
||||
Rect { x: 500.0, y: 0.0, w: 400.0, h: 400.0 },
|
||||
vec![n(20, "button", Some("B1"), 510.0, 10.0, true)],
|
||||
);
|
||||
let desktop = RawNode::structural(
|
||||
DESKTOP_ROLE,
|
||||
None,
|
||||
Rect { x: 0.0, y: 0.0, w: 1920.0, h: 1080.0 },
|
||||
vec![win_a, win_b],
|
||||
);
|
||||
let (entries, _, _, _) = build_entries(&desktop, 12, 120);
|
||||
assert_eq!(entries.len(), 2);
|
||||
// A1 (window A, listed first) gets [1] even though B1 is higher on screen.
|
||||
assert_eq!(entries[0].name.as_deref(), Some("A1"));
|
||||
assert_eq!(entries[0].r#ref, 1);
|
||||
assert_eq!(entries[1].name.as_deref(), Some("B1"));
|
||||
assert_eq!(entries[1].r#ref, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_tree_shows_desktop_windows_and_refs() {
|
||||
let win = RawNode::structural(
|
||||
WINDOW_ROLE,
|
||||
Some("Notepad".into()),
|
||||
Rect { x: 0.0, y: 0.0, w: 800.0, h: 600.0 },
|
||||
vec![n(5, "button", Some("Save"), 100.0, 50.0, true)],
|
||||
);
|
||||
let desktop = RawNode::structural(
|
||||
DESKTOP_ROLE,
|
||||
None,
|
||||
Rect { x: 0.0, y: 0.0, w: 1920.0, h: 1080.0 },
|
||||
vec![win],
|
||||
);
|
||||
let (_, _, ref_by_handle, _) = build_entries(&desktop, 12, 120);
|
||||
let tree = render_tree(&desktop, &ref_by_handle);
|
||||
assert!(tree.starts_with("desktop"), "tree:\n{tree}");
|
||||
assert!(tree.contains(r#"window "Notepad""#), "tree:\n{tree}");
|
||||
assert!(tree.contains(r#"[1] button "Save""#), "tree:\n{tree}");
|
||||
assert!(tree.contains("[action: click]"), "tree:\n{tree}");
|
||||
assert!(tree.contains("(120,58)"), "center of Save; tree:\n{tree}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_tree_promotes_named_container_and_prunes_empty_one() {
|
||||
// A named toolbar with an actionable child → promoted to a branch with
|
||||
// the button numbered under it. An empty named group (no emittable
|
||||
// descendant) → pruned.
|
||||
let named_toolbar = RawNode {
|
||||
handle_idx: 100,
|
||||
role: "toolbar".into(),
|
||||
name: Some("Formatting".into()),
|
||||
value: None,
|
||||
states: vec![],
|
||||
bounds: Rect { x: 0.0, y: 0.0, w: 800.0, h: 40.0 },
|
||||
actionable: false,
|
||||
scrollable: false,
|
||||
offscreen: false,
|
||||
children: vec![n(1, "button", Some("Bold"), 10.0, 10.0, true)],
|
||||
};
|
||||
let empty_group = RawNode {
|
||||
handle_idx: 101,
|
||||
role: "group".into(),
|
||||
name: Some("Empty".into()),
|
||||
value: None,
|
||||
states: vec![],
|
||||
bounds: Rect { x: 0.0, y: 100.0, w: 800.0, h: 40.0 },
|
||||
actionable: false,
|
||||
scrollable: false,
|
||||
offscreen: false,
|
||||
children: vec![n(2, "group", None, 0.0, 0.0, false)], // non-emittable
|
||||
};
|
||||
let win = RawNode::structural(
|
||||
WINDOW_ROLE,
|
||||
Some("App".into()),
|
||||
Rect { x: 0.0, y: 0.0, w: 800.0, h: 600.0 },
|
||||
vec![named_toolbar, empty_group],
|
||||
);
|
||||
let desktop = RawNode::structural(DESKTOP_ROLE, None, win.bounds, vec![win]);
|
||||
let (entries, _, ref_by_handle, _) = build_entries(&desktop, 12, 120);
|
||||
// Only the Bold button is emittable (toolbar/group are structural).
|
||||
assert_eq!(entries.len(), 1, "entries: {entries:?}");
|
||||
let tree = render_tree(&desktop, &ref_by_handle);
|
||||
assert!(tree.contains(r#"toolbar "Formatting""#), "tree:\n{tree}");
|
||||
assert!(tree.contains(r#"[1] button "Bold""#), "tree:\n{tree}");
|
||||
assert!(!tree.contains("Empty"), "empty container must be pruned; tree:\n{tree}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_tree_collapses_transparent_wrappers() {
|
||||
// unnamed pane wrapper → its child attaches to the window directly.
|
||||
let mut wrapper = n(1, "pane", None, 0.0, 0.0, false);
|
||||
wrapper.bounds = Rect { x: 0.0, y: 0.0, w: 800.0, h: 600.0 };
|
||||
wrapper.children = vec![n(2, "button", Some("Go"), 10.0, 10.0, true)];
|
||||
let win = RawNode::structural(WINDOW_ROLE, Some("W".into()), wrapper.bounds, vec![wrapper]);
|
||||
let desktop = RawNode::structural(DESKTOP_ROLE, None, win.bounds, vec![win]);
|
||||
let (_, _, ref_by_handle, _) = build_entries(&desktop, 12, 120);
|
||||
let tree = render_tree(&desktop, &ref_by_handle);
|
||||
// No "pane" line (unnamed wrapper collapsed); button present under window.
|
||||
assert!(!tree.contains("pane"), "transparent wrapper must collapse; tree:\n{tree}");
|
||||
assert!(tree.contains(r#"[1] button "Go""#), "tree:\n{tree}");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user