Update: 将子项目从 submodule 转为完整内容
- 移除 GovAI, nomifun-tauri, 算力盒子 的 submodule 引用 - 添加所有子项目的完整源代码 - 保留原始 .git 为 .git.bak 备份
This commit is contained in:
freedak
Submodule nomifun-tauri deleted from 5800e4796d
@@ -0,0 +1,47 @@
|
||||
# Temporary: disable China mirror (rsproxy.cn unreachable) — using official crates.io
|
||||
[source.crates-io]
|
||||
replace-with = "tuna"
|
||||
|
||||
[source.tuna]
|
||||
registry = "sparse+https://mirrors.tuna.tsinghua.edu.cn/crates.io-index/"
|
||||
|
||||
[net]
|
||||
git-fetch-with-cli = true
|
||||
|
||||
# Use the toolchain-bundled LLD linker on Windows MSVC — linking dominates
|
||||
# incremental build time in this 27-crate workspace and LLD is several times
|
||||
# faster than link.exe. rust-lld.exe ships with every rustup toolchain, so
|
||||
# this is safe across all dev machines. NOTE: changing the linker invalidates
|
||||
# cargo's fingerprints once (first build after pulling this does a full
|
||||
# rebuild); subsequent builds get the speedup.
|
||||
[target.x86_64-pc-windows-msvc]
|
||||
linker = "rust-lld.exe"
|
||||
|
||||
# Route intermediate artifacts (deps/.fingerprint/incremental — the high-churn
|
||||
# bulk of build output) into a ".noindex" directory: that suffix is macOS
|
||||
# Spotlight's per-directory opt-out, so mds stops re-indexing artifacts on
|
||||
# every rebuild (~3 CPU-seconds each, measured). Final binaries still land in
|
||||
# target/, so no tool or script path changes. On Windows/Linux it is just a
|
||||
# directory name; cargo older than the build-dir stabilization ignores the key
|
||||
# and falls back to target/. Stale artifacts in build.noindex are pruned by
|
||||
# the self-cleaning preflight (scripts/prune-build.mjs) that runs at the
|
||||
# start of every dev/build/test entry point. NOTE: first build after pulling
|
||||
# this rebuilds intermediates into the new location — run `cargo clean` once
|
||||
# to reclaim the orphaned old target/ contents.
|
||||
[build]
|
||||
build-dir = "{workspace-root}/build.noindex"
|
||||
|
||||
# Enlarge rustc's compilation-thread stack. The release `tauri build` embeds the
|
||||
# entire frontend `ui/dist` (~34 MB, hundreds of files) into the `nomifun-desktop`
|
||||
# binary via `tauri::generate_context!`, then optimizes it at opt-level=3. That
|
||||
# codegen overflows rustc's default 8 MB thread stack and the compiler dies on its
|
||||
# guard page WITHOUT a diagnostic — surfacing as a non-deterministic native crash
|
||||
# (STATUS_STACK_BUFFER_OVERRUN 0xc0000409 / STATUS_ACCESS_VIOLATION 0xc0000005).
|
||||
# Plain `cargo build` (dev) is unaffected because it serves assets from the dev
|
||||
# server and embeds nothing. 64 MB is already verified-sufficient; 128 MB leaves
|
||||
# headroom as the frontend grows (the stack is lazily committed, so the ceiling
|
||||
# costs only address space, never RAM). Applies to every build path — dev, test,
|
||||
# `tauri build`, CI — so the failure cannot regress on a machine that forgets it.
|
||||
[env]
|
||||
RUST_MIN_STACK = "134217728"
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
# cargo-nextest 配置。
|
||||
#
|
||||
# 真 Chrome 集成/端到端测试的串行化（browser-use 自研引擎）：
|
||||
# nomi-browser-engine / nomi-browser 的 `#[ignore]` 真 Chrome 用例每个都 fork 一个
|
||||
# 完整 Chrome 进程。nextest 默认跨二进制并行调度,`--run-ignored all` 会同时拉起十几个
|
||||
# Chrome → 机器过载,个别会话在启动/早期 navigate 阶段掉线,报
|
||||
# "browser session lost (recoverable=false)"。这是**并行负载竞争的伪失败,非产品/测试
|
||||
# 代码 bug**——串行(`-j1`)下 100% 绿(实测 15/15)。
|
||||
#
|
||||
# 本组把这些真 Chrome 用例钉在单线程,使默认并行的 `--run-ignored all` 也稳定,避免
|
||||
# CI/他人因伪失败误判、损套件可信度。匹配(scoped 到两个 browser crate,对其余 workspace
|
||||
# 零影响):
|
||||
# - 所有 integration_* / *_e2e 集成二进制(按构造全是真 Chrome;不跑 --ignored 时为空,无开销);
|
||||
# - 散落在 lib 单测二进制里、以 `_real` 结尾的 facade 真 Chrome 用例。
|
||||
# 快速纯逻辑单测(不含 Chrome)不匹配 → 默认并行不受影响。
|
||||
|
||||
[test-groups]
|
||||
serial-chrome = { max-threads = 1 }
|
||||
|
||||
[[profile.default.overrides]]
|
||||
filter = '(package(nomi-browser-engine) | package(nomi-browser)) & (binary_id(/integration/) | binary_id(/_e2e$/) | test(/_real$/))'
|
||||
test-group = 'serial-chrome'
|
||||
@@ -0,0 +1,19 @@
|
||||
# Keep the Docker build context small & deterministic. The image rebuilds
|
||||
# ui/dist and the Rust binary from source, so ship only sources.
|
||||
target/
|
||||
**/target/
|
||||
node_modules/
|
||||
**/node_modules/
|
||||
ui/dist/
|
||||
dist/
|
||||
.git/
|
||||
.smoketest/
|
||||
.claude/
|
||||
*.log
|
||||
# The repo's .cargo/config.toml points crates-io at a CN mirror (rsproxy.cn),
|
||||
# which stalls non-CN Docker builds. Exclude it so the image build uses the
|
||||
# default registry; opt into a mirror with the CARGO_REGISTRY_MIRROR build-arg.
|
||||
.cargo/
|
||||
# Local runtime data must never leak into the image.
|
||||
data/
|
||||
**/data/
|
||||
@@ -0,0 +1,32 @@
|
||||
# 行尾统一：仓库与工作区一律 LF
|
||||
# 根治 autocrlf 幻影改动（索引 LF + 工作区 CRLF 预期 + 工具写回 LF → status 永远脏）。
|
||||
# .gitattributes 优先级高于各机器的 core.autocrlf —— mac（autocrlf=input/false）与
|
||||
# Windows（autocrlf=true）从此行为一致，不再依赖本地配置。
|
||||
# 要求 git >= 2.10（text=auto + eol 组合正确跳过二进制；现代 mac/Windows git 均远高于此）。
|
||||
* text=auto eol=lf
|
||||
|
||||
# Unix shell 脚本必须 LF（mac/Linux 可执行；全局规则已覆盖，此处显式自文档化）
|
||||
*.sh text eol=lf
|
||||
|
||||
# Windows 脚本必须 CRLF（当前仓库无此类文件，前瞻防御）
|
||||
*.bat text eol=crlf
|
||||
*.cmd text eol=crlf
|
||||
*.ps1 text eol=crlf
|
||||
|
||||
# 二进制显式声明（防御任何 git 版本/环境的文本误判与行尾改写）
|
||||
*.png binary
|
||||
*.ico binary
|
||||
*.icns binary
|
||||
*.pptx binary
|
||||
*.jpg binary
|
||||
*.jpeg binary
|
||||
*.gif binary
|
||||
*.webp binary
|
||||
*.woff binary
|
||||
*.woff2 binary
|
||||
*.ttf binary
|
||||
*.otf binary
|
||||
*.zip binary
|
||||
*.p12 binary
|
||||
*.p8 binary
|
||||
*.key binary
|
||||
@@ -0,0 +1,57 @@
|
||||
name: Bug Report
|
||||
description: Report a reproducible problem in NomiFun.
|
||||
title: "[Bug]: "
|
||||
labels: ["bug"]
|
||||
body:
|
||||
- type: textarea
|
||||
id: summary
|
||||
attributes:
|
||||
label: Summary
|
||||
description: What happened?
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: steps
|
||||
attributes:
|
||||
label: Steps To Reproduce
|
||||
description: List exact steps, commands, or UI actions.
|
||||
placeholder: |
|
||||
1. Open ...
|
||||
2. Click ...
|
||||
3. See ...
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: expected
|
||||
attributes:
|
||||
label: Expected Behavior
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: actual
|
||||
attributes:
|
||||
label: Actual Behavior
|
||||
validations:
|
||||
required: true
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: Version Or Commit
|
||||
placeholder: "v0.x.x or git SHA"
|
||||
- type: dropdown
|
||||
id: surface
|
||||
attributes:
|
||||
label: Surface
|
||||
options:
|
||||
- Desktop app
|
||||
- Web server
|
||||
- Agent engine
|
||||
- MCP / skills
|
||||
- Packaging / updater
|
||||
- Documentation
|
||||
- Other
|
||||
- type: textarea
|
||||
id: logs
|
||||
attributes:
|
||||
label: Logs Or Screenshots
|
||||
description: Paste relevant logs. Remove secrets first.
|
||||
@@ -0,0 +1,5 @@
|
||||
blank_issues_enabled: true
|
||||
contact_links:
|
||||
- name: Security issue
|
||||
url: https://github.com/
|
||||
about: Please follow SECURITY.md instead of opening a public issue.
|
||||
@@ -0,0 +1,35 @@
|
||||
name: Feature Request
|
||||
description: Suggest a product or engineering improvement.
|
||||
title: "[Feature]: "
|
||||
labels: ["enhancement"]
|
||||
body:
|
||||
- type: textarea
|
||||
id: problem
|
||||
attributes:
|
||||
label: Problem
|
||||
description: What user problem should this solve?
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: proposal
|
||||
attributes:
|
||||
label: Proposed Solution
|
||||
description: Describe the smallest useful version.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: alternatives
|
||||
attributes:
|
||||
label: Alternatives Considered
|
||||
- type: dropdown
|
||||
id: surface
|
||||
attributes:
|
||||
label: Surface
|
||||
options:
|
||||
- Desktop app
|
||||
- Web server
|
||||
- Agent engine
|
||||
- MCP / skills
|
||||
- Packaging / updater
|
||||
- Documentation
|
||||
- Other
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
## Summary
|
||||
|
||||
-
|
||||
|
||||
## Verification
|
||||
|
||||
- [ ] Tests or checks run:
|
||||
- [ ] Documentation updated when behavior changed
|
||||
- [ ] No secrets, local-only paths, or proprietary assets added
|
||||
|
||||
## Notes
|
||||
|
||||
-
|
||||
@@ -0,0 +1,53 @@
|
||||
# Rust
|
||||
/target/
|
||||
**/target/
|
||||
/build.noindex/
|
||||
Cargo.lock.bak
|
||||
|
||||
# Packaging / runtime artifacts
|
||||
/dist/
|
||||
# Legacy dev data dirs from before all hosts shared one per-user default
|
||||
# (%LOCALAPPDATA%\NomiFun\Nomi etc.) — kept so stale checkouts don't commit them.
|
||||
/data/
|
||||
/data-verify/
|
||||
/.dev-data/
|
||||
# Per-crate runtime data (app state written at runtime, e.g. extension-states.json)
|
||||
crates/backend/nomifun-app/data/
|
||||
|
||||
# Node / Bun
|
||||
node_modules/
|
||||
**/node_modules/
|
||||
ui/dist/
|
||||
.bun/
|
||||
*.tsbuildinfo
|
||||
|
||||
# Tauri
|
||||
apps/desktop/gen/
|
||||
apps/desktop/target/
|
||||
|
||||
# Tauri updater signing keys — NEVER commit private keys
|
||||
apps/desktop/.tauri/
|
||||
*.key
|
||||
|
||||
# macOS code signing / notarization secrets — NEVER commit
|
||||
apps/desktop/signing/.env.signing
|
||||
*.p8
|
||||
*.p12
|
||||
*.cer
|
||||
*.mobileprovision
|
||||
|
||||
# Logs / OS
|
||||
*.log
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Editor
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
|
||||
# Claude Code 会话产物（临时 worktree、本地设置），不入库
|
||||
.claude/
|
||||
|
||||
# Visual brainstorming companion (transient mockups)
|
||||
.superpowers/
|
||||
@@ -0,0 +1,24 @@
|
||||
# Changelog
|
||||
|
||||
NomiFun is pre-1.0. Until the first public release, this file records release
|
||||
notes at a high level rather than a complete historical log.
|
||||
|
||||
## Unreleased
|
||||
|
||||
- Documentation overhaul for public website and open-source preparation.
|
||||
- Clarified desktop, web, remote access, AutoWork, scheduled tasks, and
|
||||
packaging documentation.
|
||||
- Removed proprietary PDF skill assets from the bundled built-in skills.
|
||||
|
||||
## Release Note Policy
|
||||
|
||||
Every public release should include:
|
||||
|
||||
- User-facing changes.
|
||||
- Breaking configuration or data migration notes.
|
||||
- Security-relevant changes.
|
||||
- Packaging and updater notes.
|
||||
- Known limitations.
|
||||
|
||||
Use calendar dates or semantic versions consistently once public releases
|
||||
begin.
|
||||
@@ -0,0 +1,27 @@
|
||||
# Code of Conduct
|
||||
|
||||
NomiFun is intended to be a practical, respectful engineering community.
|
||||
|
||||
## Expected Behavior
|
||||
|
||||
- Be direct, factual, and respectful.
|
||||
- Assume good faith, but accept correction when evidence shows otherwise.
|
||||
- Keep discussions focused on the work: bugs, design tradeoffs, documentation,
|
||||
tests, releases, and user impact.
|
||||
- Credit upstream projects and contributors.
|
||||
|
||||
## Unacceptable Behavior
|
||||
|
||||
- Harassment, threats, personal attacks, or discriminatory language.
|
||||
- Publishing private information without permission.
|
||||
- Repeatedly derailing technical discussion after maintainers ask you to stop.
|
||||
- Abusing issue trackers, discussions, or review comments for spam.
|
||||
|
||||
## Enforcement
|
||||
|
||||
Maintainers may edit or delete comments, lock threads, close issues, reject
|
||||
contributions, or block accounts when behavior harms the project or community.
|
||||
|
||||
Report conduct concerns privately to the maintainers listed in the repository
|
||||
metadata or security policy. Include links, screenshots, and context where
|
||||
possible.
|
||||
@@ -0,0 +1,68 @@
|
||||
# Contributing to NomiFun
|
||||
|
||||
NomiFun is a Rust + Tauri + React monorepo. This file is the open-source entry
|
||||
point for contributors; the detailed engineering docs live under `docs/`.
|
||||
|
||||
## Start Here
|
||||
|
||||
- [Project structure](docs/contributing/project-structure.md)
|
||||
- [Development workflow](docs/contributing/development.md)
|
||||
- [Building and packaging](docs/contributing/building-and-packaging.md)
|
||||
- [Architecture overview](docs/architecture/overview.md)
|
||||
- [Code of conduct](CODE_OF_CONDUCT.md)
|
||||
- [Security policy](SECURITY.md)
|
||||
|
||||
## Local Setup
|
||||
|
||||
```bash
|
||||
bun install
|
||||
bun run dev
|
||||
```
|
||||
|
||||
Use `bun run dev:web` for the browser-only development loop and
|
||||
`bun run serve:web` to run the headless server after building the SPA.
|
||||
|
||||
## Checks
|
||||
|
||||
Run the narrowest check that covers your change:
|
||||
|
||||
```bash
|
||||
bun run help --check
|
||||
bun run typecheck
|
||||
bun run check
|
||||
cargo test
|
||||
```
|
||||
|
||||
For UI-only work, prefer package-relative Bun commands from `ui/` or
|
||||
`bun run --filter=./ui ...` from the repo root. For Rust-only work, use the
|
||||
specific crate or test target where possible before running the full suite.
|
||||
|
||||
## Documentation Changes
|
||||
|
||||
- Keep current docs under `docs/getting-started`, `docs/guides`,
|
||||
`docs/architecture`, `docs/reference`, and `docs/contributing`.
|
||||
- Design and audit history is not kept in the repo; consult git history for past
|
||||
decisions rather than re-adding dated design docs.
|
||||
- Do not document redirected legacy routes as primary navigation.
|
||||
- Keep English and Simplified Chinese siblings in sync when both exist.
|
||||
|
||||
## Pull Request Expectations
|
||||
|
||||
- Keep changes scoped and explain user-visible behavior.
|
||||
- Include screenshots for visible UI changes when practical.
|
||||
- Update docs and screenshot manifest rows when routes, setup steps, or
|
||||
feature names change.
|
||||
- Do not commit local data directories, generated build output, credentials, or
|
||||
machine-specific configuration.
|
||||
- Do not add third-party assets or vendored skills unless their redistribution
|
||||
license is compatible with this repository.
|
||||
|
||||
## Releases
|
||||
|
||||
Maintainer release steps live in [RELEASING.md](RELEASING.md). User-facing
|
||||
changes should be summarized in [CHANGELOG.md](CHANGELOG.md).
|
||||
|
||||
## License
|
||||
|
||||
By contributing, you agree that your contribution is licensed under the
|
||||
Apache-2.0 license used by this repository.
|
||||
@@ -0,0 +1,17 @@
|
||||
# TLS + reverse proxy for nomifun-web (Caddy auto-provisions HTTPS certs).
|
||||
#
|
||||
# The app provides its own login screen, so NO basic_auth is needed here —
|
||||
# Caddy's job is just TLS termination and proxying. The /ws WebSocket upgrade
|
||||
# passes through automatically.
|
||||
#
|
||||
# IMPORTANT: when serving over HTTPS, set NOMIFUN_HTTPS=true on the nomifun
|
||||
# service so the session cookie gets the Secure flag.
|
||||
#
|
||||
# Replace the domain below with yours. For a LAN-only host without a public
|
||||
# domain you can use an internal name + `tls internal`, or just skip Caddy and
|
||||
# publish port 8787 directly (the in-app login still protects it).
|
||||
|
||||
your.domain.com {
|
||||
encode zstd gzip
|
||||
reverse_proxy nomifun:8787
|
||||
}
|
||||
Generated
+12195
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,241 @@
|
||||
[workspace]
|
||||
resolver = "3"
|
||||
members = ["crates/agent/*", "crates/backend/*", "crates/shared/*", "apps/web", "apps/desktop"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "local/nomifun-tauri"
|
||||
|
||||
[workspace.dependencies]
|
||||
# --- Internal crates: backend (from nomifun-be-rs) ---
|
||||
nomifun-common = { path = "crates/backend/nomifun-common" }
|
||||
nomifun-assets = { path = "crates/backend/nomifun-assets" }
|
||||
nomifun-db = { path = "crates/backend/nomifun-db" }
|
||||
nomifun-api-types = { path = "crates/backend/nomifun-api-types" }
|
||||
nomifun-realtime = { path = "crates/backend/nomifun-realtime" }
|
||||
nomifun-runtime = { path = "crates/backend/nomifun-runtime" }
|
||||
nomifun-auth = { path = "crates/backend/nomifun-auth" }
|
||||
nomifun-system = { path = "crates/backend/nomifun-system" }
|
||||
nomifun-file = { path = "crates/backend/nomifun-file" }
|
||||
nomifun-office = { path = "crates/backend/nomifun-office" }
|
||||
nomifun-shell = { path = "crates/backend/nomifun-shell" }
|
||||
nomifun-ai-agent = { path = "crates/backend/nomifun-ai-agent" }
|
||||
nomifun-mcp = { path = "crates/backend/nomifun-mcp" }
|
||||
nomifun-conversation = { path = "crates/backend/nomifun-conversation" }
|
||||
nomifun-extension = { path = "crates/backend/nomifun-extension" }
|
||||
nomifun-channel = { path = "crates/backend/nomifun-channel" }
|
||||
nomifun-team = { path = "crates/backend/nomifun-team" }
|
||||
nomifun-cron = { path = "crates/backend/nomifun-cron" }
|
||||
nomifun-requirement = { path = "crates/backend/nomifun-requirement" }
|
||||
nomifun-idmm = { path = "crates/backend/nomifun-idmm" }
|
||||
nomifun-knowledge = { path = "crates/backend/nomifun-knowledge" }
|
||||
nomifun-companion = { path = "crates/backend/nomifun-companion" }
|
||||
nomifun-gateway = { path = "crates/backend/nomifun-gateway" }
|
||||
nomifun-public = { path = "crates/backend/nomifun-public" }
|
||||
nomifun-webhook = { path = "crates/backend/nomifun-webhook" }
|
||||
nomifun-terminal = { path = "crates/backend/nomifun-terminal" }
|
||||
nomifun-assistant = { path = "crates/backend/nomifun-assistant" }
|
||||
nomifun-secret = { path = "crates/backend/nomifun-secret" }
|
||||
nomifun-app = { path = "crates/backend/nomifun-app" }
|
||||
|
||||
# --- Internal crates: shared ---
|
||||
nomifun-net = { path = "crates/shared/nomifun-net" }
|
||||
nomi-redact = { path = "crates/shared/nomi-redact" }
|
||||
|
||||
# --- Internal crates: agent (from nomifun-agent-rs, now local) ---
|
||||
nomi-types = { path = "crates/agent/nomi-types" }
|
||||
nomi-protocol = { path = "crates/agent/nomi-protocol" }
|
||||
nomi-compact = { path = "crates/agent/nomi-compact" }
|
||||
nomi-config = { path = "crates/agent/nomi-config" }
|
||||
nomi-providers = { path = "crates/agent/nomi-providers" }
|
||||
nomi-tools = { path = "crates/agent/nomi-tools" }
|
||||
nomi-mcp = { path = "crates/agent/nomi-mcp" }
|
||||
nomi-skills = { path = "crates/agent/nomi-skills" }
|
||||
nomi-memory = { path = "crates/agent/nomi-memory" }
|
||||
nomi-agent = { path = "crates/agent/nomi-agent" }
|
||||
nomi-computer = { path = "crates/agent/nomi-computer" }
|
||||
nomi-a11y = { path = "crates/agent/nomi-a11y" }
|
||||
nomi-browser-engine = { path = "crates/agent/nomi-browser-engine" }
|
||||
nomi-browser = { path = "crates/agent/nomi-browser" }
|
||||
|
||||
# --- Core framework ---
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
# CancellationToken (tokio_util::sync) — exposed in default features on 0.7.x.
|
||||
# Hierarchical parent→child cancellation is the Rust idiom for Playwright's
|
||||
# LongStandingScope (page.close / frame.detach cancels all in-flight ops below).
|
||||
tokio-util = { version = "0.7" }
|
||||
axum = { version = "0.8", features = ["multipart", "ws"] }
|
||||
tower = { version = "0.5" }
|
||||
tower-http = { version = "0.6", features = ["cors", "trace", "limit", "fs"] }
|
||||
http = "1"
|
||||
futures = "0.3"
|
||||
futures-util = "0.3"
|
||||
|
||||
# --- Serialization ---
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
serde_yaml = "0.9"
|
||||
|
||||
# --- Async / error / log ---
|
||||
async-trait = "0.1"
|
||||
thiserror = "2"
|
||||
anyhow = "1"
|
||||
tracing = "0.1"
|
||||
tracing-appender = "0.2"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter", "json", "registry"] }
|
||||
|
||||
# --- CLI ---
|
||||
clap = { version = "4", features = ["derive", "env"] }
|
||||
|
||||
# --- Database ---
|
||||
sqlx = { version = "0.8", features = ["runtime-tokio", "sqlite"] }
|
||||
rusqlite = { version = "0.32", features = ["bundled"] }
|
||||
|
||||
# --- Auth / Crypto ---
|
||||
# NOTE: nomi-providers uses jsonwebtoken 10; nomifun-auth pins 9 in its own
|
||||
# manifest (the only be-rs consumer). Keep the workspace default at 10.
|
||||
jsonwebtoken = "10"
|
||||
bcrypt = "0.17"
|
||||
aes-gcm = "0.10"
|
||||
ed25519-dalek = { version = "2", features = ["rand_core"] }
|
||||
# Public Suffix List (compile-time embedded, offline) for eTLD+1 domain binding
|
||||
# in nomifun-secret. Correctly handles multi-level suffixes (co.uk, com.cn).
|
||||
psl = "2"
|
||||
|
||||
uuid = { version = "1", features = ["v7"] }
|
||||
reqwest = { version = "0.12", features = ["json", "multipart", "stream", "rustls-tls", "socks", "system-proxy"], default-features = false }
|
||||
dashmap = "6"
|
||||
tokio-tungstenite = { version = "0.26", features = ["rustls-tls-native-roots"] }
|
||||
prost = "0.13"
|
||||
semver = "1"
|
||||
dirs = "6"
|
||||
regex = "1"
|
||||
|
||||
# --- File system ---
|
||||
include_dir = "0.7"
|
||||
notify = "8"
|
||||
walkdir = "2"
|
||||
ignore = "0.4"
|
||||
# Unified-diff computation for the knowledge-base inbox review panel (P4).
|
||||
similar = "2"
|
||||
zip = "2"
|
||||
# Node portable-runtime archive extraction (provisioner): tar.gz on unix/macos.
|
||||
flate2 = "1"
|
||||
tar = "0.4"
|
||||
git2 = { version = "0.20", default-features = false }
|
||||
mime_guess = "2"
|
||||
rust-embed = "8"
|
||||
fs2 = "0.4"
|
||||
zstd = "0.13"
|
||||
glob = "0.3"
|
||||
|
||||
# --- Cron / time ---
|
||||
cron = "0.15"
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
chrono-tz = "0.10"
|
||||
|
||||
# --- PTY / shell ---
|
||||
portable-pty = "0.8"
|
||||
# `shellexecute-on-windows`: launch via real ShellExecuteExW on Windows instead
|
||||
# of the default `cmd /c start "" <target>` fallback. The `start` builtin
|
||||
# mis-parses URLs/paths as window titles, runs with no console (CREATE_NO_WINDOW)
|
||||
# so its failure dialog can't surface, and pops a blocking "Windows cannot find
|
||||
# '\\'" modal. ShellExecuteExW delegates to the interactive shell (like the
|
||||
# `microsoft-edge:` protocol path that works) — no cmd, no console, no dialog.
|
||||
# Windows-only feature; macOS/Linux paths are unaffected.
|
||||
open = { version = "5", features = ["shellexecute-on-windows"] }
|
||||
which = "7"
|
||||
|
||||
# --- Office ---
|
||||
calamine = "0.26"
|
||||
rust_xlsxwriter = "0.82"
|
||||
sha1 = "0.10"
|
||||
sha2 = "0.10"
|
||||
hmac = "0.12"
|
||||
|
||||
# --- AWS ---
|
||||
aws-config = { version = "1", features = ["behavior-version-latest"] }
|
||||
aws-sdk-bedrock = { version = "1", default-features = false, features = ["rt-tokio", "default-https-client"] }
|
||||
aws-sigv4 = "1"
|
||||
aws-credential-types = { version = "1", features = ["hardcoded-credentials"] }
|
||||
|
||||
# --- TLS ---
|
||||
rustls = "0.23"
|
||||
rustls-native-certs = "0.8"
|
||||
|
||||
# --- OAuth / config ---
|
||||
oauth2 = "5.0.0-rc.1"
|
||||
# NOTE: nomi-config uses toml 1.0; be-rs (nomifun-mcp/runtime) uses 0.8 which is
|
||||
# the workspace default. nomi-config pins 1.0 in its own manifest.
|
||||
toml = "0.8"
|
||||
|
||||
# --- Encoding / misc ---
|
||||
base64 = "0.22"
|
||||
http-body-util = "0.1"
|
||||
getrandom = "0.2"
|
||||
hex = "0.4"
|
||||
# Nostr protocol (keys, NIP-04 DMs, event signing) — pure crypto, no net/TLS deps.
|
||||
nostr = { version = "0.37", default-features = false, features = ["std", "nip04"] }
|
||||
# HTML → Markdown (URL knowledge-source snapshots)
|
||||
htmd = "0.5"
|
||||
libc = "0.2"
|
||||
lru = "0.18"
|
||||
url = "2"
|
||||
unicode-width = "0.2.2"
|
||||
crossterm = "0.29"
|
||||
is-terminal = "0.4"
|
||||
|
||||
# --- Computer / browser use ---
|
||||
image = { version = "0.25", default-features = false, features = ["png"] }
|
||||
xcap = "0.9"
|
||||
enigo = "0.6"
|
||||
# In-process self-hosted CDP browser engine (nomi-browser-engine). 0.9.1 has NO
|
||||
# `tokio-runtime` feature — its transport rides `async-tungstenite` and is
|
||||
# runtime-agnostic, so the core CDP engine needs NO extra features here.
|
||||
# Deliberately default-features-only: we do NOT use chromiumoxide's built-in
|
||||
# Chrome *fetcher*. Task 6 ships a self-contained, proxy-aware download
|
||||
# (workspace `zip` + `nomifun_net::http_client`) instead — the fetcher uses a
|
||||
# non-proxy-aware reqwest and would break the proxy-aware install promise, so its
|
||||
# `rustls`/`native-tls`/`zip*` features stay OFF (enabling a TLS feature without
|
||||
# `zip0`/`zip8` also won't compile in 0.9.1). CDP generated types arrive
|
||||
# transitively via `chromiumoxide_cdp`, so no separate workspace entry.
|
||||
chromiumoxide = { version = "0.9" }
|
||||
|
||||
# --- Testing ---
|
||||
tempfile = "3"
|
||||
wiremock = "0.6"
|
||||
tokio-test = "0.4"
|
||||
mockall = "0.14"
|
||||
rstest = "0.26"
|
||||
serial_test = "3"
|
||||
# 快照测试：注入侧 aria 契约（observe_fixtures.rs）+ 序列化层契约（T6）防 PW 升级漂移。
|
||||
# yaml/json/redactions feature 用于结构化快照与随机字段归一。
|
||||
insta = { version = "1.48", features = ["yaml", "json", "redactions"] }
|
||||
|
||||
# Minimal debuginfo to shrink build output. Incremental compilation stays at
|
||||
# the Cargo default (on for dev/test): it only applies to workspace members and
|
||||
# cuts touched-crate rebuilds 2-3x, which is the hot path of local iteration.
|
||||
# The disk cost lives in the build dir (see .cargo/config.toml build-dir) and
|
||||
# is reclaimed by the self-cleaning preflight in scripts/prune-build.mjs, which
|
||||
# runs at the start of every dev/build/test entry point (package.json scripts).
|
||||
# CI, if added, should set CARGO_INCREMENTAL=0 instead of hardcoding it here.
|
||||
[profile.dev]
|
||||
debug = "line-tables-only"
|
||||
|
||||
# Dependencies don't need debuginfo — you never single-step into them. Dropping
|
||||
# it for the whole dependency graph (not workspace crates) is where most of the
|
||||
# build-dir size and file count lived: under macOS unpacked split-debuginfo each
|
||||
# dep emits many object/.dwo files, and high-frequency rebuilds never GC the old
|
||||
# ones. Workspace members keep line-tables above for usable stack traces; deps
|
||||
# compile once, so there is no rebuild-speed cost. Stale artifacts are pruned by
|
||||
# the self-cleaning preflight (scripts/prune-build.mjs) at each build start.
|
||||
[profile.dev.package."*"]
|
||||
debug = false
|
||||
|
||||
[profile.test]
|
||||
debug = "line-tables-only"
|
||||
|
||||
[profile.test.package."*"]
|
||||
debug = false
|
||||
@@ -0,0 +1,79 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
|
||||
# ============================================================================
|
||||
# nomifun-web — headless WebUI server image (no GUI / WebView; runs anywhere)
|
||||
# Stage 1 builds the React SPA (ui/dist)
|
||||
# Stage 2 compiles the nomifun-web Rust binary
|
||||
# Stage 3 slim runtime with bun (required by the agent engine)
|
||||
#
|
||||
# Authentication is ON by default, but first-run setup is a claim window: the
|
||||
# first reachable browser creates the admin account unless NOMIFUN_ADMIN_PASSWORD
|
||||
# pre-seeds it. Bind on trusted networks only until setup is complete, and put
|
||||
# TLS in front (see Caddyfile) for anything internet-facing.
|
||||
# ============================================================================
|
||||
|
||||
# ---- Stage 1: build the SPA -------------------------------------------------
|
||||
FROM oven/bun:1 AS ui
|
||||
WORKDIR /app
|
||||
# Install deps first for layer caching (only re-runs when manifests change).
|
||||
COPY package.json bun.lock ./
|
||||
COPY ui/package.json ui/package.json
|
||||
RUN bun install --frozen-lockfile
|
||||
COPY . .
|
||||
RUN bun run build:ui
|
||||
# -> /app/ui/dist
|
||||
|
||||
# ---- Stage 2: compile nomifun-web ------------------------------------------
|
||||
FROM rust:1-bookworm AS rust
|
||||
# Native build deps: rusqlite(bundled) needs cc; rustls/aws-lc-rs needs cmake+clang;
|
||||
# libgit2-sys needs cmake. If a first build fails on a *-sys crate, add its dep here.
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential cmake clang pkg-config perl git \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /src
|
||||
|
||||
# Optional cargo registry mirror for faster dependency fetches (e.g. in CN):
|
||||
# docker build --build-arg CARGO_REGISTRY_MIRROR=https://rsproxy.cn/index/ .
|
||||
# (The repo's own .cargo/ is .dockerignore'd, so the default is crates.io.)
|
||||
ARG CARGO_REGISTRY_MIRROR=""
|
||||
RUN if [ -n "$CARGO_REGISTRY_MIRROR" ]; then \
|
||||
printf '[source.crates-io]\nreplace-with = "mirror"\n[source.mirror]\nregistry = "sparse+%s"\n' \
|
||||
"$CARGO_REGISTRY_MIRROR" > "${CARGO_HOME:-/usr/local/cargo}/config.toml"; \
|
||||
fi
|
||||
|
||||
COPY . .
|
||||
# BuildKit cache mounts persist the cargo registry + compiled artifacts across
|
||||
# rebuilds, so a one-line source change recompiles in seconds, not minutes. The
|
||||
# binary is copied OUT of the (ephemeral) target cache mount into a real layer.
|
||||
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,target=/src/target \
|
||||
cargo build --release -p nomifun-web \
|
||||
&& cp target/release/nomifun-web /usr/local/bin/nomifun-web
|
||||
# -> /usr/local/bin/nomifun-web
|
||||
|
||||
# ---- Stage 3: slim runtime --------------------------------------------------
|
||||
FROM debian:bookworm-slim
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates git ripgrep \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
# bun is a hard runtime dependency of the agent engine (>= 1.3.13).
|
||||
COPY --from=oven/bun:1 /usr/local/bin/bun /usr/local/bin/bun
|
||||
# Optional: user-configured MCP stdio servers often launch via `npx`.
|
||||
# RUN apt-get update && apt-get install -y --no-install-recommends nodejs npm \
|
||||
# && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=rust /usr/local/bin/nomifun-web /usr/local/bin/nomifun-web
|
||||
COPY --from=ui /app/ui/dist /opt/nomifun/web
|
||||
|
||||
ENV NOMIFUN_WEB_HOST=0.0.0.0 \
|
||||
NOMIFUN_WEB_PORT=8787 \
|
||||
NOMIFUN_DATA_DIR=/data \
|
||||
NOMIFUN_WEB_DIST=/opt/nomifun/web \
|
||||
SHELL=/bin/bash
|
||||
# Set NOMIFUN_HTTPS=true when a TLS proxy fronts the app (makes cookies Secure).
|
||||
# Set NOMIFUN_ADMIN_PASSWORD (+ NOMIFUN_ADMIN_USERNAME) to pre-seed the admin
|
||||
# and skip the interactive first-run setup.
|
||||
|
||||
VOLUME /data
|
||||
EXPOSE 8787
|
||||
CMD ["nomifun-web"]
|
||||
@@ -0,0 +1,201 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright 2025-2026 NomiFun (nomifun.com)
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@@ -0,0 +1,20 @@
|
||||
NomiFun
|
||||
Copyright 2025-2026 NomiFun (nomifun.com)
|
||||
|
||||
This product is licensed under the Apache License, Version 2.0 (see LICENSE).
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Third-party attributions
|
||||
----------------------------------------------------------------------
|
||||
|
||||
This product began as a fork of, and incorporates software from, the
|
||||
AionUi project, which has since been substantially refactored and
|
||||
extended:
|
||||
|
||||
AionUi
|
||||
https://github.com/iOfficeAI/AionUi
|
||||
Copyright iOfficeAI
|
||||
Licensed under the Apache License, Version 2.0.
|
||||
|
||||
Our sincere thanks to the AionUi maintainers and community, and to the
|
||||
many other excellent open-source projects NomiFun builds on.
|
||||
@@ -0,0 +1,432 @@
|
||||
<a name="top"></a>
|
||||
|
||||
<div align="center">
|
||||
|
||||
<a href="https://www.nomifun.com">
|
||||
<img src="docs/images/brand/og-cover.svg" alt="NomiFun — Fully open-source, local-first super AI workstation" width="820">
|
||||
</a>
|
||||
|
||||
<h3>A no-holds-barred, fully open-source, <em>local-first</em> super AI workstation.</h3>
|
||||
|
||||
<p>
|
||||
Rich, inventive capabilities and serious productivity gains —<br/>
|
||||
with <b>all your data staying on your own machine</b>. Safe for individuals and enterprises, free to commercialize, open to audit.
|
||||
</p>
|
||||
|
||||
<p>
|
||||
<a href="LICENSE"><img alt="License: Apache-2.0" src="https://img.shields.io/badge/License-Apache_2.0-FF6F91?style=for-the-badge"></a>
|
||||
<img alt="Platform" src="https://img.shields.io/badge/Platform-macOS%20%7C%20Windows%20%7C%20Linux-7583B2?style=for-the-badge">
|
||||
<img alt="Status" src="https://img.shields.io/badge/Status-pre--1.0-FBBF24?style=for-the-badge">
|
||||
<a href="https://www.nomifun.com"><img alt="Website" src="https://img.shields.io/badge/Website-nomifun.com-FF6F91?style=for-the-badge"></a>
|
||||
</p>
|
||||
|
||||
<p>
|
||||
<img alt="Built with Tauri 2" src="https://img.shields.io/badge/Tauri-2-24C8DB?style=flat-square&logo=tauri&logoColor=white">
|
||||
<img alt="Rust 2024" src="https://img.shields.io/badge/Rust-edition_2024-CE412B?style=flat-square&logo=rust&logoColor=white">
|
||||
<img alt="React 19" src="https://img.shields.io/badge/React-19-61DAFB?style=flat-square&logo=react&logoColor=white">
|
||||
<a href="https://github.com/nomifun/nomifun-tauri/stargazers"><img alt="Stars" src="https://img.shields.io/github/stars/nomifun/nomifun-tauri?style=flat-square&color=FF6F91"></a>
|
||||
</p>
|
||||
|
||||
<p>
|
||||
<b>English</b> · <a href="README.zh-CN.md">简体中文</a>
|
||||
</p>
|
||||
|
||||
<p>
|
||||
<a href="https://www.nomifun.com">🌐 Website</a> ·
|
||||
<a href="docs/README.md">📖 Docs</a> ·
|
||||
<a href="#-getting-started">🚀 Get started</a> ·
|
||||
<a href="https://github.com/nomifun/nomifun-tauri/releases">📦 Releases</a> ·
|
||||
<a href="#-contact--community">💬 Community</a>
|
||||
</p>
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
**NomiFun** is everything you imagine an AI workstation to be — and it runs on your terms. One React frontend and one Rust backend give you an evolving desktop companion, an unattended automation platform, a unified knowledge base, native computer- and browser-use, and an open capability bus that any agent can drive. No cloud account. No telemetry. No subscription. Your data never leaves your machine except for the LLM calls **you** configure.
|
||||
|
||||
> The product name is **NomiFun**. Lowercase `nomifun` is used only for code identifiers, crate names, environment variables, and repository paths.
|
||||
|
||||
---
|
||||
|
||||
## ✨ Why NomiFun
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| 🔓 **Open & local** | Source fully open, no reservations. Data lives on your machine and is never sent out on its own. Free for personal **and** commercial use. Open to audit. |
|
||||
| 🐾 **Evolving companions** | The most complete companion-growth system we know of — it learns how you work and gets better over time. Not just a buddy, a genuine productivity partner. |
|
||||
| 🤖 **Unattended automation** | Manage requirements, then just give the order. AutoWork + IDMM keep your sessions alive and working reliably while you're away. |
|
||||
| 🌐 **Open capability ecosystem** | Everything is here, everything connects, everything cooperates — and *any* agent can borrow NomiFun's powers over MCP / REST. |
|
||||
| 🧩 **Config once, use anywhere** | Unified management of knowledge bases, skills, agents, MCP servers, and models — defined once, reused across every surface. |
|
||||
| 🖥️ **Truly native** | In-process, self-built **computer use** and **browser use** as native tools — more capable, faster, and cheaper on tokens. |
|
||||
| 🚀 **Built for productivity** | Designed from real needs, with a lot of inventive capabilities. And many delightful features are still on the way. |
|
||||
|
||||
---
|
||||
|
||||
## 🔒 Local-first, by design
|
||||
|
||||
Data security is not a setting in NomiFun — it is the architecture.
|
||||
|
||||
- **All data is local.** NomiFun never proactively sends your data anywhere. The **only** outbound network calls are the LLM requests you explicitly configure to your chosen model provider. There is no other third-party service integration phoning home.
|
||||
- **Safe for anyone who cares about data.** Individuals and enterprises with strict data-handling requirements can use it with confidence. The code is **fully open and open to audit**.
|
||||
- **We cut features to keep this promise.** To guarantee your data stays yours, we deliberately dropped several advanced, genuinely fun feature designs. Everything here is in service of letting users — and developers — relax.
|
||||
- **No ads. No commercialization. No membership tiers.** We promise to *never* charge for any feature of this project. The only thing that costs money is your LLM provider's tokens, which is outside our control. (If finding/serving models is painful, [reach out](#-contact--community) — we're happy to help build a unified model gateway.)
|
||||
|
||||
See [`SECURITY.md`](SECURITY.md) for the deployment threat model and responsible-disclosure policy.
|
||||
|
||||
---
|
||||
|
||||
## 🖼️ A look inside
|
||||
|
||||
<div align="center">
|
||||
|
||||
<p>
|
||||
🎬 <b>Intro video:</b> <a href="https://www.youtube.com/watch?v=Z28XyhvNh_E">https://www.youtube.com/watch?v=Z28XyhvNh_E</a>
|
||||
</p>
|
||||
|
||||
<p>
|
||||
<img src="docs/images/readme-01-workbench-overview.png" alt="NomiFun desktop workbench with conversation, companion, and project metrics" width="100%">
|
||||
<br/><sub><b>Desktop workbench: conversation, companion, and live session metrics</b></sub>
|
||||
</p>
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td width="50%"><img src="docs/images/gs-01-introduction-hero.png" alt="Home / new session"><br/><sub><b>Home & sessions</b></sub></td>
|
||||
<td width="50%"><img src="docs/images/channels-01-overview.png" alt="Companion remote channels"><br/><sub><b>Companion · IM channels</b></sub></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="50%"><img src="docs/images/autowork-03-kanban.png" alt="Requirements board"><br/><sub><b>Requirements · AutoWork board</b></sub></td>
|
||||
<td width="50%"><img src="docs/images/webui-01-settings-overview.png" alt="Open capabilities"><br/><sub><b>Open capability bus</b></sub></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="50%"><img src="docs/images/terminal-03-driving-session.png" alt="Terminal session"><br/><sub><b>Agent-driven terminal</b></sub></td>
|
||||
<td width="50%"><img src="docs/images/webui-04-qr-login-phone.png" alt="Phone QR login"><br/><sub><b>WebUI · scan-to-connect</b></sub></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<sub>Real in-app captures. See <a href="docs/images/SCREENSHOTS.md">the screenshot manifest</a> for the full set and capture method.</sub>
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Feature highlights
|
||||
|
||||
### 🐾 Desktop Companion — it grows with you
|
||||
|
||||
> Guide: [`docs/guides/companions.md`](docs/guides/companions.md)
|
||||
|
||||
The companion you talk to every day quietly becomes the assistant who *gets* you.
|
||||
|
||||
- **Make it yours.** Upload a custom companion figure (DIY), or pick from an independent figure library decoupled from any single companion.
|
||||
- **One brain, many faces.** Run multiple companions that share a common memory hub, while each keeps its own **private** memory and can mount different domain knowledge bases. Teach *one* companion well, then have it teach the others.
|
||||
- **It learns you (opt-in, on by default after a one-time consent).** A background learner distills your usage into durable memories; a deterministic evolution engine mines your recurring multi-step tool sequences into **draft skills** it proposes for your review. Memory is fully **visible and editable**.
|
||||
- **Skills that spread.** Companions generate their own skills, discuss them with you, and can **gift** a skill to another companion (the recipient gets a copy) — opt-in shared learning across your roster.
|
||||
- **A super gateway, not just a buddy.** Each companion is a complete, independent individual that can connect to multiple IM channels. From anywhere with a network and a chat app, message your companion to drive your computer for you. Each companion can fully operate the desktop's capabilities.
|
||||
|
||||
### 🤖 Unattended automation — Requirements + AutoWork + IDMM
|
||||
|
||||
> Guides: [`autowork-requirements.md`](docs/guides/autowork-requirements.md) · [`intelligent-decision.md`](docs/guides/intelligent-decision.md)
|
||||
|
||||
You give the orders; NomiFun reliably does the work.
|
||||
|
||||
- **Requirement platform** — a CRUD store with ordered rotation, a board/kanban, tags, and per-item claim.
|
||||
- **AutoWork** — claims pending requirements, drives a turn, rotates to the next, and renews leases while a turn is in flight. Targets can be **conversation agents *or* terminal PTYs**.
|
||||
- **IDMM (Intelligent Decision-Making)** — per-session supervision that keeps agents alive through provider faults and decision stalls, with a no-LLM rule tier and a sidecar backup-model tier, stacking on top of AutoWork.
|
||||
- **Notify out** — completion notifications to **Lark/Feishu** custom bots, **Slack**, and HTTP webhooks.
|
||||
|
||||
### 📚 Unified Knowledge Base
|
||||
|
||||
> Guide: [`docs/guides/mcp-and-skills.md`](docs/guides/mcp-and-skills.md)
|
||||
|
||||
Pull the knowledge scattered across your system into one managed, trackable place.
|
||||
|
||||
- **Centralized management & tracking** — create, mount, and track consumers across conversations, terminals, and companions.
|
||||
- **Safe write-back** — a code-enforced, per-surface write policy. By default, writes are **staged into a review inbox** with unified-diff preview and merge/discard — so agents never scribble into the wrong place.
|
||||
- **Real-time URL snapshot** — turn any web page into a knowledge source (SSRF-guarded fetch, HTML→Markdown), in *snapshot* (persisted, re-fetchable) or *live* mode.
|
||||
- **Scoped retrieval** — agents call a `knowledge_search` tool whose scope is decided server-side and cannot be widened.
|
||||
|
||||
### 🖥️ Native Computer Use & Browser Use *(desktop build)*
|
||||
|
||||
> Guide: [`docs/guides/computer-browser-use.md`](docs/guides/computer-browser-use.md)
|
||||
|
||||
Self-built, **in-process Rust** — no Playwright, no Node, no third-party automation daemon. More capable, faster, and far cheaper on tokens, with fine-grained control and fully open source for you to extend.
|
||||
|
||||
- **Computer use** — accessibility tree + Set-of-Marks overlay + OCR, steering the model to act on real UI elements instead of guessing pixels. macOS (AXUIElement + Vision OCR) and Windows (UI Automation) are complete; Linux (AT-SPI2) is partial.
|
||||
- **Browser use** — an in-process Chromium CDP engine with ARIA observation, an egress **firewall** with out-of-band approval, and an origin-bound secret vault so credentials never reach the LLM.
|
||||
- **Guarded by design** — every action carries a danger × surface approval matrix; irreversible actions wait for explicit confirmation.
|
||||
|
||||
> ℹ️ Computer/browser control ship with the **desktop app**. The headless web/server host omits them by design.
|
||||
|
||||
### 🌐 Open capability bus — MCP + REST
|
||||
|
||||
> Guides: [`remote-capability-api.md`](docs/guides/remote-capability-api.md) · [`remote-capability-api-examples.md`](docs/guides/remote-capability-api-examples.md)
|
||||
|
||||
Every capability NomiFun has is exposed through a single, typed capability registry — **~20 domains and 150+ tools** — so you can wire NomiFun into anything.
|
||||
|
||||
- **MCP front door** at `/mcp` (authenticated, Streamable-HTTP). Point **Claude Code, Cursor, or your own agent** at it and they operate NomiFun exactly as the desktop companion does.
|
||||
- **REST + OpenAPI** at `/v1/tools`, with streaming and an auto-generated `/v1/openapi.json`.
|
||||
- Adding a capability to the bus makes it appear on MCP **and** REST automatically — no drift.
|
||||
|
||||
### 🧩 Bring your own agents — or use the built-in one
|
||||
|
||||
> Guide: [`docs/guides/model-routing.md`](docs/guides/model-routing.md)
|
||||
|
||||
- **Built-in `nomi` agent** — no extra install. Works with **26+ model providers/presets** (OpenAI, Anthropic, Gemini + Vertex AI, AWS Bedrock, DeepSeek, OpenRouter, Moonshot/Kimi, Qwen/Dashscope, Zhipu/GLM, MiniMax, SiliconFlow, xAI, Volcengine/Doubao, and more) across **4 wire protocols**, plus the **New API** aggregator gateway.
|
||||
- **~19 external agents over ACP** — connect Claude Code, Codex, Gemini, Qwen, Kimi, Cursor, Copilot, Goose, OpenCode, Droid, and more, and NomiFun feeds them models *and* its native capabilities (computer/browser/knowledge/gateway) over injected MCP bridges.
|
||||
- **Everywhere** — the native capabilities are available to the built-in agent, to ACP agents, in the chat UI, **and** in the terminal.
|
||||
|
||||
### 💻 Terminal mode
|
||||
|
||||
> Guide: [`docs/guides/terminal.md`](docs/guides/terminal.md)
|
||||
|
||||
Run agent CLIs inside in-app PTY sessions (or the standalone `nomi` CLI). NomiFun injects native capabilities — knowledge search, requirement completion, and lifecycle hooks — into known CLIs through their *own* native config, so you keep full fidelity and OAuth.
|
||||
|
||||
### 📱 WebUI remote control — scan, and you're in
|
||||
|
||||
> Guide: [`docs/guides/webui-remote-access.md`](docs/guides/webui-remote-access.md)
|
||||
|
||||
No social platform required. One-tap **QR pairing** connects your phone or tablet to your computer over the LAN (one-time token, realtime over WebSocket) so you can drive your workstation remotely from the couch.
|
||||
|
||||
### ⚙️ Config once, use anywhere
|
||||
|
||||
Central hubs for **Knowledge**, **Assistants & Skills**, **MCP**, **Models**, and **Open Capabilities** — define them once, then select per conversation, terminal, channel, or companion. One source of truth, reused everywhere.
|
||||
|
||||
### 💬 11 IM channels
|
||||
|
||||
> Guide: [`docs/guides/channels.md`](docs/guides/channels.md)
|
||||
|
||||
Bind a companion to any of these and drive it from where you already chat:
|
||||
|
||||
`Telegram` · `Lark / 飞书` · `DingTalk / 钉钉` · `WeChat / 微信` · `Discord` · `Slack` · `Matrix` · `Mattermost` · `Twitch` · `Nostr` · `QQ Bot`
|
||||
|
||||
---
|
||||
|
||||
## 🏗️ Architecture
|
||||
|
||||
One React frontend, one Rust backend, **two host modes** — and the same backend runs in-process in both.
|
||||
|
||||
| | `nomifun-desktop` | `nomifun-web` |
|
||||
|---|---|---|
|
||||
| **Shell** | Tauri 2 desktop app | Standalone axum server |
|
||||
| **Backend** | Embedded in-process, private loopback port | Same backend, in-process |
|
||||
| **Auth** | Local-trust token injected into the webview | Login required by default |
|
||||
| **Serves** | Native desktop UI + tray + companion windows | API + `/ws` + built SPA on one port |
|
||||
| **Computer / browser use** | ✅ Included | ❌ Headless (omitted) |
|
||||
|
||||
There is no Electron shell, no Node web host, and no prebuilt backend handoff.
|
||||
|
||||
<details>
|
||||
<summary><b>Repository map</b></summary>
|
||||
|
||||
```text
|
||||
apps/
|
||||
desktop/ Tauri 2 shell and desktop-only commands
|
||||
web/ standalone web host for API + SPA
|
||||
crates/
|
||||
agent/ 15 nomi-* crates: engine, providers, tools, MCP, skills, memory,
|
||||
browser/computer use, and the standalone nomi CLI
|
||||
backend/ 29 nomifun-* crates: app composition, auth, database, sessions,
|
||||
MCP, knowledge, requirements, terminal, companion, gateway, etc.
|
||||
shared/ 2 cross-layer crates: nomifun-net and nomi-redact
|
||||
ui/ React 19 + Vite SPA shared by desktop and web
|
||||
docs/ technical docs, user/operator guides, architecture notes
|
||||
packaging/ Linux deployment support for the web host
|
||||
```
|
||||
|
||||
Start with [`docs/architecture/overview.md`](docs/architecture/overview.md) for the full system map. The Cargo workspace is defined in [`Cargo.toml`](Cargo.toml).
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Getting started
|
||||
|
||||
> ℹ️ There are **no prebuilt installers yet** — install from source or run the server with Docker. Watch [Releases](https://github.com/nomifun/nomifun-tauri/releases) for binaries.
|
||||
|
||||
**Prerequisites**
|
||||
|
||||
- [Rust](https://rustup.rs) — stable toolchain, edition 2024
|
||||
- [Bun](https://bun.sh) ≥ 1.3.13
|
||||
- Recommended on PATH for full agent tooling: `node` / `npm` / `npx`, `git`, `ripgrep`
|
||||
|
||||
**Desktop app (from source)**
|
||||
|
||||
```bash
|
||||
git clone https://github.com/nomifun/nomifun-tauri.git
|
||||
cd nomifun-tauri
|
||||
bun install
|
||||
|
||||
bun run dev # develop with hot reload
|
||||
bun run build # build a desktop bundle for your OS
|
||||
```
|
||||
|
||||
**Web server (self-host)**
|
||||
|
||||
```bash
|
||||
bun run build:ui && bun run serve:web
|
||||
# serves API + SPA on http://127.0.0.1:8787 (login required)
|
||||
```
|
||||
|
||||
**Docker (self-host the server)**
|
||||
|
||||
```bash
|
||||
docker compose up -d --build
|
||||
# then open http://<server-ip>:8787 — pair with the bundled Caddyfile for TLS
|
||||
```
|
||||
|
||||
See [`docs/getting-started/installation.md`](docs/getting-started/installation.md) and [`docs/guides/web-server-deployment.md`](docs/guides/web-server-deployment.md) for details.
|
||||
|
||||
---
|
||||
|
||||
## 🛠️ Development
|
||||
|
||||
```bash
|
||||
bun install # install dependencies (one-time)
|
||||
bun run dev # desktop app development (hot reload)
|
||||
bun run dev:web # web host + Vite development
|
||||
bun run build:ui # build the SPA
|
||||
bun run check # frontend typecheck + i18n + theme + script-registry gate
|
||||
bun run test # Rust tests (use test:fast for nextest)
|
||||
```
|
||||
|
||||
Prefer the scripted entry points over plain `cargo`/`vite` — they include build-dir pruning and consistency checks. New to the codebase? Read [`CONTRIBUTING.md`](CONTRIBUTING.md) and [`docs/contributing/development.md`](docs/contributing/development.md).
|
||||
|
||||
<details>
|
||||
<summary><b>Full script catalog</b></summary>
|
||||
|
||||
<!-- BEGIN GENERATED SCRIPTS (bun run help --readme) -->
|
||||
|
||||
| 脚本 | 说明 |
|
||||
| --- | --- |
|
||||
| **开发（热重载）** | |
|
||||
| `bun run dev` | 启动桌面应用开发（tauri dev，热重载） |
|
||||
| `bun run dev:web` | 启动 Web 全栈开发（后端 API + 前端 vite） |
|
||||
| `bun run dev:ui` | 仅启动前端开发服务器（纯 vite，无后端） |
|
||||
| **构建（出制品）** | |
|
||||
| `bun run build` | 为当前操作系统打桌面安装包 |
|
||||
| `bun run build:signed` | 打桌面包并签名+公证（仅 macOS） |
|
||||
| `bun run build:updater` | 打桌面包并产出自更新 .sig 制品 |
|
||||
| `bun run build:ui` | 前端生产构建 → ui/dist |
|
||||
| **运行（组装好的应用）** | |
|
||||
| `bun run serve:web` | 启动 Web 服务器，托管已构建的前端 |
|
||||
| **测试** | |
|
||||
| `bun run test` | 运行全部 Rust 测试（含 doctest） |
|
||||
| `bun run test:fast` | 用 nextest 快速跑 Rust 测试（日常） |
|
||||
| **静态检查 / 门禁** | |
|
||||
| `bun run check` | 聚合静态门禁：typecheck + i18n + 主题契约 + 脚本登记 |
|
||||
| `bun run typecheck` | 前端 TypeScript 类型检查（tsc --noEmit） |
|
||||
| `bun run check:i18n` | 校验 i18n 类型与 locale 键是否一致 |
|
||||
| `bun run check:theme` | 校验预设 CSS 主题契约 |
|
||||
| **格式化** | |
|
||||
| `bun run fmt` | 格式化 Rust 代码（cargo fmt） |
|
||||
| `bun run fmt:check` | 校验 Rust 代码格式（cargo fmt --check） |
|
||||
| **代码生成** | |
|
||||
| `bun run gen:i18n` | 由 locale 重新生成 i18n 类型声明 |
|
||||
| **维护 / 工具** | |
|
||||
| `bun run clean` | 深度回收构建空间（debug 产物 + flycheck + 旧安装包） |
|
||||
| `bun run seed:dev` | 用生产数据目录播种 dev 数据目录 |
|
||||
| `bun run help` | 打印脚本目录（--check 校验登记 / --readme 生成 README 表） |
|
||||
|
||||
<!-- END GENERATED SCRIPTS -->
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
## 📖 Documentation
|
||||
|
||||
- [`docs/README.md`](docs/README.md) — documentation index
|
||||
- [`docs/getting-started/`](docs/getting-started) — installation and first run
|
||||
- [`docs/guides/`](docs/guides) — user & operator guides (companions, channels, AutoWork, knowledge, computer/browser use, terminal, remote API, …)
|
||||
- [`docs/architecture/`](docs/architecture) — technical architecture
|
||||
- [`docs/reference/`](docs/reference) — configuration, API overview, FAQ, troubleshooting
|
||||
|
||||
Docs are bilingual: every page has an English `*.md` and a Simplified-Chinese `*.zh.md` sibling.
|
||||
|
||||
---
|
||||
|
||||
## 🗺️ Coming soon
|
||||
|
||||
NomiFun is **pre-1.0** and built part-time, so there's a lot still in flight. On the horizon: prebuilt installers, inbound issue-tracker / requirement sources, more knowledge connectors (Feishu, and beyond), official desktop binaries — plus a few surprises we're genuinely excited about. **Stay tuned.** ✨
|
||||
|
||||
---
|
||||
|
||||
## 🤝 Contributing & community
|
||||
|
||||
NomiFun very much needs your help to grow — code contributions, community building, and evangelism are all hugely welcome. If you have passion for this project, please [reach out](#-contact--community) and build the NomiFun ecosystem with us.
|
||||
|
||||
- Read [`CONTRIBUTING.md`](CONTRIBUTING.md) to get set up and learn the check ladder.
|
||||
- Be excellent to each other — see [`CODE_OF_CONDUCT.md`](CODE_OF_CONDUCT.md).
|
||||
- Found a vulnerability? Follow [`SECURITY.md`](SECURITY.md).
|
||||
- Browse [open issues](https://github.com/nomifun/nomifun-tauri/issues) for a place to start.
|
||||
|
||||
---
|
||||
|
||||
## 💛 A note from the author
|
||||
|
||||
> This is a part-time effort with limited bandwidth, and many delightful features are still on the way. If this resonates with you, join in any way you like — a line of code, a suggestion, a reshare all mean a lot.
|
||||
|
||||
NomiFun is **completely open source, with nothing held back**. Individuals and enterprises are free to build on it and use it commercially.
|
||||
|
||||
- **Forks & commercial use are welcome.** They're also at your own risk — the author and contributors assume no liability for downstream use. Apache-2.0 requires no permission from us.
|
||||
- **A friendly heads-up is appreciated, not required.** If you fork or commercialize NomiFun, we'd love a note — *not* as a license condition, simply because knowing the project is valued is the kind of recognition that keeps it going.
|
||||
- **Some features were intentionally left out of the open-source release** to keep the local-data promise airtight — without the people and funding to guarantee everyone's data security, removing them was the responsible choice. As time and resources allow, we hope to bring more of them to you.
|
||||
|
||||
Thank you for being here. 🙏
|
||||
|
||||
---
|
||||
|
||||
## 🔗 Friendly links
|
||||
|
||||
Projects and products we appreciate:
|
||||
|
||||
| Product | What it does |
|
||||
|---|---|
|
||||
| [Saytive](http://saytive.ai/) | **Be Creative, Be Saytive.** A voice input method for creative workers, using strong models and thoughtful product design to sense your work context and deliver fast, accurate, scene-aware transcription. |
|
||||
| [Fast](https://fast.saien.pro) | **Search, one tap away.** Type, click, and jump straight to search results across RED, Douyin, Meituan, and dozens of mainstream apps. No feed distraction, just search. |
|
||||
| [AionUi](https://github.com/iOfficeAI/AionUi) | AionUi ships with a complete AI agent engine. Unlike tools that require separate CLI-agent installs, AionUi works the moment you install it. |
|
||||
|
||||
---
|
||||
|
||||
## 📬 Contact & community
|
||||
|
||||
We'd love to hear from you. The fastest way to reach us is GitHub; the social channels below are all official.
|
||||
|
||||
| Channel | Where |
|
||||
|---|---|
|
||||
| 🌐 **Website** | [www.nomifun.com](https://www.nomifun.com) |
|
||||
| 🐙 **GitHub** | [nomifun/nomifun-tauri](https://github.com/nomifun/nomifun-tauri) · [Issues](https://github.com/nomifun/nomifun-tauri/issues) · [Releases](https://github.com/nomifun/nomifun-tauri/releases) |
|
||||
| ✉️ **Email** | `hello@nomifun.com` <sub>(provisional — being finalized)</sub> |
|
||||
| 📕 **小红书 / RED** | [NomiFun](https://xhslink.com/m/4x6ti8n6cA1) |
|
||||
| 📺 **Bilibili** | [NomiFun](https://b23.tv/0UhgKDh) |
|
||||
| 🎵 **抖音 / Douyin** | [NomiFun](https://v.douyin.com/MDT5QVdYaJk/) |
|
||||
| ▶️ **YouTube** | [@NomiFun-o2y](https://www.youtube.com/@NomiFun-o2y) |
|
||||
| 𝕏 **X (Twitter)** | [@colir0](https://x.com/colir0) |
|
||||
| 🎬 **TikTok** | [@colir0luo](https://www.tiktok.com/@colir0luo) |
|
||||
|
||||
**Join the chat groups** — scan to join:
|
||||
|
||||
<div align="center">
|
||||
<table>
|
||||
<tr>
|
||||
<td align="center"><img src="docs/images/contact/wechat-group-qr.png" alt="WeChat group QR" width="220"><br/><sub><b>WeChat group / 微信群</b></sub></td>
|
||||
<td align="center"><img src="docs/images/contact/qq-group-qr.png" alt="QQ group QR" width="220"><br/><sub><b>QQ group / QQ 群</b></sub></td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ⚖️ License
|
||||
|
||||
[Apache-2.0](LICENSE) © 2025–2026 NomiFun.
|
||||
|
||||
See [`NOTICE`](NOTICE) for third-party attributions, including the [AionUi](https://github.com/iOfficeAI/AionUi) project that NomiFun originally forked from before its current Tauri/Rust architecture.
|
||||
|
||||
<div align="center">
|
||||
<br/>
|
||||
<sub>Built with 💛 for people who want AI on their own terms.</sub>
|
||||
<br/><br/>
|
||||
<a href="#top">⬆ Back to top</a>
|
||||
</div>
|
||||
@@ -0,0 +1,430 @@
|
||||
<a name="top"></a>
|
||||
|
||||
<div align="center">
|
||||
|
||||
<a href="https://www.nomifun.com">
|
||||
<img src="docs/images/brand/og-cover.svg" alt="NomiFun — 完全开源 · 本地优先的超级 AI 工作站" width="820">
|
||||
</a>
|
||||
|
||||
<h3>一项毫无保留、<em>本地优先</em>的超级 AI 工作站。</h3>
|
||||
|
||||
<p>
|
||||
丰富的创新能力，极高的生产提效 ——<br/>
|
||||
而你的<b>数据始终留在自己的电脑上</b>。个人与企业都能放心使用、自由商用、接受审计。
|
||||
</p>
|
||||
|
||||
<p>
|
||||
<a href="LICENSE"><img alt="License: Apache-2.0" src="https://img.shields.io/badge/License-Apache_2.0-FF6F91?style=for-the-badge"></a>
|
||||
<img alt="Platform" src="https://img.shields.io/badge/平台-macOS%20%7C%20Windows%20%7C%20Linux-7583B2?style=for-the-badge">
|
||||
<img alt="Status" src="https://img.shields.io/badge/状态-pre--1.0-FBBF24?style=for-the-badge">
|
||||
<a href="https://www.nomifun.com"><img alt="Website" src="https://img.shields.io/badge/官网-nomifun.com-FF6F91?style=for-the-badge"></a>
|
||||
</p>
|
||||
|
||||
<p>
|
||||
<img alt="Built with Tauri 2" src="https://img.shields.io/badge/Tauri-2-24C8DB?style=flat-square&logo=tauri&logoColor=white">
|
||||
<img alt="Rust 2024" src="https://img.shields.io/badge/Rust-edition_2024-CE412B?style=flat-square&logo=rust&logoColor=white">
|
||||
<img alt="React 19" src="https://img.shields.io/badge/React-19-61DAFB?style=flat-square&logo=react&logoColor=white">
|
||||
<a href="https://github.com/nomifun/nomifun-tauri/stargazers"><img alt="Stars" src="https://img.shields.io/github/stars/nomifun/nomifun-tauri?style=flat-square&color=FF6F91"></a>
|
||||
</p>
|
||||
|
||||
<p>
|
||||
<a href="README.md">English</a> · <b>简体中文</b>
|
||||
</p>
|
||||
|
||||
<p>
|
||||
<a href="https://www.nomifun.com">🌐 官网</a> ·
|
||||
<a href="docs/README.zh.md">📖 文档</a> ·
|
||||
<a href="#-快速开始">🚀 快速开始</a> ·
|
||||
<a href="https://github.com/nomifun/nomifun-tauri/releases">📦 下载</a> ·
|
||||
<a href="#-联系我们--社区">💬 社区</a>
|
||||
</p>
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
**NomiFun** 满足你对 AI 工作站的全部想象 —— 而且一切由你做主。一套 React 前端 + 一套 Rust 后端，为你带来会成长的桌面伙伴、无人值守的自动化平台、统一知识库、原生的 computer / browser use，以及任何智能体都能驱动的开放能力总线。无需云账号、无遥测、无订阅。除了**你自己配置**的大模型调用，你的数据绝不离开本机。
|
||||
|
||||
> 产品名是 **NomiFun**；小写 `nomifun` 仅用于代码标识符、crate 名、环境变量与仓库路径。
|
||||
|
||||
---
|
||||
|
||||
## ✨ 为什么选 NomiFun
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| 🔓 **开放 · 本地** | 源码完全开放，毫无保留。数据全在本地、绝不主动外发。个人与企业**均可**免费商用，接受审计。 |
|
||||
| 🐾 **超级伙伴，智能进化** | 我们所知最完整的伙伴养成体系 —— 越用越懂你。不只是伙伴，更是真正的生产力工具。 |
|
||||
| 🤖 **智能值守，需求管理** | 你只管指挥。AutoWork + IDMM 高可靠保活，在你离开时持续、可靠地为你工作。 |
|
||||
| 🌐 **开放能力，超级生态** | 什么都有、什么都能用、什么都能配合 —— 而且*任意*智能体都能经 MCP / REST 拥有它的能力。 |
|
||||
| 🧩 **无限搭配，config one** | 知识库、skill、agent、MCP、模型统一管理 —— 配置一次，处处复用。 |
|
||||
| 🖥️ **更 native 的实现** | 进程内、自研的 **computer use** 与 **browser use** 作为原生工具 —— 更强、更快、更省 token。 |
|
||||
| 🚀 **专为提效设计** | 从实际需求出发，用心打磨，海量创新能力。更多惊喜功能，敬请期待。 |
|
||||
|
||||
---
|
||||
|
||||
## 🔒 本地优先，是底层设计
|
||||
|
||||
在 NomiFun 里，数据安全不是一个开关，而是架构本身。
|
||||
|
||||
- **数据全在本地。** NomiFun 绝不主动向外发送任何数据。**唯一**的出站网络请求，是你自己明确配置、调用所选模型厂商的大模型请求；除此之外，没有任何第三方服务的网络对接。
|
||||
- **关注数据安全的个体与企业都可放心使用。** 代码**完全开源、接受审计**。
|
||||
- **为了这个承诺，我们砍掉了不少功能。** 为了保障你的数据安全，我们刻意舍弃了很多先进、有趣的功能设计 —— 一切都是为了让用户、也让开发者更放心。
|
||||
- **无广告、无商业化、无会员制。** 我们承诺：永远不对本项目的任何功能收费。唯一花钱的地方是模型供应商的 token，这是我们无法替你解决的客观成本。（如果你在寻找 / 搭建模型上遇到困难，欢迎[联系我们](#-联系我们--社区)，我们很乐意帮忙搭建统一的模型网关。）
|
||||
|
||||
部署威胁模型与漏洞披露策略见 [`SECURITY.md`](SECURITY.md)。
|
||||
|
||||
---
|
||||
|
||||
## 🖼️ 先睹为快
|
||||
|
||||
<div align="center">
|
||||
|
||||
<p>
|
||||
🎬 <b>宣传视频：</b><a href="https://www.youtube.com/watch?v=Z28XyhvNh_E">https://www.youtube.com/watch?v=Z28XyhvNh_E</a>
|
||||
</p>
|
||||
|
||||
<p>
|
||||
<img src="docs/images/readme-01-workbench-overview.png" alt="NomiFun 桌面工作台，会话、桌面伙伴与项目指标同屏展示" width="100%">
|
||||
<br/><sub><b>桌面工作台：会话、伙伴与实时会话指标</b></sub>
|
||||
</p>
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td width="50%"><img src="docs/images/gs-01-introduction-hero.png" alt="首页 / 新建会话"><br/><sub><b>首页与会话</b></sub></td>
|
||||
<td width="50%"><img src="docs/images/channels-01-overview.png" alt="伙伴 IM 渠道"><br/><sub><b>伙伴 · IM 渠道</b></sub></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="50%"><img src="docs/images/autowork-03-kanban.png" alt="需求看板"><br/><sub><b>需求 · AutoWork 看板</b></sub></td>
|
||||
<td width="50%"><img src="docs/images/webui-01-settings-overview.png" alt="开放能力"><br/><sub><b>开放能力总线</b></sub></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="50%"><img src="docs/images/terminal-03-driving-session.png" alt="终端会话"><br/><sub><b>智能体驱动的终端</b></sub></td>
|
||||
<td width="50%"><img src="docs/images/webui-04-qr-login-phone.png" alt="手机扫码登录"><br/><sub><b>WebUI · 扫码即连</b></sub></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<sub>均为真实应用内截图。完整截图清单与采集方式见 <a href="docs/images/SCREENSHOTS.md">截图 manifest</a>。</sub>
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## 🚀 功能亮点
|
||||
|
||||
### 🐾 桌面伙伴 —— 越用越懂你
|
||||
|
||||
> 指南：[`docs/guides/companions.zh.md`](docs/guides/companions.zh.md)
|
||||
|
||||
每天与你对话的伙伴，会悄悄变成那个最懂你的助理。
|
||||
|
||||
- **专属形象。** 上传自定义伙伴形象（DIY），或从与具体伙伴解耦的独立**形象库**中挑选。
|
||||
- **一脑多面。** 运行多个伙伴，共享统一记忆中枢，同时各自保留**专属**私有记忆，并可挂载不同领域的知识库。你只需教好*一个*伙伴，再让它去教其他伙伴。
|
||||
- **它在学你（默认开启，首启动一次性确认）。** 后台 Learner 把你的使用蒸馏为长期记忆；确定性的进化引擎从你反复出现的多步工具序列中挖掘出 **skill 草稿**，提交给你审阅。记忆**完全可见、可编辑**。
|
||||
- **会传播的 skill。** 伙伴自动总结、生成 skill 并与你商议，还能把 skill **赠予**另一个伙伴（对方得到一份副本）—— 开启跨伙伴的共享学习。
|
||||
- **不只是伙伴，更是超级网关。** 每个伙伴都是完整、独立的个体，可连接多个 IM 渠道。只要有网络和社交平台，随时随地一条消息，就能指挥伙伴帮你操作电脑。每个伙伴都能完整驱动桌面的系统能力。
|
||||
|
||||
### 🤖 智能值守 —— 需求平台 + AutoWork + IDMM
|
||||
|
||||
> 指南：[`autowork-requirements.zh.md`](docs/guides/autowork-requirements.zh.md) · [`intelligent-decision.zh.md`](docs/guides/intelligent-decision.zh.md)
|
||||
|
||||
你只管下令，NomiFun 可靠地把活干完。
|
||||
|
||||
- **需求平台** —— 带有序轮转的 CRUD 存储、看板、标签与逐项 claim。
|
||||
- **AutoWork** —— 自动 claim 待办需求、驱动一个回合、轮转到下一个，并在回合进行中续租保活。目标可以是**会话智能体**，也可以是**终端 PTY**。
|
||||
- **IDMM（智能决策）** —— 逐会话的守护，穿越供应商故障与决策停滞维持会话存活；无 LLM 的规则层 + 旁路备用模型层，叠加在 AutoWork 之上。
|
||||
- **出站通知** —— 完成通知可推送到**飞书/Lark** 自定义机器人、**Slack** 与 HTTP webhook。
|
||||
|
||||
### 📚 统一知识库
|
||||
|
||||
> 指南：[`docs/guides/mcp-and-skills.zh.md`](docs/guides/mcp-and-skills.zh.md)
|
||||
|
||||
把散落在系统各处的知识，收拢到一个可管理、可追踪的地方。
|
||||
|
||||
- **集中管理与追踪** —— 创建、挂载，并跨会话、终端、伙伴追踪消费方。
|
||||
- **安全回写** —— 代码强制、按使用面分级的写策略。默认把写入**暂存到审阅收件箱**，提供 unified-diff 预览与合并/丢弃 —— 智能体绝不会把内容写错地方。
|
||||
- **实时 URL 快照** —— 把任意网页变成知识来源（带 SSRF 防护抓取、HTML→Markdown），支持*快照*（持久化、可重抓）与*实时*两种模式。
|
||||
- **作用域受控的检索** —— 智能体调用 `knowledge_search` 工具，其作用域由服务端裁定、无法被擅自放大。
|
||||
|
||||
### 🖥️ 原生 Computer Use 与 Browser Use *（桌面版）*
|
||||
|
||||
> 指南：[`docs/guides/computer-browser-use.zh.md`](docs/guides/computer-browser-use.zh.md)
|
||||
|
||||
自研、**进程内 Rust** 实现 —— 不依赖 Playwright、不依赖 Node、不依赖第三方自动化守护进程。能力更强、速度更快、token 更省，提供细粒度控制，且完全开源供你增强。
|
||||
|
||||
- **Computer use** —— 无障碍树 + Set-of-Marks 叠层 + OCR，引导模型操作真实 UI 元素而非猜像素。macOS（AXUIElement + Vision OCR）与 Windows（UI Automation）已完整，Linux（AT-SPI2）为部分支持。
|
||||
- **Browser use** —— 进程内 Chromium CDP 引擎，含 ARIA 观察、带带外审批的出站**防火墙**，以及与来源绑定的密钥保险库，凭据绝不进入 LLM。
|
||||
- **生而受控** —— 每个动作都带 danger × surface 审批矩阵，不可逆操作须显式确认。
|
||||
|
||||
> ℹ️ computer/browser 控制随**桌面应用**提供；无头的 web/server 宿主按设计不含。
|
||||
|
||||
### 🌐 开放能力总线 —— MCP + REST
|
||||
|
||||
> 指南：[`remote-capability-api.zh.md`](docs/guides/remote-capability-api.zh.md) · [`remote-capability-api-examples.zh.md`](docs/guides/remote-capability-api-examples.zh.md)
|
||||
|
||||
NomiFun 的每一项能力都经由单一、强类型的能力注册表对外开放 —— **约 20 个域、150+ 个工具** —— 让你能把 NomiFun 接进任何地方。
|
||||
|
||||
- **MCP 前门** 位于 `/mcp`（鉴权，Streamable-HTTP）。把 **Claude Code、Cursor 或你自己的智能体**指向它，它们就能像桌面伙伴一样操作 NomiFun。
|
||||
- **REST + OpenAPI** 位于 `/v1/tools`，支持流式，并自动生成 `/v1/openapi.json`。
|
||||
- 在总线上新增一项能力，会自动同时出现在 MCP **与** REST 上 —— 不漂移。
|
||||
|
||||
### 🧩 自带智能体，也能接入你的
|
||||
|
||||
> 指南：[`docs/guides/model-routing.zh.md`](docs/guides/model-routing.zh.md)
|
||||
|
||||
- **内置 `nomi` 智能体** —— 无需额外安装。支持 **26+ 模型供应商/预设**（OpenAI、Anthropic、Gemini + Vertex AI、AWS Bedrock、DeepSeek、OpenRouter、Moonshot/Kimi、通义千问/Dashscope、智谱/GLM、MiniMax、SiliconFlow、xAI、火山/豆包 等），覆盖 **4 种线缆协议**，并支持 **New API** 聚合网关。
|
||||
- **经 ACP 直连约 19 个外部智能体** —— Claude Code、Codex、Gemini、Qwen、Kimi、Cursor、Copilot、Goose、OpenCode、Droid 等，NomiFun 为它们提供模型*以及*自家的原生能力（computer/browser/knowledge/gateway，经注入的 MCP 桥）。
|
||||
- **处处可用** —— 这些原生能力对内置智能体、ACP 智能体、聊天界面**以及**终端一律可用。
|
||||
|
||||
### 💻 终端模式
|
||||
|
||||
> 指南：[`docs/guides/terminal.zh.md`](docs/guides/terminal.zh.md)
|
||||
|
||||
在应用内 PTY 会话里运行各种 agent CLI（或独立的 `nomi` CLI）。NomiFun 会把原生能力 —— 知识检索、需求完成、生命周期 hooks —— 经各 CLI *自己的*原生配置注入进去，从而保留完整保真度与 OAuth。
|
||||
|
||||
### 📱 WebUI 远程操控 —— 一扫即用
|
||||
|
||||
> 指南：[`docs/guides/webui-remote-access.zh.md`](docs/guides/webui-remote-access.zh.md)
|
||||
|
||||
不用任何社交平台。一键**扫码配对**，就能让手机或平板经局域网连上电脑（一次性令牌，实时走 WebSocket），让你窝在沙发上也能远程操控你的工作站。
|
||||
|
||||
### ⚙️ config one，use anywhere
|
||||
|
||||
**知识库**、**Assistants & Skills**、**MCP**、**模型**、**开放能力**的集中管理中枢 —— 配置一次，再按会话、终端、渠道或伙伴逐一选用。单一事实源，处处复用。
|
||||
|
||||
### 💬 11 个 IM 渠道
|
||||
|
||||
> 指南：[`docs/guides/channels.zh.md`](docs/guides/channels.zh.md)
|
||||
|
||||
把伙伴绑定到下列任意渠道，从你已经在用的聊天工具里指挥它：
|
||||
|
||||
`Telegram` · `飞书 / Lark` · `钉钉 / DingTalk` · `微信 / WeChat` · `Discord` · `Slack` · `Matrix` · `Mattermost` · `Twitch` · `Nostr` · `QQ Bot`
|
||||
|
||||
---
|
||||
|
||||
## 🏗️ 架构
|
||||
|
||||
一套 React 前端、一套 Rust 后端，**两种宿主模式** —— 同一套后端在两者中均为进程内运行。
|
||||
|
||||
| | `nomifun-desktop` | `nomifun-web` |
|
||||
|---|---|---|
|
||||
| **外壳** | Tauri 2 桌面应用 | 独立 axum 服务器 |
|
||||
| **后端** | 进程内嵌入，私有回环端口 | 同一后端，进程内 |
|
||||
| **鉴权** | 注入 webview 的本地信任令牌 | 默认需要登录 |
|
||||
| **提供** | 原生桌面 UI + 托盘 + 伙伴窗口 | 单端口提供 API + `/ws` + 已构建 SPA |
|
||||
| **Computer / browser use** | ✅ 含 | ❌ 无头（不含） |
|
||||
|
||||
没有 Electron 外壳，没有 Node web 宿主，也没有预编译后端交接。
|
||||
|
||||
<details>
|
||||
<summary><b>仓库结构</b></summary>
|
||||
|
||||
```text
|
||||
apps/
|
||||
desktop/ Tauri 2 外壳与桌面专属命令
|
||||
web/ API + SPA 的独立 web 宿主
|
||||
crates/
|
||||
agent/ 15 个 nomi-* crate：引擎、供应商、工具、MCP、skills、记忆、
|
||||
browser/computer use，以及独立 nomi CLI
|
||||
backend/ 29 个 nomifun-* crate：应用组装、鉴权、数据库、会话、
|
||||
MCP、知识库、需求、终端、伙伴、网关等
|
||||
shared/ 2 个跨层 crate：nomifun-net 与 nomi-redact
|
||||
ui/ 桌面与 web 共用的 React 19 + Vite SPA
|
||||
docs/ 技术文档、用户/运维指南、架构说明
|
||||
packaging/ web 宿主的 Linux 部署支持
|
||||
```
|
||||
|
||||
系统全景从 [`docs/architecture/overview.zh.md`](docs/architecture/overview.zh.md) 入门。Cargo 工作区定义见 [`Cargo.toml`](Cargo.toml)。
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
## 🚀 快速开始
|
||||
|
||||
> ℹ️ **目前还没有预编译安装包** —— 请从源码安装，或用 Docker 跑服务器。安装包发布请关注 [Releases](https://github.com/nomifun/nomifun-tauri/releases)。
|
||||
|
||||
**前置依赖**
|
||||
|
||||
- [Rust](https://rustup.rs) —— stable 工具链，edition 2024
|
||||
- [Bun](https://bun.sh) ≥ 1.3.13
|
||||
- 建议在 PATH 中具备（以获得完整 agent 工具链）：`node` / `npm` / `npx`、`git`、`ripgrep`
|
||||
|
||||
**桌面应用（源码）**
|
||||
|
||||
```bash
|
||||
git clone https://github.com/nomifun/nomifun-tauri.git
|
||||
cd nomifun-tauri
|
||||
bun install
|
||||
|
||||
bun run dev # 热重载开发
|
||||
bun run build # 为当前操作系统打桌面安装包
|
||||
```
|
||||
|
||||
**Web 服务器（自托管）**
|
||||
|
||||
```bash
|
||||
bun run build:ui && bun run serve:web
|
||||
# 单端口提供 API + SPA：http://127.0.0.1:8787（需登录）
|
||||
```
|
||||
|
||||
**Docker（自托管服务器）**
|
||||
|
||||
```bash
|
||||
docker compose up -d --build
|
||||
# 然后打开 http://<服务器IP>:8787 — 配合自带的 Caddyfile 启用 TLS
|
||||
```
|
||||
|
||||
详见 [`docs/getting-started/installation.zh.md`](docs/getting-started/installation.zh.md) 与 [`docs/guides/web-server-deployment.zh.md`](docs/guides/web-server-deployment.zh.md)。
|
||||
|
||||
---
|
||||
|
||||
## 🛠️ 开发
|
||||
|
||||
```bash
|
||||
bun install # 安装依赖（一次性）
|
||||
bun run dev # 桌面应用开发（热重载）
|
||||
bun run dev:web # web 宿主 + Vite 开发
|
||||
bun run build:ui # 构建 SPA
|
||||
bun run check # 前端 typecheck + i18n + 主题 + 脚本登记 门禁
|
||||
bun run test # Rust 测试（日常可用 test:fast 跑 nextest）
|
||||
```
|
||||
|
||||
优先使用脚本入口而非裸 `cargo`/`vite` —— 它们附带了构建目录清理与一致性检查。第一次接触代码库？请读 [`CONTRIBUTING.md`](CONTRIBUTING.md) 与 [`docs/contributing/development.zh.md`](docs/contributing/development.zh.md)。
|
||||
|
||||
<details>
|
||||
<summary><b>完整脚本目录</b></summary>
|
||||
|
||||
| 脚本 | 说明 |
|
||||
| --- | --- |
|
||||
| **开发（热重载）** | |
|
||||
| `bun run dev` | 启动桌面应用开发（tauri dev，热重载） |
|
||||
| `bun run dev:web` | 启动 Web 全栈开发（后端 API + 前端 vite） |
|
||||
| `bun run dev:ui` | 仅启动前端开发服务器（纯 vite，无后端） |
|
||||
| **构建（出制品）** | |
|
||||
| `bun run build` | 为当前操作系统打桌面安装包 |
|
||||
| `bun run build:signed` | 打桌面包并签名+公证（仅 macOS） |
|
||||
| `bun run build:updater` | 打桌面包并产出自更新 .sig 制品 |
|
||||
| `bun run build:ui` | 前端生产构建 → ui/dist |
|
||||
| **运行（组装好的应用）** | |
|
||||
| `bun run serve:web` | 启动 Web 服务器，托管已构建的前端 |
|
||||
| **测试** | |
|
||||
| `bun run test` | 运行全部 Rust 测试（含 doctest） |
|
||||
| `bun run test:fast` | 用 nextest 快速跑 Rust 测试（日常） |
|
||||
| **静态检查 / 门禁** | |
|
||||
| `bun run check` | 聚合静态门禁：typecheck + i18n + 主题契约 + 脚本登记 |
|
||||
| `bun run typecheck` | 前端 TypeScript 类型检查（tsc --noEmit） |
|
||||
| `bun run check:i18n` | 校验 i18n 类型与 locale 键是否一致 |
|
||||
| `bun run check:theme` | 校验预设 CSS 主题契约 |
|
||||
| **格式化** | |
|
||||
| `bun run fmt` | 格式化 Rust 代码（cargo fmt） |
|
||||
| `bun run fmt:check` | 校验 Rust 代码格式（cargo fmt --check） |
|
||||
| **代码生成** | |
|
||||
| `bun run gen:i18n` | 由 locale 重新生成 i18n 类型声明 |
|
||||
| **维护 / 工具** | |
|
||||
| `bun run clean` | 深度回收构建空间（debug 产物 + flycheck + 旧安装包） |
|
||||
| `bun run seed:dev` | 用生产数据目录播种 dev 数据目录 |
|
||||
| `bun run help` | 打印脚本目录（--check 校验登记 / --readme 生成 README 表） |
|
||||
|
||||
<sub>此表的英文权威版由 <code>bun run help --readme</code> 在 <a href="README.md">README.md</a> 中自动维护。</sub>
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
## 📖 文档
|
||||
|
||||
- [`docs/README.zh.md`](docs/README.zh.md) —— 文档索引
|
||||
- [`docs/getting-started/`](docs/getting-started) —— 安装与首次运行
|
||||
- [`docs/guides/`](docs/guides) —— 用户与运维指南（伙伴、渠道、AutoWork、知识库、computer/browser use、终端、远程 API……）
|
||||
- [`docs/architecture/`](docs/architecture) —— 技术架构
|
||||
- [`docs/reference/`](docs/reference) —— 配置、API 概览、FAQ、排障
|
||||
|
||||
文档为双语：每篇都有英文 `*.md` 与简体中文 `*.zh.md` 两份。
|
||||
|
||||
---
|
||||
|
||||
## 🗺️ 敬请期待
|
||||
|
||||
NomiFun 目前处于 **pre-1.0**，且为兼职开发，所以还有很多正在路上：预编译安装包、入站 issue / 需求来源接入、更多知识库连接器（飞书及更多）、官方桌面安装包 —— 以及几个我们非常期待的惊喜。**敬请期待。** ✨
|
||||
|
||||
---
|
||||
|
||||
## 🤝 贡献与社区
|
||||
|
||||
NomiFun 非常需要你的加入来壮大 —— 代码贡献、社区运营、技术布道都热烈欢迎。如果你对这个项目有热情，请[联系我们](#-联系我们--社区)，与我们一起共建 NomiFun 的生态。
|
||||
|
||||
- 阅读 [`CONTRIBUTING.md`](CONTRIBUTING.md) 完成环境搭建、了解检查阶梯。
|
||||
- 友善相待 —— 见 [`CODE_OF_CONDUCT.md`](CODE_OF_CONDUCT.md)。
|
||||
- 发现漏洞？请按 [`SECURITY.md`](SECURITY.md) 操作。
|
||||
- 从 [open issues](https://github.com/nomifun/nomifun-tauri/issues) 找一个起点。
|
||||
|
||||
---
|
||||
|
||||
## 💛 写在最后（来自作者）
|
||||
|
||||
> 开发者兼职、精力有限，很多惊喜功能还在路上。如果你认同这件事，欢迎以任何方式加入 —— 一行代码、一条建议、一次转发，都是莫大的鼓励。
|
||||
|
||||
NomiFun **完全开源、毫无保留**。个人与企业都可以在它之上二次开发并商用。
|
||||
|
||||
- **欢迎二次开发与商用。** 同时，这些行为风险自担 —— 作者与贡献者不承担后续一切法律责任。Apache-2.0 无需我们另行授权。
|
||||
- **告知一声，是渴望而非要求。** 如果你二次开发或商用 NomiFun，希望你能留言告知我们 —— 这*不是*授权条件，只是因为「知道项目被认可」这份肯定，正是让它走下去的动力。
|
||||
- **部分功能被刻意排除在开源版之外** —— 为了让本地数据的承诺滴水不漏。在没有足够人力与资金保障每位用户数据安全的前提下，移除它们是负责任的选择。等条件允许，我们希望把更多功能奉上给大家。
|
||||
|
||||
谢谢你来到这里。🙏
|
||||
|
||||
---
|
||||
|
||||
## 🔗 友情链接
|
||||
|
||||
这些是我们欣赏的产品与项目：
|
||||
|
||||
| 产品 | 简介 |
|
||||
|---|---|
|
||||
| [Saytive](http://saytive.ai/) | **Be Creative, Be Saytive.** Saytive 是一款专为创意工作者打造的语音输入法，它通过顶级模型和产品设计，自动感知你的工作上下文，提供快速准确而符合场景的转写体验。 |
|
||||
| [Fast](https://fast.saien.pro) | **搜索，一触即达。** 你只需输入文字并点击，即可直达小红书、抖音、美团等数十个主流应用的搜索结果页面。拒绝信息流，专注搜索本身，搜索本该如此简单。 |
|
||||
| [AionUi](https://github.com/iOfficeAI/AionUi) | AionUi 内置完整的 AI agent 引擎。不同于需要你额外安装 CLI agent 的工具，AionUi 安装后即可使用。 |
|
||||
|
||||
---
|
||||
|
||||
## 📬 联系我们 / 社区
|
||||
|
||||
我们很想听到你的声音。最快的方式是 GitHub；下列社交渠道均为官方。
|
||||
|
||||
| 渠道 | 入口 |
|
||||
|---|---|
|
||||
| 🌐 **官网** | [www.nomifun.com](https://www.nomifun.com) |
|
||||
| 🐙 **GitHub** | [nomifun/nomifun-tauri](https://github.com/nomifun/nomifun-tauri) · [Issues](https://github.com/nomifun/nomifun-tauri/issues) · [Releases](https://github.com/nomifun/nomifun-tauri/releases) |
|
||||
| ✉️ **邮箱** | `hello@nomifun.com` <sub>（占位 · 待确认）</sub> |
|
||||
| 📕 **小红书** | [NomiFun](https://xhslink.com/m/4x6ti8n6cA1) |
|
||||
| 📺 **哔哩哔哩** | [NomiFun](https://b23.tv/0UhgKDh) |
|
||||
| 🎵 **抖音** | [NomiFun](https://v.douyin.com/MDT5QVdYaJk/) |
|
||||
| ▶️ **YouTube** | [@NomiFun-o2y](https://www.youtube.com/@NomiFun-o2y) |
|
||||
| 𝕏 **X (Twitter)** | [@colir0](https://x.com/colir0) |
|
||||
| 🎬 **TikTok** | [@colir0luo](https://www.tiktok.com/@colir0luo) |
|
||||
|
||||
**加入交流群** —— 扫码即可：
|
||||
|
||||
<div align="center">
|
||||
<table>
|
||||
<tr>
|
||||
<td align="center"><img src="docs/images/contact/wechat-group-qr.png" alt="微信群二维码" width="220"><br/><sub><b>微信群</b></sub></td>
|
||||
<td align="center"><img src="docs/images/contact/qq-group-qr.png" alt="QQ 群二维码" width="220"><br/><sub><b>QQ 群</b></sub></td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ⚖️ 许可证
|
||||
|
||||
[Apache-2.0](LICENSE) © 2025–2026 NomiFun。
|
||||
|
||||
第三方署名见 [`NOTICE`](NOTICE)，其中包括 NomiFun 在迁移到当前 Tauri/Rust 架构之前最初 fork 自的 [AionUi](https://github.com/iOfficeAI/AionUi) 项目。
|
||||
|
||||
<div align="center">
|
||||
<br/>
|
||||
<sub>用 💛 打造，献给希望以自己的方式拥有 AI 的人。</sub>
|
||||
<br/><br/>
|
||||
<a href="#top">⬆ 回到顶部</a>
|
||||
</div>
|
||||
@@ -0,0 +1,42 @@
|
||||
# Releasing NomiFun
|
||||
|
||||
This checklist is for maintainers preparing a public release.
|
||||
|
||||
## Before Tagging
|
||||
|
||||
1. Update `CHANGELOG.md`.
|
||||
2. Run the documented verification commands for the changed surface.
|
||||
3. Confirm `docs/`, `README.md`, `STATUS.md`, and packaging guides match the
|
||||
release behavior.
|
||||
4. Confirm no private keys, local paths, proprietary assets, or internal-only
|
||||
roadmap claims are included.
|
||||
5. Confirm third-party licenses and attributions are current.
|
||||
|
||||
## Desktop Release
|
||||
|
||||
1. Build unsigned bundles with `bun run build`.
|
||||
2. For macOS public distribution, use `bun run build:signed` with the
|
||||
release-owner Developer ID credentials.
|
||||
3. For updater artifacts, configure the release-owner Tauri updater key and run
|
||||
`bun run build:updater`.
|
||||
4. Publish installers and signatures to the release host.
|
||||
5. Publish a signed `latest.json` only after artifacts are uploaded.
|
||||
|
||||
Updater signing and OS code signing are separate. See:
|
||||
|
||||
- `apps/desktop/updater/README.md`
|
||||
- `apps/desktop/signing/README.md`
|
||||
|
||||
## Server Release
|
||||
|
||||
1. Build `nomifun-web` and the SPA.
|
||||
2. Build and smoke-test the Docker image.
|
||||
3. Verify first-run admin setup and `NOMIFUN_ADMIN_PASSWORD` pre-seeding.
|
||||
4. Verify `127.0.0.1` default binding and explicit `0.0.0.0` deployment docs.
|
||||
|
||||
## After Release
|
||||
|
||||
1. Create a GitHub release with notes from `CHANGELOG.md`.
|
||||
2. Attach platform artifacts.
|
||||
3. Update website/download links.
|
||||
4. Watch issues for install, updater, and migration regressions.
|
||||
@@ -0,0 +1,40 @@
|
||||
# Security Policy
|
||||
|
||||
NomiFun can execute local tools, shell commands, browser automation, desktop
|
||||
automation, and remote capability calls. Treat an authenticated NomiFun instance
|
||||
as a high-privilege local automation surface.
|
||||
|
||||
## Reporting Vulnerabilities
|
||||
|
||||
Please report suspected vulnerabilities privately before opening a public issue.
|
||||
If the project has not published a dedicated security contact yet, contact the
|
||||
maintainers through the repository owner channel and include:
|
||||
|
||||
- affected version or commit,
|
||||
- operating system and deployment mode (`nomifun-desktop`, `nomifun-web`, or
|
||||
standalone `nomicore`),
|
||||
- reproduction steps,
|
||||
- impact assessment,
|
||||
- logs or screenshots with secrets redacted.
|
||||
|
||||
Do not include live tokens, passwords, provider keys, private conversation
|
||||
content, or proprietary workspace files in reports.
|
||||
|
||||
## Supported Versions
|
||||
|
||||
The project is pre-1.0. Security fixes target the current default branch unless
|
||||
a release branch explicitly says it is supported.
|
||||
|
||||
## Deployment Guidance
|
||||
|
||||
- Do not expose the embedded desktop backend port directly. Use WebUI Remote
|
||||
Access or `nomifun-web`, both of which provide authenticated surfaces.
|
||||
- Use TLS when exposing `nomifun-web` or remote capability APIs over a network.
|
||||
- Treat companion access tokens as full-control credentials for the scoped
|
||||
companion and its enabled capabilities.
|
||||
- Prefer least-privilege provider keys, MCP servers, and workspace paths.
|
||||
- Review full-auto terminal permissions before binding them to AutoWork.
|
||||
|
||||
See [docs/reference/troubleshooting.md](docs/reference/troubleshooting.md) and
|
||||
[docs/guides/remote-capability-api.md](docs/guides/remote-capability-api.md)
|
||||
for related operational details.
|
||||
@@ -0,0 +1,70 @@
|
||||
# Current Technical Status
|
||||
|
||||
Updated: 2026-06-24.
|
||||
|
||||
This file is a compact current-state snapshot. Historical P0-P5 migration notes
|
||||
were removed from the active status because they described the 2026-06-08
|
||||
transition plan, not the product shape in this repository now.
|
||||
|
||||
## Current Architecture
|
||||
|
||||
- One Cargo workspace:
|
||||
- `crates/agent/*`: 15 `nomi-*` crates.
|
||||
- `crates/backend/*`: 29 `nomifun-*` crates.
|
||||
- `crates/shared/*`: 2 cross-layer crates.
|
||||
- `apps/web` and `apps/desktop`.
|
||||
- One frontend: `ui/`, a React 19 + Vite SPA.
|
||||
- Two host modes:
|
||||
- Desktop: `apps/desktop`, Tauri 2 shell, embedded backend on loopback,
|
||||
local-trust header injected into `fetch` and `XMLHttpRequest`.
|
||||
- Web: `apps/web`, standalone server, authenticated by default, serves API,
|
||||
`/ws`, and `ui/dist` on one port.
|
||||
- One backend composition root: `nomifun-app`, assembled through
|
||||
`AppServices`, `build_module_states`, and `create_router`.
|
||||
|
||||
## Active Product Surfaces
|
||||
|
||||
The current frontend route map lives in
|
||||
`ui/src/renderer/components/layout/Router.tsx`. Active top-level surfaces are:
|
||||
|
||||
- `/guid` and `/conversation/:id`
|
||||
- `/terminal-new` and `/terminal/:id`
|
||||
- `/models`
|
||||
- `/assistants`
|
||||
- `/mcp`
|
||||
- `/open-capabilities`
|
||||
- `/requirements`, `/requirements/extensions`, `/requirements/sources`
|
||||
- `/scheduled` and `/scheduled/:job_id`
|
||||
- `/nomi`
|
||||
- `/knowledge` and `/knowledge/:id`
|
||||
- `/settings/system` plus system sub-sections routed through that page
|
||||
|
||||
Several legacy paths still exist only as redirects. Do not document them as
|
||||
primary navigation.
|
||||
|
||||
## Commands
|
||||
|
||||
Use the root script catalog:
|
||||
|
||||
```bash
|
||||
bun run help
|
||||
bun run dev
|
||||
bun run dev:web
|
||||
bun run build:ui
|
||||
bun run check
|
||||
bun run test
|
||||
```
|
||||
|
||||
For packaging and signing, see:
|
||||
|
||||
- `docs/contributing/building-and-packaging.md`
|
||||
- `apps/desktop/signing/README.md`
|
||||
- `apps/desktop/updater/README.md`
|
||||
- `packaging/linux/README.md`
|
||||
|
||||
## Known Documentation Policy
|
||||
|
||||
The active docs are `README.md`, `STATUS.md`, and the non-archive sections under
|
||||
`docs/`. Dated design specs, audits, and Superpowers implementation plans are
|
||||
historical records. They can explain why code exists, but they must not be used
|
||||
as current product or operator instructions without re-checking the source.
|
||||
@@ -0,0 +1,57 @@
|
||||
[package]
|
||||
name = "nomifun-desktop"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
# NOTE: this crate is EXCLUDED from the root workspace (see root Cargo.toml)
|
||||
# until P2 wires the Tauri toolchain. It is the scaffold/target for the
|
||||
# "Tauri shell embeds the backend in-process" spike.
|
||||
|
||||
[build-dependencies]
|
||||
tauri-build = { version = "2", features = [] }
|
||||
|
||||
[dependencies]
|
||||
# The unified backend, linked in-process (no spawned binary, no bundled-nomicore).
|
||||
# computer-use: the desktop host is the one place screen/input control is sane.
|
||||
# browser-use: likewise, the desktop host is where a managed/connected Chromium
|
||||
# is sane (the in-process self-hosted CDP engine).
|
||||
nomifun-app = { workspace = true, features = ["computer-use", "browser-use"] }
|
||||
nomifun-runtime.workspace = true
|
||||
clap.workspace = true
|
||||
|
||||
# `macos-private-api` is required for the transparent nomi companion window on macOS.
|
||||
# `tray-icon` powers the system tray (close-to-tray default + tray Show/Quit menu).
|
||||
tauri = { version = "2", features = ["macos-private-api", "tray-icon"] }
|
||||
# `deep-link` feature is REQUIRED for runtime deep links on Windows/Linux: those
|
||||
# OSes deliver `nomifun://` as argv to a second process, which single-instance
|
||||
# intercepts. Only with this feature does single-instance forward that argv into
|
||||
# the deep-link plugin so `on_open_url` fires while the app is already running
|
||||
# (macOS uses Apple Events and is unaffected). Without it the link is silently
|
||||
# dropped. single-instance must also be registered FIRST (see main.rs), which it is.
|
||||
tauri-plugin-single-instance = { version = "2", features = ["deep-link"] }
|
||||
tauri-plugin-deep-link = "2"
|
||||
tauri-plugin-dialog = "2"
|
||||
tauri-plugin-notification = "2"
|
||||
tauri-plugin-updater = "2"
|
||||
tauri-plugin-process = "2"
|
||||
tauri-plugin-autostart = "2"
|
||||
|
||||
# 跨平台保持唤醒:阻止系统空闲休眠(macOS=IOPMAssertion via objc2-io-kit,
|
||||
# Windows=SetThreadExecutionState via windows crate)。仅阻止系统休眠,不阻止显示器息屏。
|
||||
keepawake = "0.6"
|
||||
|
||||
tokio.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
anyhow.workspace = true
|
||||
tracing.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
|
||||
# Tests build a real NTFS junction inside the fake legacy tree (the same
|
||||
# primitive nomifun-extension's skill_service materializes workspace skill
|
||||
# links with) to prove the data relocation skips links instead of failing.
|
||||
[target.'cfg(windows)'.dev-dependencies]
|
||||
junction = "1"
|
||||
@@ -0,0 +1,3 @@
|
||||
fn main() {
|
||||
tauri_build::build()
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"$schema": "../gen/schemas/desktop-schema.json",
|
||||
"identifier": "default",
|
||||
"description": "Default capability for the NomiFun desktop window.",
|
||||
"windows": ["main", "companion-*"],
|
||||
"permissions": [
|
||||
"core:default",
|
||||
"core:window:default",
|
||||
"core:window:allow-minimize",
|
||||
"core:window:allow-maximize",
|
||||
"core:window:allow-unmaximize",
|
||||
"core:window:allow-close",
|
||||
"core:window:allow-is-maximized",
|
||||
"core:window:allow-start-dragging",
|
||||
"core:window:allow-internal-toggle-maximize",
|
||||
"core:window:allow-show",
|
||||
"core:window:allow-hide",
|
||||
"core:window:allow-set-position",
|
||||
"core:window:allow-set-size",
|
||||
"core:window:allow-set-focus",
|
||||
"core:window:allow-outer-position",
|
||||
"core:window:allow-cursor-position",
|
||||
"core:window:allow-set-ignore-cursor-events",
|
||||
"core:webview:default",
|
||||
"core:webview:allow-set-webview-zoom",
|
||||
"core:app:default",
|
||||
"dialog:default",
|
||||
"dialog:allow-open",
|
||||
"dialog:allow-save",
|
||||
"notification:default",
|
||||
"notification:allow-notify",
|
||||
"notification:allow-is-permission-granted",
|
||||
"notification:allow-request-permission",
|
||||
"deep-link:default",
|
||||
"updater:default",
|
||||
"updater:allow-check",
|
||||
"updater:allow-download-and-install",
|
||||
"process:default",
|
||||
"autostart:default"
|
||||
]
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 7.3 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 15 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 6.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 8.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 8.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 17 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 19 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.8 KiB |
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 26 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 32 KiB |
@@ -0,0 +1,46 @@
|
||||
# ============================================================================
|
||||
# macOS 代码签名 + 公证(notarization)配置 —— 模板(可入库,无密钥)
|
||||
# ----------------------------------------------------------------------------
|
||||
# 用法:
|
||||
# 1. 复制本文件为同目录的 .env.signing(真实文件,已被 .gitignore,绝不入库):
|
||||
# cp apps/desktop/signing/.env.signing.example apps/desktop/signing/.env.signing
|
||||
# 2. 填入下面的真实值(来源见同目录 README.md)。
|
||||
# 3. 出带签名 + 公证的安装包:
|
||||
# bun run build:signed
|
||||
#
|
||||
# 普通 `bun run build` 不读这些变量,保持 ad-hoc 签名,任何人都能照常构建。
|
||||
# ============================================================================
|
||||
|
||||
|
||||
# ── 1) 签名身份(下面两种二选一)─────────────────────────────────────────
|
||||
|
||||
# 【推荐 / 本机开发】证书已导入「登录」钥匙串后,只填身份全名即可。
|
||||
# 取值:终端执行 security find-identity -v -p codesigning
|
||||
# 复制引号里的全名,形如:Developer ID Application: Your Name (TEAMID1234)
|
||||
# ⚠️ 必须是 "Developer ID Application" 类型,不能是 "Apple Development" / "Mac Developer"。
|
||||
APPLE_SIGNING_IDENTITY="Developer ID Application: YOUR NAME (YOURTEAMID)"
|
||||
|
||||
# 【或 / CI、无钥匙串】直接给 .p12 的 base64 内容(此时把上面的 IDENTITY 留空/注释掉)。
|
||||
# 导出:钥匙串里右键证书 → 导出为 .p12(设个导出口令)→ base64 -i DeveloperID.p12 | pbcopy
|
||||
# APPLE_CERTIFICATE=""
|
||||
# APPLE_CERTIFICATE_PASSWORD=""
|
||||
|
||||
|
||||
# ── 2) 公证 notarization(下面两种二选一)───────────────────────────────
|
||||
# 不配公证 → 别人下载后仍会报「无法验证开发者」(比「已损坏」好,但仍打不开)。
|
||||
# 要彻底解决,必须配公证。
|
||||
|
||||
# 【推荐】App Store Connect API Key 方式:
|
||||
# 生成:App Store Connect → Users and Access → Integrations → Keys
|
||||
# → 生成一个 "Developer" 角色的 Key → 下载 AuthKey_XXXX.p8(只能下一次!)
|
||||
# .p8 放到「仓库外」或下面这个已 gitignore 的目录;路径可相对仓库根,也可绝对路径。
|
||||
APPLE_API_ISSUER="00000000-0000-0000-0000-000000000000" # Issuer ID(keys 表格上方那串)
|
||||
APPLE_API_KEY="ABCDE12345" # Key ID(表格 "Key ID" 列)
|
||||
APPLE_API_KEY_PATH="apps/desktop/.tauri/AuthKey_ABCDE12345.p8" # .p8 文件路径
|
||||
|
||||
# 【或】Apple ID 方式(此时把上面三个 APPLE_API_* 留空/注释掉):
|
||||
# APPLE_PASSWORD 是「App 专用密码」,不是你的登录密码。
|
||||
# 生成:appleid.apple.com → 登录与安全 → App 专用密码 → 生成。
|
||||
# APPLE_ID="you@example.com"
|
||||
# APPLE_PASSWORD="xxxx-xxxx-xxxx-xxxx"
|
||||
# APPLE_TEAM_ID="YOURTEAMID"
|
||||
@@ -0,0 +1,113 @@
|
||||
# NomiFun 桌面 macOS 代码签名 + 公证(Gatekeeper)
|
||||
|
||||
> 解决「把安装包发给别人,对方打开提示**已损坏,无法打开**」的问题。
|
||||
>
|
||||
> 这跟 `updater/`(自动更新签名)是**两套完全不同的密钥**,别混。本目录只管
|
||||
> Apple 的 **Developer ID 签名 + 公证(notarization)**,让 App 在任何 Mac 上双击即开。
|
||||
|
||||
## 发布责任边界
|
||||
|
||||
本仓库只提供签名脚本和无密钥模板。正式发布必须使用发布方自己的 Apple
|
||||
Developer 账号、Developer ID 证书、App Store Connect API Key，以及独立的
|
||||
自动更新签名密钥。不要把 fork、本地开发机或历史测试密钥当成官方发布凭据。
|
||||
|
||||
## 为什么会「已损坏」
|
||||
|
||||
默认 `bun run build` 产出的 App 只是 **ad-hoc 签名**(`Signature=adhoc`,无
|
||||
`TeamIdentifier`)。别人下载/传输后,文件被打上 `com.apple.quarantine` 隔离标记;在
|
||||
Apple 芯片 Mac 上,被隔离 + 未正规签名公证的 App,Gatekeeper 直接判为「已损坏」。
|
||||
|
||||
**根治办法只有一个**:用 **Developer ID Application** 证书签名 → 提交 Apple **公证** →
|
||||
**staple** 把公证票据钉进 App。之后任何人下载双击即开,无任何提示。
|
||||
|
||||
## 密钥绝不入库(本仓库的约定)
|
||||
|
||||
| 东西 | 放哪 | 是否入库 |
|
||||
|---|---|---|
|
||||
| 模板 `.env.signing.example` | 本目录 | ✅ 入库(无密钥) |
|
||||
| 真实 `.env.signing`(身份名 / Key ID / 路径) | 本目录 | ❌ 已 gitignore |
|
||||
| App Store Connect API Key `AuthKey_*.p8` | `apps/desktop/.tauri/` 或仓库外 | ❌ 已 gitignore |
|
||||
| Developer ID 证书私钥 | macOS **登录钥匙串**(不是文件) | ❌ 不在仓库里 |
|
||||
|
||||
构建脚本 `scripts/desktop-build-signed.sh`(可入库,无密钥)在运行时 `source` 本地
|
||||
`.env.signing` 注入环境变量,Tauri 据此签名 + 公证。
|
||||
|
||||
---
|
||||
|
||||
## 一次性准备(在 Apple 侧)
|
||||
|
||||
### 1. 生成 Developer ID Application 证书并装进钥匙串
|
||||
- 最简单:用 **Xcode**(Settings → Accounts → 选中团队 → Manage Certificates → `+` →
|
||||
**Developer ID Application**),它会自动装进登录钥匙串。
|
||||
- 或 developer.apple.com → Certificates → `+` → **Developer ID Application** → 按引导用
|
||||
CSR 生成 → 下载 `.cer` 双击导入钥匙串。
|
||||
- 验证已就位:
|
||||
```bash
|
||||
security find-identity -v -p codesigning
|
||||
# 应能看到: "Developer ID Application: Your Name (TEAMID1234)"
|
||||
```
|
||||
把引号里的**全名**填到 `.env.signing` 的 `APPLE_SIGNING_IDENTITY`。
|
||||
|
||||
### 2. 生成 App Store Connect API Key(用于公证,推荐)
|
||||
- App Store Connect → **Users and Access** → **Integrations** → **Keys** → 生成一个
|
||||
**Developer** 角色的 Key。
|
||||
- 下载 `AuthKey_XXXX.p8`(**只能下载一次**),放到 `apps/desktop/.tauri/`(已 gitignore)
|
||||
或仓库外的安全目录。
|
||||
- 记下两个值填进 `.env.signing`:
|
||||
- **Issuer ID** = keys 表格**上方**那串 UUID → `APPLE_API_ISSUER`
|
||||
- **Key ID** = 表格 "Key ID" 列 → `APPLE_API_KEY`
|
||||
- `.p8` 路径 → `APPLE_API_KEY_PATH`
|
||||
|
||||
> 不想用 API Key 也可用 Apple ID 方式:`APPLE_ID` + `APPLE_PASSWORD`(App 专用密码,
|
||||
> 在 appleid.apple.com 生成)+ `APPLE_TEAM_ID`。三选一组,二者填其一即可。
|
||||
|
||||
---
|
||||
|
||||
## 本地配置 + 构建
|
||||
|
||||
```bash
|
||||
# 1. 复制模板(真实文件不入库)
|
||||
cp apps/desktop/signing/.env.signing.example apps/desktop/signing/.env.signing
|
||||
|
||||
# 2. 按上面拿到的值填写 .env.signing,并把 AuthKey_*.p8 放到对应路径
|
||||
|
||||
# 3. 出带签名 + 公证的安装包(公证联网,首次几分钟,耐心等)
|
||||
bun run build:signed
|
||||
```
|
||||
|
||||
产物在 `target/release/bundle/{macos,dmg}/`。构建末尾会先由 Tauri 公证并 staple
|
||||
`.app`,随后脚本会对最终分发用的 `.dmg` 再提交一次公证并 staple。
|
||||
|
||||
## 验证(发出去前自检)
|
||||
|
||||
```bash
|
||||
APP=target/release/bundle/macos/NomiFun.app
|
||||
DMG=target/release/bundle/dmg/NomiFun_0.1.0_aarch64.dmg
|
||||
|
||||
codesign -dvv "$APP" # 期望: Authority=Developer ID Application: ...
|
||||
codesign --verify --deep --strict -v "$APP" # 期望: valid on disk / satisfies Designated Requirement
|
||||
xcrun stapler validate "$APP" # 期望: The validate action worked!
|
||||
spctl -a -vvv "$APP" # 期望: source=Notarized Developer ID → accepted
|
||||
|
||||
codesign --verify --strict -v "$DMG" # 期望: valid on disk / satisfies Designated Requirement
|
||||
xcrun stapler validate "$DMG" # 期望: The validate action worked!
|
||||
spctl -a -vvv -t open --context context:primary-signature "$DMG" # 期望: accepted
|
||||
```
|
||||
|
||||
这些验证全过,就可以放心分发 DMG——别人下载双击即开,不再报「已损坏」。
|
||||
|
||||
## 常见报错
|
||||
|
||||
- **`The binary is not signed with a valid Developer ID certificate`**:钥匙串里没有
|
||||
Developer ID Application 证书,或 `APPLE_SIGNING_IDENTITY` 名字写错。重看准备步骤 1。
|
||||
- **`ambiguous (matches ... login.keychain-db and ... System.keychain)`**:同名
|
||||
Developer ID Application 证书同时存在于多个钥匙串。删除多余副本,或把
|
||||
`security find-identity -v -p codesigning` 输出中的 SHA-1 哈希填入
|
||||
`APPLE_SIGNING_IDENTITY`。
|
||||
- **`APPLE_API_KEY_PATH 必须指向 AuthKey_*.p8`**:`APPLE_API_KEY_PATH` 是 App Store
|
||||
Connect API Key 路径,不要填 Developer ID `.p12` 证书路径。
|
||||
- **公证被拒 / `Invalid` 状态**:多为「未启用 hardened runtime」或缺 entitlements。
|
||||
Tauri 用 Developer ID 签名时默认开启 hardened runtime;若 App 需要特殊能力(JIT、
|
||||
加载第三方动态库等),在 `tauri.conf.json` 的 `bundle.macOS.entitlements` 指定 plist。
|
||||
查看具体原因:`xcrun notarytool log <submission-id> --key ... --key-id ... --issuer ...`。
|
||||
- **只签名没公证**:别人会看到「无法验证开发者」(不是「已损坏」)。补上公证变量即可。
|
||||
@@ -0,0 +1,809 @@
|
||||
// Prevent an extra console window on Windows in release builds. Debug builds
|
||||
// keep the console so backend `tracing` logs are visible during development.
|
||||
#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
|
||||
|
||||
//! `nomifun-desktop` — the Tauri shell (replaces the Electron shell).
|
||||
//!
|
||||
//! Core idea (the whole point of this rewrite): there is NO spawned backend
|
||||
//! binary. The unified Rust backend (`nomifun-app`, ex-`nomicore`) is linked
|
||||
//! into THIS process and started in-process on a localhost port. The webview
|
||||
//! loads the bundled SPA (`ui/dist`) and talks to `http://127.0.0.1:<port>/api`
|
||||
//! exactly as it does today — so the renderer's ~295 HTTP calls are unchanged.
|
||||
//!
|
||||
//! ┌── nomifun-desktop (this process) ──────────────────────────┐
|
||||
//! │ Tauri shell (window/tray/dialog/deep-link/updater) │
|
||||
//! │ └─ tokio task: nomifun_app embedded axum on 127.0.0.1:<p> │
|
||||
//! │ WebView2/WKWebView/WebKitGTK ── HTTP ──▶ 127.0.0.1:<p>/api │
|
||||
//! └────────────────────────────────────────────────────────────┘
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use clap::Parser;
|
||||
use nomifun_app::{DesktopServer, WebUiStatus};
|
||||
use tauri::menu::{Menu, MenuItem};
|
||||
use tauri::tray::{MouseButton, MouseButtonState, TrayIconBuilder, TrayIconEvent};
|
||||
use tauri::{Emitter, Manager};
|
||||
use tauri_plugin_deep_link::DeepLinkExt;
|
||||
|
||||
mod relocate;
|
||||
|
||||
/// Build the webview initialization script. Injects the loopback backend port
|
||||
/// (`window.__backendPort`), the OS tag, and the per-boot local-trust secret
|
||||
/// (`window.__nomiLocalTrust`). Also installs `fetch` AND `XMLHttpRequest`
|
||||
/// interceptors that attach the trust header to EVERY request bound for the
|
||||
/// backend — so any code path (httpBridge, configService, raw `fetch`/XHR,
|
||||
/// multipart uploads with progress events, …) is trusted without per-call
|
||||
/// instrumentation, while requests to external origins are untouched (the
|
||||
/// secret never leaks off-box). Runs before any page script.
|
||||
fn webui_init_script(port: u16, trust_secret: &str) -> String {
|
||||
// `{:?}` emits properly quoted/escaped JS string literals.
|
||||
format!(
|
||||
r#"window.__backendPort = {port}; window.__os = {os:?}; window.__nomiLocalTrust = {secret:?};
|
||||
(function () {{
|
||||
var secret = {secret:?};
|
||||
var origin = "http://127.0.0.1:" + {port};
|
||||
if (!secret) return;
|
||||
function isBackend(url) {{
|
||||
url = url || "";
|
||||
return url.indexOf(origin) === 0 || url.charAt(0) === "/";
|
||||
}}
|
||||
if (window.fetch && !window.__nomiFetchPatched) {{
|
||||
window.__nomiFetchPatched = true;
|
||||
var origFetch = window.fetch.bind(window);
|
||||
window.fetch = function (input, init) {{
|
||||
try {{
|
||||
var url = typeof input === "string" ? input : (input && input.url) || "";
|
||||
if (isBackend(url)) {{
|
||||
init = init || {{}};
|
||||
var h = new Headers((init && init.headers) || undefined);
|
||||
if (!h.has("x-nomi-local-trust")) h.set("x-nomi-local-trust", secret);
|
||||
init.headers = h;
|
||||
if (typeof input !== "string") input = url;
|
||||
}}
|
||||
}} catch (e) {{}}
|
||||
return origFetch(input, init);
|
||||
}};
|
||||
}}
|
||||
var XHR = window.XMLHttpRequest;
|
||||
if (XHR && XHR.prototype && !window.__nomiXhrPatched) {{
|
||||
window.__nomiXhrPatched = true;
|
||||
var proto = XHR.prototype;
|
||||
var origOpen = proto.open;
|
||||
var origSetHeader = proto.setRequestHeader;
|
||||
var origSend = proto.send;
|
||||
proto.open = function (method, url) {{
|
||||
this.__nomiUrl = url;
|
||||
return origOpen.apply(this, arguments);
|
||||
}};
|
||||
proto.setRequestHeader = function (name) {{
|
||||
try {{ (this.__nomiHeaders || (this.__nomiHeaders = {{}}))[String(name).toLowerCase()] = true; }} catch (e) {{}}
|
||||
return origSetHeader.apply(this, arguments);
|
||||
}};
|
||||
proto.send = function () {{
|
||||
try {{
|
||||
if (isBackend(this.__nomiUrl) && !(this.__nomiHeaders && this.__nomiHeaders["x-nomi-local-trust"])) {{
|
||||
this.setRequestHeader("x-nomi-local-trust", secret);
|
||||
}}
|
||||
}} catch (e) {{}}
|
||||
return origSend.apply(this, arguments);
|
||||
}};
|
||||
}}
|
||||
}})();"#,
|
||||
os = std::env::consts::OS,
|
||||
secret = trust_secret,
|
||||
port = port,
|
||||
)
|
||||
}
|
||||
|
||||
/// Resolve the bundled SPA directory (`ui/dist`) served to remote browsers by
|
||||
/// the LAN listener. Probes a `NOMIFUN_WEBUI_DIST` override, several
|
||||
/// resource-dir layouts (production bundle), then dev-tree relatives. A
|
||||
/// candidate must contain `index.html` to be accepted; `None` means remote
|
||||
/// browsers get the API only (logged as a warning).
|
||||
fn resolve_webui_spa_dir(app: &tauri::App) -> Option<PathBuf> {
|
||||
if let Some(p) = std::env::var_os("NOMIFUN_WEBUI_DIST") {
|
||||
let p = PathBuf::from(p);
|
||||
if p.join("index.html").is_file() {
|
||||
return Some(p);
|
||||
}
|
||||
}
|
||||
let mut candidates: Vec<PathBuf> = Vec::new();
|
||||
if let Ok(res) = app.path().resource_dir() {
|
||||
candidates.push(res.join("webui-dist"));
|
||||
candidates.push(res.join("dist"));
|
||||
candidates.push(res.join("ui").join("dist"));
|
||||
// Tauri encodes `..` segments of a resource path as `_up_`.
|
||||
candidates.push(res.join("_up_").join("_up_").join("ui").join("dist"));
|
||||
}
|
||||
candidates.push(PathBuf::from("ui/dist"));
|
||||
candidates.push(PathBuf::from("../../ui/dist"));
|
||||
candidates.push(PathBuf::from("../ui/dist"));
|
||||
candidates.into_iter().find(|c| c.join("index.html").is_file())
|
||||
}
|
||||
|
||||
/// Data root resolution, in priority order:
|
||||
///
|
||||
/// 1. `NOMIFUN_DATA_DIR` env — explicit override; the shell appends `/Nomi`
|
||||
/// (semantics unchanged since the Electron era).
|
||||
/// 2. The shared per-host default from `nomifun_app::cli::default_data_dir()`:
|
||||
/// `%LOCALAPPDATA%\NomiFun\Nomi` on Windows, `~/Library/Application
|
||||
/// Support/NomiFun/Nomi` on macOS, `$XDG_DATA_HOME/NomiFun/Nomi` on Linux,
|
||||
/// with the historic `<system temp>/nomifun-data/Nomi` as the extreme
|
||||
/// fallback (installs that used to land there are auto-relocated, see
|
||||
/// `relocate.rs`). The web host and the `nomicore` bin resolve to the SAME
|
||||
/// directory, so dev loops and the installed app share one state.
|
||||
fn default_data_dir() -> PathBuf {
|
||||
if let Some(dir) = std::env::var_os("NOMIFUN_DATA_DIR") {
|
||||
return PathBuf::from(dir).join("Nomi");
|
||||
}
|
||||
nomifun_app::cli::default_data_dir()
|
||||
}
|
||||
|
||||
/// Updater scaffold: ask the configured update endpoint whether a newer signed
|
||||
/// release is available. Invoked from the renderer via
|
||||
/// `invoke("check_for_updates")`. Returns the new version string, or `null` if
|
||||
/// up to date. Inert until `plugins.updater.endpoints` in tauri.conf.json serves
|
||||
/// a valid `latest.json` signed with the project key
|
||||
/// (see apps/desktop/updater/README.md).
|
||||
#[tauri::command]
|
||||
async fn check_for_updates(app: tauri::AppHandle) -> Result<Option<String>, String> {
|
||||
use tauri_plugin_updater::UpdaterExt;
|
||||
let updater = app.updater().map_err(|e| e.to_string())?;
|
||||
match updater.check().await {
|
||||
Ok(Some(update)) => Ok(Some(update.version)),
|
||||
Ok(None) => Ok(None),
|
||||
Err(e) => Err(e.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Desired desktop-companion window, one per companion (multi-companion, spec §4.6).
|
||||
#[derive(serde::Deserialize)]
|
||||
struct CompanionWindowSpec {
|
||||
companion_id: String,
|
||||
enabled: bool,
|
||||
}
|
||||
|
||||
// ---- WebUI / LAN remote-access lifecycle commands -------------------------
|
||||
// The embedded backend always serves the app's own webview on loopback. These
|
||||
// commands toggle the SEPARATE on-demand LAN listener (`0.0.0.0`) so remote
|
||||
// browsers on the same network can reach the app by IP (with login). They are
|
||||
// async so they run on Tauri's async runtime — never blocking the main thread.
|
||||
|
||||
/// Current WebUI/LAN serving status (running, port, LAN IP, URL).
|
||||
#[tauri::command]
|
||||
fn webui_get_status(server: tauri::State<'_, Arc<DesktopServer>>) -> WebUiStatus {
|
||||
server.status()
|
||||
}
|
||||
|
||||
/// Start LAN serving (bind `0.0.0.0:25808`, fallback port if taken).
|
||||
#[tauri::command]
|
||||
async fn webui_start(server: tauri::State<'_, Arc<DesktopServer>>) -> Result<WebUiStatus, String> {
|
||||
let server = server.inner().clone();
|
||||
Ok(server.start_lan().await)
|
||||
}
|
||||
|
||||
/// Stop LAN serving (the loopback listener / desktop webview are unaffected).
|
||||
#[tauri::command]
|
||||
async fn webui_stop(server: tauri::State<'_, Arc<DesktopServer>>) -> Result<WebUiStatus, String> {
|
||||
let server = server.inner().clone();
|
||||
Ok(server.stop_lan().await)
|
||||
}
|
||||
|
||||
/// 持有当前生效的系统防休眠 assertion;`None`=允许休眠。
|
||||
/// Drop `KeepAwake` 即释放 assertion,所以进程退出/关闭开关都能干净恢复正常电源行为。
|
||||
/// Managed state holding the active OS sleep-inhibitor assertion (None = sleep allowed).
|
||||
struct AwakeState(Mutex<Option<keepawake::KeepAwake>>);
|
||||
|
||||
/// 获取"保持唤醒"的 OS assertion:仅阻止系统空闲休眠(PreventUserIdleSystemSleep),
|
||||
/// **不**阻止显示器空闲关闭 —— 等价 `caffeinate -i`(而非 `-di`)。电脑保持活动时屏幕仍可正常熄屏,
|
||||
/// 既省电也避免长时间常亮对屏幕(尤其 OLED)的损耗;熄屏不影响定时任务运行。
|
||||
/// `set_keep_awake` 与回归测试共用此单一来源。
|
||||
/// Acquire the keep-awake assertion: inhibit system idle sleep only, while letting the display
|
||||
/// sleep normally (≈ `caffeinate -i`, not `-di`) — saves power and avoids screen wear, and the
|
||||
/// display turning off does NOT pause scheduled tasks. Single source shared with the test.
|
||||
fn acquire_keep_awake() -> Result<keepawake::KeepAwake, String> {
|
||||
keepawake::Builder::default()
|
||||
.display(false) // 不持有 PreventUserIdleDisplaySleep:允许显示器空闲关闭(省电 + 护屏)
|
||||
.idle(true) // PreventUserIdleSystemSleep:系统保持唤醒;电池供电时同样生效
|
||||
.sleep(false) // PreventSystemSleep:已废弃 + 电池下被忽略,显式关闭
|
||||
.reason("NomiFun keep-awake enabled")
|
||||
.app_name("NomiFun")
|
||||
.app_reverse_domain("com.nomifun.desktop")
|
||||
.create()
|
||||
.map_err(|e| format!("failed to acquire keep-awake assertion: {e}"))
|
||||
}
|
||||
|
||||
/// 开启/关闭"保持唤醒":开盖状态下阻止系统空闲休眠,但允许显示器照常熄屏(等价 `caffeinate -i`)。
|
||||
/// macOS 硬限制:合盖属于"强制休眠"(forced sleep),任何 IOKit assertion 都拦不住(参见 Apple QA1340);
|
||||
/// 合盖仍要运行,只能 clamshell 模式(插电 + 外接显示器 + 外接键鼠)或 root 级 `pmset disablesleep 1`。
|
||||
/// 早先还持有 PreventUserIdleDisplaySleep(display=true)强制屏幕常亮,会阻止显示器关闭、徒增屏幕损耗,
|
||||
/// 现已去掉;PreventSystemSleep(sleep)自 macOS 10.9 起已废弃且电池下被忽略,同样不用。
|
||||
/// Keep-awake: with the lid OPEN, inhibit idle system sleep but let the display sleep (~`caffeinate -i`).
|
||||
/// Lid-close is forced sleep that no assertion can block; the old display-on assertion (which blocked
|
||||
/// the monitor from turning off) and the deprecated PreventSystemSleep are both gone.
|
||||
#[tauri::command]
|
||||
fn set_keep_awake(enabled: bool, state: tauri::State<'_, AwakeState>) -> Result<(), String> {
|
||||
let mut guard = state.0.lock().map_err(|e| e.to_string())?;
|
||||
if enabled {
|
||||
if guard.is_none() {
|
||||
*guard = Some(acquire_keep_awake()?);
|
||||
}
|
||||
} else {
|
||||
*guard = None; // Drop 释放 assertion / Drop releases the assertion.
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(all(test, target_os = "macos"))]
|
||||
mod keep_awake_tests {
|
||||
use super::acquire_keep_awake;
|
||||
use std::process::Command;
|
||||
use std::thread::sleep;
|
||||
use std::time::Duration;
|
||||
|
||||
/// 回归测试:保持唤醒必须只阻止系统空闲休眠,绝不阻止显示器关闭。
|
||||
/// 用真实 IOKit assertion + `pmset -g assertions` 验证 —— 只看本测试进程(按 pid 过滤)
|
||||
/// 自己持有的 assertion,因此不受同时运行的 App 实例或 `caffeinate` 干扰。
|
||||
/// Regression: keep-awake must hold PreventUserIdleSystemSleep but NOT
|
||||
/// PreventUserIdleDisplaySleep (the latter is what stops the monitor from turning off).
|
||||
#[test]
|
||||
fn holds_system_idle_assertion_but_not_display() {
|
||||
let handle = acquire_keep_awake().expect("acquire keep-awake assertion");
|
||||
let owner = format!("pid {}(", std::process::id());
|
||||
|
||||
// assertion 注册是同步的,但留一点重试余量以防极偶发的可见性延迟。
|
||||
let mut ours: Vec<String> = Vec::new();
|
||||
for _ in 0..10 {
|
||||
let out = Command::new("pmset")
|
||||
.args(["-g", "assertions"])
|
||||
.output()
|
||||
.expect("run `pmset -g assertions`");
|
||||
ours = String::from_utf8_lossy(&out.stdout)
|
||||
.lines()
|
||||
.filter(|l| l.contains(&owner))
|
||||
.map(str::to_owned)
|
||||
.collect();
|
||||
if ours.iter().any(|l| l.contains("PreventUserIdleSystemSleep")) {
|
||||
break;
|
||||
}
|
||||
sleep(Duration::from_millis(50));
|
||||
}
|
||||
|
||||
assert!(
|
||||
ours.iter().any(|l| l.contains("PreventUserIdleSystemSleep")),
|
||||
"keep-awake should hold PreventUserIdleSystemSleep; our assertions: {ours:?}"
|
||||
);
|
||||
assert!(
|
||||
!ours.iter().any(|l| l.contains("PreventUserIdleDisplaySleep")),
|
||||
"keep-awake must NOT hold PreventUserIdleDisplaySleep (it blocks the display from \
|
||||
turning off); our assertions: {ours:?}"
|
||||
);
|
||||
|
||||
drop(handle);
|
||||
}
|
||||
}
|
||||
|
||||
/// 关闭=收到托盘的护栏标志。仅在「真正退出」(托盘「退出」/`app.exit`)前置真,届时主窗口的
|
||||
/// `CloseRequested` 处理停止拦截、放行关闭;默认关闭手势(标题栏 ×、系统关闭、Alt+F4)保持假,
|
||||
/// 因此一律隐藏到托盘而非退出进程。
|
||||
/// Set true just before a real quit so the main window's CloseRequested handler stops
|
||||
/// intercepting; default close gestures leave it false and therefore hide to tray.
|
||||
struct QuitFlag(AtomicBool);
|
||||
|
||||
/// 托盘菜单两项的句柄,留存以便前端在 UI 语言就绪后本地化标签(见 `set_tray_labels`)。
|
||||
/// 创建时用英文兜底,确保渲染层挂载前托盘已可用。
|
||||
/// Handles to the two tray menu items so the renderer can localize their labels once the
|
||||
/// UI locale is known; built with English fallbacks so the tray works before the UI mounts.
|
||||
struct TrayMenuItems {
|
||||
show: MenuItem<tauri::Wry>,
|
||||
quit: MenuItem<tauri::Wry>,
|
||||
}
|
||||
|
||||
/// 把主窗口从托盘(或其他窗口背后)唤回:隐藏则显示、最小化则还原,并聚焦。
|
||||
/// Bring the main window back from the tray: show if hidden, restore if minimized, then focus.
|
||||
fn show_main_window(app: &tauri::AppHandle) {
|
||||
if let Some(window) = app.get_webview_window("main") {
|
||||
let _ = window.show();
|
||||
let _ = window.unminimize();
|
||||
let _ = window.set_focus();
|
||||
}
|
||||
}
|
||||
|
||||
fn should_show_main_window_for_macos_reopen(_has_visible_windows: bool) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn handle_run_event(app: &tauri::AppHandle, event: tauri::RunEvent) {
|
||||
match event {
|
||||
// Real app exit: tray-quit's `app.exit(0)`, the `Destroyed`→`exit(0)`
|
||||
// path, macOS Cmd-Q, and last-window-closed all surface here. Close-to-tray
|
||||
// uses `api.prevent_close()` in the `CloseRequested` handler so the window
|
||||
// is merely hidden and this event NEVER fires for it — which makes it safe
|
||||
// to wipe every terminal session here (kill PTYs + delete rows) with no
|
||||
// QuitFlag guard. Blocks briefly (≤3s) so the wipe finishes before exit.
|
||||
tauri::RunEvent::ExitRequested { .. } => {
|
||||
if let Some(server) = app.try_state::<Arc<DesktopServer>>() {
|
||||
server.shutdown_terminals_blocking();
|
||||
}
|
||||
}
|
||||
#[cfg(target_os = "macos")]
|
||||
tauri::RunEvent::Reopen { has_visible_windows, .. } => {
|
||||
if should_show_main_window_for_macos_reopen(has_visible_windows) {
|
||||
show_main_window(app);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
let _ = app;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 本地化原生托盘菜单。渲染层在挂载时及语言切换时调用,传入翻译后的 `tray.showWindow` /
|
||||
/// `tray.quit` 文案——Rust 侧无法自行解析 i18n,故创建时用英文兜底,随后采纳这些标签。
|
||||
/// Localize the native tray menu: the renderer hands over translated labels on mount and on
|
||||
/// language change (Rust can't resolve i18n itself, so it ships English fallbacks first).
|
||||
#[tauri::command]
|
||||
fn set_tray_labels(
|
||||
show: String,
|
||||
quit: String,
|
||||
items: tauri::State<'_, TrayMenuItems>,
|
||||
) -> Result<(), String> {
|
||||
items.show.set_text(show).map_err(|e| e.to_string())?;
|
||||
items.quit.set_text(quit).map_err(|e| e.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reconcile the native desktop-companion window set (labels `companion-{companion_id}`) against
|
||||
/// the desired specs sent by the main window (useCompanionWindowsSync):
|
||||
/// - `companion-*` windows whose companion is gone or disabled → close;
|
||||
/// - enabled companions without a window → create, hidden — the companion page shows the
|
||||
/// window itself once its config loads (window autonomy, unchanged);
|
||||
/// - windows already matching a desired spec are left untouched.
|
||||
/// Async on purpose: creating a webview from a *sync* command can deadlock on
|
||||
/// Windows (wry limitation).
|
||||
#[tauri::command]
|
||||
async fn sync_companion_windows(
|
||||
app: tauri::AppHandle,
|
||||
server: tauri::State<'_, Arc<DesktopServer>>,
|
||||
specs: Vec<CompanionWindowSpec>,
|
||||
) -> Result<(), String> {
|
||||
use std::collections::HashSet;
|
||||
|
||||
let known: HashSet<String> = specs
|
||||
.iter()
|
||||
.map(|s| format!("companion-{}", s.companion_id))
|
||||
.collect();
|
||||
let desired: HashSet<String> = specs
|
||||
.iter()
|
||||
.filter(|s| s.enabled)
|
||||
.map(|s| format!("companion-{}", s.companion_id))
|
||||
.collect();
|
||||
|
||||
// Reconcile existing companion windows. Disabling a companion HIDES (keeps)
|
||||
// its window rather than closing it: destroy-then-recreate raced with the
|
||||
// async close/lingering and could leave a re-enabled companion with NO
|
||||
// visible window ("隐藏后再点显示，桌面伙伴再也起不来"). Only a companion that no
|
||||
// longer exists at all (deleted) is closed/destroyed.
|
||||
for (label, window) in app.webview_windows() {
|
||||
if !label.starts_with("companion-") {
|
||||
continue;
|
||||
}
|
||||
if !known.contains(&label) {
|
||||
if let Err(e) = window.close() {
|
||||
tracing::warn!(error = %e, label = %label, "failed to close removed companion window");
|
||||
}
|
||||
} else if !desired.contains(&label) {
|
||||
if let Err(e) = window.hide() {
|
||||
tracing::warn!(error = %e, label = %label, "failed to hide disabled companion window");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure every enabled companion has a VISIBLE window: show the existing one
|
||||
// (it may be hidden from a previous disable / right-click hide), or create it
|
||||
// (hidden; the companion page shows itself once its config loads).
|
||||
let init_script = webui_init_script(server.loopback_port(), server.local_trust_secret());
|
||||
for spec in specs.iter().filter(|s| s.enabled) {
|
||||
let label = format!("companion-{}", spec.companion_id);
|
||||
if let Some(window) = app.get_webview_window(&label) {
|
||||
// Only show a window that is actually HIDDEN. Tauri's `show()` maps to
|
||||
// tao `set_visible(true)` → `makeKeyAndOrderFront`, which makes the
|
||||
// window the macOS *key* window — stealing keyboard focus from the
|
||||
// main window — and it does this even when the window is already
|
||||
// visible (no `isVisible()` short-circuit anywhere in the chain). This
|
||||
// sync also fires on the MAIN window's own `focus` event
|
||||
// (useCompanionWindowsSync), so an unconditional `show()` turned every
|
||||
// main-window refocus into a focus-steal loop ("点按钮/打字总被夺焦").
|
||||
// Re-showing an already-visible companion has no visible effect anyway,
|
||||
// so skipping it is purely the removal of the unwanted re-key. An
|
||||
// `is_visible()` error biases toward showing (visibility correctness
|
||||
// outweighs a rare extra steal — a companion that won't appear is the
|
||||
// worse bug).
|
||||
if !window.is_visible().unwrap_or(false) {
|
||||
if let Err(e) = window.show() {
|
||||
tracing::warn!(error = %e, label = %label, "failed to show enabled companion window");
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let url = format!("index.html#/companion?companionId={}", spec.companion_id);
|
||||
let builder =
|
||||
tauri::WebviewWindowBuilder::new(&app, &label, tauri::WebviewUrl::App(url.into()))
|
||||
// Placeholder title for the brief pre-load frame; the companion page
|
||||
// overwrites it with the companion's custom name once its profile loads
|
||||
// (see setTitle in pages/companion/index.tsx). Never the lowercase engine id.
|
||||
.title("NomiFun")
|
||||
// Matches DEFAULT_DESK (characters/index.ts): figure + minimal chrome,
|
||||
// no reserved bubble headroom (the page grows the window on demand).
|
||||
// Keeping these in sync avoids a visible startup resize for built-ins.
|
||||
.inner_size(240.0, 214.0)
|
||||
.resizable(false)
|
||||
.decorations(false)
|
||||
.transparent(true)
|
||||
.always_on_top(true)
|
||||
.skip_taskbar(true)
|
||||
.shadow(false)
|
||||
.visible(false)
|
||||
.initialization_script(&init_script);
|
||||
// Show the freshly-built window from here rather than relying SOLELY on
|
||||
// the companion page's self-show (applyWindowState): on a FIRST enable
|
||||
// (no window existed, so we land in this create branch) the page init
|
||||
// can stall/fail (transient boot 5xx in its Promise.all, the configReady
|
||||
// gate, a wry build hiccup) and — since no later event re-enters the
|
||||
// create branch — the window would stay hidden forever ("开启桌面显示但
|
||||
// 桌面伙伴不出现"). The page's own show() is idempotent and still handles
|
||||
// position correction + later enable/disable toggles. A failed build
|
||||
// self-heals on the next sync.
|
||||
match builder.build() {
|
||||
Ok(window) => {
|
||||
// 创建即默认整窗穿透（安全侧）：JS 点击穿透轮询(useCompanionClickThrough)
|
||||
// 还没起、或 stale/dev 下 outerPosition 抛错卡死时，透明窗也不挡底层点击。
|
||||
// 命中立绘时由轮询切回 false。
|
||||
if let Err(e) = window.set_ignore_cursor_events(true) {
|
||||
tracing::warn!(error = %e, label = %label, "failed to set ignore-cursor on companion window");
|
||||
}
|
||||
if let Err(e) = window.show() {
|
||||
tracing::warn!(error = %e, label = %label, "failed to show created companion window");
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::warn!(error = %e, label = %label, "failed to create companion window"),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn main() -> std::process::ExitCode {
|
||||
// If an ACP agent CLI spawned this shell as an MCP stdio bridge
|
||||
// (`current_exe() mcp-requirement-stdio` etc.), run that helper and exit
|
||||
// BEFORE any runtime init, single-instance handling, or window creation.
|
||||
// Every host binary must honor these or the injected declaration tools
|
||||
// (requirement_complete / team / guide) never appear in the agent's session.
|
||||
if let Some(code) = nomifun_app::commands::run_mcp_stdio_subcommand_if_present() {
|
||||
return code;
|
||||
}
|
||||
|
||||
// Env mutation + runtime init BEFORE Tauri builds its runtime/threads,
|
||||
// mirroring the nomicore bin's ordering. The relocation (legacy temp dir →
|
||||
// per-user app-data, one-shot) must run first of all: everything below —
|
||||
// runtime cache, backend cli, embedded server — keys off the data dir it
|
||||
// returns. On relocation failure it falls back to the legacy dir.
|
||||
let data_dir = relocate::effective_data_dir(default_data_dir());
|
||||
nomifun_runtime::init(&data_dir);
|
||||
// SAFETY: no worker threads exist yet (Tauri's runtime is built by .run()).
|
||||
let merged_path = unsafe { nomifun_runtime::enhance_process_path() };
|
||||
|
||||
// Backend config. The desktop does NOT use `--local`: `DesktopServer::start`
|
||||
// runs the backend under `TrustLocalToken` (trusts only its own webview via
|
||||
// a per-boot secret) so the LAN listener can require login. Only the data
|
||||
// dir + log level flow from here; the listeners bind their own ports.
|
||||
let mut cli = nomifun_app::cli::Cli::parse_from(["nomifun-desktop"]);
|
||||
cli.data_dir = data_dir;
|
||||
// Opt-in verbose backend logging without a custom build, e.g.
|
||||
// NOMI_LOG_LEVEL=debug (everything)
|
||||
// NOMI_LOG_LEVEL=info (default)
|
||||
// At `debug`, the `nomi_providers` target logs the outgoing request body and
|
||||
// each SSE chunk, and `nomi_mcp` logs MCP connect results — exactly what is
|
||||
// needed to diagnose a provider/gateway stall. Console output appears in the
|
||||
// terminal that launched `tauri dev`; it is also written to the log files
|
||||
// under {data-dir}/logs/.
|
||||
if let Ok(level) = std::env::var("NOMI_LOG_LEVEL") {
|
||||
let level = level.trim();
|
||||
if !level.is_empty() {
|
||||
cli.log_level = Some(level.to_owned());
|
||||
}
|
||||
}
|
||||
|
||||
let app = tauri::Builder::default()
|
||||
// single-instance MUST be the first plugin. With its `deep-link` feature
|
||||
// enabled (see Cargo.toml), it forwards a second instance's argv into the
|
||||
// deep-link plugin BEFORE invoking this callback, so `on_open_url` (wired
|
||||
// in setup) fires on its own. We still use the callback to surface the
|
||||
// existing window: a second launch usually means the app is hidden in the
|
||||
// tray (close-to-tray), so bring it back instead of silently no-op'ing.
|
||||
// (Schemes are statically configured in tauri.conf.json, so there is no
|
||||
// need to re-parse argv here.)
|
||||
.plugin(tauri_plugin_single_instance::init(|app, _argv, _cwd| {
|
||||
show_main_window(app);
|
||||
}))
|
||||
.plugin(tauri_plugin_dialog::init())
|
||||
.plugin(tauri_plugin_notification::init())
|
||||
.plugin(tauri_plugin_updater::Builder::new().build())
|
||||
.plugin(tauri_plugin_process::init())
|
||||
.plugin(tauri_plugin_autostart::init(
|
||||
tauri_plugin_autostart::MacosLauncher::LaunchAgent,
|
||||
None::<Vec<&str>>,
|
||||
))
|
||||
.plugin(tauri_plugin_deep_link::init())
|
||||
.setup(move |app| {
|
||||
// Resolve the bundled SPA dir for serving the app shell to remote
|
||||
// browsers over the LAN listener (loopback webview loads via the
|
||||
// Tauri asset protocol and does not need this).
|
||||
let spa_dir = resolve_webui_spa_dir(app);
|
||||
// In dev, the desktop webview loads the live vite dev server; serving
|
||||
// the (stale) bundled `ui/dist` to remote browsers would desync them
|
||||
// from the desktop. So in dev the LAN listener proxies the SPA to vite
|
||||
// instead. In production this is None and the bundled dist is served.
|
||||
let dev_frontend_url: Option<String> = if tauri::is_dev() {
|
||||
app.config()
|
||||
.build
|
||||
.dev_url
|
||||
.as_ref()
|
||||
.map(|u| u.to_string())
|
||||
.or_else(|| Some("http://localhost:5173".to_string()))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if spa_dir.is_none() && dev_frontend_url.is_none() {
|
||||
tracing::warn!(
|
||||
"WebUI SPA directory not found — remote browsers would receive the API but no app shell"
|
||||
);
|
||||
}
|
||||
|
||||
// Hand the backend control handle (port + trust secret + LAN
|
||||
// lifecycle) back to this thread once the loopback listener is bound.
|
||||
let (boot_tx, boot_rx) = std::sync::mpsc::channel::<Arc<DesktopServer>>();
|
||||
let backend_err_handle = app.handle().clone();
|
||||
let status_emit_handle = app.handle().clone();
|
||||
std::thread::Builder::new()
|
||||
.name("nomifun-backend".into())
|
||||
.spawn(move || {
|
||||
let run = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| -> anyhow::Result<()> {
|
||||
let rt = tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.map_err(|e| anyhow::anyhow!("failed to build backend runtime: {e}"))?;
|
||||
rt.block_on(async move {
|
||||
let (server, _keep_alive) =
|
||||
DesktopServer::start(&cli, &merged_path, spa_dir, dev_frontend_url).await?;
|
||||
// Unblock the main thread's window build.
|
||||
let _ = boot_tx.send(server.clone());
|
||||
// Forward LAN status changes to the renderer. Holding
|
||||
// `server` + `_keep_alive` here keeps the backend (and
|
||||
// this runtime) alive for the process lifetime.
|
||||
let mut rx = server.subscribe_status();
|
||||
while rx.changed().await.is_ok() {
|
||||
let status = rx.borrow().clone();
|
||||
let _ = status_emit_handle.emit("webui://status-changed", status);
|
||||
}
|
||||
drop(_keep_alive);
|
||||
Ok::<(), anyhow::Error>(())
|
||||
})
|
||||
}));
|
||||
let error = match run {
|
||||
Ok(Ok(())) => return, // clean shutdown
|
||||
Ok(Err(e)) => format!("{e:#}"),
|
||||
Err(panic) => panic
|
||||
.downcast_ref::<&str>()
|
||||
.map(|s| (*s).to_owned())
|
||||
.or_else(|| panic.downcast_ref::<String>().cloned())
|
||||
.unwrap_or_else(|| "backend thread panicked".to_owned()),
|
||||
};
|
||||
tracing::error!(error = %error, "embedded backend exited with error");
|
||||
use tauri_plugin_dialog::{DialogExt, MessageDialogKind};
|
||||
backend_err_handle
|
||||
.dialog()
|
||||
.message(error)
|
||||
.title("NomiFun backend failed to start")
|
||||
.kind(MessageDialogKind::Error)
|
||||
.blocking_show();
|
||||
backend_err_handle.exit(1);
|
||||
})
|
||||
.expect("failed to spawn backend thread");
|
||||
|
||||
// Wait for the backend to bind its loopback listener (or fail) before
|
||||
// building the window — the init script needs the port + trust
|
||||
// secret. A recv error means the backend failed; it has already shown
|
||||
// a dialog and will exit, so we just stop building the window.
|
||||
let Ok(server) = boot_rx.recv() else {
|
||||
return Ok(());
|
||||
};
|
||||
let loopback_port = server.loopback_port();
|
||||
|
||||
// Build the main window programmatically so we can inject the backend
|
||||
// port + local-trust secret via an INITIALIZATION SCRIPT — it runs
|
||||
// before any page script, so the renderer's first `getBaseUrl()` (and
|
||||
// its trust-header attach) always see them. Race-free (unlike
|
||||
// eval-after-load).
|
||||
//
|
||||
// Frameless on Windows/Linux: the React titlebar draws its own
|
||||
// min/max/close (via @tauri-apps/api/window) on the same row as the
|
||||
// app's nav buttons. macOS keeps native traffic-light buttons via the
|
||||
// Overlay title-bar style, with content extending under the bar.
|
||||
// resizable defaults to true, so edge-resize + Snap are retained even
|
||||
// without decorations on Windows.
|
||||
let init_script = webui_init_script(loopback_port, server.local_trust_secret());
|
||||
app.manage(server);
|
||||
let win_builder =
|
||||
tauri::WebviewWindowBuilder::new(app, "main", tauri::WebviewUrl::App("index.html".into()))
|
||||
.title("NomiFun")
|
||||
.inner_size(1280.0, 832.0)
|
||||
.min_inner_size(880.0, 600.0)
|
||||
.initialization_script(&init_script);
|
||||
// macOS: Overlay makes the titlebar transparent + extends content under
|
||||
// it, but it does NOT hide the native title text. With the title still
|
||||
// set to "NomiFun", AppKit draws that string next to the traffic lights,
|
||||
// overlapping the React sidebar toggle. `hidden_title(true)` maps to
|
||||
// `setTitleVisibility(Hidden)` so the OS keeps the title for menus /
|
||||
// Mission Control while leaving the titlebar visually empty.
|
||||
//
|
||||
// Vertically center the traffic lights on the React toolbar's button
|
||||
// line. The React titlebar (`.app-titlebar--mac`, height 45px in
|
||||
// ui/.../titlebar.css) centers its 36px buttons at y≈22.5px from the
|
||||
// window top, but AppKit's default places the 16px lights at center
|
||||
// y≈16px — ~6.5px too high. tao's `inset_traffic_lights` (view.rs)
|
||||
// makes `y` the height of the button *container* (16 + y) and
|
||||
// bottom-anchors the lights in it with a ~10px margin, so the lights'
|
||||
// center-from-top works out to (y - 2). To land the center at 22.5px:
|
||||
// y = 24.5 (empirically verified via the Accessibility API).
|
||||
//
|
||||
// Horizontally, `x` is the left edge of the close button's frame
|
||||
// (tao sets `rect.origin.x = x` per button). That frame is 14px
|
||||
// wide with the visible 12px circle centered in it (1px each
|
||||
// side), so the circle's left gap from the window edge is x + 1.
|
||||
// Balance that gap with the vertical whitespace around the lights:
|
||||
// (45 - 12) / 2 = 16.5px above/below the circle, hence
|
||||
// x = 16.5 - 1 = 15.5. (AppKit's native ~8px inset assumes a 28px
|
||||
// titlebar and looks glued to the corner in a 45px one; Apple's
|
||||
// own apps use ~16-20px in tall toolbars.) The lights then span up
|
||||
// to zoom's right edge at 15.5 + 2*20 + 14 = 69.5px, still clear
|
||||
// of the React menu, which starts at 84px (8px titlebar padding +
|
||||
// 76px margin-left in Titlebar/index.tsx).
|
||||
// (`traffic_light_position` requires Overlay + decorations:true, both set.)
|
||||
#[cfg(target_os = "macos")]
|
||||
let win_builder = win_builder
|
||||
.title_bar_style(tauri::TitleBarStyle::Overlay)
|
||||
.hidden_title(true)
|
||||
.traffic_light_position(tauri::LogicalPosition::new(15.5, 24.5));
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
let win_builder = win_builder.decorations(false);
|
||||
win_builder.build()?;
|
||||
|
||||
// System tray. Closing the main window HIDES it here instead of
|
||||
// quitting (see the CloseRequested handler in on_window_event); the
|
||||
// process truly exits only via the tray's "退出" item. Left-click the
|
||||
// icon to bring the window back; right-click for the Show/Quit menu.
|
||||
// Labels are English fallbacks, adopted from the renderer's locale via
|
||||
// `set_tray_labels` once it mounts (the renderer always loads before
|
||||
// the user can close, so the first menu open is already localized).
|
||||
let tray_show = MenuItem::with_id(app, "tray-show", "Show NomiFun", true, None::<&str>)?;
|
||||
let tray_quit = MenuItem::with_id(app, "tray-quit", "Quit", true, None::<&str>)?;
|
||||
let tray_menu = Menu::with_items(app, &[&tray_show, &tray_quit])?;
|
||||
app.manage(TrayMenuItems {
|
||||
show: tray_show.clone(),
|
||||
quit: tray_quit.clone(),
|
||||
});
|
||||
let mut tray_builder = TrayIconBuilder::with_id("nomi-tray")
|
||||
.tooltip("NomiFun")
|
||||
.menu(&tray_menu)
|
||||
// Left-click is reserved for "surface the window"; the menu is
|
||||
// right-click only (otherwise a left-click would both pop the menu
|
||||
// AND try to show the window).
|
||||
.show_menu_on_left_click(false)
|
||||
.on_menu_event(|app, event| match event.id.as_ref() {
|
||||
"tray-show" => show_main_window(app),
|
||||
"tray-quit" => {
|
||||
// Arm the quit guard FIRST, then exit — the CloseRequested
|
||||
// handler checks this flag and stops hiding-to-tray.
|
||||
app.state::<QuitFlag>().0.store(true, Ordering::SeqCst);
|
||||
app.exit(0);
|
||||
}
|
||||
_ => {}
|
||||
})
|
||||
.on_tray_icon_event(|tray, event| {
|
||||
if let TrayIconEvent::Click {
|
||||
button: MouseButton::Left,
|
||||
button_state: MouseButtonState::Up,
|
||||
..
|
||||
} = event
|
||||
{
|
||||
show_main_window(tray.app_handle());
|
||||
}
|
||||
});
|
||||
// Reuse the app's bundled window icon for the tray (no extra asset).
|
||||
if let Some(icon) = app.default_window_icon() {
|
||||
tray_builder = tray_builder.icon(icon.clone());
|
||||
}
|
||||
tray_builder.build(app)?;
|
||||
|
||||
// Desktop-companion windows are NOT created here anymore. They are
|
||||
// multi-companion and dynamic: the main window's useCompanionWindowsSync hook
|
||||
// invokes `sync_companion_windows` (above) on boot and on companion
|
||||
// created/deleted/config-updated events, reconciling one
|
||||
// transparent always-on-top `companion-{companion_id}` window per enabled companion.
|
||||
|
||||
// Wire deep-link open-url events to a Tauri event the renderer can
|
||||
// `listen()` to. `register_all()` is best-effort (some platforms /
|
||||
// dev contexts need it; ignore the error if it fails).
|
||||
let handle = app.handle().clone();
|
||||
let _ = app.deep_link().register_all();
|
||||
app.deep_link().on_open_url(move |event| {
|
||||
let urls: Vec<String> = event.urls().iter().map(|u| u.to_string()).collect();
|
||||
let _ = handle.emit("deep-link://received", urls);
|
||||
});
|
||||
Ok(())
|
||||
})
|
||||
// The ~38 OS-shell commands (window controls, tray, zoom, get-path,
|
||||
// feedback, auto-update status) register here as #[tauri::command]s (P3).
|
||||
.manage(AwakeState(Mutex::new(None)))
|
||||
.manage(QuitFlag(AtomicBool::new(false)))
|
||||
.invoke_handler(tauri::generate_handler![
|
||||
check_for_updates,
|
||||
sync_companion_windows,
|
||||
webui_get_status,
|
||||
webui_start,
|
||||
webui_stop,
|
||||
set_keep_awake,
|
||||
set_tray_labels
|
||||
])
|
||||
// Close-to-tray is now the DEFAULT (and only) close behavior. Closing the
|
||||
// main window (titlebar ×, OS close, Alt+F4) hides it to the tray instead
|
||||
// of quitting — the agent, scheduled tasks, and companions keep running in
|
||||
// the background. The process exits ONLY via the tray's "退出" item, which
|
||||
// arms QuitFlag and calls app.exit(0); with the flag set we let the close
|
||||
// proceed and the Destroyed arm tears the process down (the always-on-top
|
||||
// companion windows would otherwise keep it — and a floating companion —
|
||||
// alive after the main window is gone).
|
||||
.on_window_event(|window, event| {
|
||||
if window.label() != "main" {
|
||||
return;
|
||||
}
|
||||
match event {
|
||||
tauri::WindowEvent::CloseRequested { api, .. } => {
|
||||
let quitting = window.app_handle().state::<QuitFlag>().0.load(Ordering::SeqCst);
|
||||
if !quitting {
|
||||
api.prevent_close();
|
||||
let _ = window.hide();
|
||||
}
|
||||
}
|
||||
tauri::WindowEvent::Destroyed => {
|
||||
window.app_handle().exit(0);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
})
|
||||
.build(tauri::generate_context!())
|
||||
.expect("error while building tauri application");
|
||||
|
||||
// `Builder::run(context)` installs an empty app-level event callback. Build
|
||||
// manually so a Dock click after close-to-tray can surface the hidden main window.
|
||||
app.run(handle_run_event);
|
||||
|
||||
std::process::ExitCode::SUCCESS
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn macos_reopen_surfaces_main_window_when_no_windows_are_visible() {
|
||||
assert!(should_show_main_window_for_macos_reopen(false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn macos_reopen_surfaces_main_window_even_when_companion_window_is_visible() {
|
||||
assert!(should_show_main_window_for_macos_reopen(true));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"$schema": "https://schema.tauri.app/config/2",
|
||||
"productName": "NomiFun",
|
||||
"version": "0.1.0",
|
||||
"identifier": "com.nomifun.desktop",
|
||||
"build": {
|
||||
"frontendDist": "../../ui/dist",
|
||||
"devUrl": "http://localhost:5173",
|
||||
"beforeDevCommand": "bun run --filter=./ui dev",
|
||||
"beforeBuildCommand": "bun scripts/prune-build.mjs --pre && bun run --filter=./ui build"
|
||||
},
|
||||
"app": {
|
||||
"windows": [],
|
||||
"macOSPrivateApi": true,
|
||||
"security": {
|
||||
"csp": null
|
||||
}
|
||||
},
|
||||
"bundle": {
|
||||
"active": true,
|
||||
"targets": "all",
|
||||
"resources": ["../../ui/dist"],
|
||||
"icon": [
|
||||
"icons/32x32.png",
|
||||
"icons/128x128.png",
|
||||
"icons/128x128@2x.png",
|
||||
"icons/icon.icns",
|
||||
"icons/icon.ico"
|
||||
],
|
||||
"windows": {
|
||||
"webviewInstallMode": {
|
||||
"type": "downloadBootstrapper",
|
||||
"silent": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"plugins": {
|
||||
"deep-link": {
|
||||
"desktop": {
|
||||
"schemes": ["nomifun"]
|
||||
}
|
||||
},
|
||||
"updater": {
|
||||
"endpoints": [
|
||||
"https://REPLACE-WITH-YOUR-HOST/nomifun/updates/latest.json"
|
||||
],
|
||||
"pubkey": "dW50cnVzdGVkIGNvbW1lbnQ6IG1pbmlzaWduIHB1YmxpYyBrZXk6IDhGOTlCMjEwMzg2MDFFRDgKUldUWUhtQTRFTEtaajNCVlBFS3ZvVzFheVo5RkttMnJnNnk4b3gycS95MFZNdnlvRjhMRG5nUDcK"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"$schema": "https://schema.tauri.app/config/2",
|
||||
"productName": "NomiFun Dev",
|
||||
"identifier": "com.nomifun.desktop.dev",
|
||||
"plugins": {
|
||||
"deep-link": {
|
||||
"desktop": {
|
||||
"schemes": ["nomifun-dev"]
|
||||
}
|
||||
},
|
||||
"updater": {
|
||||
"endpoints": [
|
||||
"http://127.0.0.1:59999/nomifun-dev-channel-has-no-update-feed/latest.json"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
# NomiFun Desktop Updater
|
||||
|
||||
This directory documents the Tauri updater wiring for `nomifun-desktop`.
|
||||
|
||||
The updater is a scaffold, not a ready production release channel. The plugin
|
||||
is installed and the desktop command exists, but public releases still require
|
||||
release-owner credentials, a real HTTPS endpoint, and a signing key pair that
|
||||
is controlled outside the repository.
|
||||
|
||||
## Current State
|
||||
|
||||
- Rust plugin: `apps/desktop/Cargo.toml` includes `tauri-plugin-updater`.
|
||||
- Frontend package: `ui/package.json` includes `@tauri-apps/plugin-updater`.
|
||||
- Tauri config: `apps/desktop/tauri.conf.json` contains
|
||||
`plugins.updater.endpoints` and `plugins.updater.pubkey`.
|
||||
- Desktop command: `check_for_updates` is exposed through Tauri invoke and
|
||||
returns a version string or `null`.
|
||||
- Build script: `bun run build:updater` produces updater signatures (`.sig`)
|
||||
next to release installers when the signing environment variables are set.
|
||||
|
||||
The configured endpoint is a placeholder. The configured pubkey must be treated
|
||||
as a development value unless the release owner has explicitly replaced it and
|
||||
stored the matching private key in release infrastructure.
|
||||
|
||||
## Required Before Public Release
|
||||
|
||||
1. Generate an updater signing key pair owned by the release owner:
|
||||
|
||||
```bash
|
||||
bun x tauri signer generate -w <private-key-output-path>
|
||||
```
|
||||
|
||||
2. Put the printed public key in `plugins.updater.pubkey` in
|
||||
`apps/desktop/tauri.conf.json`.
|
||||
3. Store the private key and password in CI or another release-secret store.
|
||||
Never commit them.
|
||||
4. Replace `plugins.updater.endpoints` with a real HTTPS URL that serves
|
||||
`latest.json`.
|
||||
5. Build release artifacts with updater signing enabled.
|
||||
6. Upload installers and signatures to your release hosting.
|
||||
7. Publish `latest.json` with the correct URLs and signatures.
|
||||
|
||||
Updater signing is separate from OS code signing. macOS Developer ID signing and
|
||||
notarization are documented in `apps/desktop/signing/README.md`. Windows
|
||||
SmartScreen reputation still requires an external code-signing certificate and
|
||||
publisher reputation.
|
||||
|
||||
## Build A Signed Update Artifact
|
||||
|
||||
Set the private key content, not a path:
|
||||
|
||||
```bash
|
||||
export TAURI_SIGNING_PRIVATE_KEY="$(cat <private-key-output-path>)"
|
||||
export TAURI_SIGNING_PRIVATE_KEY_PASSWORD="<private-key-password-or-empty>"
|
||||
bun run build:updater
|
||||
```
|
||||
|
||||
Artifacts land under `target/release/bundle/`. Each installer that supports
|
||||
updates gets a sibling `.sig` file, for example:
|
||||
|
||||
```text
|
||||
target/release/bundle/nsis/NomiFun_0.1.1_x64-setup.exe
|
||||
target/release/bundle/nsis/NomiFun_0.1.1_x64-setup.exe.sig
|
||||
```
|
||||
|
||||
Copy the full `.sig` file content into the matching platform entry in
|
||||
`latest.json`.
|
||||
|
||||
## `latest.json`
|
||||
|
||||
The Tauri updater expects a manifest similar to:
|
||||
|
||||
```json
|
||||
{
|
||||
"version": "0.1.1",
|
||||
"notes": "Release notes for users.",
|
||||
"pub_date": "2026-06-24T00:00:00Z",
|
||||
"platforms": {
|
||||
"windows-x86_64": {
|
||||
"signature": "<contents-of-.sig>",
|
||||
"url": "https://example.com/downloads/NomiFun_0.1.1_x64-setup.exe"
|
||||
},
|
||||
"darwin-aarch64": {
|
||||
"signature": "<contents-of-.sig>",
|
||||
"url": "https://example.com/downloads/NomiFun_0.1.1_aarch64.dmg"
|
||||
},
|
||||
"linux-x86_64": {
|
||||
"signature": "<contents-of-.sig>",
|
||||
"url": "https://example.com/downloads/nomifun_0.1.1_amd64.AppImage"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Common platform keys are:
|
||||
|
||||
- `windows-x86_64`
|
||||
- `darwin-x86_64`
|
||||
- `darwin-aarch64`
|
||||
- `linux-x86_64`
|
||||
|
||||
## Client Behavior
|
||||
|
||||
The current command only checks for an available update:
|
||||
|
||||
```ts
|
||||
import { invoke } from "@tauri-apps/api/core";
|
||||
|
||||
const newVersion = await invoke<string | null>("check_for_updates");
|
||||
```
|
||||
|
||||
Download/install UX can be implemented either in the frontend with
|
||||
`@tauri-apps/plugin-updater` (`check`, `downloadAndInstall`) or in the Rust
|
||||
command by calling `download_and_install` after a user confirms the action.
|
||||
|
||||
## Safety Checklist
|
||||
|
||||
- Private updater key is stored only in release secrets.
|
||||
- `plugins.updater.pubkey` matches the private key used by CI.
|
||||
- `plugins.updater.endpoints` points to HTTPS.
|
||||
- `latest.json` contains real installer URLs and exact signature contents.
|
||||
- OS code signing and notarization are handled separately for each platform.
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"version": "0.1.1",
|
||||
"notes": "示例发布说明 — 替换为真实更新内容。",
|
||||
"pub_date": "2026-06-08T00:00:00Z",
|
||||
"platforms": {
|
||||
"windows-x86_64": {
|
||||
"signature": "<<粘贴 .exe.sig 内容>>",
|
||||
"url": "https://REPLACE-WITH-YOUR-HOST/nomifun/updates/NomiFun_0.1.1_x64-setup.exe"
|
||||
},
|
||||
"darwin-x86_64": {
|
||||
"signature": "<<粘贴 .app.tar.gz.sig 内容>>",
|
||||
"url": "https://REPLACE-WITH-YOUR-HOST/nomifun/updates/NomiFun_0.1.1_x64.app.tar.gz"
|
||||
},
|
||||
"darwin-aarch64": {
|
||||
"signature": "<<粘贴 .app.tar.gz.sig 内容>>",
|
||||
"url": "https://REPLACE-WITH-YOUR-HOST/nomifun/updates/NomiFun_0.1.1_aarch64.app.tar.gz"
|
||||
},
|
||||
"linux-x86_64": {
|
||||
"signature": "<<粘贴 .AppImage.sig 内容>>",
|
||||
"url": "https://REPLACE-WITH-YOUR-HOST/nomifun/updates/NomiFun_0.1.1_amd64.AppImage"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
[package]
|
||||
name = "nomifun-web"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "nomifun-web"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
# The unified backend, consumed in-process as a library (no spawned binary).
|
||||
# P2 will expose nomifun_app's boot entry (init_environment / data layer /
|
||||
# AppServices) so this host can mount the real /api router. For now this host
|
||||
# serves the built SPA (ui/dist) and a health endpoint.
|
||||
nomifun-app.workspace = true
|
||||
nomifun-runtime.workspace = true
|
||||
|
||||
tokio.workspace = true
|
||||
axum.workspace = true
|
||||
tower-http = { workspace = true, features = ["fs", "trace"] }
|
||||
tracing.workspace = true
|
||||
anyhow.workspace = true
|
||||
clap.workspace = true
|
||||
@@ -0,0 +1,213 @@
|
||||
//! `nomifun-web` — the standalone Web host for the browser deployment.
|
||||
//!
|
||||
//! In the unified architecture there is ONE Rust backend (`nomifun-app`, the
|
||||
//! former `nomicore`). It runs in two host modes:
|
||||
//! * embedded in the Tauri desktop shell (`apps/desktop`), started in-process
|
||||
//! on a localhost port in `--local` (no-auth) mode — the shell IS the trust
|
||||
//! boundary, so no login is required;
|
||||
//! * here, as a standalone server that ALSO serves the built SPA (`ui/dist`)
|
||||
//! so browsers hit the same HTTP API. This replaces the old Node `web-host`.
|
||||
//!
|
||||
//! Unlike the desktop shell, this host is reachable over the network, so it
|
||||
//! boots the backend in AUTHENTICATED mode by default (login required). On a
|
||||
//! fresh data dir it provisions the first admin out-of-band (see
|
||||
//! `ensure_admin_credentials`), because the in-band setup endpoints are
|
||||
//! local-only. `--insecure-no-auth` opts back into desktop-style no-auth for a
|
||||
//! host that is only reachable over loopback / a trusted private network.
|
||||
//!
|
||||
//! This host boots the backend **in-process** (same binary), composes its `/api`
|
||||
//! router with a static `ServeDir` fallback for the SPA, and serves both on one
|
||||
//! port. Env mutation + runtime init happen before the tokio runtime starts,
|
||||
//! mirroring the `nomicore` bin's ordering.
|
||||
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::path::PathBuf;
|
||||
use std::process::ExitCode;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use clap::Parser;
|
||||
use tower_http::services::{ServeDir, ServeFile};
|
||||
use tower_http::trace::TraceLayer;
|
||||
|
||||
/// Env var that, when truthy, opts into `--insecure-no-auth` without the flag.
|
||||
const ENV_INSECURE_NO_AUTH: &str = "NOMIFUN_WEB_INSECURE_NO_AUTH";
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(
|
||||
name = "nomifun-web",
|
||||
about = "NomiFun unified Web host (SPA + backend API)"
|
||||
)]
|
||||
struct Args {
|
||||
/// Host/IP address to bind on. Defaults to loopback; use `0.0.0.0` to accept
|
||||
/// connections from other machines (do so behind a trusted gateway / private
|
||||
/// network — see `--insecure-no-auth`).
|
||||
#[arg(long, env = "NOMIFUN_WEB_HOST", default_value = "127.0.0.1")]
|
||||
host: String,
|
||||
/// Port to listen on (serves both the API and the SPA).
|
||||
#[arg(long, env = "NOMIFUN_WEB_PORT", default_value_t = 8787)]
|
||||
port: u16,
|
||||
/// Data directory for the backend (db + storage). Defaults to the same
|
||||
/// per-user dir as the desktop shell (`%LOCALAPPDATA%\NomiFun\Nomi` on
|
||||
/// Windows, see `nomifun_app::cli::default_data_dir`) so every host and
|
||||
/// dev loop shares one state by default. The env value is taken literally
|
||||
/// (no `/Nomi` suffix) — production deployments (Docker `/data`, systemd
|
||||
/// `/var/lib/nomifun`) rely on that.
|
||||
#[arg(
|
||||
long,
|
||||
env = "NOMIFUN_DATA_DIR",
|
||||
default_value_os_t = nomifun_app::cli::default_data_dir(),
|
||||
value_parser = nomifun_app::cli::parse_non_empty_path
|
||||
)]
|
||||
data_dir: PathBuf,
|
||||
/// Directory containing the built SPA (ui/dist).
|
||||
#[arg(long, env = "NOMIFUN_WEB_DIST", default_value = "../../ui/dist")]
|
||||
dist: PathBuf,
|
||||
/// DANGER: run the backend in local mode — authentication is fully DISABLED
|
||||
/// and every client acts as a privileged user with shell/file/agent access.
|
||||
/// Only for a host reachable solely over loopback or a trusted private
|
||||
/// network. Without this flag the web host requires a login (safe default).
|
||||
/// Can also be enabled via `NOMIFUN_WEB_INSECURE_NO_AUTH=true`.
|
||||
#[arg(long)]
|
||||
insecure_no_auth: bool,
|
||||
/// Initial admin username provisioned on first run (authenticated mode only).
|
||||
/// Ignored once an admin exists.
|
||||
#[arg(long, env = "NOMIFUN_ADMIN_USERNAME", default_value = "admin")]
|
||||
admin_user: String,
|
||||
/// Initial admin password provisioned on first run (authenticated mode only).
|
||||
/// If omitted, no admin is pre-seeded: the install is left uninitialised and
|
||||
/// the first WebUI visitor creates the admin interactively via first-run
|
||||
/// setup (`POST /api/auth/setup`). Ignored once an admin exists.
|
||||
#[arg(long, env = "NOMIFUN_ADMIN_PASSWORD")]
|
||||
admin_password: Option<String>,
|
||||
}
|
||||
|
||||
/// Parse a truthy env value (`1`/`true`/`yes`/`on`, case-insensitive).
|
||||
fn env_flag(name: &str) -> bool {
|
||||
std::env::var(name)
|
||||
.map(|v| {
|
||||
matches!(
|
||||
v.trim().to_ascii_lowercase().as_str(),
|
||||
"1" | "true" | "yes" | "on"
|
||||
)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn main() -> Result<ExitCode> {
|
||||
// If an ACP agent CLI spawned this binary as an MCP stdio bridge
|
||||
// (`current_exe() mcp-requirement-stdio` etc.), run that helper and exit
|
||||
// BEFORE clap parses our own Args (which would reject the subcommand) and
|
||||
// before any backend/server init. Every host binary must honor these or the
|
||||
// injected declaration tools (requirement_complete / team / guide) never
|
||||
// appear in the agent's session.
|
||||
if let Some(code) = nomifun_app::commands::run_mcp_stdio_subcommand_if_present() {
|
||||
return Ok(code);
|
||||
}
|
||||
|
||||
let args = Args::parse();
|
||||
|
||||
// Authentication is ON by default; `--insecure-no-auth` (or the env var)
|
||||
// opts into the desktop-style no-auth local mode. The env is read manually
|
||||
// to avoid clap's bool+env flag ambiguity.
|
||||
let insecure_no_auth = args.insecure_no_auth || env_flag(ENV_INSECURE_NO_AUTH);
|
||||
|
||||
// Build a fully-defaulted backend CLI without touching this process's argv,
|
||||
// then override the bits this host owns. `parse_from` gives a defaulted Cli.
|
||||
let mut cli = nomifun_app::cli::Cli::parse_from(["nomifun-web"]);
|
||||
cli.host = args.host.clone();
|
||||
cli.port = args.port;
|
||||
cli.data_dir = args.data_dir.clone();
|
||||
cli.local = insecure_no_auth;
|
||||
|
||||
// Same ordering as the nomicore bin: runtime init + PATH enhancement BEFORE
|
||||
// any worker thread / tokio runtime exists.
|
||||
nomifun_runtime::init(&cli.data_dir);
|
||||
// SAFETY: called before the tokio runtime (and its threads) is built.
|
||||
let merged_path = unsafe { nomifun_runtime::enhance_process_path() };
|
||||
|
||||
let runtime = tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()?;
|
||||
runtime.block_on(serve(cli, merged_path, args))
|
||||
}
|
||||
|
||||
async fn serve(cli: nomifun_app::cli::Cli, merged_path: String, args: Args) -> Result<ExitCode> {
|
||||
// Resolve the bind address up front so a bad --host fails fast with a clear
|
||||
// message instead of a cryptic socket error.
|
||||
let ip: IpAddr = args.host.parse().with_context(|| {
|
||||
format!(
|
||||
"invalid --host '{}': expected an IP like 127.0.0.1 or 0.0.0.0",
|
||||
args.host
|
||||
)
|
||||
})?;
|
||||
if !ip.is_loopback() && cli.local {
|
||||
tracing::warn!(
|
||||
%ip,
|
||||
"binding a non-loopback address with --insecure-no-auth: authentication is DISABLED — \
|
||||
anyone who can reach this port gets full host access (shell, files, agents). \
|
||||
Put a trusted gateway in front, use a private network, or drop --insecure-no-auth."
|
||||
);
|
||||
}
|
||||
|
||||
// Boot the backend in-process (env → data layer → services), then mount the
|
||||
// real API router with the SPA as the fallback for non-/api routes.
|
||||
let env = nomifun_app::bootstrap::init_environment(&cli, &merged_path)?;
|
||||
let database = nomifun_app::bootstrap::init_data_layer(&env.config).await?;
|
||||
let services = nomifun_app::AppServices::from_config(database, &env.config).await?;
|
||||
|
||||
// First-run admin provisioning. No-op in local mode and once an admin
|
||||
// exists; otherwise a fresh authenticated install would have no way to set
|
||||
// the first password (the in-band setup routes are local-only). Returns
|
||||
// whether the install still awaits interactive first-run setup.
|
||||
let needs_first_run_setup = nomifun_app::bootstrap::ensure_admin_credentials(
|
||||
&services,
|
||||
nomifun_app::bootstrap::AdminBootstrap {
|
||||
username: Some(args.admin_user.clone()),
|
||||
password: args.admin_password.clone(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
if needs_first_run_setup && !ip.is_loopback() {
|
||||
tracing::warn!(
|
||||
%ip,
|
||||
"first-run setup is OPEN on a non-loopback address: the NEXT client to reach this \
|
||||
port will create the admin account. Complete setup over a trusted network/tunnel \
|
||||
first, or pre-seed with NOMIFUN_ADMIN_PASSWORD."
|
||||
);
|
||||
}
|
||||
|
||||
let api = nomifun_app::create_router(&services).await;
|
||||
let app = api
|
||||
.fallback_service(
|
||||
ServeDir::new(&args.dist)
|
||||
.append_index_html_on_directories(true)
|
||||
.fallback(ServeFile::new(args.dist.join("index.html"))),
|
||||
)
|
||||
.layer(TraceLayer::new_for_http());
|
||||
|
||||
let addr = SocketAddr::new(ip, args.port);
|
||||
tracing::info!(
|
||||
requested = %addr,
|
||||
auth = if cli.local { "disabled (insecure-no-auth)" } else { "required" },
|
||||
dist = ?args.dist,
|
||||
"nomifun-web: embedded backend + SPA on one port"
|
||||
);
|
||||
// Port failover: if `args.port` is taken, bind a bounded-scan neighbour (or
|
||||
// an ephemeral port) instead of hard-failing, then announce the actually
|
||||
// bound port via `{data_dir}/port.json` + stdout so the operator/launcher
|
||||
// can re-point clients — a browser cannot self-discover a moved port.
|
||||
let (actual_port, listener) = nomifun_app::bootstrap::bind_with_fallback(ip, args.port).await?;
|
||||
if actual_port != args.port {
|
||||
tracing::warn!(
|
||||
requested = args.port,
|
||||
actual = actual_port,
|
||||
"preferred port was busy; bound a fallback port"
|
||||
);
|
||||
}
|
||||
nomifun_app::bootstrap::announce_bound_port(&cli.data_dir, &args.host, actual_port);
|
||||
axum::serve(listener, app).await?;
|
||||
|
||||
services.database.close().await;
|
||||
drop(env);
|
||||
Ok(ExitCode::SUCCESS)
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Executable
+27
@@ -0,0 +1,27 @@
|
||||
#!/bin/bash
|
||||
|
||||
# 彻底清理数据库文件脚本
|
||||
|
||||
DB_DIR="$HOME/Library/Application Support/NomiFun/Nomi"
|
||||
|
||||
echo "正在清理数据库文件..."
|
||||
echo "目录: $DB_DIR"
|
||||
echo ""
|
||||
|
||||
if [ -d "$DB_DIR" ]; then
|
||||
echo "找到数据库目录，列出当前文件："
|
||||
ls -la "$DB_DIR" | grep nomifun-backend
|
||||
echo ""
|
||||
|
||||
echo "删除所有数据库相关文件..."
|
||||
rm -f "$DB_DIR"/nomifun-backend.db*
|
||||
|
||||
echo ""
|
||||
echo "清理完成！剩余文件："
|
||||
ls -la "$DB_DIR" | grep nomifun-backend || echo "（无）"
|
||||
else
|
||||
echo "数据库目录不存在"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "现在可以重新启动应用了"
|
||||
@@ -0,0 +1,35 @@
|
||||
# crates/agent
|
||||
|
||||
AI agent engine crates. Package names use the `nomi-*` prefix.
|
||||
|
||||
Current crates:
|
||||
|
||||
| Crate | Role |
|
||||
| --- | --- |
|
||||
| `nomi-types` | Provider-neutral data types. |
|
||||
| `nomi-protocol` | Host/agent command and event protocol. |
|
||||
| `nomi-compact` | Conversation compaction and context shaping. |
|
||||
| `nomi-config` | Provider, auth, hook, and runtime configuration. |
|
||||
| `nomi-providers` | LLM provider clients and streaming logic. |
|
||||
| `nomi-tools` | Built-in tool registry. |
|
||||
| `nomi-mcp` | MCP client, config, transports, and tool proxying. |
|
||||
| `nomi-skills` | Skill discovery, loading, and execution support. |
|
||||
| `nomi-memory` | Long-term project/user memory. |
|
||||
| `nomi-agent` | Core session engine and orchestration. |
|
||||
| `nomi-cli` | Standalone `nomi` CLI. |
|
||||
| `nomi-computer` | Desktop computer-use tool implementation. |
|
||||
| `nomi-a11y` | Accessibility helpers used by computer-use flows. |
|
||||
| `nomi-browser-engine` | Self-hosted browser/CDP automation engine. |
|
||||
| `nomi-browser` | Browser-use tool layer. |
|
||||
|
||||
## Boundary
|
||||
|
||||
- `crates/agent` must not depend on `nomifun-*` backend crates.
|
||||
- Backend access to the agent layer should pass through
|
||||
`crates/backend/nomifun-ai-agent`.
|
||||
- Shared utilities that genuinely belong on both sides live under
|
||||
`crates/shared`.
|
||||
|
||||
The old extraction checklist in `docs/specs/agent-extraction-checklist.md` is a
|
||||
historical aid. Re-check it against the current crate list before using it as an
|
||||
execution plan.
|
||||
@@ -0,0 +1,76 @@
|
||||
[package]
|
||||
name = "nomi-a11y"
|
||||
description = "Cross-platform accessibility-tree + Set-of-Marks engine for Nomi computer-use (macOS AX / Windows UIA / Linux AT-SPI)"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
nomi-types.workspace = true
|
||||
|
||||
tracing.workspace = true
|
||||
thiserror.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
image.workspace = true
|
||||
base64.workspace = true
|
||||
|
||||
# --- macOS backend: Accessibility (AXUIElement) + CoreGraphics actuation +
|
||||
# AppKit (NSRunningApplication focus) + Vision (OCR/text fusion). macOS only. ---
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
core-foundation = "0.10"
|
||||
core-graphics = "0.25"
|
||||
objc2 = "0.6"
|
||||
objc2-foundation = { version = "0.3", features = [
|
||||
"NSData",
|
||||
"NSArray",
|
||||
"NSString",
|
||||
"NSError",
|
||||
"NSDictionary",
|
||||
"NSValue",
|
||||
] }
|
||||
objc2-core-foundation = { version = "0.3", features = ["CFCGTypes"] }
|
||||
objc2-vision = { version = "0.3", features = [
|
||||
"VNRequest",
|
||||
"VNRecognizeTextRequest",
|
||||
"VNRequestHandler",
|
||||
"VNObservation",
|
||||
"VNTypes",
|
||||
"VNGeometry",
|
||||
"objc2-core-foundation",
|
||||
] }
|
||||
|
||||
# --- Windows backend: UI Automation (via the `uiautomation` high-level wrapper)
|
||||
# for the accessibility tree + actuation, and `windows` (windows-rs) for
|
||||
# Win32 foreground/window management + `Windows.Media.Ocr` text fusion.
|
||||
# Windows only. ---
|
||||
[target.'cfg(target_os = "windows")'.dependencies]
|
||||
# High-level IUIAutomation wrapper (COM init via new_direct + MTA, tree walkers,
|
||||
# patterns, control types). leexgone/uiautomation-rs.
|
||||
uiautomation = "0.25"
|
||||
# windows-rs: foreground window (Win32) + on-device OCR (Windows.Media.Ocr).
|
||||
# 0.61 aligns with the version xcap already pulls into Cargo.lock.
|
||||
windows = { version = "0.61", features = [
|
||||
"Win32_Foundation",
|
||||
"Win32_System_Com",
|
||||
"Win32_System_Threading",
|
||||
"Win32_UI_WindowsAndMessaging",
|
||||
"Media_Ocr",
|
||||
"Graphics_Imaging",
|
||||
"Storage_Streams",
|
||||
"Foundation",
|
||||
"Foundation_Collections",
|
||||
"Globalization",
|
||||
] }
|
||||
|
||||
# --- Linux backend: AT-SPI2 (pure-Rust, async via zbus over D-Bus). Compiled
|
||||
# only on Linux. No C deps — the actor thread runs a current-thread tokio
|
||||
# runtime and block_on's the async atspi calls. ---
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
atspi = "0.30"
|
||||
zbus = { version = "5", features = ["tokio"] }
|
||||
tokio = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
image.workspace = true
|
||||
@@ -0,0 +1,25 @@
|
||||
# Linux a11y dev/CI image for nomi-a11y.
|
||||
#
|
||||
# Purpose: develop + verify the Linux (AT-SPI2) backend from a non-Linux host.
|
||||
# - Native Rust build inside the container (no cross-linker needed).
|
||||
# - Headless AT-SPI test harness: Xvfb (virtual X11 display) + a private D-Bus
|
||||
# session + at-spi2-core (the a11y bus) + a real accessible GTK app
|
||||
# (gtk3-widget-factory) to `observe` and `do_action` against.
|
||||
#
|
||||
# Build: docker build -t nomi-a11y-linux:dev -f crates/agent/nomi-a11y/dev/Dockerfile.linux-a11y .
|
||||
# Use: see crates/agent/nomi-a11y/dev/run-linux-a11y.sh
|
||||
#
|
||||
# The `atspi`/`zbus` crates are pure Rust (no C deps), so nomi-a11y itself needs
|
||||
# no system libs to compile; the X11/dbus dev libs below are only so the wider
|
||||
# computer-use chain (xcap/enigo) can also be built for Linux if desired.
|
||||
FROM rust:1-bookworm
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
at-spi2-core \
|
||||
xvfb dbus dbus-x11 x11-utils \
|
||||
gtk-3-examples \
|
||||
libx11-dev libxtst-dev libxi-dev libxcb1-dev libxkbcommon-dev \
|
||||
libdbus-1-dev pkg-config ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /work
|
||||
@@ -0,0 +1,85 @@
|
||||
# Linux Accessibility Backend Validation
|
||||
|
||||
This directory contains helper assets for validating the Linux AT-SPI backend
|
||||
from `nomi-a11y`.
|
||||
|
||||
Use the lightest path that proves the behavior you are changing.
|
||||
|
||||
## Path A: Type And API Check
|
||||
|
||||
This catches most portability issues without linking or running Linux binaries.
|
||||
|
||||
```bash
|
||||
rustup target add x86_64-unknown-linux-gnu
|
||||
cargo check --target x86_64-unknown-linux-gnu \
|
||||
-p nomi-a11y --examples --tests
|
||||
```
|
||||
|
||||
## Path B: Docker
|
||||
|
||||
The Dockerfile installs Rust, AT-SPI, Xvfb, D-Bus, and GTK example widgets so
|
||||
the smoke test can run headlessly.
|
||||
|
||||
```bash
|
||||
docker build -t nomi-a11y-linux:dev \
|
||||
-f crates/agent/nomi-a11y/dev/Dockerfile.linux-a11y .
|
||||
|
||||
crates/agent/nomi-a11y/dev/run-linux-a11y.sh test
|
||||
crates/agent/nomi-a11y/dev/run-linux-a11y.sh smoke
|
||||
```
|
||||
|
||||
If the environment cannot pull base images, run the same commands inside any
|
||||
Linux VM with the dependencies below installed.
|
||||
|
||||
## Path C: Native Linux VM
|
||||
|
||||
Install the runtime dependencies:
|
||||
|
||||
```bash
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y \
|
||||
at-spi2-core gtk-3-examples xvfb dbus-x11 \
|
||||
build-essential pkg-config
|
||||
```
|
||||
|
||||
Build the smoke example with a VM-local target directory so host builds do not
|
||||
share artifacts:
|
||||
|
||||
```bash
|
||||
CARGO_TARGET_DIR="$HOME/nomi-a11y-target" \
|
||||
CARGO_BUILD_BUILD_DIR="$HOME/nomi-a11y-build" \
|
||||
cargo build -p nomi-a11y --example linux_smoke
|
||||
```
|
||||
|
||||
Run the smoke test under Xvfb and a private D-Bus session:
|
||||
|
||||
```bash
|
||||
export DISPLAY=:99
|
||||
Xvfb :99 -screen 0 1280x900x24 -nolisten tcp >/tmp/xvfb.log 2>&1 &
|
||||
|
||||
dbus-run-session -- bash -uc '
|
||||
export QT_LINUX_ACCESSIBILITY_ALWAYS_ON=1
|
||||
export GTK_MODULES=atk-bridge
|
||||
export NO_AT_BRIDGE=0
|
||||
export DISPLAY=:99
|
||||
gtk3-widget-factory >/tmp/app.log 2>&1 &
|
||||
sleep 4
|
||||
./target/debug/examples/linux_smoke
|
||||
'
|
||||
```
|
||||
|
||||
Set `NOMI_A11Y_CLICK=<substring>` to ask `linux_smoke` to invoke the first
|
||||
matching element action.
|
||||
|
||||
## Coverage Notes
|
||||
|
||||
- The smoke test validates an X11 session. Wayland support can legitimately
|
||||
degrade for synthetic pixel input while semantic actions remain available;
|
||||
check the reported `capabilities`.
|
||||
- KDE/Qt apps may require `QT_LINUX_ACCESSIBILITY_ALWAYS_ON=1`.
|
||||
- Electron apps often require `--force-renderer-accessibility`.
|
||||
- Sandboxed apps such as Flatpak may not expose a complete accessibility tree.
|
||||
|
||||
`atspi` / `zbus` are pure Rust dependencies, so cross-checking the Linux backend
|
||||
from a non-Linux host is practical. Behavior validation still needs a Linux
|
||||
runtime.
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build / test / smoke the nomi-a11y Linux backend inside the dev container.
|
||||
# Native Linux build (no cross-linker). Named volumes cache target + registry
|
||||
# so incremental builds are fast across runs.
|
||||
#
|
||||
# ./run-linux-a11y.sh build # cargo build -p nomi-a11y
|
||||
# ./run-linux-a11y.sh test # cargo test -p nomi-a11y
|
||||
# ./run-linux-a11y.sh check-chain # cargo check -p nomi-agent --features computer-use
|
||||
# ./run-linux-a11y.sh smoke # headless AT-SPI behavioral run (Xvfb+dbus+gtk app)
|
||||
# ./run-linux-a11y.sh shell # interactive shell in the container
|
||||
set -euo pipefail
|
||||
REPO="$(cd "$(dirname "$0")/../../../.." && pwd)"
|
||||
IMG=nomi-a11y-linux:dev
|
||||
COMMON=(--rm -v "$REPO":/work -w /work
|
||||
-v nomi-a11y-target:/target -e CARGO_TARGET_DIR=/target
|
||||
-v nomi-a11y-cargo-registry:/usr/local/cargo/registry)
|
||||
|
||||
case "${1:-test}" in
|
||||
build) docker run "${COMMON[@]}" "$IMG" cargo build -p nomi-a11y ;;
|
||||
test) docker run "${COMMON[@]}" "$IMG" cargo test -p nomi-a11y ;;
|
||||
check-chain) docker run "${COMMON[@]}" "$IMG" cargo check -p nomi-agent --features computer-use ;;
|
||||
smoke) docker run "${COMMON[@]}" "$IMG" bash crates/agent/nomi-a11y/dev/smoke.sh ;;
|
||||
shell) docker run -it "${COMMON[@]}" "$IMG" bash ;;
|
||||
*) echo "usage: $0 {build|test|check-chain|smoke|shell}" >&2; exit 1 ;;
|
||||
esac
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env bash
|
||||
# Headless AT-SPI behavioral harness (runs INSIDE the dev container):
|
||||
# Xvfb virtual display + private D-Bus session + at-spi2 a11y bus + a real
|
||||
# accessible GTK app, then run the nomi-a11y `linux_smoke` example against it.
|
||||
set -uo pipefail
|
||||
|
||||
export DISPLAY=:99
|
||||
Xvfb :99 -screen 0 1280x900x24 -nolisten tcp >/tmp/xvfb.log 2>&1 &
|
||||
XVFB_PID=$!
|
||||
sleep 1
|
||||
|
||||
dbus-run-session -- bash -u -c '
|
||||
export QT_LINUX_ACCESSIBILITY_ALWAYS_ON=1
|
||||
export GTK_MODULES="${GTK_MODULES:-}:atk-bridge"
|
||||
export NO_AT_BRIDGE=0
|
||||
# Launch the AT-SPI registry/bus (path differs across Debian versions; try both).
|
||||
for d in /usr/libexec /usr/lib/at-spi2-core /usr/lib/at-spi2; do
|
||||
[ -x "$d/at-spi-bus-launcher" ] && ( "$d/at-spi-bus-launcher" --launch-immediately >/tmp/atspi-bus.log 2>&1 & )
|
||||
[ -x "$d/at-spi2-registryd" ] && ( "$d/at-spi2-registryd" >/tmp/atspi-reg.log 2>&1 & )
|
||||
done
|
||||
sleep 1
|
||||
gtk3-widget-factory >/tmp/app.log 2>&1 &
|
||||
APP_PID=$!
|
||||
sleep 3
|
||||
echo "=== AT-SPI bus address: ${AT_SPI_BUS_ADDRESS:-<unset>} ==="
|
||||
echo "=== running nomi-a11y linux_smoke example ==="
|
||||
CARGO_TARGET_DIR=/target cargo run -p nomi-a11y --example linux_smoke
|
||||
rc=$?
|
||||
kill "$APP_PID" 2>/dev/null || true
|
||||
exit $rc
|
||||
'
|
||||
rc=$?
|
||||
kill "$XVFB_PID" 2>/dev/null || true
|
||||
exit $rc
|
||||
@@ -0,0 +1,56 @@
|
||||
//! Headless smoke for the Linux AT-SPI backend (run inside the dev container's
|
||||
//! `smoke.sh`: Xvfb + dbus + at-spi2 + a GTK app). Connects, observes the
|
||||
//! focused window, prints the element list, and optionally activates the
|
||||
//! element whose name matches $NOMI_A11Y_CLICK (to exercise `do_action`).
|
||||
//!
|
||||
//! Usage: `cargo run -p nomi-a11y --example linux_smoke`
|
||||
|
||||
fn main() {
|
||||
let engine = match nomi_a11y::create_engine() {
|
||||
Ok(e) => e,
|
||||
Err(e) => {
|
||||
eprintln!("create_engine failed: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
|
||||
println!("capabilities: {:?}", engine.capabilities());
|
||||
|
||||
let snap = match engine.observe(&nomi_a11y::ObserveOpts::default()) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
eprintln!("observe failed: {e}");
|
||||
std::process::exit(2);
|
||||
}
|
||||
};
|
||||
|
||||
println!(
|
||||
"observed {} interactable element(s) app={:?} window={:?} truncated={}",
|
||||
snap.entries.len(),
|
||||
snap.app_name,
|
||||
snap.window_title,
|
||||
snap.truncated,
|
||||
);
|
||||
println!("--- element list ---\n{}", snap.text);
|
||||
|
||||
// Optional: activate the first element whose name contains $NOMI_A11Y_CLICK.
|
||||
if let Ok(needle) = std::env::var("NOMI_A11Y_CLICK") {
|
||||
if let Some(e) = snap
|
||||
.entries
|
||||
.iter()
|
||||
.find(|e| e.name.as_deref().is_some_and(|n| n.contains(&needle)))
|
||||
{
|
||||
println!("activating element [{}] {:?}…", e.r#ref, e.name);
|
||||
match engine.invoke(
|
||||
&nomi_a11y::Target::Ref(e.r#ref),
|
||||
snap.generation,
|
||||
nomi_a11y::ElementAction::LeftClick,
|
||||
) {
|
||||
Ok(eff) => println!("invoke ok: {}", eff.message),
|
||||
Err(err) => eprintln!("invoke failed: {err}"),
|
||||
}
|
||||
} else {
|
||||
eprintln!("no element matching {needle:?} to click");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
//! Real-machine smoke test for the Windows UIA backend.
|
||||
//!
|
||||
//! Run with: cargo run -p nomi-a11y --example winsmoke
|
||||
//! Optional: cargo run -p nomi-a11y --example winsmoke -- <pid>
|
||||
//!
|
||||
//! Opens Notepad on a temp file with known (Chinese) content, then exercises
|
||||
//! the engine: observe → numbered element list + bounds + app_name → read the
|
||||
//! document's text back via the Text pattern (validates the TextPattern value
|
||||
//! path) → attempt SetValue → stale-generation guard. Prints a numbered
|
||||
//! Set-of-Marks-style listing so coordinates can be eyeballed against the window.
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
fn main() {
|
||||
use std::{thread::sleep, time::Duration};
|
||||
|
||||
use nomi_a11y::{ElementAction, ObserveOpts, Snapshot, Target};
|
||||
|
||||
fn dump(snap: &Snapshot) {
|
||||
println!(
|
||||
" app={:?} window={:?} pid={:?} entries={} truncated={}",
|
||||
snap.app_name,
|
||||
snap.window_title,
|
||||
snap.pid,
|
||||
snap.entries.len(),
|
||||
snap.truncated
|
||||
);
|
||||
for e in &snap.entries {
|
||||
let b = e.bounds;
|
||||
println!(
|
||||
" [{:>2}] {:<11} name={:?} value={:?} states={:?} @ ({:.0},{:.0}) {:.0}x{:.0}",
|
||||
e.r#ref, e.role, e.name, e.value, e.states, b.x, b.y, b.w, b.h
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let arg_pid: Option<i32> = std::env::args().nth(1).and_then(|s| s.parse().ok());
|
||||
|
||||
// A temp file with known content lets us verify the TextPattern read path
|
||||
// (the Win11 RichEdit Notepad exposes text via TextPattern, not ValuePattern).
|
||||
const MARKER: &str = "你好世界";
|
||||
let tmp = std::env::temp_dir().join("nomi_winsmoke.txt");
|
||||
let _ = std::fs::write(&tmp, format!("NomiFun TextPattern 验证 Hello {MARKER}\n第二行 line two\n"));
|
||||
|
||||
let mut child = None;
|
||||
let target_pid = match arg_pid {
|
||||
Some(p) => {
|
||||
println!("== using provided pid {p} ==");
|
||||
Some(p)
|
||||
}
|
||||
None => {
|
||||
println!("== launching notepad.exe on {} ==", tmp.display());
|
||||
match std::process::Command::new("notepad.exe").arg(&tmp).spawn() {
|
||||
Ok(c) => {
|
||||
let p = c.id() as i32;
|
||||
println!(" spawned notepad, launcher pid = {p}");
|
||||
child = Some(c);
|
||||
sleep(Duration::from_millis(2500)); // allow the file to load
|
||||
Some(p)
|
||||
}
|
||||
Err(e) => {
|
||||
println!(" failed to launch notepad: {e}");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let engine = match nomi_a11y::create_engine() {
|
||||
Ok(e) => e,
|
||||
Err(e) => {
|
||||
println!("FATAL: create_engine failed: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
println!("capabilities: {:?}", engine.capabilities());
|
||||
|
||||
println!("\n== observe(foreground) ==");
|
||||
let t0 = std::time::Instant::now();
|
||||
let fg = engine.observe(&ObserveOpts::default());
|
||||
let elapsed = t0.elapsed();
|
||||
match &fg {
|
||||
Ok(s) => {
|
||||
println!(
|
||||
" observe latency: {:?} ({} entries, truncated={})",
|
||||
elapsed,
|
||||
s.entries.len(),
|
||||
s.truncated
|
||||
);
|
||||
dump(s);
|
||||
println!("\n -- semantic tree (snap.text) --");
|
||||
for line in s.text.lines() {
|
||||
println!(" {line}");
|
||||
}
|
||||
}
|
||||
Err(e) => println!(" observe(foreground) error: {e} (after {elapsed:?})"),
|
||||
}
|
||||
|
||||
let mut pid_snap = None;
|
||||
if let Some(pid) = target_pid {
|
||||
println!("\n== observe(pid={pid}) ==");
|
||||
match engine.observe(&ObserveOpts {
|
||||
pid: Some(pid),
|
||||
..Default::default()
|
||||
}) {
|
||||
Ok(s) => {
|
||||
dump(&s);
|
||||
pid_snap = Some(s);
|
||||
}
|
||||
Err(e) => {
|
||||
println!(" observe(pid) error: {e} (Store Notepad reparents; using foreground)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let snap = pid_snap
|
||||
.filter(|s| !s.entries.is_empty())
|
||||
.or_else(|| fg.ok().filter(|s| !s.entries.is_empty()));
|
||||
|
||||
let Some(snap) = snap else {
|
||||
println!("\nNo usable snapshot with elements; aborting actuation phase.");
|
||||
if let Some(mut c) = child {
|
||||
let _ = c.kill();
|
||||
}
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
return;
|
||||
};
|
||||
|
||||
// --- TextPattern value read: the document should show the file's text ---
|
||||
let doc = snap
|
||||
.entries
|
||||
.iter()
|
||||
.find(|e| matches!(e.role.as_str(), "edit" | "document"));
|
||||
println!("\n== TextPattern value read ==");
|
||||
match doc.and_then(|e| e.value.clone()) {
|
||||
Some(v) => println!(
|
||||
" TEXTPATTERN READ: {} (value={:?})",
|
||||
if v.contains(MARKER) { "PASS ✔" } else { "got text but no marker" },
|
||||
v
|
||||
),
|
||||
None => println!(" document value = None (no text read)"),
|
||||
}
|
||||
|
||||
// --- SetValue actuation (RichEdit Notepad often no-ops ValuePattern.SetValue;
|
||||
// the tool layer types instead — we just confirm the call is honest) ---
|
||||
if let Some(e) = doc {
|
||||
let text = "NomiFun SetValue 测试".to_string();
|
||||
println!("\n== invoke SetValue on [{}] {} ==", e.r#ref, e.role);
|
||||
match engine.invoke(
|
||||
&Target::Ref(e.r#ref),
|
||||
snap.generation,
|
||||
ElementAction::SetValue(text),
|
||||
) {
|
||||
Ok(eff) => println!(" ok: {}", eff.message),
|
||||
Err(err) => println!(" SetValue error (data, not a panic): {err}"),
|
||||
}
|
||||
}
|
||||
|
||||
// --- press_chain demonstration: focus then activate a button via the chain ---
|
||||
if let Some(btn) = snap.entries.iter().find(|e| e.role == "button") {
|
||||
println!("\n== invoke Focus on [{}] button {:?} ==", btn.r#ref, btn.name);
|
||||
match engine.invoke(&Target::Ref(btn.r#ref), snap.generation, ElementAction::Focus) {
|
||||
Ok(eff) => println!(" ok: {}", eff.message),
|
||||
Err(err) => println!(" error: {err}"),
|
||||
}
|
||||
}
|
||||
|
||||
// --- stale-ref guard ---
|
||||
println!("\n== stale generation guard ==");
|
||||
let stale = engine.invoke(
|
||||
&Target::Ref(1),
|
||||
nomi_a11y::SnapshotGen(snap.generation.0.wrapping_sub(1)),
|
||||
ElementAction::Focus,
|
||||
);
|
||||
println!(" invoke with old generation → {stale:?}");
|
||||
|
||||
if let Some(mut c) = child {
|
||||
sleep(Duration::from_millis(300));
|
||||
let _ = c.kill();
|
||||
println!("\n(killed spawned notepad)");
|
||||
}
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
println!("\n== done ==");
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
fn main() {
|
||||
eprintln!("winsmoke is a Windows-only example.");
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
//! Platform-neutral engine types + the `A11yEngine` trait every OS backend
|
||||
//! implements.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use nomi_types::tool::ToolImage;
|
||||
|
||||
use crate::selector::Selector;
|
||||
|
||||
/// Monotonic snapshot generation. A `ref` (index into a snapshot's element
|
||||
/// list) is only valid against the generation it was produced in; backends use
|
||||
/// this to reject stale references instead of acting on a moved element.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
pub struct SnapshotGen(pub u64);
|
||||
|
||||
/// An opaque, generation-tagged handle to an element in a backend's registry.
|
||||
/// The raw OS handle (AXUIElement / IUIAutomationElement / AT-SPI Accessible)
|
||||
/// never crosses the engine boundary — only this token and serializable data.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ElementId {
|
||||
pub generation: SnapshotGen,
|
||||
pub index: u32,
|
||||
}
|
||||
|
||||
/// A rectangle. Backends return element bounds in **OS accessibility
|
||||
/// coordinates** (e.g. macOS global screen points, top-left origin); mapping to
|
||||
/// screenshot-pixel space for overlays/pixel-fallback is the caller's job (see
|
||||
/// the design's AX-points→pixel conversion).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Rect {
|
||||
pub x: f64,
|
||||
pub y: f64,
|
||||
pub w: f64,
|
||||
pub h: f64,
|
||||
}
|
||||
|
||||
impl Rect {
|
||||
pub fn center(&self) -> (f64, f64) {
|
||||
(self.x + self.w / 2.0, self.y + self.h / 2.0)
|
||||
}
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.w <= 0.0 || self.h <= 0.0
|
||||
}
|
||||
}
|
||||
|
||||
/// Where an element entry came from. Set-of-Marks fuses these.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Source {
|
||||
/// Native accessibility tree (most reliable).
|
||||
A11y,
|
||||
/// OCR text recognition (fallback where a11y is thin).
|
||||
Ocr,
|
||||
/// Vision/icon classification (fallback).
|
||||
Vision,
|
||||
}
|
||||
|
||||
/// One interactable element exposed to the model as `[ref]`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ElementEntry {
|
||||
/// The number the model targets: "click element [ref]".
|
||||
pub r#ref: u32,
|
||||
pub role: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub name: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub value: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub states: Vec<String>,
|
||||
pub bounds: Rect,
|
||||
pub source: Source,
|
||||
}
|
||||
|
||||
/// A line of text recognized by OCR, with bounds in screenshot-pixel space
|
||||
/// (top-left origin). Fused into the Set-of-Marks list where the accessibility
|
||||
/// tree is thin (Electron/canvas/games).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct OcrLine {
|
||||
pub text: String,
|
||||
pub bounds: Rect,
|
||||
}
|
||||
|
||||
/// How synthetic input is delivered on this platform/session.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum InputKind {
|
||||
/// Native event posting (macOS CGEvent / Windows SendInput / AT-SPI action).
|
||||
Native,
|
||||
/// X11 XTest.
|
||||
X11,
|
||||
/// Wayland xdg-desktop-portal RemoteDesktop (per-session consent).
|
||||
WaylandPortal,
|
||||
/// No reliable synthetic-input path in this session.
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
/// What the engine can actually do this session — injected into the system
|
||||
/// prompt so the model knows its real abilities up front.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Capabilities {
|
||||
pub os: String,
|
||||
/// Can read the accessibility tree (`observe`).
|
||||
pub tree_read: bool,
|
||||
/// Can capture a screenshot for the Set-of-Marks overlay.
|
||||
pub screenshot: bool,
|
||||
/// Can perform semantic actions (AXPress / Invoke / do_action) on elements.
|
||||
pub semantic_action: bool,
|
||||
pub synthetic_input: InputKind,
|
||||
/// Can move/resize/raise windows.
|
||||
pub window_management: bool,
|
||||
}
|
||||
|
||||
/// A completed `observe`: the filtered interactable elements + an optional
|
||||
/// Set-of-Marks overlay image, plus the indented text rendering for the model.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Snapshot {
|
||||
pub generation: SnapshotGen,
|
||||
pub entries: Vec<ElementEntry>,
|
||||
/// Set-of-Marks overlay (numbered boxes on the screenshot), when produced.
|
||||
pub overlay: Option<ToolImage>,
|
||||
/// Indented text rendering: `[14] button "Submit" enabled`.
|
||||
pub text: String,
|
||||
/// True if the tree exceeded the node budget and was truncated.
|
||||
pub truncated: bool,
|
||||
/// Process id of the observed application (for `focus_window`).
|
||||
pub pid: Option<i32>,
|
||||
pub app_name: Option<String>,
|
||||
pub window_title: Option<String>,
|
||||
}
|
||||
|
||||
/// How the model addresses an element. Three mutually-exclusive modes, shared
|
||||
/// with the browser tool's contract.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum Target {
|
||||
/// A `[ref]` from the most recent snapshot.
|
||||
Ref(u32),
|
||||
/// A deterministic selector (`role:Button && name:Save`).
|
||||
Selector(Selector),
|
||||
/// Last-resort absolute screen coordinates (pixel fallback).
|
||||
Pixel { x: i32, y: i32 },
|
||||
}
|
||||
|
||||
/// A semantic action to perform on a resolved element.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum ElementAction {
|
||||
/// The element's default action (AXPress / Invoke / do_action).
|
||||
Press,
|
||||
LeftClick,
|
||||
RightClick,
|
||||
DoubleClick,
|
||||
Focus,
|
||||
SetValue(String),
|
||||
}
|
||||
|
||||
/// The observed effect of an action, for closed-loop verification.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Effect {
|
||||
pub changed: bool,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
/// Options controlling an `observe` tree walk.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ObserveOpts {
|
||||
/// Maximum tree depth to traverse.
|
||||
pub max_depth: usize,
|
||||
/// Stop after this many interactable elements (then set `truncated`).
|
||||
pub node_budget: usize,
|
||||
/// Restrict to a specific process; `None` = the frontmost app.
|
||||
pub pid: Option<i32>,
|
||||
}
|
||||
|
||||
impl Default for ObserveOpts {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_depth: 12,
|
||||
node_budget: 120,
|
||||
pid: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Errors are data the model reads and routes around — never a panic, never a
|
||||
/// silent no-op.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum A11yError {
|
||||
#[error("not supported ({capability}): {hint}")]
|
||||
Unsupported { capability: String, hint: String },
|
||||
#[error("element not found: {0}")]
|
||||
NotFound(String),
|
||||
#[error("stale reference: {0}")]
|
||||
Stale(String),
|
||||
#[error("permission required: {0}")]
|
||||
Permission(String),
|
||||
#[error("accessibility backend error: {0}")]
|
||||
Backend(String),
|
||||
}
|
||||
|
||||
/// The contract every OS backend implements. Methods are synchronous; callers
|
||||
/// invoke them from `spawn_blocking`. macOS marshals each call to a single
|
||||
/// CFRunLoop actor thread internally, so the engine is `Send + Sync`.
|
||||
pub trait A11yEngine: Send + Sync {
|
||||
/// Honest report of what this session can do.
|
||||
fn capabilities(&self) -> Capabilities;
|
||||
|
||||
/// Read the frontmost (or `opts.pid`) window's accessibility tree, filter to
|
||||
/// interactable elements, and return them numbered as a Set-of-Marks
|
||||
/// snapshot. Element `bounds` are in OS accessibility coordinates.
|
||||
fn observe(&self, opts: &ObserveOpts) -> Result<Snapshot, A11yError>;
|
||||
|
||||
/// Perform `action` on the element addressed by `target`. `Ref` targets are
|
||||
/// validated against `generation` and rejected if stale.
|
||||
fn invoke(
|
||||
&self,
|
||||
target: &Target,
|
||||
generation: SnapshotGen,
|
||||
action: ElementAction,
|
||||
) -> Result<Effect, A11yError>;
|
||||
|
||||
/// Raise/activate a window by its owning process id.
|
||||
fn focus_window(&self, pid: i32) -> Result<Effect, A11yError>;
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
//! Cross-platform accessibility-tree + Set-of-Marks engine for Nomi computer-use.
|
||||
//!
|
||||
//! The platform-neutral layer (engine trait/types, selector grammar, tree
|
||||
//! model + filtering, Set-of-Marks overlay) compiles on every target. Per-OS
|
||||
//! backends live behind `#[cfg(target_os = …)]`:
|
||||
//! - macOS: AXUIElement via a dedicated CFRunLoop actor thread (implemented).
|
||||
//! - Windows: UI Automation via a dedicated MTA actor thread (implemented).
|
||||
//! - Linux: AT-SPI2 over D-Bus (implemented).
|
||||
//!
|
||||
//! Backends report honest `Capabilities`; unimplemented operations return
|
||||
//! `A11yError::Unsupported { capability, hint }` (never panic, never a silent
|
||||
//! no-op) so the agent can route around them.
|
||||
|
||||
pub mod engine;
|
||||
pub mod overlay;
|
||||
pub mod selector;
|
||||
pub mod tree;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos;
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
mod windows;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
|
||||
pub use engine::{
|
||||
A11yEngine, A11yError, Capabilities, Effect, ElementAction, ElementEntry, ElementId,
|
||||
InputKind, ObserveOpts, OcrLine, Rect, Snapshot, SnapshotGen, Source, Target,
|
||||
};
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
/// Process-wide label for the host application, woven into permission-error
|
||||
/// guidance so the message names the *actual* app the user must grant (and
|
||||
/// restart) instead of a generic "this app". On a desktop host that ambiguity
|
||||
/// is actively harmful: computer-use runs IN-PROCESS inside the host app, but a
|
||||
/// model reading "this app" reliably misattributes it to the terminal/editor it
|
||||
/// imagines is hosting the session and sends the user to grant the wrong
|
||||
/// process. The host sets this once at startup (the desktop shell sets
|
||||
/// "NomiFun"); library/headless embeddings leave it unset and get "this app".
|
||||
static HOST_APP_LABEL: RwLock<Option<String>> = RwLock::new(None);
|
||||
|
||||
/// Set the host-application label used in permission-error guidance (e.g.
|
||||
/// "NomiFun"). Last writer wins; call once early in host startup. A poisoned
|
||||
/// lock is ignored — the default ("this app") is a safe fallback, never a panic.
|
||||
pub fn set_host_app_label(label: impl Into<String>) {
|
||||
if let Ok(mut guard) = HOST_APP_LABEL.write() {
|
||||
*guard = Some(label.into());
|
||||
}
|
||||
}
|
||||
|
||||
/// The host-application label for permission guidance, or `"this app"` when the
|
||||
/// host has not set one. Always returns an owned, non-empty string.
|
||||
pub fn host_app_label() -> String {
|
||||
HOST_APP_LABEL
|
||||
.read()
|
||||
.ok()
|
||||
.and_then(|g| g.clone())
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or_else(|| "this app".to_string())
|
||||
}
|
||||
|
||||
/// Construct the platform's accessibility engine, or report why it is
|
||||
/// unavailable. The returned engine is `Send + Sync` and its methods are
|
||||
/// synchronous (call them from `spawn_blocking`); macOS marshals every call to
|
||||
/// a single CFRunLoop actor thread internally.
|
||||
pub fn create_engine() -> Result<Arc<dyn A11yEngine>, A11yError> {
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
let engine = macos::MacEngine::start()?;
|
||||
Ok(Arc::new(engine))
|
||||
}
|
||||
#[cfg(target_os = "windows")]
|
||||
{
|
||||
let engine = windows::WinEngine::start()?;
|
||||
Ok(Arc::new(engine))
|
||||
}
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let engine = linux::LinuxEngine::start()?;
|
||||
Ok(Arc::new(engine))
|
||||
}
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))]
|
||||
{
|
||||
Err(A11yError::Unsupported {
|
||||
capability: "accessibility engine".to_string(),
|
||||
hint: "The accessibility-tree backend is implemented on macOS, Windows, and Linux. \
|
||||
Pixel-based computer-use still works."
|
||||
.to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod host_label_tests {
|
||||
use super::{host_app_label, set_host_app_label};
|
||||
|
||||
// The only test in this crate that touches the process-global label, so its
|
||||
// steps observe each other deterministically under the parallel runner.
|
||||
#[test]
|
||||
fn label_defaults_then_reflects_set_and_ignores_empty() {
|
||||
assert_eq!(host_app_label(), "this app", "default before any host sets it");
|
||||
set_host_app_label("NomiFun");
|
||||
assert_eq!(host_app_label(), "NomiFun");
|
||||
// An empty label is ignored so a mis-set never blanks the guidance.
|
||||
set_host_app_label("");
|
||||
assert_eq!(host_app_label(), "this app");
|
||||
}
|
||||
}
|
||||
|
||||
/// Recognize on-screen text in a screenshot via the OS OCR engine (macOS:
|
||||
/// Vision.framework `VNRecognizeTextRequest`, on-device, with CJK support).
|
||||
/// `langs` are BCP-47 hints (e.g. `["zh-Hans", "en-US"]`). Bounds are in the
|
||||
/// image's pixel space (top-left origin). Used to fuse text into Set-of-Marks
|
||||
/// where the accessibility tree is thin. Returns `Unsupported` off macOS.
|
||||
pub fn ocr_screenshot(img: &image::RgbaImage, langs: &[String]) -> Result<Vec<OcrLine>, A11yError> {
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
macos::ocr_screenshot(img, langs)
|
||||
}
|
||||
#[cfg(target_os = "windows")]
|
||||
{
|
||||
windows::ocr_screenshot(img, langs)
|
||||
}
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
linux::ocr_screenshot(img, langs)
|
||||
}
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))]
|
||||
{
|
||||
let _ = (img, langs);
|
||||
Err(A11yError::Unsupported {
|
||||
capability: "OCR".to_string(),
|
||||
hint: "OCR fusion is implemented on macOS (Vision.framework) and Windows \
|
||||
(Windows.Media.Ocr)."
|
||||
.to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,519 @@
|
||||
//! Linux AT-SPI actor: a dedicated thread owns a current-thread tokio runtime
|
||||
//! and the `AccessibilityConnection`; the synchronous `A11yEngine` methods
|
||||
//! marshal here over a command channel and `block_on` the async AT-SPI calls.
|
||||
//!
|
||||
//! `invoke` prefers AT-SPI semantic actions (Action.do_action / EditableText /
|
||||
//! grab_focus) — coordinate-free and reliable on both X11 and Wayland. Element
|
||||
//! bounds come from Component.get_extents(Screen) (valid pixels on X11; often
|
||||
//! unavailable on Wayland, where the tool's pixel fallback is degraded anyway).
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::mpsc::{Sender, channel};
|
||||
|
||||
use atspi::connection::AccessibilityConnection;
|
||||
use atspi::proxy::accessible::ObjectRefExt;
|
||||
use atspi::proxy::action::ActionProxy;
|
||||
use atspi::proxy::component::ComponentProxy;
|
||||
use atspi::proxy::editable_text::EditableTextProxy;
|
||||
use atspi::{CoordType, ObjectRefOwned, State as AtspiState};
|
||||
|
||||
use crate::engine::{
|
||||
A11yError, Capabilities, Effect, ElementAction, ElementEntry, InputKind, ObserveOpts, Rect,
|
||||
Snapshot, SnapshotGen, Source, Target,
|
||||
};
|
||||
use crate::tree::format_entries;
|
||||
|
||||
enum Cmd {
|
||||
Capabilities(Sender<Capabilities>),
|
||||
Observe(ObserveOpts, Sender<Result<Snapshot, A11yError>>),
|
||||
Invoke(
|
||||
Target,
|
||||
SnapshotGen,
|
||||
ElementAction,
|
||||
Sender<Result<Effect, A11yError>>,
|
||||
),
|
||||
Focus(i32, Sender<Result<Effect, A11yError>>),
|
||||
}
|
||||
|
||||
pub struct ActorHandle {
|
||||
tx: Mutex<Sender<Cmd>>,
|
||||
}
|
||||
|
||||
struct State {
|
||||
gen_counter: u64,
|
||||
current_gen: SnapshotGen,
|
||||
registry: HashMap<u32, ObjectRefOwned>,
|
||||
}
|
||||
|
||||
/// Session probe → honest `Capabilities`. AT-SPI tree-read works on X11 +
|
||||
/// Wayland; input/coordinates/window-mgmt degrade on Wayland.
|
||||
fn detect_caps() -> Capabilities {
|
||||
let session = std::env::var("XDG_SESSION_TYPE").unwrap_or_default();
|
||||
let wayland =
|
||||
session.eq_ignore_ascii_case("wayland") || std::env::var_os("WAYLAND_DISPLAY").is_some();
|
||||
let x11 = !wayland
|
||||
&& (session.eq_ignore_ascii_case("x11") || std::env::var_os("DISPLAY").is_some());
|
||||
Capabilities {
|
||||
os: "linux".to_string(),
|
||||
tree_read: true,
|
||||
screenshot: true,
|
||||
semantic_action: true,
|
||||
synthetic_input: if x11 {
|
||||
InputKind::X11
|
||||
} else {
|
||||
// No reliable persistent unattended Wayland input without a portal grant.
|
||||
InputKind::Unsupported
|
||||
},
|
||||
window_management: x11,
|
||||
}
|
||||
}
|
||||
|
||||
// ---- AT-SPI helpers (run on the actor thread's runtime) ------------------
|
||||
|
||||
/// Map an AT-SPI role to a stable lowercase name (Debug form, e.g. `pushbutton`,
|
||||
/// `entry`, `text`). The model just needs readable, stable role names.
|
||||
fn role_name(acc_role: Option<atspi::Role>) -> String {
|
||||
match acc_role {
|
||||
Some(r) => format!("{r:?}").to_lowercase(),
|
||||
None => "element".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_click_action(name: &str) -> bool {
|
||||
matches!(
|
||||
name.to_lowercase().as_str(),
|
||||
"click" | "activate" | "press" | "jump" | "open" | "do default" | "default"
|
||||
)
|
||||
}
|
||||
|
||||
/// Find the focused (Active) toplevel window by scanning each application's
|
||||
/// children. Returns its object reference.
|
||||
async fn find_active_window(conn: &AccessibilityConnection) -> Result<ObjectRefOwned, A11yError> {
|
||||
let zconn = conn.connection();
|
||||
let root = conn
|
||||
.root_accessible_on_registry()
|
||||
.await
|
||||
.map_err(|e| A11yError::Backend(format!("cannot read the AT-SPI registry root: {e}")))?;
|
||||
let apps = root.get_children().await.map_err(|e| {
|
||||
A11yError::Backend(format!("cannot list accessible applications: {e}"))
|
||||
})?;
|
||||
|
||||
let mut first_window: Option<ObjectRefOwned> = None;
|
||||
for app in apps {
|
||||
let Ok(app_acc) = app.as_accessible_proxy(zconn).await else {
|
||||
continue;
|
||||
};
|
||||
let Ok(windows) = app_acc.get_children().await else {
|
||||
continue;
|
||||
};
|
||||
for win in windows {
|
||||
let Ok(win_acc) = win.as_accessible_proxy(zconn).await else {
|
||||
continue;
|
||||
};
|
||||
let states = win_acc.get_state().await.unwrap_or_default();
|
||||
if states.contains(AtspiState::Active) {
|
||||
return Ok(win);
|
||||
}
|
||||
if first_window.is_none() {
|
||||
first_window = Some(win);
|
||||
}
|
||||
}
|
||||
}
|
||||
// No window reported Active (common headless) — fall back to the first one.
|
||||
first_window.ok_or_else(|| {
|
||||
A11yError::NotFound(
|
||||
"no accessible window found. Ensure the app exposes accessibility \
|
||||
(KDE: QT_LINUX_ACCESSIBILITY_ALWAYS_ON=1; Electron: --force-renderer-accessibility)."
|
||||
.to_string(),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
struct Collected {
|
||||
obj: ObjectRefOwned,
|
||||
role: String,
|
||||
name: Option<String>,
|
||||
value: Option<String>,
|
||||
states: Vec<String>,
|
||||
bounds: Rect,
|
||||
}
|
||||
|
||||
/// Walk the window subtree (iteratively, to avoid async recursion), collecting
|
||||
/// interactable elements with screen-pixel bounds.
|
||||
async fn walk_window(
|
||||
conn: &AccessibilityConnection,
|
||||
window: ObjectRefOwned,
|
||||
opts: &ObserveOpts,
|
||||
) -> (Vec<Collected>, bool) {
|
||||
let zconn = conn.connection();
|
||||
let mut out: Vec<Collected> = Vec::new();
|
||||
let mut truncated = false;
|
||||
let mut stack: Vec<(ObjectRefOwned, usize)> = vec![(window, 0)];
|
||||
|
||||
while let Some((obj, depth)) = stack.pop() {
|
||||
if out.len() >= opts.node_budget {
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
let Ok(acc) = obj.as_accessible_proxy(zconn).await else {
|
||||
continue;
|
||||
};
|
||||
let dest = acc.inner().destination().to_owned();
|
||||
let path = acc.inner().path().to_owned();
|
||||
|
||||
let role = acc.get_role().await.ok();
|
||||
let name = acc.name().await.ok().filter(|s| !s.trim().is_empty());
|
||||
let states = acc.get_state().await.unwrap_or_default();
|
||||
|
||||
let bounds = match ComponentProxy::builder(zconn)
|
||||
.destination(dest.clone())
|
||||
.and_then(|b| b.path(path.clone()))
|
||||
{
|
||||
Ok(builder) => match builder.build().await {
|
||||
Ok(comp) => comp.get_extents(CoordType::Screen).await.ok().map(|(x, y, w, h)| {
|
||||
Rect {
|
||||
x: x as f64,
|
||||
y: y as f64,
|
||||
w: w as f64,
|
||||
h: h as f64,
|
||||
}
|
||||
}),
|
||||
Err(_) => None,
|
||||
},
|
||||
Err(_) => None,
|
||||
};
|
||||
|
||||
let has_action = match ActionProxy::builder(zconn)
|
||||
.destination(dest.clone())
|
||||
.and_then(|b| b.path(path.clone()))
|
||||
{
|
||||
Ok(builder) => match builder.build().await {
|
||||
Ok(act) => act.n_actions().await.map(|n| n > 0).unwrap_or(false),
|
||||
Err(_) => false,
|
||||
},
|
||||
Err(_) => false,
|
||||
};
|
||||
|
||||
let focusable = states.contains(AtspiState::Focusable);
|
||||
let editable = states.contains(AtspiState::Editable);
|
||||
let enabled = states.contains(AtspiState::Enabled) || states.contains(AtspiState::Sensitive);
|
||||
|
||||
if let Some(b) = bounds {
|
||||
if b.w > 0.0 && b.h > 0.0 && (has_action || focusable || editable) {
|
||||
let mut st = Vec::new();
|
||||
if !enabled {
|
||||
st.push("disabled".to_string());
|
||||
}
|
||||
if states.contains(AtspiState::Focused) {
|
||||
st.push("focused".to_string());
|
||||
}
|
||||
out.push(Collected {
|
||||
obj: obj.clone(),
|
||||
role: role_name(role),
|
||||
name,
|
||||
value: None,
|
||||
states: st,
|
||||
bounds: b,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if depth < opts.max_depth {
|
||||
if let Ok(children) = acc.get_children().await {
|
||||
for c in children {
|
||||
stack.push((c, depth + 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
(out, truncated)
|
||||
}
|
||||
|
||||
async fn do_observe(
|
||||
conn: &AccessibilityConnection,
|
||||
opts: &ObserveOpts,
|
||||
state: &mut State,
|
||||
) -> Result<Snapshot, A11yError> {
|
||||
let window = find_active_window(conn).await?;
|
||||
let window_title = match window.as_accessible_proxy(conn.connection()).await {
|
||||
Ok(w) => w.name().await.ok().filter(|s| !s.trim().is_empty()),
|
||||
Err(_) => None,
|
||||
};
|
||||
|
||||
let (mut collected, truncated) = walk_window(conn, window, opts).await;
|
||||
|
||||
// Reading order: top-to-bottom, left-to-right.
|
||||
collected.sort_by(|a, b| {
|
||||
(a.bounds.y.round() as i64, a.bounds.x.round() as i64)
|
||||
.cmp(&(b.bounds.y.round() as i64, b.bounds.x.round() as i64))
|
||||
});
|
||||
|
||||
state.gen_counter += 1;
|
||||
let generation = SnapshotGen(state.gen_counter);
|
||||
state.current_gen = generation;
|
||||
state.registry.clear();
|
||||
|
||||
let mut entries = Vec::with_capacity(collected.len());
|
||||
for (i, c) in collected.into_iter().enumerate() {
|
||||
let r = i as u32 + 1;
|
||||
state.registry.insert(r, c.obj);
|
||||
entries.push(ElementEntry {
|
||||
r#ref: r,
|
||||
role: c.role,
|
||||
name: c.name,
|
||||
value: c.value,
|
||||
states: c.states,
|
||||
bounds: c.bounds,
|
||||
source: Source::A11y,
|
||||
});
|
||||
}
|
||||
|
||||
let text = format_entries(&entries);
|
||||
Ok(Snapshot {
|
||||
generation,
|
||||
entries,
|
||||
overlay: None,
|
||||
text,
|
||||
truncated,
|
||||
pid: None,
|
||||
app_name: None,
|
||||
window_title,
|
||||
})
|
||||
}
|
||||
|
||||
async fn do_invoke(
|
||||
conn: &AccessibilityConnection,
|
||||
target: &Target,
|
||||
generation: SnapshotGen,
|
||||
action: &ElementAction,
|
||||
state: &State,
|
||||
) -> Result<Effect, A11yError> {
|
||||
let r = match target {
|
||||
Target::Ref(r) => *r,
|
||||
Target::Selector(_) => {
|
||||
return Err(A11yError::Unsupported {
|
||||
capability: "selector targeting".to_string(),
|
||||
hint: "Resolve a selector against the latest observe() result and act by [ref]."
|
||||
.to_string(),
|
||||
});
|
||||
}
|
||||
Target::Pixel { .. } => {
|
||||
return Err(A11yError::Unsupported {
|
||||
capability: "pixel targeting".to_string(),
|
||||
hint: "Pixel fallback is handled by the computer tool's input layer.".to_string(),
|
||||
});
|
||||
}
|
||||
};
|
||||
if generation != state.current_gen {
|
||||
return Err(A11yError::Stale(format!(
|
||||
"ref [{r}] is from an older snapshot; re-run observe and use a fresh [ref]"
|
||||
)));
|
||||
}
|
||||
let obj = state
|
||||
.registry
|
||||
.get(&r)
|
||||
.cloned()
|
||||
.ok_or_else(|| A11yError::NotFound(format!("no element [{r}] in the latest snapshot")))?;
|
||||
|
||||
let zconn = conn.connection();
|
||||
let acc = obj
|
||||
.as_accessible_proxy(zconn)
|
||||
.await
|
||||
.map_err(|e| A11yError::Backend(format!("cannot resolve [{r}]: {e}")))?;
|
||||
let dest = acc.inner().destination().to_owned();
|
||||
let path = acc.inner().path().to_owned();
|
||||
|
||||
match action {
|
||||
ElementAction::Press | ElementAction::LeftClick | ElementAction::DoubleClick => {
|
||||
let act = ActionProxy::builder(zconn)
|
||||
.destination(dest)
|
||||
.and_then(|b| b.path(path))
|
||||
.map_err(|e| A11yError::Backend(format!("action proxy: {e}")))?
|
||||
.build()
|
||||
.await
|
||||
.map_err(|e| A11yError::Backend(format!("action proxy: {e}")))?;
|
||||
let actions = act.get_actions().await.unwrap_or_default();
|
||||
let idx = actions
|
||||
.iter()
|
||||
.position(|a| is_click_action(&a.name))
|
||||
.unwrap_or(0);
|
||||
if act.n_actions().await.unwrap_or(0) <= 0 {
|
||||
return Err(A11yError::Backend(format!(
|
||||
"element [{r}] exposes no AT-SPI action; fall back to a pixel click"
|
||||
)));
|
||||
}
|
||||
match act.do_action(idx as i32).await {
|
||||
Ok(true) => Ok(Effect {
|
||||
changed: true,
|
||||
message: format!("performed action {idx} on element [{r}]"),
|
||||
}),
|
||||
Ok(false) => Err(A11yError::Backend(format!(
|
||||
"do_action on [{r}] returned false; try a pixel click"
|
||||
))),
|
||||
Err(e) => Err(A11yError::Backend(format!("do_action on [{r}] failed: {e}"))),
|
||||
}
|
||||
}
|
||||
ElementAction::RightClick => Err(A11yError::Unsupported {
|
||||
capability: "right click".to_string(),
|
||||
hint: "AT-SPI has no standard right-click action; use a pixel right-click.".to_string(),
|
||||
}),
|
||||
ElementAction::Focus => {
|
||||
let comp = ComponentProxy::builder(zconn)
|
||||
.destination(dest)
|
||||
.and_then(|b| b.path(path))
|
||||
.map_err(|e| A11yError::Backend(format!("component proxy: {e}")))?
|
||||
.build()
|
||||
.await
|
||||
.map_err(|e| A11yError::Backend(format!("component proxy: {e}")))?;
|
||||
match comp.grab_focus().await {
|
||||
Ok(_) => Ok(Effect {
|
||||
changed: true,
|
||||
message: format!("focused element [{r}]"),
|
||||
}),
|
||||
Err(e) => Err(A11yError::Backend(format!("grab_focus on [{r}] failed: {e}"))),
|
||||
}
|
||||
}
|
||||
ElementAction::SetValue(v) => {
|
||||
let et = EditableTextProxy::builder(zconn)
|
||||
.destination(dest)
|
||||
.and_then(|b| b.path(path))
|
||||
.map_err(|e| A11yError::Backend(format!("editable-text proxy: {e}")))?
|
||||
.build()
|
||||
.await
|
||||
.map_err(|e| A11yError::Backend(format!("editable-text proxy: {e}")))?;
|
||||
match et.set_text_contents(v).await {
|
||||
Ok(_) => Ok(Effect {
|
||||
changed: true,
|
||||
message: format!("set value of element [{r}]"),
|
||||
}),
|
||||
Err(e) => Err(A11yError::Backend(format!(
|
||||
"set_text_contents on [{r}] failed ({e}); fall back to focus + type"
|
||||
))),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn do_focus(_pid: i32) -> Result<Effect, A11yError> {
|
||||
Err(A11yError::Unsupported {
|
||||
capability: "window activation".to_string(),
|
||||
hint: "Cross-application window activation is not wired on Linux yet (X11 EWMH / Wayland \
|
||||
has no portable protocol); the focused window is used by observe."
|
||||
.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
// ---- thread + channel plumbing ------------------------------------------
|
||||
|
||||
impl ActorHandle {
|
||||
pub fn spawn() -> Result<Self, A11yError> {
|
||||
let (tx, rx) = channel::<Cmd>();
|
||||
let (ready_tx, ready_rx) = channel::<Result<(), A11yError>>();
|
||||
|
||||
std::thread::Builder::new()
|
||||
.name("nomi-a11y-linux".to_string())
|
||||
.spawn(move || {
|
||||
let rt = match tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
{
|
||||
Ok(rt) => rt,
|
||||
Err(e) => {
|
||||
let _ =
|
||||
ready_tx.send(Err(A11yError::Backend(format!("tokio runtime: {e}"))));
|
||||
return;
|
||||
}
|
||||
};
|
||||
let conn = match rt.block_on(AccessibilityConnection::new()) {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
let _ = ready_tx.send(Err(A11yError::Permission(format!(
|
||||
"cannot connect to the AT-SPI accessibility bus: {e}. Ensure \
|
||||
at-spi2-core is running; KDE needs QT_LINUX_ACCESSIBILITY_ALWAYS_ON=1, \
|
||||
Electron apps need --force-renderer-accessibility."
|
||||
))));
|
||||
return;
|
||||
}
|
||||
};
|
||||
let _ = ready_tx.send(Ok(()));
|
||||
|
||||
let mut state = State {
|
||||
gen_counter: 0,
|
||||
current_gen: SnapshotGen(0),
|
||||
registry: HashMap::new(),
|
||||
};
|
||||
|
||||
while let Ok(cmd) = rx.recv() {
|
||||
match cmd {
|
||||
Cmd::Capabilities(reply) => {
|
||||
let _ = reply.send(detect_caps());
|
||||
}
|
||||
Cmd::Observe(opts, reply) => {
|
||||
let _ = reply.send(rt.block_on(do_observe(&conn, &opts, &mut state)));
|
||||
}
|
||||
Cmd::Invoke(target, generation, action, reply) => {
|
||||
let _ = reply.send(rt.block_on(do_invoke(
|
||||
&conn,
|
||||
&target,
|
||||
generation,
|
||||
&action,
|
||||
&state,
|
||||
)));
|
||||
}
|
||||
Cmd::Focus(pid, reply) => {
|
||||
let _ = reply.send(do_focus(pid));
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.map_err(|e| A11yError::Backend(format!("failed to start AT-SPI actor thread: {e}")))?;
|
||||
|
||||
ready_rx
|
||||
.recv()
|
||||
.map_err(|_| A11yError::Backend("AT-SPI actor died at startup".to_string()))??;
|
||||
Ok(Self { tx: Mutex::new(tx) })
|
||||
}
|
||||
|
||||
fn send(&self, cmd: Cmd) -> Result<(), A11yError> {
|
||||
self.tx
|
||||
.lock()
|
||||
.map_err(|_| A11yError::Backend("AT-SPI actor lock poisoned".to_string()))?
|
||||
.send(cmd)
|
||||
.map_err(|_| A11yError::Backend("AT-SPI actor thread is gone".to_string()))
|
||||
}
|
||||
|
||||
pub fn capabilities(&self) -> Capabilities {
|
||||
let (tx, rx) = channel();
|
||||
if self.send(Cmd::Capabilities(tx)).is_err() {
|
||||
return detect_caps();
|
||||
}
|
||||
rx.recv().unwrap_or_else(|_| detect_caps())
|
||||
}
|
||||
|
||||
pub fn observe(&self, opts: ObserveOpts) -> Result<Snapshot, A11yError> {
|
||||
let (tx, rx) = channel();
|
||||
self.send(Cmd::Observe(opts, tx))?;
|
||||
rx.recv()
|
||||
.map_err(|_| A11yError::Backend("AT-SPI actor dropped the reply".to_string()))?
|
||||
}
|
||||
|
||||
pub fn invoke(
|
||||
&self,
|
||||
target: Target,
|
||||
generation: SnapshotGen,
|
||||
action: ElementAction,
|
||||
) -> Result<Effect, A11yError> {
|
||||
let (tx, rx) = channel();
|
||||
self.send(Cmd::Invoke(target, generation, action, tx))?;
|
||||
rx.recv()
|
||||
.map_err(|_| A11yError::Backend("AT-SPI actor dropped the reply".to_string()))?
|
||||
}
|
||||
|
||||
pub fn focus_window(&self, pid: i32) -> Result<Effect, A11yError> {
|
||||
let (tx, rx) = channel();
|
||||
self.send(Cmd::Focus(pid, tx))?;
|
||||
rx.recv()
|
||||
.map_err(|_| A11yError::Backend("AT-SPI actor dropped the reply".to_string()))?
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
//! Linux AT-SPI2 backend.
|
||||
//!
|
||||
//! AT-SPI2 is a D-Bus protocol; the `atspi` crate is a pure-Rust (zbus) async
|
||||
//! client. We mirror the macOS actor: a dedicated thread owns a current-thread
|
||||
//! tokio runtime + the `AccessibilityConnection`, and the synchronous
|
||||
//! `A11yEngine` methods `block_on` async AT-SPI calls via a command channel.
|
||||
//!
|
||||
//! Status: skeleton (reports honest capabilities; observe/invoke wired in
|
||||
//! `actor.rs`). Compiled only on Linux.
|
||||
|
||||
use crate::engine::{
|
||||
A11yEngine, A11yError, Capabilities, Effect, ElementAction, ObserveOpts, OcrLine, Snapshot,
|
||||
SnapshotGen, Target,
|
||||
};
|
||||
|
||||
mod actor;
|
||||
|
||||
pub struct LinuxEngine {
|
||||
inner: actor::ActorHandle,
|
||||
}
|
||||
|
||||
impl LinuxEngine {
|
||||
pub fn start() -> Result<Self, A11yError> {
|
||||
let inner = actor::ActorHandle::spawn()?;
|
||||
Ok(Self { inner })
|
||||
}
|
||||
}
|
||||
|
||||
impl A11yEngine for LinuxEngine {
|
||||
fn capabilities(&self) -> Capabilities {
|
||||
self.inner.capabilities()
|
||||
}
|
||||
fn observe(&self, opts: &ObserveOpts) -> Result<Snapshot, A11yError> {
|
||||
self.inner.observe(opts.clone())
|
||||
}
|
||||
fn invoke(
|
||||
&self,
|
||||
target: &Target,
|
||||
generation: SnapshotGen,
|
||||
action: ElementAction,
|
||||
) -> Result<Effect, A11yError> {
|
||||
self.inner.invoke(target.clone(), generation, action)
|
||||
}
|
||||
fn focus_window(&self, pid: i32) -> Result<Effect, A11yError> {
|
||||
self.inner.focus_window(pid)
|
||||
}
|
||||
}
|
||||
|
||||
/// Linux has no OS-native OCR (unlike macOS Vision / Windows.Media.Ocr). The
|
||||
/// tool layer handles this `Unsupported` gracefully (it just skips OCR fusion).
|
||||
/// A `tesseract`-backed path could be added behind a cargo feature later.
|
||||
pub fn ocr_screenshot(_img: &image::RgbaImage, _langs: &[String]) -> Result<Vec<OcrLine>, A11yError> {
|
||||
Err(A11yError::Unsupported {
|
||||
capability: "OCR".to_string(),
|
||||
hint: "Linux has no built-in OCR engine; accessibility-tree targeting still works, and \
|
||||
a11y-thin content falls back to pixel actions."
|
||||
.to_string(),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,796 @@
|
||||
//! The macOS AX actor: a single dedicated thread that owns every AXUIElement
|
||||
//! and is the sole caller of the Accessibility C API. The public engine sends
|
||||
//! commands over a channel and blocks on a per-command reply, so AX handles
|
||||
//! (which are not `Send` and have thread affinity) never leave this thread.
|
||||
//!
|
||||
//! Raw FFI is used (rather than a higher-level AX crate) so the whole backend
|
||||
//! pins to one CoreFoundation version (0.10, shared with core-graphics 0.25)
|
||||
//! and we control retain/release precisely. Attribute names are plain CFStrings
|
||||
//! ("AXRole", "AXTitle", …) so no framework string constants need linking.
|
||||
//!
|
||||
//! The actor thread runs a CFRunLoop (polled via `recv_timeout` + a
|
||||
//! non-blocking `CFRunLoopRunInMode`, so it never hot-spins) and owns an
|
||||
//! AXObserver on the frontmost app. Change notifications flip a `dirty` flag so
|
||||
//! `observe` re-serves the cached snapshot when nothing has changed and
|
||||
//! re-walks the tree otherwise. Every mutating command also marks `dirty`, so
|
||||
//! the cache is never stale after one of our own actions. OCR/vision fusion
|
||||
//! lives one layer up (the computer tool fuses `nomi_a11y::ocr_screenshot`).
|
||||
|
||||
// This whole module is FFI against the Accessibility C API; every helper is an
|
||||
// `unsafe fn` that is only valid on the actor thread. We keep the pre-2024
|
||||
// "unsafe fn body is unsafe" ergonomics rather than wrapping each FFI call.
|
||||
#![allow(unsafe_op_in_unsafe_fn)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::ffi::c_void;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::mpsc::{Sender, channel};
|
||||
use std::time::Duration;
|
||||
|
||||
use core_foundation::base::TCFType;
|
||||
use core_foundation::string::{CFString, CFStringRef};
|
||||
use core_graphics::geometry::{CGPoint, CGSize};
|
||||
|
||||
use crate::engine::{
|
||||
A11yError, Effect, ElementAction, ElementEntry, ObserveOpts, Rect, Snapshot, SnapshotGen,
|
||||
Source, Target,
|
||||
};
|
||||
use crate::tree::{format_entries, normalize_role};
|
||||
|
||||
// ---- FFI ---------------------------------------------------------------
|
||||
|
||||
#[link(name = "CoreFoundation", kind = "framework")]
|
||||
unsafe extern "C" {
|
||||
fn CFRetain(cf: *const c_void) -> *const c_void;
|
||||
fn CFRelease(cf: *const c_void);
|
||||
fn CFGetTypeID(cf: *const c_void) -> usize;
|
||||
fn CFStringGetTypeID() -> usize;
|
||||
fn CFBooleanGetTypeID() -> usize;
|
||||
fn CFBooleanGetValue(b: *const c_void) -> u8;
|
||||
fn CFArrayGetCount(arr: *const c_void) -> isize;
|
||||
fn CFArrayGetValueAtIndex(arr: *const c_void, idx: isize) -> *const c_void;
|
||||
fn CFRunLoopGetCurrent() -> *mut c_void;
|
||||
fn CFRunLoopRunInMode(mode: CFStringRef, seconds: f64, return_after_source_handled: u8) -> i32;
|
||||
fn CFRunLoopAddSource(rl: *mut c_void, source: *const c_void, mode: CFStringRef);
|
||||
fn CFRunLoopRemoveSource(rl: *mut c_void, source: *const c_void, mode: CFStringRef);
|
||||
}
|
||||
|
||||
/// AXObserver notification callback: flips the `dirty` flag (passed as `refcon`)
|
||||
/// so the next `observe` re-walks instead of re-serving a stale snapshot. Runs
|
||||
/// on the actor thread (the run loop that owns the observer source).
|
||||
unsafe extern "C" fn observer_callback(
|
||||
_observer: *mut c_void,
|
||||
_element: *const c_void,
|
||||
_notification: CFStringRef,
|
||||
refcon: *mut c_void,
|
||||
) {
|
||||
if !refcon.is_null() {
|
||||
(*(refcon as *const AtomicBool)).store(true, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
type AXObserverCallback = unsafe extern "C" fn(*mut c_void, *const c_void, CFStringRef, *mut c_void);
|
||||
|
||||
#[link(name = "ApplicationServices", kind = "framework")]
|
||||
unsafe extern "C" {
|
||||
fn AXUIElementCreateSystemWide() -> *const c_void;
|
||||
fn AXUIElementCreateApplication(pid: i32) -> *const c_void;
|
||||
fn AXUIElementCopyAttributeValue(
|
||||
el: *const c_void,
|
||||
attr: CFStringRef,
|
||||
out: *mut *const c_void,
|
||||
) -> i32;
|
||||
fn AXUIElementSetAttributeValue(el: *const c_void, attr: CFStringRef, val: *const c_void)
|
||||
-> i32;
|
||||
fn AXUIElementCopyActionNames(el: *const c_void, out: *mut *const c_void) -> i32;
|
||||
fn AXUIElementPerformAction(el: *const c_void, action: CFStringRef) -> i32;
|
||||
fn AXUIElementGetPid(el: *const c_void, out: *mut i32) -> i32;
|
||||
fn AXValueGetValue(value: *const c_void, the_type: u32, out: *mut c_void) -> u8;
|
||||
fn AXObserverCreate(
|
||||
application: i32,
|
||||
callback: AXObserverCallback,
|
||||
out: *mut *mut c_void,
|
||||
) -> i32;
|
||||
fn AXObserverAddNotification(
|
||||
observer: *mut c_void,
|
||||
element: *const c_void,
|
||||
notification: CFStringRef,
|
||||
refcon: *mut c_void,
|
||||
) -> i32;
|
||||
fn AXObserverRemoveNotification(
|
||||
observer: *mut c_void,
|
||||
element: *const c_void,
|
||||
notification: CFStringRef,
|
||||
) -> i32;
|
||||
fn AXObserverGetRunLoopSource(observer: *mut c_void) -> *const c_void;
|
||||
}
|
||||
|
||||
const AX_VALUE_CGPOINT: u32 = 1;
|
||||
const AX_VALUE_CGSIZE: u32 = 2;
|
||||
|
||||
// ---- AxElem: RAII owner of one AXUIElement (thread-confined, !Send) -----
|
||||
|
||||
struct AxElem(*const c_void);
|
||||
|
||||
impl AxElem {
|
||||
/// Take ownership of a +1 reference (from a Create/Copy call).
|
||||
unsafe fn from_create(p: *const c_void) -> Option<Self> {
|
||||
if p.is_null() {
|
||||
None
|
||||
} else {
|
||||
Some(AxElem(p))
|
||||
}
|
||||
}
|
||||
/// Retain a borrowed (+0) reference and own the new count.
|
||||
unsafe fn from_borrowed(p: *const c_void) -> Option<Self> {
|
||||
if p.is_null() {
|
||||
None
|
||||
} else {
|
||||
Some(AxElem(CFRetain(p)))
|
||||
}
|
||||
}
|
||||
fn ptr(&self) -> *const c_void {
|
||||
self.0
|
||||
}
|
||||
fn retain(&self) -> AxElem {
|
||||
unsafe { AxElem(CFRetain(self.0)) }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for AxElem {
|
||||
fn drop(&mut self) {
|
||||
unsafe { CFRelease(self.0) }
|
||||
}
|
||||
}
|
||||
|
||||
// ---- low-level attribute helpers (call only on the actor thread) --------
|
||||
|
||||
unsafe fn copy_attr_raw(el: *const c_void, name: &str) -> *const c_void {
|
||||
let attr = CFString::new(name);
|
||||
let mut out: *const c_void = std::ptr::null();
|
||||
let err = AXUIElementCopyAttributeValue(el, attr.as_concrete_TypeRef(), &mut out);
|
||||
if err != 0 {
|
||||
std::ptr::null()
|
||||
} else {
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn copy_str_attr(el: *const c_void, name: &str) -> Option<String> {
|
||||
let out = copy_attr_raw(el, name);
|
||||
if out.is_null() {
|
||||
return None;
|
||||
}
|
||||
if CFGetTypeID(out) == CFStringGetTypeID() {
|
||||
// Take the +1 directly as a CFString and let it release on drop.
|
||||
Some(CFString::wrap_under_create_rule(out as CFStringRef).to_string())
|
||||
} else {
|
||||
CFRelease(out);
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn copy_bool_attr(el: *const c_void, name: &str) -> Option<bool> {
|
||||
let out = copy_attr_raw(el, name);
|
||||
if out.is_null() {
|
||||
return None;
|
||||
}
|
||||
let r = if CFGetTypeID(out) == CFBooleanGetTypeID() {
|
||||
Some(CFBooleanGetValue(out) != 0)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
CFRelease(out);
|
||||
r
|
||||
}
|
||||
|
||||
unsafe fn copy_elem_attr(el: *const c_void, name: &str) -> Option<AxElem> {
|
||||
AxElem::from_create(copy_attr_raw(el, name))
|
||||
}
|
||||
|
||||
unsafe fn copy_children(el: *const c_void) -> Vec<AxElem> {
|
||||
let out = copy_attr_raw(el, "AXChildren");
|
||||
if out.is_null() {
|
||||
return Vec::new();
|
||||
}
|
||||
let n = CFArrayGetCount(out);
|
||||
let mut v = Vec::with_capacity(n.max(0) as usize);
|
||||
for i in 0..n {
|
||||
let item = CFArrayGetValueAtIndex(out, i);
|
||||
if let Some(e) = AxElem::from_borrowed(item) {
|
||||
v.push(e);
|
||||
}
|
||||
}
|
||||
CFRelease(out);
|
||||
v
|
||||
}
|
||||
|
||||
unsafe fn copy_point(el: *const c_void, name: &str) -> Option<(f64, f64)> {
|
||||
let out = copy_attr_raw(el, name);
|
||||
if out.is_null() {
|
||||
return None;
|
||||
}
|
||||
let mut p = CGPoint { x: 0.0, y: 0.0 };
|
||||
let ok = AXValueGetValue(out, AX_VALUE_CGPOINT, &mut p as *mut _ as *mut c_void);
|
||||
CFRelease(out);
|
||||
if ok != 0 {
|
||||
Some((p.x, p.y))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn copy_size(el: *const c_void, name: &str) -> Option<(f64, f64)> {
|
||||
let out = copy_attr_raw(el, name);
|
||||
if out.is_null() {
|
||||
return None;
|
||||
}
|
||||
let mut s = CGSize {
|
||||
width: 0.0,
|
||||
height: 0.0,
|
||||
};
|
||||
let ok = AXValueGetValue(out, AX_VALUE_CGSIZE, &mut s as *mut _ as *mut c_void);
|
||||
CFRelease(out);
|
||||
if ok != 0 {
|
||||
Some((s.width, s.height))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn copy_actions(el: *const c_void) -> Vec<String> {
|
||||
let mut out: *const c_void = std::ptr::null();
|
||||
let err = AXUIElementCopyActionNames(el, &mut out);
|
||||
if err != 0 || out.is_null() {
|
||||
return Vec::new();
|
||||
}
|
||||
let n = CFArrayGetCount(out);
|
||||
let mut v = Vec::new();
|
||||
for i in 0..n {
|
||||
let item = CFArrayGetValueAtIndex(out, i);
|
||||
if !item.is_null() && CFGetTypeID(item) == CFStringGetTypeID() {
|
||||
v.push(CFString::wrap_under_get_rule(item as CFStringRef).to_string());
|
||||
}
|
||||
}
|
||||
CFRelease(out);
|
||||
v
|
||||
}
|
||||
|
||||
unsafe fn pid_of(el: *const c_void) -> Option<i32> {
|
||||
let mut p = 0i32;
|
||||
if AXUIElementGetPid(el, &mut p) == 0 {
|
||||
Some(p)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn perform(el: *const c_void, action: &str) -> i32 {
|
||||
let a = CFString::new(action);
|
||||
AXUIElementPerformAction(el, a.as_concrete_TypeRef())
|
||||
}
|
||||
|
||||
unsafe fn set_string_value(el: *const c_void, val: &str) -> i32 {
|
||||
let attr = CFString::new("AXValue");
|
||||
let v = CFString::new(val);
|
||||
AXUIElementSetAttributeValue(
|
||||
el,
|
||||
attr.as_concrete_TypeRef(),
|
||||
v.as_concrete_TypeRef() as *const c_void,
|
||||
)
|
||||
}
|
||||
|
||||
fn is_action_actionable(actions: &[String]) -> bool {
|
||||
actions.iter().any(|a| {
|
||||
matches!(
|
||||
a.as_str(),
|
||||
"AXPress" | "AXConfirm" | "AXOpen" | "AXShowMenu" | "AXPick" | "AXIncrement"
|
||||
| "AXDecrement"
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
// ---- collection: walk the focused window into numbered entries ----------
|
||||
|
||||
struct Collected {
|
||||
elem: AxElem,
|
||||
role: String,
|
||||
name: Option<String>,
|
||||
value: Option<String>,
|
||||
states: Vec<String>,
|
||||
bounds: Rect,
|
||||
}
|
||||
|
||||
unsafe fn walk(
|
||||
el: &AxElem,
|
||||
depth: usize,
|
||||
opts: &ObserveOpts,
|
||||
out: &mut Vec<Collected>,
|
||||
truncated: &mut bool,
|
||||
) {
|
||||
if out.len() >= opts.node_budget {
|
||||
*truncated = true;
|
||||
return;
|
||||
}
|
||||
let role = copy_str_attr(el.ptr(), "AXRole");
|
||||
let name = copy_str_attr(el.ptr(), "AXTitle")
|
||||
.or_else(|| copy_str_attr(el.ptr(), "AXDescription"))
|
||||
.filter(|s| !s.trim().is_empty());
|
||||
let value = copy_str_attr(el.ptr(), "AXValue").filter(|s| !s.trim().is_empty());
|
||||
let pos = copy_point(el.ptr(), "AXPosition");
|
||||
let size = copy_size(el.ptr(), "AXSize");
|
||||
let actions = copy_actions(el.ptr());
|
||||
let actionable = is_action_actionable(&actions);
|
||||
let enabled = copy_bool_attr(el.ptr(), "AXEnabled").unwrap_or(true);
|
||||
|
||||
if let (Some((x, y)), Some((w, h))) = (pos, size) {
|
||||
let bounds = Rect { x, y, w, h };
|
||||
if !bounds.is_empty() && (actionable || name.is_some()) {
|
||||
let mut states = Vec::new();
|
||||
if !enabled {
|
||||
states.push("disabled".to_string());
|
||||
}
|
||||
if copy_bool_attr(el.ptr(), "AXFocused").unwrap_or(false) {
|
||||
states.push("focused".to_string());
|
||||
}
|
||||
out.push(Collected {
|
||||
elem: el.retain(),
|
||||
role: role.clone().unwrap_or_else(|| "element".to_string()),
|
||||
name,
|
||||
value,
|
||||
states,
|
||||
bounds,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if depth >= opts.max_depth {
|
||||
return;
|
||||
}
|
||||
for child in copy_children(el.ptr()) {
|
||||
if out.len() >= opts.node_budget {
|
||||
*truncated = true;
|
||||
return;
|
||||
}
|
||||
walk(&child, depth + 1, opts, out, truncated);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- actor thread state + command handling ------------------------------
|
||||
|
||||
/// A registered AXObserver watching one application for change notifications.
|
||||
/// Dropping it removes the run-loop source and notifications before the `dirty`
|
||||
/// flag it points at can be freed (see `State` field order).
|
||||
struct AxObserver {
|
||||
observer: *mut c_void,
|
||||
app: AxElem,
|
||||
runloop: *mut c_void,
|
||||
notifications: Vec<CFString>,
|
||||
}
|
||||
|
||||
impl Drop for AxObserver {
|
||||
fn drop(&mut self) {
|
||||
unsafe {
|
||||
let src = AXObserverGetRunLoopSource(self.observer);
|
||||
if !src.is_null() {
|
||||
let mode = CFString::new("kCFRunLoopDefaultMode");
|
||||
CFRunLoopRemoveSource(self.runloop, src, mode.as_concrete_TypeRef());
|
||||
}
|
||||
for n in &self.notifications {
|
||||
AXObserverRemoveNotification(self.observer, self.app.ptr(), n.as_concrete_TypeRef());
|
||||
}
|
||||
CFRelease(self.observer as *const c_void);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Register a change observer for `pid` on the run loop, with `refcon` pointing
|
||||
/// at the `dirty` flag. Returns `None` (caller then never caches) on failure.
|
||||
unsafe fn register_observer(
|
||||
pid: i32,
|
||||
app: &AxElem,
|
||||
runloop: *mut c_void,
|
||||
refcon: *mut c_void,
|
||||
) -> Option<AxObserver> {
|
||||
let mut obs: *mut c_void = std::ptr::null_mut();
|
||||
if AXObserverCreate(pid, observer_callback, &mut obs) != 0 || obs.is_null() {
|
||||
return None;
|
||||
}
|
||||
const NOTIFS: &[&str] = &[
|
||||
"AXValueChanged",
|
||||
"AXUIElementDestroyed",
|
||||
"AXFocusedUIElementChanged",
|
||||
"AXMainWindowChanged",
|
||||
"AXFocusedWindowChanged",
|
||||
"AXWindowResized",
|
||||
"AXWindowMoved",
|
||||
"AXCreated",
|
||||
"AXLayoutChanged",
|
||||
"AXSelectedChildrenChanged",
|
||||
"AXRowCountChanged",
|
||||
"AXTitleChanged",
|
||||
"AXMenuOpened",
|
||||
"AXMenuClosed",
|
||||
];
|
||||
let mut notifications = Vec::new();
|
||||
for name in NOTIFS {
|
||||
let cf = CFString::new(name);
|
||||
// Not every notification applies to every app element; ignore failures.
|
||||
if AXObserverAddNotification(obs, app.ptr(), cf.as_concrete_TypeRef(), refcon) == 0 {
|
||||
notifications.push(cf);
|
||||
}
|
||||
}
|
||||
let src = AXObserverGetRunLoopSource(obs);
|
||||
if src.is_null() {
|
||||
CFRelease(obs as *const c_void);
|
||||
return None;
|
||||
}
|
||||
let mode = CFString::new("kCFRunLoopDefaultMode");
|
||||
CFRunLoopAddSource(runloop, src, mode.as_concrete_TypeRef());
|
||||
Some(AxObserver {
|
||||
observer: obs,
|
||||
app: app.retain(),
|
||||
runloop,
|
||||
notifications,
|
||||
})
|
||||
}
|
||||
|
||||
/// The last walk, kept so repeated `observe`s on an unchanged window re-serve
|
||||
/// instead of re-walking the tree.
|
||||
struct CachedWalk {
|
||||
entries: Vec<ElementEntry>,
|
||||
app_name: Option<String>,
|
||||
window_title: Option<String>,
|
||||
pid: Option<i32>,
|
||||
truncated: bool,
|
||||
}
|
||||
|
||||
struct State {
|
||||
gen_counter: u64,
|
||||
current_gen: SnapshotGen,
|
||||
registry: HashMap<u32, AxElem>,
|
||||
/// This thread's run loop (observer sources are attached to it).
|
||||
runloop: *mut c_void,
|
||||
/// MUST be declared before `dirty`: dropping the observer removes its
|
||||
/// callback source before the `dirty` flag it references is freed.
|
||||
observer: Option<AxObserver>,
|
||||
/// Boxed for a stable address (the observer's `refcon`). Set true by the
|
||||
/// observer callback and by every mutating command; cleared on a fresh walk.
|
||||
dirty: Box<AtomicBool>,
|
||||
observed_pid: Option<i32>,
|
||||
cached: Option<CachedWalk>,
|
||||
}
|
||||
|
||||
unsafe fn focused_app() -> Result<AxElem, A11yError> {
|
||||
let sw = AxElem::from_create(AXUIElementCreateSystemWide()).ok_or_else(|| {
|
||||
A11yError::Backend("AXUIElementCreateSystemWide returned null".to_string())
|
||||
})?;
|
||||
copy_elem_attr(sw.ptr(), "AXFocusedApplication").ok_or_else(|| {
|
||||
let app = crate::host_app_label();
|
||||
A11yError::Permission(format!(
|
||||
"No focused application is readable — Accessibility permission is not in effect for \
|
||||
{app}. Grant it in System Settings → Privacy & Security → Accessibility (the entry is \
|
||||
named \"{app}\"), then COMPLETELY quit and reopen {app} — macOS does not apply this \
|
||||
permission to an already-running process. Computer-use runs inside {app} itself, so \
|
||||
do not grant a terminal or editor."
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
unsafe fn focused_window(app: &AxElem) -> Option<AxElem> {
|
||||
copy_elem_attr(app.ptr(), "AXFocusedWindow")
|
||||
.or_else(|| copy_elem_attr(app.ptr(), "AXMainWindow"))
|
||||
.or_else(|| copy_children(app.ptr()).into_iter().next())
|
||||
}
|
||||
|
||||
fn do_observe(opts: &ObserveOpts, state: &mut State) -> Result<Snapshot, A11yError> {
|
||||
unsafe {
|
||||
let app = match opts.pid {
|
||||
Some(pid) => AxElem::from_create(AXUIElementCreateApplication(pid))
|
||||
.ok_or_else(|| A11yError::NotFound(format!("no app for pid {pid}")))?,
|
||||
None => focused_app()?,
|
||||
};
|
||||
let app_pid = pid_of(app.ptr());
|
||||
|
||||
// Cache re-serve: frontmost app unchanged, an observer is watching it,
|
||||
// and nothing has dirtied the snapshot since the last walk. (Explicit-pid
|
||||
// observes always re-walk.)
|
||||
if opts.pid.is_none()
|
||||
&& app_pid.is_some()
|
||||
&& state.observed_pid == app_pid
|
||||
&& state.observer.is_some()
|
||||
&& !state.dirty.load(Ordering::Relaxed)
|
||||
{
|
||||
if let Some(c) = &state.cached {
|
||||
return Ok(Snapshot {
|
||||
generation: state.current_gen,
|
||||
entries: c.entries.clone(),
|
||||
overlay: None,
|
||||
text: format_entries(&c.entries),
|
||||
truncated: c.truncated,
|
||||
pid: c.pid,
|
||||
app_name: c.app_name.clone(),
|
||||
window_title: c.window_title.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let app_name = copy_str_attr(app.ptr(), "AXTitle");
|
||||
let window = focused_window(&app).ok_or_else(|| {
|
||||
A11yError::NotFound("the focused application has no readable window".to_string())
|
||||
})?;
|
||||
let window_title = copy_str_attr(window.ptr(), "AXTitle");
|
||||
|
||||
let mut collected = Vec::new();
|
||||
let mut truncated = false;
|
||||
walk(&window, 0, opts, &mut collected, &mut truncated);
|
||||
|
||||
// Reading order: top-to-bottom, left-to-right.
|
||||
collected.sort_by(|a, b| {
|
||||
(a.bounds.y.round() as i64, a.bounds.x.round() as i64)
|
||||
.cmp(&(b.bounds.y.round() as i64, b.bounds.x.round() as i64))
|
||||
});
|
||||
|
||||
state.gen_counter += 1;
|
||||
let generation = SnapshotGen(state.gen_counter);
|
||||
state.current_gen = generation;
|
||||
state.registry.clear();
|
||||
|
||||
let mut entries = Vec::with_capacity(collected.len());
|
||||
for (i, c) in collected.into_iter().enumerate() {
|
||||
let r = i as u32 + 1;
|
||||
state.registry.insert(r, c.elem);
|
||||
entries.push(ElementEntry {
|
||||
r#ref: r,
|
||||
role: normalize_role(&c.role),
|
||||
name: c.name,
|
||||
value: c.value,
|
||||
states: c.states,
|
||||
bounds: c.bounds,
|
||||
source: Source::A11y,
|
||||
});
|
||||
}
|
||||
|
||||
// (Re)register the change observer if the frontmost app changed.
|
||||
if state.observed_pid != app_pid {
|
||||
state.observer = None; // drop the old observer first (removes its source)
|
||||
if let Some(p) = app_pid {
|
||||
let refcon = (&*state.dirty as *const AtomicBool) as *mut c_void;
|
||||
state.observer = register_observer(p, &app, state.runloop, refcon);
|
||||
}
|
||||
state.observed_pid = app_pid;
|
||||
}
|
||||
state.dirty.store(false, Ordering::Relaxed);
|
||||
|
||||
let text = format_entries(&entries);
|
||||
state.cached = Some(CachedWalk {
|
||||
entries: entries.clone(),
|
||||
app_name: app_name.clone(),
|
||||
window_title: window_title.clone(),
|
||||
pid: app_pid,
|
||||
truncated,
|
||||
});
|
||||
Ok(Snapshot {
|
||||
generation,
|
||||
entries,
|
||||
overlay: None, // the tool captures the screenshot + draws the overlay
|
||||
text,
|
||||
truncated,
|
||||
pid: app_pid,
|
||||
app_name,
|
||||
window_title,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn do_invoke(
|
||||
target: &Target,
|
||||
generation: SnapshotGen,
|
||||
action: &ElementAction,
|
||||
state: &State,
|
||||
) -> Result<Effect, A11yError> {
|
||||
let r = match target {
|
||||
Target::Ref(r) => *r,
|
||||
Target::Selector(_) => {
|
||||
return Err(A11yError::Unsupported {
|
||||
capability: "selector targeting".to_string(),
|
||||
hint: "Resolve a selector against the latest observe() result and act by [ref]; \
|
||||
direct selector actuation is not yet implemented."
|
||||
.to_string(),
|
||||
})
|
||||
}
|
||||
Target::Pixel { .. } => {
|
||||
return Err(A11yError::Unsupported {
|
||||
capability: "pixel targeting".to_string(),
|
||||
hint: "Pixel fallback is handled by the computer tool's input layer, not the \
|
||||
accessibility engine."
|
||||
.to_string(),
|
||||
})
|
||||
}
|
||||
};
|
||||
if generation != state.current_gen {
|
||||
return Err(A11yError::Stale(format!(
|
||||
"ref [{r}] is from an older snapshot (the UI may have changed); re-run observe and \
|
||||
use a fresh [ref]"
|
||||
)));
|
||||
}
|
||||
let elem = state
|
||||
.registry
|
||||
.get(&r)
|
||||
.ok_or_else(|| A11yError::NotFound(format!("no element [{r}] in the latest snapshot")))?;
|
||||
|
||||
unsafe {
|
||||
let err = match action {
|
||||
ElementAction::Press | ElementAction::LeftClick | ElementAction::DoubleClick => {
|
||||
perform(elem.ptr(), "AXPress")
|
||||
}
|
||||
ElementAction::RightClick => perform(elem.ptr(), "AXShowMenu"),
|
||||
ElementAction::Focus => {
|
||||
let attr = CFString::new("AXFocused");
|
||||
let t = core_foundation::boolean::CFBoolean::true_value();
|
||||
AXUIElementSetAttributeValue(
|
||||
elem.ptr(),
|
||||
attr.as_concrete_TypeRef(),
|
||||
t.as_concrete_TypeRef() as *const c_void,
|
||||
)
|
||||
}
|
||||
ElementAction::SetValue(v) => set_string_value(elem.ptr(), v),
|
||||
};
|
||||
if err == 0 {
|
||||
Ok(Effect {
|
||||
changed: true,
|
||||
message: format!("performed {action:?} on element [{r}]"),
|
||||
})
|
||||
} else {
|
||||
Err(A11yError::Backend(format!(
|
||||
"AX action on [{r}] failed (AXError {err}); the element may be a web view \
|
||||
(AXWebArea) that ignores AXPress — fall back to a pixel click"
|
||||
)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn do_focus(pid: i32) -> Result<Effect, A11yError> {
|
||||
unsafe {
|
||||
let app = AxElem::from_create(AXUIElementCreateApplication(pid))
|
||||
.ok_or_else(|| A11yError::NotFound(format!("no app for pid {pid}")))?;
|
||||
let attr = CFString::new("AXFrontmost");
|
||||
let t = core_foundation::boolean::CFBoolean::true_value();
|
||||
let err = AXUIElementSetAttributeValue(
|
||||
app.ptr(),
|
||||
attr.as_concrete_TypeRef(),
|
||||
t.as_concrete_TypeRef() as *const c_void,
|
||||
);
|
||||
if let Some(win) = focused_window(&app) {
|
||||
let _ = perform(win.ptr(), "AXRaise");
|
||||
}
|
||||
if err == 0 {
|
||||
Ok(Effect {
|
||||
changed: true,
|
||||
message: format!("brought pid {pid} to the front"),
|
||||
})
|
||||
} else {
|
||||
Err(A11yError::Backend(format!(
|
||||
"could not activate pid {pid} (AXError {err})"
|
||||
)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- command plumbing ---------------------------------------------------
|
||||
|
||||
enum Cmd {
|
||||
Observe(ObserveOpts, Sender<Result<Snapshot, A11yError>>),
|
||||
Invoke(
|
||||
Target,
|
||||
SnapshotGen,
|
||||
ElementAction,
|
||||
Sender<Result<Effect, A11yError>>,
|
||||
),
|
||||
Focus(i32, Sender<Result<Effect, A11yError>>),
|
||||
}
|
||||
|
||||
pub struct ActorHandle {
|
||||
// mpsc::Sender is not Sync; the Mutex makes the handle Sync (calls are
|
||||
// serialized anyway — the tool marks the tool non-concurrency-safe).
|
||||
tx: Mutex<Sender<Cmd>>,
|
||||
}
|
||||
|
||||
impl ActorHandle {
|
||||
pub fn spawn() -> Result<Self, A11yError> {
|
||||
let (tx, rx) = channel::<Cmd>();
|
||||
std::thread::Builder::new()
|
||||
.name("nomi-a11y-macos".to_string())
|
||||
.spawn(move || {
|
||||
let runloop = unsafe { CFRunLoopGetCurrent() };
|
||||
let mut state = State {
|
||||
gen_counter: 0,
|
||||
current_gen: SnapshotGen(0),
|
||||
registry: HashMap::new(),
|
||||
runloop,
|
||||
observer: None,
|
||||
dirty: Box::new(AtomicBool::new(true)),
|
||||
observed_pid: None,
|
||||
cached: None,
|
||||
};
|
||||
let mode = CFString::new("kCFRunLoopDefaultMode");
|
||||
loop {
|
||||
// Block for a command. Before there are any observer sources
|
||||
// (first observe not run yet), the run loop has nothing to
|
||||
// wait on, so we must NOT spin on CFRunLoopRunInMode — block
|
||||
// on the channel instead and pump callbacks non-blocking.
|
||||
match rx.recv_timeout(Duration::from_millis(100)) {
|
||||
Ok(cmd) => {
|
||||
// Flush any pending observer callbacks (→ `dirty`)
|
||||
// before handling, so `observe` sees the freshest state.
|
||||
unsafe {
|
||||
CFRunLoopRunInMode(mode.as_concrete_TypeRef(), 0.0, 0);
|
||||
}
|
||||
match cmd {
|
||||
Cmd::Observe(opts, reply) => {
|
||||
let _ = reply.send(do_observe(&opts, &mut state));
|
||||
}
|
||||
Cmd::Invoke(target, generation, action, reply) => {
|
||||
let r = do_invoke(&target, generation, &action, &state);
|
||||
// A mutating action invalidates the cache even
|
||||
// before the observer notification arrives.
|
||||
state.dirty.store(true, Ordering::Relaxed);
|
||||
let _ = reply.send(r);
|
||||
}
|
||||
Cmd::Focus(pid, reply) => {
|
||||
let r = do_focus(pid);
|
||||
state.dirty.store(true, Ordering::Relaxed);
|
||||
let _ = reply.send(r);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
|
||||
// Periodically service the observer source so pending
|
||||
// notifications don't pile up while idle.
|
||||
unsafe {
|
||||
CFRunLoopRunInMode(mode.as_concrete_TypeRef(), 0.0, 0);
|
||||
}
|
||||
}
|
||||
Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => return,
|
||||
}
|
||||
}
|
||||
})
|
||||
.map_err(|e| A11yError::Backend(format!("failed to start AX actor thread: {e}")))?;
|
||||
Ok(Self { tx: Mutex::new(tx) })
|
||||
}
|
||||
|
||||
fn send(&self, cmd: Cmd) -> Result<(), A11yError> {
|
||||
self.tx
|
||||
.lock()
|
||||
.map_err(|_| A11yError::Backend("AX actor lock poisoned".to_string()))?
|
||||
.send(cmd)
|
||||
.map_err(|_| A11yError::Backend("AX actor thread is gone".to_string()))
|
||||
}
|
||||
|
||||
pub fn observe(&self, opts: ObserveOpts) -> Result<Snapshot, A11yError> {
|
||||
let (tx, rx) = channel();
|
||||
self.send(Cmd::Observe(opts, tx))?;
|
||||
rx.recv()
|
||||
.map_err(|_| A11yError::Backend("AX actor dropped the reply".to_string()))?
|
||||
}
|
||||
|
||||
pub fn invoke(
|
||||
&self,
|
||||
target: Target,
|
||||
generation: SnapshotGen,
|
||||
action: ElementAction,
|
||||
) -> Result<Effect, A11yError> {
|
||||
let (tx, rx) = channel();
|
||||
self.send(Cmd::Invoke(target, generation, action, tx))?;
|
||||
rx.recv()
|
||||
.map_err(|_| A11yError::Backend("AX actor dropped the reply".to_string()))?
|
||||
}
|
||||
|
||||
pub fn focus_window(&self, pid: i32) -> Result<Effect, A11yError> {
|
||||
let (tx, rx) = channel();
|
||||
self.send(Cmd::Focus(pid, tx))?;
|
||||
rx.recv()
|
||||
.map_err(|_| A11yError::Backend("AX actor dropped the reply".to_string()))?
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
//! macOS accessibility backend (AXUIElement).
|
||||
//!
|
||||
//! Threading model: AXUIElement / AXObserver have CFRunLoop / main-thread
|
||||
//! affinity, so all AX calls are marshaled to a single dedicated actor thread
|
||||
//! that owns a CFRunLoop and is the sole caller of the AX APIs. The public
|
||||
//! `MacEngine` is a `Send + Sync` handle that sends commands to that actor and
|
||||
//! blocks on a reply channel. Raw `AXUIElement` handles never cross the actor
|
||||
//! boundary — only serializable `Snapshot` / `Effect` data does.
|
||||
//!
|
||||
//! Status: the actor scaffolding + capabilities are in place; the AX tree walk
|
||||
//! and actuation are wired in `actor.rs` (see below). This module is compiled
|
||||
//! only on macOS.
|
||||
|
||||
use crate::engine::{
|
||||
A11yEngine, A11yError, Capabilities, Effect, ElementAction, InputKind, ObserveOpts, Snapshot,
|
||||
SnapshotGen, Target,
|
||||
};
|
||||
|
||||
pub struct MacEngine {
|
||||
inner: actor::ActorHandle,
|
||||
}
|
||||
|
||||
impl MacEngine {
|
||||
pub fn start() -> Result<Self, A11yError> {
|
||||
let inner = actor::ActorHandle::spawn()?;
|
||||
Ok(Self { inner })
|
||||
}
|
||||
}
|
||||
|
||||
impl A11yEngine for MacEngine {
|
||||
fn capabilities(&self) -> Capabilities {
|
||||
Capabilities {
|
||||
os: "macos".to_string(),
|
||||
tree_read: true,
|
||||
screenshot: true,
|
||||
semantic_action: true,
|
||||
synthetic_input: InputKind::Native,
|
||||
window_management: true,
|
||||
}
|
||||
}
|
||||
|
||||
fn observe(&self, opts: &ObserveOpts) -> Result<Snapshot, A11yError> {
|
||||
self.inner.observe(opts.clone())
|
||||
}
|
||||
|
||||
fn invoke(
|
||||
&self,
|
||||
target: &Target,
|
||||
generation: SnapshotGen,
|
||||
action: ElementAction,
|
||||
) -> Result<Effect, A11yError> {
|
||||
self.inner.invoke(target.clone(), generation, action)
|
||||
}
|
||||
|
||||
fn focus_window(&self, pid: i32) -> Result<Effect, A11yError> {
|
||||
self.inner.focus_window(pid)
|
||||
}
|
||||
}
|
||||
|
||||
mod actor;
|
||||
mod ocr;
|
||||
|
||||
pub use ocr::ocr_screenshot;
|
||||
@@ -0,0 +1,112 @@
|
||||
//! macOS OCR via Vision.framework (`VNRecognizeTextRequest`) — on-device, with
|
||||
//! CJK support (essential for a Chinese-first product). The screenshot is fed
|
||||
//! in as PNG data (`VNImageRequestHandler initWithData:`) so we avoid
|
||||
//! hand-building a CGImage. Vision returns normalized bounding boxes with a
|
||||
//! bottom-left origin; we convert them to pixel rectangles with a top-left
|
||||
//! origin to match the screenshot space the overlay/tool use.
|
||||
//!
|
||||
//! OCR has no main-thread/run-loop affinity, so this runs on whatever thread
|
||||
//! the caller uses (the tool calls it from `spawn_blocking`).
|
||||
|
||||
use objc2::AnyThread;
|
||||
use objc2::rc::Retained;
|
||||
use objc2::runtime::AnyObject;
|
||||
use objc2_foundation::{NSArray, NSData, NSDictionary, NSString};
|
||||
use objc2_vision::{
|
||||
VNImageRequestHandler, VNRecognizeTextRequest, VNRequest, VNRequestTextRecognitionLevel,
|
||||
};
|
||||
|
||||
use crate::engine::{A11yError, OcrLine, Rect};
|
||||
|
||||
pub fn ocr_screenshot(img: &image::RgbaImage, langs: &[String]) -> Result<Vec<OcrLine>, A11yError> {
|
||||
let (w, h) = img.dimensions();
|
||||
if w == 0 || h == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
// Encode to PNG so VNImageRequestHandler can decode it directly.
|
||||
let mut png = Vec::new();
|
||||
image::DynamicImage::ImageRgba8(img.clone())
|
||||
.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png)
|
||||
.map_err(|e| A11yError::Backend(format!("OCR: PNG encode failed: {e}")))?;
|
||||
|
||||
unsafe {
|
||||
let data = NSData::with_bytes(&png);
|
||||
|
||||
let request = VNRecognizeTextRequest::new();
|
||||
request.setRecognitionLevel(VNRequestTextRecognitionLevel::Accurate);
|
||||
request.setUsesLanguageCorrection(true);
|
||||
if !langs.is_empty() {
|
||||
let ns: Vec<Retained<NSString>> =
|
||||
langs.iter().map(|l| NSString::from_str(l)).collect();
|
||||
let arr = NSArray::from_retained_slice(&ns);
|
||||
request.setRecognitionLanguages(&arr);
|
||||
}
|
||||
|
||||
let options: Retained<NSDictionary<NSString, AnyObject>> = NSDictionary::new();
|
||||
let handler = VNImageRequestHandler::initWithData_options(
|
||||
VNImageRequestHandler::alloc(),
|
||||
&data,
|
||||
&options,
|
||||
);
|
||||
|
||||
let req_ref: &VNRequest = &request;
|
||||
let requests = NSArray::from_slice(&[req_ref]);
|
||||
handler
|
||||
.performRequests_error(&requests)
|
||||
.map_err(|e| A11yError::Backend(format!("OCR perform failed: {e:?}")))?;
|
||||
|
||||
let Some(results) = request.results() else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
|
||||
let mut lines = Vec::new();
|
||||
for obs in results.iter() {
|
||||
let top = obs.topCandidates(1);
|
||||
let Some(text) = top.firstObject() else {
|
||||
continue;
|
||||
};
|
||||
let s = text.string().to_string();
|
||||
if s.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
// Normalized (0..1), bottom-left origin → pixel, top-left origin.
|
||||
let bb = obs.boundingBox();
|
||||
let px = bb.origin.x * w as f64;
|
||||
let pw = bb.size.width * w as f64;
|
||||
let ph = bb.size.height * h as f64;
|
||||
let py = (1.0 - bb.origin.y - bb.size.height) * h as f64;
|
||||
lines.push(OcrLine {
|
||||
text: s,
|
||||
bounds: Rect {
|
||||
x: px,
|
||||
y: py,
|
||||
w: pw,
|
||||
h: ph,
|
||||
},
|
||||
});
|
||||
}
|
||||
Ok(lines)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn ocr_blank_image_runs_without_error() {
|
||||
// Exercises the full Vision FFI path (compile + link + run). A blank
|
||||
// image yields no recognized text; OCR needs no TCC permission.
|
||||
let img = image::RgbaImage::from_pixel(80, 40, image::Rgba([255, 255, 255, 255]));
|
||||
let lines = ocr_screenshot(&img, &["en-US".to_string()]).expect("ocr should not error");
|
||||
assert!(lines.iter().all(|l| !l.text.trim().is_empty()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ocr_empty_image_is_empty() {
|
||||
let img = image::RgbaImage::new(0, 0);
|
||||
assert!(ocr_screenshot(&img, &[]).unwrap().is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
//! Set-of-Marks overlay: draw numbered boxes for each interactable element onto
|
||||
//! the screenshot the model sees. Self-contained (only the `image` crate) — a
|
||||
//! tiny embedded 3×5 bitmap digit font renders the `[ref]` labels, so no font
|
||||
//! asset or extra dependency is needed.
|
||||
//!
|
||||
//! Element `bounds` MUST already be in the image's pixel space (the caller
|
||||
//! converts OS accessibility coordinates → screenshot pixels before calling).
|
||||
|
||||
use image::{Rgba, RgbaImage};
|
||||
|
||||
use crate::engine::ElementEntry;
|
||||
|
||||
/// Distinct, high-contrast mark colors cycled by ref so neighbors differ.
|
||||
const PALETTE: [[u8; 3]; 6] = [
|
||||
[255, 59, 48], // red
|
||||
[0, 122, 255], // blue
|
||||
[52, 199, 89], // green
|
||||
[255, 149, 0], // orange
|
||||
[175, 82, 222], // purple
|
||||
[255, 45, 85], // pink
|
||||
];
|
||||
|
||||
/// 3×5 bitmap font, digits 0-9. Each row's low 3 bits are pixels (MSB = left).
|
||||
const DIGITS: [[u8; 5]; 10] = [
|
||||
[0b111, 0b101, 0b101, 0b101, 0b111], // 0
|
||||
[0b010, 0b110, 0b010, 0b010, 0b111], // 1
|
||||
[0b111, 0b001, 0b111, 0b100, 0b111], // 2
|
||||
[0b111, 0b001, 0b111, 0b001, 0b111], // 3
|
||||
[0b101, 0b101, 0b111, 0b001, 0b001], // 4
|
||||
[0b111, 0b100, 0b111, 0b001, 0b111], // 5
|
||||
[0b111, 0b100, 0b111, 0b101, 0b111], // 6
|
||||
[0b111, 0b001, 0b010, 0b010, 0b010], // 7
|
||||
[0b111, 0b101, 0b111, 0b101, 0b111], // 8
|
||||
[0b111, 0b101, 0b111, 0b001, 0b111], // 9
|
||||
];
|
||||
|
||||
const SCALE: i64 = 3; // pixels per font cell
|
||||
const DIGIT_W: i64 = 3 * SCALE;
|
||||
const DIGIT_H: i64 = 5 * SCALE;
|
||||
const GAP: i64 = SCALE;
|
||||
const PAD: i64 = SCALE;
|
||||
|
||||
/// Draw a numbered box for each entry.
|
||||
pub fn draw_set_of_marks(img: &mut RgbaImage, entries: &[ElementEntry]) {
|
||||
let (iw, ih) = img.dimensions();
|
||||
for e in entries {
|
||||
if e.bounds.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let color = PALETTE[(e.r#ref as usize) % PALETTE.len()];
|
||||
let x = e.bounds.x.round() as i64;
|
||||
let y = e.bounds.y.round() as i64;
|
||||
let w = e.bounds.w.round() as i64;
|
||||
let h = e.bounds.h.round() as i64;
|
||||
draw_rect_border(img, x, y, w, h, color, 2, iw, ih);
|
||||
draw_label(img, x, y, e.r#ref, color, iw, ih);
|
||||
}
|
||||
}
|
||||
|
||||
fn put(img: &mut RgbaImage, x: i64, y: i64, c: [u8; 3], iw: u32, ih: u32) {
|
||||
if x < 0 || y < 0 || x >= iw as i64 || y >= ih as i64 {
|
||||
return;
|
||||
}
|
||||
img.put_pixel(x as u32, y as u32, Rgba([c[0], c[1], c[2], 255]));
|
||||
}
|
||||
|
||||
fn fill_rect(img: &mut RgbaImage, x: i64, y: i64, w: i64, h: i64, c: [u8; 3], iw: u32, ih: u32) {
|
||||
for dy in 0..h {
|
||||
for dx in 0..w {
|
||||
put(img, x + dx, y + dy, c, iw, ih);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn draw_rect_border(
|
||||
img: &mut RgbaImage,
|
||||
x: i64,
|
||||
y: i64,
|
||||
w: i64,
|
||||
h: i64,
|
||||
c: [u8; 3],
|
||||
t: i64,
|
||||
iw: u32,
|
||||
ih: u32,
|
||||
) {
|
||||
for k in 0..t {
|
||||
// top / bottom
|
||||
for dx in 0..w {
|
||||
put(img, x + dx, y + k, c, iw, ih);
|
||||
put(img, x + dx, y + h - 1 - k, c, iw, ih);
|
||||
}
|
||||
// left / right
|
||||
for dy in 0..h {
|
||||
put(img, x + k, y + dy, c, iw, ih);
|
||||
put(img, x + w - 1 - k, y + dy, c, iw, ih);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn label_size(n: u32) -> (i64, i64) {
|
||||
let digits = n.max(1).to_string().len() as i64;
|
||||
let w = PAD * 2 + digits * DIGIT_W + (digits - 1) * GAP;
|
||||
let h = PAD * 2 + DIGIT_H;
|
||||
(w, h)
|
||||
}
|
||||
|
||||
fn draw_label(img: &mut RgbaImage, ex: i64, ey: i64, n: u32, bg: [u8; 3], iw: u32, ih: u32) {
|
||||
let (lw, lh) = label_size(n);
|
||||
// Prefer just above the element's top-left; if no room, place inside.
|
||||
let lx = ex.max(0);
|
||||
let ly = if ey - lh >= 0 { ey - lh } else { ey };
|
||||
fill_rect(img, lx, ly, lw, lh, bg, iw, ih);
|
||||
|
||||
let fg = [255u8, 255, 255]; // white digits on the colored chip
|
||||
let mut cx = lx + PAD;
|
||||
let cy = ly + PAD;
|
||||
for ch in n.to_string().chars() {
|
||||
let d = ch.to_digit(10).unwrap_or(0) as usize;
|
||||
draw_digit(img, cx, cy, DIGITS[d], fg, iw, ih);
|
||||
cx += DIGIT_W + GAP;
|
||||
}
|
||||
}
|
||||
|
||||
fn draw_digit(img: &mut RgbaImage, x: i64, y: i64, glyph: [u8; 5], c: [u8; 3], iw: u32, ih: u32) {
|
||||
for (row, bits) in glyph.iter().enumerate() {
|
||||
for col in 0..3i64 {
|
||||
// MSB is the leftmost column.
|
||||
if bits & (1 << (2 - col)) != 0 {
|
||||
fill_rect(
|
||||
img,
|
||||
x + col * SCALE,
|
||||
y + row as i64 * SCALE,
|
||||
SCALE,
|
||||
SCALE,
|
||||
c,
|
||||
iw,
|
||||
ih,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::engine::{Rect, Source};
|
||||
|
||||
fn entry(r: u32, x: f64, y: f64) -> ElementEntry {
|
||||
ElementEntry {
|
||||
r#ref: r,
|
||||
role: "button".into(),
|
||||
name: Some("x".into()),
|
||||
value: None,
|
||||
states: vec![],
|
||||
bounds: Rect { x, y, w: 40.0, h: 20.0 },
|
||||
source: Source::A11y,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn draws_marks_without_panicking_at_edges() {
|
||||
let mut img = RgbaImage::from_pixel(100, 100, Rgba([0, 0, 0, 255]));
|
||||
// One in-bounds, one clipped at the top edge (label would go off-screen).
|
||||
draw_set_of_marks(&mut img, &[entry(1, 30.0, 40.0), entry(12, 0.0, 0.0)]);
|
||||
// Some pixels must now be non-black (a border or label was drawn).
|
||||
let changed = img.pixels().any(|p| p[0] > 2 || p[1] > 2 || p[2] > 2);
|
||||
assert!(changed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn label_size_grows_with_digits() {
|
||||
assert!(label_size(7).0 < label_size(42).0);
|
||||
assert!(label_size(42).0 < label_size(123).0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
//! A focused, platform-independent selector grammar for addressing
|
||||
//! accessibility elements deterministically — the durable way to re-locate an
|
||||
//! element across snapshots (vs a `[ref]`, which is snapshot-scoped).
|
||||
//!
|
||||
//! Grammar (v1 subset; positional/relative combinators are a planned
|
||||
//! extension): `prefix:value` terms joined by `&&` / `||`, each optionally
|
||||
//! negated with a leading `!`.
|
||||
//!
|
||||
//! ```text
|
||||
//! role:Button && name:Save
|
||||
//! name:Save || name:Submit
|
||||
//! role:Button && !name:Cancel
|
||||
//! role:Button && name:Item && nth:2
|
||||
//! ```
|
||||
//!
|
||||
//! Prefixes: `role:` `name:` `text:` `nth:`. A bare term with no prefix is
|
||||
//! treated as `name:`. `name`/`role` match case-insensitively as substrings;
|
||||
//! `text` matches case-sensitively (visible-text semantics).
|
||||
|
||||
use crate::engine::ElementEntry;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Selector {
|
||||
/// Accessibility role (case-insensitive substring).
|
||||
Role(String),
|
||||
/// Accessible name/label (case-insensitive substring).
|
||||
Name(String),
|
||||
/// Visible text / value (case-sensitive substring).
|
||||
Text(String),
|
||||
/// Pick the Nth (0-based) of the otherwise-matching set.
|
||||
Nth(usize),
|
||||
/// All must match.
|
||||
And(Vec<Selector>),
|
||||
/// Any may match.
|
||||
Or(Vec<Selector>),
|
||||
/// Must not match.
|
||||
Not(Box<Selector>),
|
||||
}
|
||||
|
||||
impl Selector {
|
||||
/// Parse a selector expression. Returns a human-readable error on malformed
|
||||
/// input (which the caller surfaces to the model, not a panic).
|
||||
pub fn parse(input: &str) -> Result<Selector, String> {
|
||||
let s = input.trim();
|
||||
if s.is_empty() {
|
||||
return Err("empty selector".to_string());
|
||||
}
|
||||
parse_or(s)
|
||||
}
|
||||
|
||||
/// True if this selector (ignoring any positional `Nth`) matches `e`.
|
||||
pub fn matches(&self, e: &ElementEntry) -> bool {
|
||||
match self {
|
||||
Selector::Role(r) => contains_ci(&e.role, r),
|
||||
Selector::Name(n) => e.name.as_deref().is_some_and(|v| contains_ci(v, n)),
|
||||
Selector::Text(t) => {
|
||||
e.name.as_deref().is_some_and(|v| v.contains(t.as_str()))
|
||||
|| e.value.as_deref().is_some_and(|v| v.contains(t.as_str()))
|
||||
}
|
||||
Selector::Nth(_) => true, // positional; applied in `select`
|
||||
Selector::And(parts) => parts.iter().all(|p| p.matches(e)),
|
||||
Selector::Or(parts) => parts.iter().any(|p| p.matches(e)),
|
||||
Selector::Not(inner) => !inner.matches(e),
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve against a snapshot's entries: filter by the match predicate, then
|
||||
/// apply any top-level `Nth` positional pick. Returns matching refs in order.
|
||||
pub fn select<'a>(&self, entries: &'a [ElementEntry]) -> Vec<&'a ElementEntry> {
|
||||
let matched: Vec<&ElementEntry> = entries.iter().filter(|e| self.matches(e)).collect();
|
||||
match self.find_nth() {
|
||||
Some(n) => matched.into_iter().skip(n).take(1).collect(),
|
||||
None => matched,
|
||||
}
|
||||
}
|
||||
|
||||
/// Find a top-level `Nth` index if present (directly or inside a top `And`).
|
||||
fn find_nth(&self) -> Option<usize> {
|
||||
match self {
|
||||
Selector::Nth(n) => Some(*n),
|
||||
Selector::And(parts) => parts.iter().find_map(|p| match p {
|
||||
Selector::Nth(n) => Some(*n),
|
||||
_ => None,
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn contains_ci(haystack: &str, needle: &str) -> bool {
|
||||
haystack.to_lowercase().contains(&needle.to_lowercase())
|
||||
}
|
||||
|
||||
fn parse_or(s: &str) -> Result<Selector, String> {
|
||||
let parts = split_top(s, "||");
|
||||
if parts.len() == 1 {
|
||||
return parse_and(parts[0]);
|
||||
}
|
||||
let parsed: Result<Vec<_>, _> = parts.iter().map(|p| parse_and(p)).collect();
|
||||
Ok(Selector::Or(parsed?))
|
||||
}
|
||||
|
||||
fn parse_and(s: &str) -> Result<Selector, String> {
|
||||
let parts = split_top(s, "&&");
|
||||
if parts.len() == 1 {
|
||||
return parse_term(parts[0]);
|
||||
}
|
||||
let parsed: Result<Vec<_>, _> = parts.iter().map(|p| parse_term(p)).collect();
|
||||
Ok(Selector::And(parsed?))
|
||||
}
|
||||
|
||||
fn parse_term(s: &str) -> Result<Selector, String> {
|
||||
let t = s.trim();
|
||||
if let Some(rest) = t.strip_prefix('!') {
|
||||
return Ok(Selector::Not(Box::new(parse_simple(rest.trim())?)));
|
||||
}
|
||||
parse_simple(t)
|
||||
}
|
||||
|
||||
fn parse_simple(s: &str) -> Result<Selector, String> {
|
||||
let t = s.trim();
|
||||
if t.is_empty() {
|
||||
return Err("empty selector term".to_string());
|
||||
}
|
||||
let (prefix, value) = match t.split_once(':') {
|
||||
Some((p, v)) => (p.trim().to_lowercase(), v.trim().to_string()),
|
||||
None => ("name".to_string(), t.to_string()),
|
||||
};
|
||||
if value.is_empty() && prefix != "nth" {
|
||||
return Err(format!("selector term `{t}` has an empty value"));
|
||||
}
|
||||
match prefix.as_str() {
|
||||
"role" => Ok(Selector::Role(value)),
|
||||
"name" => Ok(Selector::Name(value)),
|
||||
"text" => Ok(Selector::Text(value)),
|
||||
"nth" => value
|
||||
.parse::<usize>()
|
||||
.map(Selector::Nth)
|
||||
.map_err(|_| format!("`nth:` expects a non-negative integer, got `{value}`")),
|
||||
other => Err(format!(
|
||||
"unknown selector prefix `{other}:` (supported: role, name, text, nth)"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Split on a two-char operator at the top level. (v1 has no parentheses, so
|
||||
/// this is a plain delimiter split; selector values do not contain `&&`/`||`.)
|
||||
fn split_top<'a>(s: &'a str, op: &str) -> Vec<&'a str> {
|
||||
s.split(op).map(|p| p.trim()).filter(|p| !p.is_empty()).collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::engine::{Rect, Source};
|
||||
|
||||
fn entry(r: u32, role: &str, name: Option<&str>) -> ElementEntry {
|
||||
ElementEntry {
|
||||
r#ref: r,
|
||||
role: role.to_string(),
|
||||
name: name.map(|s| s.to_string()),
|
||||
value: None,
|
||||
states: vec![],
|
||||
bounds: Rect { x: 0.0, y: 0.0, w: 1.0, h: 1.0 },
|
||||
source: Source::A11y,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_role_and_name() {
|
||||
let s = Selector::parse("role:Button && name:Save").unwrap();
|
||||
assert_eq!(
|
||||
s,
|
||||
Selector::And(vec![
|
||||
Selector::Role("Button".into()),
|
||||
Selector::Name("Save".into())
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bare_term_is_name() {
|
||||
assert_eq!(Selector::parse("Submit").unwrap(), Selector::Name("Submit".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_or_and_not() {
|
||||
let s = Selector::parse("name:Save || name:Submit").unwrap();
|
||||
assert!(matches!(s, Selector::Or(_)));
|
||||
let n = Selector::parse("!name:Cancel").unwrap();
|
||||
assert!(matches!(n, Selector::Not(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_case_insensitive_substring() {
|
||||
let e = entry(1, "AXButton", Some("Save Document"));
|
||||
assert!(Selector::parse("role:button && name:save").unwrap().matches(&e));
|
||||
assert!(!Selector::parse("name:delete").unwrap().matches(&e));
|
||||
assert!(Selector::parse("role:Button && !name:Cancel").unwrap().matches(&e));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nth_picks_positionally() {
|
||||
let entries = vec![
|
||||
entry(1, "AXButton", Some("Item")),
|
||||
entry(2, "AXButton", Some("Item")),
|
||||
entry(3, "AXButton", Some("Item")),
|
||||
];
|
||||
let s = Selector::parse("role:Button && name:Item && nth:1").unwrap();
|
||||
let got = s.select(&entries);
|
||||
assert_eq!(got.len(), 1);
|
||||
assert_eq!(got[0].r#ref, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_is_error() {
|
||||
assert!(Selector::parse(" ").is_err());
|
||||
assert!(Selector::parse("bogus:x").is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
//! Neutral accessibility-tree model + interactable filtering and text
|
||||
//! formatting, shared by every OS backend. Backends build a `UiNode` tree from
|
||||
//! their native API; this module turns it into the numbered `ElementEntry`
|
||||
//! list the model consumes (and the overlay draws).
|
||||
|
||||
use crate::engine::{ElementEntry, Rect, Source};
|
||||
|
||||
/// A raw accessibility node as captured by a backend, before filtering.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct UiNode {
|
||||
pub role: String,
|
||||
pub name: Option<String>,
|
||||
pub value: Option<String>,
|
||||
pub states: Vec<String>,
|
||||
pub bounds: Option<Rect>,
|
||||
/// Backend's verdict that this node is actionable (has a default action /
|
||||
/// is a control role) — the primary interactability signal.
|
||||
pub actionable: bool,
|
||||
pub children: Vec<UiNode>,
|
||||
}
|
||||
|
||||
impl UiNode {
|
||||
pub fn leaf(role: impl Into<String>) -> Self {
|
||||
Self {
|
||||
role: role.into(),
|
||||
name: None,
|
||||
value: None,
|
||||
states: vec![],
|
||||
bounds: None,
|
||||
actionable: false,
|
||||
children: vec![],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_interactable(n: &UiNode) -> bool {
|
||||
let Some(b) = n.bounds else { return false };
|
||||
if b.is_empty() {
|
||||
return false;
|
||||
}
|
||||
// Actionable per the backend, or a control that carries a label/value worth
|
||||
// targeting even if no explicit action was reported.
|
||||
n.actionable || n.name.is_some() || n.value.is_some()
|
||||
}
|
||||
|
||||
/// Depth-first collect interactable nodes (honoring depth + budget), then number
|
||||
/// them in reading order (top-to-bottom, left-to-right). Returns
|
||||
/// `(entries, truncated)`.
|
||||
pub fn flatten_interactable(
|
||||
root: &UiNode,
|
||||
max_depth: usize,
|
||||
node_budget: usize,
|
||||
) -> (Vec<ElementEntry>, bool) {
|
||||
let mut collected: Vec<&UiNode> = Vec::new();
|
||||
let mut truncated = false;
|
||||
collect(root, 0, max_depth, node_budget, &mut collected, &mut truncated);
|
||||
|
||||
// Reading order: sort by rounded (y, x) so the model's numbering tracks the
|
||||
// visual layout. Stable so equal positions keep DFS order.
|
||||
collected.sort_by(|a, b| {
|
||||
let (ax, ay) = a.bounds.map(|r| (r.x, r.y)).unwrap_or((0.0, 0.0));
|
||||
let (bx, by) = b.bounds.map(|r| (r.x, r.y)).unwrap_or((0.0, 0.0));
|
||||
(ay.round() as i64, ax.round() as i64).cmp(&(by.round() as i64, bx.round() as i64))
|
||||
});
|
||||
|
||||
let entries = collected
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(i, n)| ElementEntry {
|
||||
r#ref: i as u32 + 1, // 1-based: matches the [ref] the model sees
|
||||
role: normalize_role(&n.role),
|
||||
name: n.name.clone().filter(|s| !s.trim().is_empty()),
|
||||
value: n.value.clone().filter(|s| !s.trim().is_empty()),
|
||||
states: n.states.clone(),
|
||||
bounds: n.bounds.unwrap_or(Rect { x: 0.0, y: 0.0, w: 0.0, h: 0.0 }),
|
||||
source: Source::A11y,
|
||||
})
|
||||
.collect();
|
||||
(entries, truncated)
|
||||
}
|
||||
|
||||
fn collect<'a>(
|
||||
node: &'a UiNode,
|
||||
depth: usize,
|
||||
max_depth: usize,
|
||||
budget: usize,
|
||||
out: &mut Vec<&'a UiNode>,
|
||||
truncated: &mut bool,
|
||||
) {
|
||||
if is_interactable(node) {
|
||||
if out.len() >= budget {
|
||||
*truncated = true;
|
||||
return;
|
||||
}
|
||||
out.push(node);
|
||||
}
|
||||
if depth >= max_depth {
|
||||
if !node.children.is_empty() {
|
||||
*truncated = true;
|
||||
}
|
||||
return;
|
||||
}
|
||||
for child in &node.children {
|
||||
if out.len() >= budget {
|
||||
*truncated = true;
|
||||
return;
|
||||
}
|
||||
collect(child, depth + 1, max_depth, budget, out, truncated);
|
||||
}
|
||||
}
|
||||
|
||||
/// Strip the platform `AX`/`UIA_` prefix and lowercase so the model sees
|
||||
/// stable cross-platform role names (`button`, `textfield`, …).
|
||||
pub fn normalize_role(role: &str) -> String {
|
||||
let r = role
|
||||
.strip_prefix("AX")
|
||||
.or_else(|| role.strip_prefix("UIA_"))
|
||||
.unwrap_or(role);
|
||||
r.to_lowercase()
|
||||
}
|
||||
|
||||
/// Render entries as a numbered text list for the model:
|
||||
/// `[14] button "Submit" enabled`.
|
||||
pub fn format_entries(entries: &[ElementEntry]) -> String {
|
||||
if entries.is_empty() {
|
||||
return "No interactable elements found in the accessibility tree.".to_string();
|
||||
}
|
||||
let mut out = String::new();
|
||||
for e in entries {
|
||||
out.push_str(&format!("[{}] {}", e.r#ref, e.role));
|
||||
if let Some(name) = &e.name {
|
||||
out.push_str(&format!(" {:?}", truncate(name, 80)));
|
||||
}
|
||||
if let Some(value) = &e.value {
|
||||
if Some(value) != e.name.as_ref() {
|
||||
out.push_str(&format!(" = {:?}", truncate(value, 60)));
|
||||
}
|
||||
}
|
||||
if !e.states.is_empty() {
|
||||
out.push_str(&format!(" [{}]", e.states.join(",")));
|
||||
}
|
||||
out.push('\n');
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn truncate(s: &str, max: usize) -> String {
|
||||
if s.chars().count() <= max {
|
||||
s.to_string()
|
||||
} else {
|
||||
let t: String = s.chars().take(max).collect();
|
||||
format!("{t}…")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn node(role: &str, name: Option<&str>, x: f64, y: f64, actionable: bool, children: Vec<UiNode>) -> UiNode {
|
||||
UiNode {
|
||||
role: role.to_string(),
|
||||
name: name.map(|s| s.to_string()),
|
||||
value: None,
|
||||
states: vec![],
|
||||
bounds: Some(Rect { x, y, w: 50.0, h: 20.0 }),
|
||||
actionable,
|
||||
children,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flattens_filters_and_numbers_in_reading_order() {
|
||||
// Root window (not actionable, no name) with two buttons out of order.
|
||||
let root = UiNode {
|
||||
role: "AXWindow".into(),
|
||||
name: None,
|
||||
value: None,
|
||||
states: vec![],
|
||||
bounds: Some(Rect { x: 0.0, y: 0.0, w: 800.0, h: 600.0 }),
|
||||
actionable: false,
|
||||
children: vec![
|
||||
node("AXButton", Some("Bottom"), 10.0, 200.0, true, vec![]),
|
||||
node("AXButton", Some("Top"), 10.0, 10.0, true, vec![]),
|
||||
],
|
||||
};
|
||||
let (entries, truncated) = flatten_interactable(&root, 12, 120);
|
||||
assert!(!truncated);
|
||||
assert_eq!(entries.len(), 2);
|
||||
// Sorted top-to-bottom: "Top" gets [1].
|
||||
assert_eq!(entries[0].name.as_deref(), Some("Top"));
|
||||
assert_eq!(entries[0].role, "button");
|
||||
assert_eq!(entries[0].r#ref, 1);
|
||||
assert_eq!(entries[1].name.as_deref(), Some("Bottom"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn budget_truncates() {
|
||||
let children: Vec<UiNode> = (0..10)
|
||||
.map(|i| node("AXButton", Some("b"), 0.0, i as f64, true, vec![]))
|
||||
.collect();
|
||||
let root = UiNode {
|
||||
role: "AXWindow".into(),
|
||||
name: None,
|
||||
value: None,
|
||||
states: vec![],
|
||||
bounds: Some(Rect { x: 0.0, y: 0.0, w: 100.0, h: 100.0 }),
|
||||
actionable: false,
|
||||
children,
|
||||
};
|
||||
let (entries, truncated) = flatten_interactable(&root, 12, 3);
|
||||
assert!(truncated);
|
||||
assert!(entries.len() <= 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn format_is_readable() {
|
||||
let entries = flatten_interactable(
|
||||
&node("AXButton", Some("Save"), 0.0, 0.0, true, vec![]),
|
||||
12,
|
||||
120,
|
||||
)
|
||||
.0;
|
||||
let text = format_entries(&entries);
|
||||
assert!(text.contains("[1] button"));
|
||||
assert!(text.contains("Save"));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,66 @@
|
||||
//! Windows accessibility backend (UI Automation).
|
||||
//!
|
||||
//! Threading model (mirrors the macOS backend): every `IUIAutomation` /
|
||||
//! `IUIAutomationElement` call has COM apartment affinity, so all UIA work is
|
||||
//! marshaled to a single dedicated actor thread that initializes COM as MTA
|
||||
//! (`CoInitializeEx(COINIT_MULTITHREADED)`) and is the sole owner of the
|
||||
//! `UIAutomation` instance and every element handle. The public `WinEngine` is
|
||||
//! a `Send + Sync` handle that sends commands over a channel and blocks on a
|
||||
//! per-command reply; raw UIA element handles never cross the actor boundary —
|
||||
//! only serializable `Snapshot` / `Effect` data does (so `WinEngine` is
|
||||
//! `Send + Sync` automatically, no `unsafe impl` needed).
|
||||
//!
|
||||
//! OCR (`Windows.Media.Ocr`) has no apartment affinity and runs on whatever
|
||||
//! thread the caller uses (the computer tool calls it from `spawn_blocking`).
|
||||
|
||||
use crate::engine::{
|
||||
A11yEngine, A11yError, Capabilities, Effect, ElementAction, InputKind, ObserveOpts, Snapshot,
|
||||
SnapshotGen, Target,
|
||||
};
|
||||
|
||||
mod actor;
|
||||
mod ocr;
|
||||
mod tree_map;
|
||||
|
||||
pub use ocr::ocr_screenshot;
|
||||
|
||||
pub struct WinEngine {
|
||||
inner: actor::ActorHandle,
|
||||
}
|
||||
|
||||
impl WinEngine {
|
||||
pub fn start() -> Result<Self, A11yError> {
|
||||
let inner = actor::ActorHandle::spawn()?;
|
||||
Ok(Self { inner })
|
||||
}
|
||||
}
|
||||
|
||||
impl A11yEngine for WinEngine {
|
||||
fn capabilities(&self) -> Capabilities {
|
||||
Capabilities {
|
||||
os: "windows".to_string(),
|
||||
tree_read: true,
|
||||
screenshot: true,
|
||||
semantic_action: true,
|
||||
synthetic_input: InputKind::Native,
|
||||
window_management: true,
|
||||
}
|
||||
}
|
||||
|
||||
fn observe(&self, opts: &ObserveOpts) -> Result<Snapshot, A11yError> {
|
||||
self.inner.observe(opts.clone())
|
||||
}
|
||||
|
||||
fn invoke(
|
||||
&self,
|
||||
target: &Target,
|
||||
generation: SnapshotGen,
|
||||
action: ElementAction,
|
||||
) -> Result<Effect, A11yError> {
|
||||
self.inner.invoke(target.clone(), generation, action)
|
||||
}
|
||||
|
||||
fn focus_window(&self, pid: i32) -> Result<Effect, A11yError> {
|
||||
self.inner.focus_window(pid)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
//! Windows OCR via `Windows.Media.Ocr` — on-device, with CJK support (essential
|
||||
//! for a Chinese-first product). Mirrors the macOS Vision backend: the
|
||||
//! screenshot is encoded to PNG, decoded into a `SoftwareBitmap` via
|
||||
//! `BitmapDecoder` (so we avoid hand-building a pixel buffer), then recognized.
|
||||
//! `OcrEngine`/`OcrResult` report word bounding boxes already in pixel space
|
||||
//! with a top-left origin — the same space the overlay/tool use — so no flip is
|
||||
//! needed.
|
||||
//!
|
||||
//! WinRT activation requires COM to be initialized on the calling thread. The
|
||||
//! tool calls this from `spawn_blocking`, whose pooled threads are not
|
||||
//! necessarily initialized, so we initialize COM (idempotently) up front.
|
||||
|
||||
use std::cell::Cell;
|
||||
use std::io::Cursor;
|
||||
|
||||
use windows::Globalization::Language;
|
||||
use windows::Graphics::Imaging::{
|
||||
BitmapAlphaMode, BitmapDecoder, BitmapPixelFormat, SoftwareBitmap,
|
||||
};
|
||||
use windows::Media::Ocr::OcrEngine;
|
||||
use windows::Storage::Streams::{DataWriter, InMemoryRandomAccessStream};
|
||||
use windows::Win32::System::Com::{COINIT_MULTITHREADED, CoInitializeEx};
|
||||
use windows::core::HSTRING;
|
||||
|
||||
use crate::engine::{A11yError, OcrLine, Rect};
|
||||
|
||||
fn win_err(ctx: &str, e: windows::core::Error) -> A11yError {
|
||||
A11yError::Backend(format!("OCR: {ctx}: {e}"))
|
||||
}
|
||||
|
||||
thread_local! {
|
||||
/// Whether this thread has already initialized COM. WinRT activation
|
||||
/// (OcrEngine, BitmapDecoder) requires COM on the calling thread; the tool
|
||||
/// calls us from pooled `spawn_blocking` threads. Initialize at most ONCE
|
||||
/// per thread and intentionally never `CoUninitialize` — these are
|
||||
/// process-lifetime pool threads, so a single MTA init is correct and a
|
||||
/// per-call init/leak is avoided.
|
||||
static COM_READY: Cell<bool> = const { Cell::new(false) };
|
||||
}
|
||||
|
||||
fn ensure_com() {
|
||||
COM_READY.with(|ready| {
|
||||
if !ready.get() {
|
||||
// Ignore S_FALSE / RPC_E_CHANGED_MODE — the thread is usable either way.
|
||||
unsafe {
|
||||
let _ = CoInitializeEx(None, COINIT_MULTITHREADED);
|
||||
}
|
||||
ready.set(true);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Build an OCR engine: prefer the caller's requested languages (e.g.
|
||||
/// `zh-Hans`) when an OCR pack for them is installed, else fall back to the
|
||||
/// user-profile languages.
|
||||
fn make_engine(langs: &[String]) -> Result<OcrEngine, A11yError> {
|
||||
for l in langs {
|
||||
if let Ok(lang) = Language::CreateLanguage(&HSTRING::from(l.as_str())) {
|
||||
if OcrEngine::IsLanguageSupported(&lang).unwrap_or(false) {
|
||||
if let Ok(engine) = OcrEngine::TryCreateFromLanguage(&lang) {
|
||||
return Ok(engine);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
OcrEngine::TryCreateFromUserProfileLanguages().map_err(|e| {
|
||||
A11yError::Backend(format!(
|
||||
"OCR engine unavailable: {e}. Install an OCR language pack (Settings → Time & \
|
||||
Language → Language → add a language and enable its Optical character recognition \
|
||||
feature)."
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn ocr_screenshot(img: &image::RgbaImage, langs: &[String]) -> Result<Vec<OcrLine>, A11yError> {
|
||||
let (w, h) = img.dimensions();
|
||||
if w == 0 || h == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
// WinRT activation needs COM on this thread (initialized at most once).
|
||||
ensure_com();
|
||||
|
||||
// Encode to PNG so BitmapDecoder can decode it into a SoftwareBitmap.
|
||||
let mut png = Vec::new();
|
||||
image::DynamicImage::ImageRgba8(img.clone())
|
||||
.write_to(&mut Cursor::new(&mut png), image::ImageFormat::Png)
|
||||
.map_err(|e| A11yError::Backend(format!("OCR: PNG encode failed: {e}")))?;
|
||||
|
||||
let engine = make_engine(langs)?;
|
||||
|
||||
// OcrEngine rejects images larger than MaxImageDimension on a side. The tool
|
||||
// already downscales screenshots well under this, but guard defensively with
|
||||
// a clear error rather than letting RecognizeAsync fail opaquely.
|
||||
if let Ok(max) = OcrEngine::MaxImageDimension() {
|
||||
if w > max || h > max {
|
||||
return Err(A11yError::Backend(format!(
|
||||
"OCR: image {w}x{h} exceeds the engine's max dimension {max} per side; downscale \
|
||||
before OCR"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
// PNG bytes → in-memory stream → SoftwareBitmap.
|
||||
let stream = InMemoryRandomAccessStream::new().map_err(|e| win_err("create stream", e))?;
|
||||
let writer = DataWriter::CreateDataWriter(&stream).map_err(|e| win_err("create writer", e))?;
|
||||
writer.WriteBytes(&png).map_err(|e| win_err("write bytes", e))?;
|
||||
writer
|
||||
.StoreAsync()
|
||||
.map_err(|e| win_err("store", e))?
|
||||
.get()
|
||||
.map_err(|e| win_err("store.get", e))?;
|
||||
writer
|
||||
.FlushAsync()
|
||||
.map_err(|e| win_err("flush", e))?
|
||||
.get()
|
||||
.map_err(|e| win_err("flush.get", e))?;
|
||||
writer
|
||||
.DetachStream()
|
||||
.map_err(|e| win_err("detach stream", e))?;
|
||||
stream.Seek(0).map_err(|e| win_err("seek", e))?;
|
||||
|
||||
let decoder = BitmapDecoder::CreateAsync(&stream)
|
||||
.map_err(|e| win_err("create decoder", e))?
|
||||
.get()
|
||||
.map_err(|e| win_err("decoder.get", e))?;
|
||||
let bitmap = decoder
|
||||
.GetSoftwareBitmapAsync()
|
||||
.map_err(|e| win_err("get bitmap", e))?
|
||||
.get()
|
||||
.map_err(|e| win_err("bitmap.get", e))?;
|
||||
// The PNG decoder auto-selects the pixel format (often Rgba8); OcrEngine
|
||||
// reliably accepts Bgra8/Premultiplied, so normalize before recognition
|
||||
// instead of relying on an undocumented accepted-format set.
|
||||
let bitmap = SoftwareBitmap::ConvertWithAlpha(
|
||||
&bitmap,
|
||||
BitmapPixelFormat::Bgra8,
|
||||
BitmapAlphaMode::Premultiplied,
|
||||
)
|
||||
.map_err(|e| win_err("convert to bgra8", e))?;
|
||||
|
||||
let result = engine
|
||||
.RecognizeAsync(&bitmap)
|
||||
.map_err(|e| win_err("recognize", e))?
|
||||
.get()
|
||||
.map_err(|e| win_err("recognize.get", e))?;
|
||||
|
||||
let mut out = Vec::new();
|
||||
let lines = result.Lines().map_err(|e| win_err("lines", e))?;
|
||||
for line in lines {
|
||||
let text = line.Text().map_err(|e| win_err("line text", e))?.to_string();
|
||||
if text.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
// Union of the line's word bounding rects (already pixel, top-left).
|
||||
let words = line.Words().map_err(|e| win_err("words", e))?;
|
||||
let (mut min_x, mut min_y) = (f64::MAX, f64::MAX);
|
||||
let (mut max_x, mut max_y) = (f64::MIN, f64::MIN);
|
||||
let mut any = false;
|
||||
for word in words {
|
||||
let r = word.BoundingRect().map_err(|e| win_err("word rect", e))?;
|
||||
min_x = min_x.min(r.X as f64);
|
||||
min_y = min_y.min(r.Y as f64);
|
||||
max_x = max_x.max((r.X + r.Width) as f64);
|
||||
max_y = max_y.max((r.Y + r.Height) as f64);
|
||||
any = true;
|
||||
}
|
||||
let bounds = if any && max_x >= min_x && max_y >= min_y {
|
||||
Rect {
|
||||
x: min_x,
|
||||
y: min_y,
|
||||
w: max_x - min_x,
|
||||
h: max_y - min_y,
|
||||
}
|
||||
} else {
|
||||
Rect {
|
||||
x: 0.0,
|
||||
y: 0.0,
|
||||
w: 0.0,
|
||||
h: 0.0,
|
||||
}
|
||||
};
|
||||
out.push(OcrLine { text, bounds });
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn ocr_blank_image_runs_without_error() {
|
||||
// Exercises the full Windows.Media.Ocr FFI path (compile + link + run).
|
||||
// A blank image yields no recognized text; OCR needs no permission, but
|
||||
// does require an OCR language pack (English ships by default on
|
||||
// Windows 10/11).
|
||||
let img = image::RgbaImage::from_pixel(80, 40, image::Rgba([255, 255, 255, 255]));
|
||||
match ocr_screenshot(&img, &["en-US".to_string()]) {
|
||||
Ok(lines) => assert!(lines.iter().all(|l| !l.text.trim().is_empty())),
|
||||
Err(A11yError::Backend(msg)) if msg.contains("language pack") => {
|
||||
eprintln!("skipping: no OCR language pack installed ({msg})");
|
||||
}
|
||||
Err(e) => panic!("OCR failed unexpectedly: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ocr_empty_image_is_empty() {
|
||||
assert!(ocr_screenshot(&image::RgbaImage::new(0, 0), &[]).unwrap().is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,715 @@
|
||||
//! Pure transform layer for the Windows UIA backend.
|
||||
//!
|
||||
//! The actor (which owns COM) captures the cached UIA tree of every target
|
||||
//! window into a `RawNode` forest of plain data — a synthetic `desktop` root
|
||||
//! whose children are per-window subtrees, each node carrying an index into a
|
||||
//! parallel handle table — and this module turns it into two products:
|
||||
//! * the numbered `ElementEntry` Set-of-Marks list (`build_entries`), grouped
|
||||
//! per window then in reading order, with a `ref → handle_idx` map so the
|
||||
//! actor can rebuild `ref → UIElement` for actuation; and
|
||||
//! * a hierarchical **semantic tree** text rendering (`render_tree`):
|
||||
//! `desktop → window → structural container → [ref] control`, pruning empty
|
||||
//! containers — the view the model reasons over.
|
||||
//!
|
||||
//! Keeping it free of any `uiautomation` / COM type makes the non-trivial logic
|
||||
//! (interactability filter, off-screen handling, per-window reading-order
|
||||
//! numbering, depth/budget truncation, the ref→handle correlation that must
|
||||
//! survive the sort, and the structural-tree pruning) unit-testable without a
|
||||
//! live UIA session; the COM reads in the actor are exercised by the `winsmoke`
|
||||
//! example instead.
|
||||
//!
|
||||
//! It is intentionally Windows-local rather than routed through the neutral
|
||||
//! `tree::flatten_interactable`: every emitted entry must map back to its live
|
||||
//! `UIElement` handle for `invoke`, which the neutral `UiNode` cannot carry, the
|
||||
//! off-screen-park filtering is UIA-specific, and the neutral layer has no
|
||||
//! multi-window / semantic-tree concept. It reuses the neutral `normalize_role`
|
||||
//! so role names stay consistent across platforms.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::engine::{ElementEntry, Rect, Source};
|
||||
use crate::tree::normalize_role;
|
||||
|
||||
/// Synthetic role for the forest root the actor builds above the per-window
|
||||
/// subtrees. Excluded from emission and rendered as the literal `desktop` line.
|
||||
pub(crate) const DESKTOP_ROLE: &str = "desktop";
|
||||
/// Role the actor stamps on each top-level window node. Excluded from emission
|
||||
/// (you target controls, not the frame) and rendered as `window "title"`.
|
||||
pub(crate) const WINDOW_ROLE: &str = "window";
|
||||
|
||||
/// One UIA element captured as plain, COM-free data. `handle_idx` indexes the
|
||||
/// actor's parallel `Vec<UIElement>` handle table, so a surviving entry can be
|
||||
/// mapped back to its element for actuation. `children` preserve document order.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct RawNode {
|
||||
pub handle_idx: usize,
|
||||
pub role: String,
|
||||
pub name: Option<String>,
|
||||
pub value: Option<String>,
|
||||
pub states: Vec<String>,
|
||||
pub bounds: Rect,
|
||||
/// Inherently actionable or keyboard-focusable (the primary interactability
|
||||
/// signal). A named/valued node is also emitted even when this is false.
|
||||
pub actionable: bool,
|
||||
/// Has a ScrollPattern with a scrollable axis — emitted as a target so the
|
||||
/// model can scroll the region even when it is not otherwise actionable.
|
||||
pub scrollable: bool,
|
||||
/// Scrolled/clipped/parked off-screen: never emitted as a target, but its
|
||||
/// children are still traversed (a visible control inside an off-screen
|
||||
/// container is still reachable).
|
||||
pub offscreen: bool,
|
||||
pub children: Vec<RawNode>,
|
||||
}
|
||||
|
||||
impl RawNode {
|
||||
/// A synthetic structural node (desktop root / window node) carrying no live
|
||||
/// handle. `handle_idx` is a sentinel that is never inserted into the handle
|
||||
/// table; such nodes are excluded from emission by role.
|
||||
pub(crate) fn structural(role: &str, name: Option<String>, bounds: Rect, children: Vec<RawNode>) -> Self {
|
||||
RawNode {
|
||||
handle_idx: usize::MAX,
|
||||
role: role.to_string(),
|
||||
name,
|
||||
value: None,
|
||||
states: Vec::new(),
|
||||
bounds,
|
||||
actionable: false,
|
||||
scrollable: false,
|
||||
offscreen: false,
|
||||
children,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Roles that are never themselves emitted as targets even when named: the
|
||||
/// synthetic forest scaffolding. (Real window-like panes inside an app still
|
||||
/// surface via their control type.)
|
||||
fn is_scaffold_role(role: &str) -> bool {
|
||||
role == DESKTOP_ROLE || role == WINDOW_ROLE
|
||||
}
|
||||
|
||||
/// True if this node should be emitted as a numbered target: on-screen, with
|
||||
/// non-empty bounds, not forest scaffolding, and either actionable, scrollable,
|
||||
/// or a named/valued leaf. A named *structural container* (toolbar/group/pane/…)
|
||||
/// is a grouping branch in the semantic tree, not a click target, so it is not
|
||||
/// emitted unless it is itself actionable or scrollable.
|
||||
fn is_emittable(n: &RawNode) -> bool {
|
||||
if n.offscreen || n.bounds.is_empty() || is_scaffold_role(&n.role) {
|
||||
return false;
|
||||
}
|
||||
if n.actionable || n.scrollable {
|
||||
return true;
|
||||
}
|
||||
(n.name.is_some() || n.value.is_some()) && !is_structural_role(&n.role)
|
||||
}
|
||||
|
||||
/// Container roles promoted to a labelled branch in the semantic tree when they
|
||||
/// carry a name and have emittable descendants (otherwise pruned). Gives the
|
||||
/// model the grouping context ("this button is inside the Formatting toolbar")
|
||||
/// without numbering the container itself.
|
||||
fn is_structural_role(role: &str) -> bool {
|
||||
matches!(
|
||||
role,
|
||||
"pane" | "group" | "toolbar" | "menubar" | "menu" | "tab" | "tree" | "list"
|
||||
| "table" | "datagrid" | "statusbar" | "titlebar" | "header" | "tabitem"
|
||||
)
|
||||
}
|
||||
|
||||
/// The verb the model performs on a control of this role — surfaced in the
|
||||
/// semantic tree as `[action: …]` so the affordance is explicit. Mirrors
|
||||
/// Windows-MCP's action map (edit→fill, checkbox→toggle, …); scrollable regions
|
||||
/// override to `scroll`.
|
||||
pub(crate) fn action_for(role: &str, scrollable: bool) -> &'static str {
|
||||
if scrollable && role != "slider" {
|
||||
return "scroll";
|
||||
}
|
||||
match role {
|
||||
"edit" => "fill",
|
||||
"checkbox" => "toggle",
|
||||
"combobox" => "select",
|
||||
"radiobutton" => "select",
|
||||
"slider" => "slide",
|
||||
"document" => "scroll",
|
||||
_ => "click",
|
||||
}
|
||||
}
|
||||
|
||||
/// Map a UIA `ToggleState` code (Off=0, On=1, Indeterminate=2) to a state label.
|
||||
/// Off yields none (the unremarkable default), keeping the list terse.
|
||||
pub(crate) fn toggle_label(code: i32) -> Option<&'static str> {
|
||||
match code {
|
||||
1 => Some("checked"),
|
||||
2 => Some("indeterminate"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Map a UIA `ExpandCollapseState` code (Collapsed=0, Expanded=1,
|
||||
/// PartiallyExpanded=2, LeafNode=3) to a state label. LeafNode (nothing to
|
||||
/// expand) yields none.
|
||||
pub(crate) fn expand_label(code: i32) -> Option<&'static str> {
|
||||
match code {
|
||||
0 => Some("collapsed"),
|
||||
1 => Some("expanded"),
|
||||
2 => Some("partially-expanded"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Filter the `RawNode` forest to emittable targets (honoring `max_depth` +
|
||||
/// `node_budget`), number them per-window then in reading order (top-to-bottom,
|
||||
/// left-to-right), and return:
|
||||
/// * the `ElementEntry` list (1-based `ref`s),
|
||||
/// * the parallel handle indices in the SAME order (to rebuild ref→`UIElement`),
|
||||
/// * a `handle_idx → ref` map (so the semantic renderer can annotate nodes),
|
||||
/// * whether the forest was truncated by depth or budget.
|
||||
///
|
||||
/// `root` is either the synthetic `desktop` forest root (children = windows) or
|
||||
/// a single window subtree (used directly).
|
||||
pub(crate) fn build_entries(
|
||||
root: &RawNode,
|
||||
max_depth: usize,
|
||||
node_budget: usize,
|
||||
) -> (Vec<ElementEntry>, Vec<usize>, HashMap<usize, u32>, bool) {
|
||||
let windows: Vec<&RawNode> = if root.role == DESKTOP_ROLE {
|
||||
root.children.iter().collect()
|
||||
} else {
|
||||
vec![root]
|
||||
};
|
||||
|
||||
// Collect (window_index, node) so refs group by window (foreground first),
|
||||
// then within a window by reading order — matching the semantic tree layout.
|
||||
let mut collected: Vec<(usize, &RawNode)> = Vec::new();
|
||||
let mut truncated = false;
|
||||
for (wi, win) in windows.iter().enumerate() {
|
||||
collect(win, wi, 0, max_depth, node_budget, &mut collected, &mut truncated);
|
||||
}
|
||||
|
||||
// Stable sort by (window, rounded y, rounded x): equal positions keep DFS
|
||||
// order. Window grouping dominates so refs never interleave across windows.
|
||||
collected.sort_by(|(awi, a), (bwi, b)| {
|
||||
(
|
||||
*awi,
|
||||
a.bounds.y.round() as i64,
|
||||
a.bounds.x.round() as i64,
|
||||
)
|
||||
.cmp(&(*bwi, b.bounds.y.round() as i64, b.bounds.x.round() as i64))
|
||||
});
|
||||
|
||||
let mut entries = Vec::with_capacity(collected.len());
|
||||
let mut handle_indices = Vec::with_capacity(collected.len());
|
||||
let mut ref_by_handle = HashMap::with_capacity(collected.len());
|
||||
for (i, (_wi, node)) in collected.iter().enumerate() {
|
||||
let r = i as u32 + 1; // 1-based: matches the [ref] the model sees
|
||||
entries.push(ElementEntry {
|
||||
r#ref: r,
|
||||
role: normalize_role(&node.role),
|
||||
name: node.name.clone().filter(|s| !s.trim().is_empty()),
|
||||
value: node.value.clone().filter(|s| !s.trim().is_empty()),
|
||||
states: node.states.clone(),
|
||||
bounds: node.bounds,
|
||||
source: Source::A11y,
|
||||
});
|
||||
handle_indices.push(node.handle_idx);
|
||||
ref_by_handle.insert(node.handle_idx, r);
|
||||
}
|
||||
(entries, handle_indices, ref_by_handle, truncated)
|
||||
}
|
||||
|
||||
/// Depth-first collect of emittable nodes within one window subtree. An
|
||||
/// off-screen / non-emittable node is not pushed, but its children are still
|
||||
/// traversed (until the depth cap), so a visible control inside an off-screen
|
||||
/// container is reached. `depth` is measured from the window root (0).
|
||||
fn collect<'a>(
|
||||
node: &'a RawNode,
|
||||
win_idx: usize,
|
||||
depth: usize,
|
||||
max_depth: usize,
|
||||
budget: usize,
|
||||
out: &mut Vec<(usize, &'a RawNode)>,
|
||||
truncated: &mut bool,
|
||||
) {
|
||||
if is_emittable(node) {
|
||||
if out.len() >= budget {
|
||||
*truncated = true;
|
||||
return;
|
||||
}
|
||||
out.push((win_idx, node));
|
||||
}
|
||||
if depth >= max_depth {
|
||||
if !node.children.is_empty() {
|
||||
*truncated = true;
|
||||
}
|
||||
return;
|
||||
}
|
||||
for child in &node.children {
|
||||
if out.len() >= budget {
|
||||
*truncated = true;
|
||||
return;
|
||||
}
|
||||
collect(child, win_idx, depth + 1, max_depth, budget, out, truncated);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- semantic tree rendering --------------------------------------------
|
||||
//
|
||||
// Two phases (mirrors Windows-MCP's SemanticNode build + prune + render):
|
||||
// 1. `build_sem` collapses the raw forest to only meaningful nodes — desktop,
|
||||
// windows, named structural containers, and emittable controls — making
|
||||
// transparent wrapper panes disappear so descendants attach to the nearest
|
||||
// meaningful ancestor.
|
||||
// 2. `render_sem` draws it with ├──/└── connectors.
|
||||
|
||||
#[derive(Debug)]
|
||||
enum SemKind {
|
||||
Desktop,
|
||||
Window,
|
||||
Structural,
|
||||
/// An emittable control, carrying its `[ref]`.
|
||||
Control(u32),
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct SemNode {
|
||||
kind: SemKind,
|
||||
role: String,
|
||||
name: String,
|
||||
/// Center coordinates (emittable controls only).
|
||||
coords: Option<(i64, i64)>,
|
||||
scrollable: bool,
|
||||
states: Vec<String>,
|
||||
children: Vec<SemNode>,
|
||||
}
|
||||
|
||||
/// Build the meaningful-node tree for `node`, appending the resulting node(s)
|
||||
/// to `parent_children`. Transparent nodes (unnamed containers, plain wrappers)
|
||||
/// contribute their children directly to the parent. Returns nothing; mutates
|
||||
/// `parent_children`.
|
||||
fn build_sem(node: &RawNode, ref_by_handle: &HashMap<usize, u32>, parent_children: &mut Vec<SemNode>) {
|
||||
let role = normalize_role(&node.role);
|
||||
|
||||
// Desktop / window scaffolding: always a branch.
|
||||
if node.role == DESKTOP_ROLE {
|
||||
let mut me = SemNode {
|
||||
kind: SemKind::Desktop,
|
||||
role,
|
||||
name: node.name.clone().unwrap_or_default(),
|
||||
coords: None,
|
||||
scrollable: false,
|
||||
states: Vec::new(),
|
||||
children: Vec::new(),
|
||||
};
|
||||
for c in &node.children {
|
||||
build_sem(c, ref_by_handle, &mut me.children);
|
||||
}
|
||||
parent_children.push(me);
|
||||
return;
|
||||
}
|
||||
if node.role == WINDOW_ROLE {
|
||||
let mut me = SemNode {
|
||||
kind: SemKind::Window,
|
||||
role,
|
||||
name: node.name.clone().unwrap_or_default(),
|
||||
coords: None,
|
||||
scrollable: false,
|
||||
states: Vec::new(),
|
||||
children: Vec::new(),
|
||||
};
|
||||
for c in &node.children {
|
||||
build_sem(c, ref_by_handle, &mut me.children);
|
||||
}
|
||||
parent_children.push(me);
|
||||
return;
|
||||
}
|
||||
|
||||
// An emittable control: a numbered leaf-or-branch.
|
||||
if let Some(&r) = ref_by_handle.get(&node.handle_idx) {
|
||||
let (cx, cy) = node.bounds.center();
|
||||
let mut me = SemNode {
|
||||
kind: SemKind::Control(r),
|
||||
role,
|
||||
name: node.name.clone().unwrap_or_default(),
|
||||
coords: Some((cx.round() as i64, cy.round() as i64)),
|
||||
scrollable: node.scrollable,
|
||||
states: node.states.clone(),
|
||||
children: Vec::new(),
|
||||
};
|
||||
for c in &node.children {
|
||||
build_sem(c, ref_by_handle, &mut me.children);
|
||||
}
|
||||
parent_children.push(me);
|
||||
return;
|
||||
}
|
||||
|
||||
// A named structural container: tentatively a branch, kept only if it ends
|
||||
// up with children (pruned below otherwise).
|
||||
let named = node.name.as_deref().map(|s| !s.trim().is_empty()).unwrap_or(false);
|
||||
if named && is_structural_role(&role) && !node.offscreen {
|
||||
let mut me = SemNode {
|
||||
kind: SemKind::Structural,
|
||||
role,
|
||||
name: node.name.clone().unwrap_or_default(),
|
||||
coords: None,
|
||||
scrollable: false,
|
||||
states: Vec::new(),
|
||||
children: Vec::new(),
|
||||
};
|
||||
for c in &node.children {
|
||||
build_sem(c, ref_by_handle, &mut me.children);
|
||||
}
|
||||
if !me.children.is_empty() {
|
||||
parent_children.push(me);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Transparent: attach children to the current parent.
|
||||
for c in &node.children {
|
||||
build_sem(c, ref_by_handle, parent_children);
|
||||
}
|
||||
}
|
||||
|
||||
fn format_sem_line(node: &SemNode) -> String {
|
||||
match &node.kind {
|
||||
SemKind::Desktop => "desktop".to_string(),
|
||||
SemKind::Window => format!("window {:?}", node.name),
|
||||
SemKind::Structural => {
|
||||
if node.name.is_empty() {
|
||||
node.role.clone()
|
||||
} else {
|
||||
format!("{} {:?}", node.role, node.name)
|
||||
}
|
||||
}
|
||||
SemKind::Control(r) => {
|
||||
let mut s = format!("[{r}] {}", node.role);
|
||||
if !node.name.is_empty() {
|
||||
s.push_str(&format!(" {:?}", truncate(&node.name, 80)));
|
||||
}
|
||||
if let Some((x, y)) = node.coords {
|
||||
s.push_str(&format!(" ({x},{y})"));
|
||||
}
|
||||
s.push_str(&format!(" [action: {}]", action_for(&node.role, node.scrollable)));
|
||||
if !node.states.is_empty() {
|
||||
s.push_str(&format!(" [{}]", node.states.join(",")));
|
||||
}
|
||||
s
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn render_sem(node: &SemNode, lines: &mut Vec<String>, prefix: &str, is_last: bool, is_root: bool) {
|
||||
if is_root {
|
||||
lines.push(format_sem_line(node));
|
||||
} else {
|
||||
let connector = if is_last { "└── " } else { "├── " };
|
||||
lines.push(format!("{prefix}{connector}{}", format_sem_line(node)));
|
||||
}
|
||||
let extension = if is_root {
|
||||
""
|
||||
} else if is_last {
|
||||
" "
|
||||
} else {
|
||||
"│ "
|
||||
};
|
||||
let child_prefix = format!("{prefix}{extension}");
|
||||
let n = node.children.len();
|
||||
for (i, child) in node.children.iter().enumerate() {
|
||||
render_sem(child, lines, &child_prefix, i == n - 1, false);
|
||||
}
|
||||
}
|
||||
|
||||
/// Render the raw forest as the hierarchical semantic tree the model reads.
|
||||
/// `root` is the synthetic `desktop` root (or a single window subtree).
|
||||
pub(crate) fn render_tree(root: &RawNode, ref_by_handle: &HashMap<usize, u32>) -> String {
|
||||
let mut tops: Vec<SemNode> = Vec::new();
|
||||
build_sem(root, ref_by_handle, &mut tops);
|
||||
let mut lines: Vec<String> = Vec::new();
|
||||
let n = tops.len();
|
||||
for (i, top) in tops.iter().enumerate() {
|
||||
render_sem(top, &mut lines, "", i == n - 1, true);
|
||||
}
|
||||
lines.join("\n")
|
||||
}
|
||||
|
||||
fn truncate(s: &str, max: usize) -> String {
|
||||
if s.chars().count() <= max {
|
||||
s.to_string()
|
||||
} else {
|
||||
let t: String = s.chars().take(max).collect();
|
||||
format!("{t}…")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn n(handle_idx: usize, role: &str, name: Option<&str>, x: f64, y: f64, actionable: bool) -> RawNode {
|
||||
RawNode {
|
||||
handle_idx,
|
||||
role: role.to_string(),
|
||||
name: name.map(|s| s.to_string()),
|
||||
value: None,
|
||||
states: vec![],
|
||||
bounds: Rect { x, y, w: 40.0, h: 16.0 },
|
||||
actionable,
|
||||
scrollable: false,
|
||||
offscreen: false,
|
||||
children: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
fn window(children: Vec<RawNode>) -> RawNode {
|
||||
RawNode {
|
||||
handle_idx: 0,
|
||||
role: "window".to_string(),
|
||||
name: Some("Test Window".to_string()),
|
||||
value: None,
|
||||
states: vec![],
|
||||
bounds: Rect { x: 0.0, y: 0.0, w: 800.0, h: 600.0 },
|
||||
actionable: false,
|
||||
scrollable: false,
|
||||
offscreen: false,
|
||||
children,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn emits_actionable_or_named_and_skips_plain_containers() {
|
||||
let root = window(vec![
|
||||
n(1, "button", None, 10.0, 50.0, true), // actionable, no name → emit
|
||||
n(2, "group", None, 10.0, 80.0, false), // not actionable, no name → skip
|
||||
n(3, "text", Some("Hello"), 10.0, 110.0, false), // named label → emit
|
||||
]);
|
||||
let (entries, handles, ref_by_handle, trunc) = build_entries(&root, 12, 120);
|
||||
assert!(!trunc);
|
||||
// window root (role "window") skipped; button + text emitted.
|
||||
assert_eq!(entries.len(), 2, "entries: {entries:?}");
|
||||
assert_eq!(handles.len(), 2);
|
||||
assert_eq!(ref_by_handle.len(), 2);
|
||||
let roles: Vec<_> = entries.iter().map(|e| e.role.as_str()).collect();
|
||||
assert!(roles.contains(&"button"));
|
||||
assert!(roles.contains(&"text"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skips_empty_bounds() {
|
||||
let mut ghost = n(1, "button", Some("Ghost"), 10.0, 10.0, true);
|
||||
ghost.bounds = Rect { x: 10.0, y: 10.0, w: 0.0, h: 0.0 };
|
||||
let (entries, _, _, _) = build_entries(&window(vec![ghost]), 12, 120);
|
||||
assert_eq!(entries.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn offscreen_node_not_emitted_but_children_traversed() {
|
||||
let mut container = n(1, "pane", Some("Panel"), 5.0, 5.0, false);
|
||||
container.offscreen = true;
|
||||
container.bounds = Rect { x: 5.0, y: 5.0, w: 300.0, h: 300.0 };
|
||||
container.children = vec![n(2, "button", Some("Deep"), 20.0, 20.0, true)];
|
||||
let (entries, handles, _, _) = build_entries(&window(vec![container]), 12, 120);
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].name.as_deref(), Some("Deep"));
|
||||
assert_eq!(handles, vec![2]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn numbers_in_reading_order_and_correlates_handles_across_sort() {
|
||||
let root = window(vec![
|
||||
n(7, "button", Some("Bottom"), 10.0, 200.0, true),
|
||||
n(9, "button", Some("Top"), 10.0, 10.0, true),
|
||||
]);
|
||||
let (entries, handles, ref_by_handle, _) = build_entries(&root, 12, 120);
|
||||
assert_eq!(entries.len(), 2);
|
||||
assert_eq!(entries[0].name.as_deref(), Some("Top"));
|
||||
assert_eq!(entries[0].r#ref, 1);
|
||||
assert_eq!(handles[0], 9);
|
||||
assert_eq!(ref_by_handle[&9], 1);
|
||||
assert_eq!(entries[1].name.as_deref(), Some("Bottom"));
|
||||
assert_eq!(entries[1].r#ref, 2);
|
||||
assert_eq!(handles[1], 7);
|
||||
assert_eq!(ref_by_handle[&7], 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn budget_truncates() {
|
||||
let kids: Vec<RawNode> = (1..=10).map(|i| n(i, "button", Some("b"), 0.0, i as f64, true)).collect();
|
||||
let (entries, handles, _, trunc) = build_entries(&window(kids), 12, 3);
|
||||
assert!(trunc);
|
||||
assert!(entries.len() <= 3);
|
||||
assert_eq!(entries.len(), handles.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn depth_cap_truncates_and_drops_deep_nodes() {
|
||||
let gc = n(2, "button", Some("Deep"), 10.0, 10.0, true);
|
||||
let mut child = n(1, "pane", None, 5.0, 5.0, false);
|
||||
child.bounds = Rect { x: 5.0, y: 5.0, w: 100.0, h: 100.0 };
|
||||
child.children = vec![gc];
|
||||
let (entries, _, _, trunc) = build_entries(&window(vec![child]), 1, 120);
|
||||
assert!(trunc);
|
||||
assert!(entries.iter().all(|e| e.name.as_deref() != Some("Deep")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn states_and_value_pass_through() {
|
||||
let mut cb = n(1, "checkbox", Some("Agree"), 10.0, 10.0, true);
|
||||
cb.states = vec!["checked".into(), "focused".into()];
|
||||
cb.value = Some("on".into());
|
||||
let (entries, _, _, _) = build_entries(&window(vec![cb]), 12, 120);
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].states, vec!["checked".to_string(), "focused".to_string()]);
|
||||
assert_eq!(entries[0].value.as_deref(), Some("on"));
|
||||
assert_eq!(entries[0].role, "checkbox");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn toggle_and_expand_labels_map_uia_codes() {
|
||||
assert_eq!(toggle_label(0), None);
|
||||
assert_eq!(toggle_label(1), Some("checked"));
|
||||
assert_eq!(toggle_label(2), Some("indeterminate"));
|
||||
assert_eq!(toggle_label(99), None);
|
||||
assert_eq!(expand_label(0), Some("collapsed"));
|
||||
assert_eq!(expand_label(1), Some("expanded"));
|
||||
assert_eq!(expand_label(2), Some("partially-expanded"));
|
||||
assert_eq!(expand_label(3), None);
|
||||
}
|
||||
|
||||
// ---- new behavior: scrollables, action map, multi-window, rendering ----
|
||||
|
||||
#[test]
|
||||
fn scrollable_container_is_emitted_even_without_name_or_action() {
|
||||
let mut scroll = n(1, "pane", None, 0.0, 0.0, false);
|
||||
scroll.scrollable = true;
|
||||
scroll.bounds = Rect { x: 0.0, y: 0.0, w: 300.0, h: 300.0 };
|
||||
let (entries, _, _, _) = build_entries(&window(vec![scroll]), 12, 120);
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].role, "pane");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn action_map_matches_role_and_scrollable() {
|
||||
assert_eq!(action_for("edit", false), "fill");
|
||||
assert_eq!(action_for("checkbox", false), "toggle");
|
||||
assert_eq!(action_for("combobox", false), "select");
|
||||
assert_eq!(action_for("radiobutton", false), "select");
|
||||
assert_eq!(action_for("slider", false), "slide");
|
||||
assert_eq!(action_for("document", false), "scroll");
|
||||
assert_eq!(action_for("button", false), "click");
|
||||
assert_eq!(action_for("pane", true), "scroll"); // scrollable overrides
|
||||
assert_eq!(action_for("slider", true), "slide"); // slider keeps slide
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_entries_groups_refs_by_window_then_reading_order() {
|
||||
// Window B sits visually ABOVE window A (smaller y), but refs must group
|
||||
// by window order (A first in the forest), not by global y.
|
||||
let win_a = RawNode::structural(
|
||||
WINDOW_ROLE,
|
||||
Some("App A".into()),
|
||||
Rect { x: 0.0, y: 100.0, w: 400.0, h: 400.0 },
|
||||
vec![n(10, "button", Some("A1"), 10.0, 300.0, true)],
|
||||
);
|
||||
let win_b = RawNode::structural(
|
||||
WINDOW_ROLE,
|
||||
Some("App B".into()),
|
||||
Rect { x: 500.0, y: 0.0, w: 400.0, h: 400.0 },
|
||||
vec![n(20, "button", Some("B1"), 510.0, 10.0, true)],
|
||||
);
|
||||
let desktop = RawNode::structural(
|
||||
DESKTOP_ROLE,
|
||||
None,
|
||||
Rect { x: 0.0, y: 0.0, w: 1920.0, h: 1080.0 },
|
||||
vec![win_a, win_b],
|
||||
);
|
||||
let (entries, _, _, _) = build_entries(&desktop, 12, 120);
|
||||
assert_eq!(entries.len(), 2);
|
||||
// A1 (window A, listed first) gets [1] even though B1 is higher on screen.
|
||||
assert_eq!(entries[0].name.as_deref(), Some("A1"));
|
||||
assert_eq!(entries[0].r#ref, 1);
|
||||
assert_eq!(entries[1].name.as_deref(), Some("B1"));
|
||||
assert_eq!(entries[1].r#ref, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_tree_shows_desktop_windows_and_refs() {
|
||||
let win = RawNode::structural(
|
||||
WINDOW_ROLE,
|
||||
Some("Notepad".into()),
|
||||
Rect { x: 0.0, y: 0.0, w: 800.0, h: 600.0 },
|
||||
vec![n(5, "button", Some("Save"), 100.0, 50.0, true)],
|
||||
);
|
||||
let desktop = RawNode::structural(
|
||||
DESKTOP_ROLE,
|
||||
None,
|
||||
Rect { x: 0.0, y: 0.0, w: 1920.0, h: 1080.0 },
|
||||
vec![win],
|
||||
);
|
||||
let (_, _, ref_by_handle, _) = build_entries(&desktop, 12, 120);
|
||||
let tree = render_tree(&desktop, &ref_by_handle);
|
||||
assert!(tree.starts_with("desktop"), "tree:\n{tree}");
|
||||
assert!(tree.contains(r#"window "Notepad""#), "tree:\n{tree}");
|
||||
assert!(tree.contains(r#"[1] button "Save""#), "tree:\n{tree}");
|
||||
assert!(tree.contains("[action: click]"), "tree:\n{tree}");
|
||||
assert!(tree.contains("(120,58)"), "center of Save; tree:\n{tree}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_tree_promotes_named_container_and_prunes_empty_one() {
|
||||
// A named toolbar with an actionable child → promoted to a branch with
|
||||
// the button numbered under it. An empty named group (no emittable
|
||||
// descendant) → pruned.
|
||||
let named_toolbar = RawNode {
|
||||
handle_idx: 100,
|
||||
role: "toolbar".into(),
|
||||
name: Some("Formatting".into()),
|
||||
value: None,
|
||||
states: vec![],
|
||||
bounds: Rect { x: 0.0, y: 0.0, w: 800.0, h: 40.0 },
|
||||
actionable: false,
|
||||
scrollable: false,
|
||||
offscreen: false,
|
||||
children: vec![n(1, "button", Some("Bold"), 10.0, 10.0, true)],
|
||||
};
|
||||
let empty_group = RawNode {
|
||||
handle_idx: 101,
|
||||
role: "group".into(),
|
||||
name: Some("Empty".into()),
|
||||
value: None,
|
||||
states: vec![],
|
||||
bounds: Rect { x: 0.0, y: 100.0, w: 800.0, h: 40.0 },
|
||||
actionable: false,
|
||||
scrollable: false,
|
||||
offscreen: false,
|
||||
children: vec![n(2, "group", None, 0.0, 0.0, false)], // non-emittable
|
||||
};
|
||||
let win = RawNode::structural(
|
||||
WINDOW_ROLE,
|
||||
Some("App".into()),
|
||||
Rect { x: 0.0, y: 0.0, w: 800.0, h: 600.0 },
|
||||
vec![named_toolbar, empty_group],
|
||||
);
|
||||
let desktop = RawNode::structural(DESKTOP_ROLE, None, win.bounds, vec![win]);
|
||||
let (entries, _, ref_by_handle, _) = build_entries(&desktop, 12, 120);
|
||||
// Only the Bold button is emittable (toolbar/group are structural).
|
||||
assert_eq!(entries.len(), 1, "entries: {entries:?}");
|
||||
let tree = render_tree(&desktop, &ref_by_handle);
|
||||
assert!(tree.contains(r#"toolbar "Formatting""#), "tree:\n{tree}");
|
||||
assert!(tree.contains(r#"[1] button "Bold""#), "tree:\n{tree}");
|
||||
assert!(!tree.contains("Empty"), "empty container must be pruned; tree:\n{tree}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_tree_collapses_transparent_wrappers() {
|
||||
// unnamed pane wrapper → its child attaches to the window directly.
|
||||
let mut wrapper = n(1, "pane", None, 0.0, 0.0, false);
|
||||
wrapper.bounds = Rect { x: 0.0, y: 0.0, w: 800.0, h: 600.0 };
|
||||
wrapper.children = vec![n(2, "button", Some("Go"), 10.0, 10.0, true)];
|
||||
let win = RawNode::structural(WINDOW_ROLE, Some("W".into()), wrapper.bounds, vec![wrapper]);
|
||||
let desktop = RawNode::structural(DESKTOP_ROLE, None, win.bounds, vec![win]);
|
||||
let (_, _, ref_by_handle, _) = build_entries(&desktop, 12, 120);
|
||||
let tree = render_tree(&desktop, &ref_by_handle);
|
||||
// No "pane" line (unnamed wrapper collapsed); button present under window.
|
||||
assert!(!tree.contains("pane"), "transparent wrapper must collapse; tree:\n{tree}");
|
||||
assert!(tree.contains(r#"[1] button "Go""#), "tree:\n{tree}");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
[package]
|
||||
name = "nomi-agent"
|
||||
description = "Agent engine, session management, output sinks, and orchestration for Nomi"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[features]
|
||||
# Desktop control (screenshots, synthetic input). Gated because xcap/enigo
|
||||
# pull native display-server libraries that headless hosts must not require.
|
||||
computer-use = ["dep:nomi-computer"]
|
||||
# Browser automation (in-process self-hosted CDP engine). Gated because it pulls
|
||||
# the chromiumoxide CDP stack + a managed/connected Chromium; headless hosts that
|
||||
# never drive a browser must not require it.
|
||||
browser-use = ["dep:nomi-browser", "dep:base64"]
|
||||
|
||||
[dependencies]
|
||||
nomi-types.workspace = true
|
||||
nomi-protocol.workspace = true
|
||||
nomi-config.workspace = true
|
||||
nomi-providers.workspace = true
|
||||
nomi-tools.workspace = true
|
||||
nomi-mcp.workspace = true
|
||||
nomi-skills.workspace = true
|
||||
nomi-memory.workspace = true
|
||||
nomi-compact.workspace = true
|
||||
nomi-redact.workspace = true
|
||||
nomi-computer = { workspace = true, optional = true }
|
||||
nomi-browser = { workspace = true, optional = true }
|
||||
# Used only by the browser-use SessionVisualLocator adapter (base64-encode the
|
||||
# screenshot PNG into a ContentBlock::Image for the vision model). Optional →
|
||||
# pulled in only with the browser-use feature.
|
||||
base64 = { workspace = true, optional = true }
|
||||
|
||||
tracing.workspace = true
|
||||
tokio.workspace = true
|
||||
tokio-util.workspace = true
|
||||
futures.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
async-trait.workspace = true
|
||||
anyhow.workspace = true
|
||||
thiserror.workspace = true
|
||||
glob.workspace = true
|
||||
chrono.workspace = true
|
||||
dirs.workspace = true
|
||||
crossterm.workspace = true
|
||||
is-terminal.workspace = true
|
||||
|
||||
|
||||
[dev-dependencies]
|
||||
nomi-mcp = { path = "../nomi-mcp", features = ["test-utils"] }
|
||||
wiremock.workspace = true
|
||||
tokio-test.workspace = true
|
||||
tempfile.workspace = true
|
||||
mockall.workspace = true
|
||||
rstest.workspace = true
|
||||
serial_test.workspace = true
|
||||
|
||||
# ── Integration test binaries ──────────────────────────────────────────────
|
||||
[[test]]
|
||||
name = "e2e"
|
||||
path = "tests/e2e/mod.rs"
|
||||
# E2E tests require real API keys; run via: cargo nextest run --profile e2e --test e2e
|
||||
|
||||
[[test]]
|
||||
name = "acceptance"
|
||||
path = "tests/acceptance/mod.rs"
|
||||
# Acceptance tests for evolution features; run via: cargo nextest run --profile e2e --test acceptance
|
||||
|
||||
@@ -0,0 +1,490 @@
|
||||
use std::collections::HashSet;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use nomi_config::config::app_config_dir;
|
||||
use nomi_skills::paths::stop_boundary;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub struct AgentsMdFile {
|
||||
pub path: PathBuf,
|
||||
pub content: String,
|
||||
pub is_global: bool,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Constants
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const MAX_INCLUDE_DEPTH: u8 = 5;
|
||||
|
||||
const ALLOWED_EXTENSIONS: &[&str] = &[".md", ".txt", ".json", ".yaml", ".yml", ".toml"];
|
||||
|
||||
const INSTRUCTION_PREAMBLE: &str = "Codebase and user instructions are shown below. \
|
||||
Be sure to adhere to these instructions. IMPORTANT: These instructions OVERRIDE any \
|
||||
default behavior and you MUST follow them exactly as written.";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Discovery
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub fn collect_agents_md(cwd: &str) -> Vec<AgentsMdFile> {
|
||||
let cwd_path = Path::new(cwd);
|
||||
let mut files = Vec::new();
|
||||
|
||||
// 1. Global: <config_dir>/nomi/AGENTS.md
|
||||
if let Some(global_path) = app_config_dir().map(|d| d.join("AGENTS.md"))
|
||||
&& let Some(file) = read_agents_md(&global_path, true)
|
||||
{
|
||||
files.push(file);
|
||||
}
|
||||
|
||||
// 2. Walk up from cwd to stop_boundary, collect AGENTS.md paths
|
||||
let boundary = stop_boundary(cwd_path);
|
||||
let mut project_paths = Vec::new();
|
||||
let mut current = cwd_path.to_path_buf();
|
||||
|
||||
loop {
|
||||
let candidate = current.join("AGENTS.md");
|
||||
if candidate.is_file() {
|
||||
project_paths.push(candidate);
|
||||
}
|
||||
|
||||
if Some(¤t) == boundary.as_ref() || current.parent().is_none() {
|
||||
break;
|
||||
}
|
||||
|
||||
match current.parent() {
|
||||
Some(parent) if parent != current.as_path() => {
|
||||
current = parent.to_path_buf();
|
||||
}
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
|
||||
// Reverse: collected deepest-first, we want root-first
|
||||
project_paths.reverse();
|
||||
|
||||
for path in project_paths {
|
||||
if let Some(file) = read_agents_md(&path, false) {
|
||||
files.push(file);
|
||||
}
|
||||
}
|
||||
|
||||
files
|
||||
}
|
||||
|
||||
fn read_agents_md(path: &Path, is_global: bool) -> Option<AgentsMdFile> {
|
||||
let raw = std::fs::read_to_string(path).ok()?;
|
||||
if raw.trim().is_empty() {
|
||||
return None;
|
||||
}
|
||||
let base_dir = path.parent()?;
|
||||
let mut seen = HashSet::new();
|
||||
if let Ok(canonical) = path.canonicalize() {
|
||||
seen.insert(canonical);
|
||||
}
|
||||
let content = expand_includes(&raw, base_dir, 0, &mut seen);
|
||||
Some(AgentsMdFile {
|
||||
path: path.to_path_buf(),
|
||||
content,
|
||||
is_global,
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Formatting
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub fn format_agents_md_section(files: &[AgentsMdFile]) -> String {
|
||||
if files.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
let mut parts = vec![INSTRUCTION_PREAMBLE.to_string()];
|
||||
|
||||
for file in files {
|
||||
let description = if file.is_global {
|
||||
"(user's global instructions for all projects)"
|
||||
} else {
|
||||
"(project instructions)"
|
||||
};
|
||||
let header = format!("Contents of {} {}:", file.path.display(), description);
|
||||
parts.push(format!("{header}\n\n{}", file.content.trim()));
|
||||
}
|
||||
|
||||
parts.join("\n\n")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// @include expansion
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn is_allowed_extension(path: &Path) -> bool {
|
||||
path.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|e| {
|
||||
let dotted = format!(".{e}");
|
||||
ALLOWED_EXTENSIONS.contains(&dotted.as_str())
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn resolve_include_path(raw: &str, base_dir: &Path) -> Option<PathBuf> {
|
||||
let path_str = raw.trim();
|
||||
if path_str.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let resolved = if let Some(rest) = path_str.strip_prefix("~/") {
|
||||
dirs::home_dir()?.join(rest)
|
||||
} else if let Some(rest) = path_str.strip_prefix("./") {
|
||||
base_dir.join(rest)
|
||||
} else if path_str.starts_with('/') {
|
||||
PathBuf::from(path_str)
|
||||
} else {
|
||||
base_dir.join(path_str)
|
||||
};
|
||||
|
||||
Some(resolved)
|
||||
}
|
||||
|
||||
fn expand_includes(
|
||||
content: &str,
|
||||
base_dir: &Path,
|
||||
depth: u8,
|
||||
seen: &mut HashSet<PathBuf>,
|
||||
) -> String {
|
||||
let mut result = Vec::new();
|
||||
let mut in_code_block = false;
|
||||
|
||||
for line in content.lines() {
|
||||
let trimmed = line.trim_start();
|
||||
if trimmed.starts_with("```") {
|
||||
in_code_block = !in_code_block;
|
||||
result.push(line.to_string());
|
||||
continue;
|
||||
}
|
||||
|
||||
if in_code_block {
|
||||
result.push(line.to_string());
|
||||
continue;
|
||||
}
|
||||
|
||||
let standalone = line.trim();
|
||||
if standalone.starts_with('@') && !standalone.contains('`') {
|
||||
let path_str = &standalone[1..];
|
||||
// Strip fragment identifiers
|
||||
let path_str = match path_str.find('#') {
|
||||
Some(i) => &path_str[..i],
|
||||
None => path_str,
|
||||
};
|
||||
|
||||
if let Some(resolved) = resolve_include_path(path_str, base_dir) {
|
||||
if !is_allowed_extension(&resolved) {
|
||||
continue;
|
||||
}
|
||||
let canonical = resolved.canonicalize().unwrap_or_else(|_| resolved.clone());
|
||||
if seen.contains(&canonical) || depth >= MAX_INCLUDE_DEPTH {
|
||||
continue;
|
||||
}
|
||||
if let Ok(included) = std::fs::read_to_string(&resolved) {
|
||||
seen.insert(canonical);
|
||||
let expanded = expand_includes(
|
||||
&included,
|
||||
resolved.parent().unwrap_or(base_dir),
|
||||
depth + 1,
|
||||
seen,
|
||||
);
|
||||
result.push(expanded);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
result.push(line.to_string());
|
||||
}
|
||||
|
||||
result.join("\n")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::fs;
|
||||
use tempfile::TempDir;
|
||||
|
||||
// --- @include expansion tests ---
|
||||
|
||||
#[test]
|
||||
fn test_no_includes_passthrough() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let mut seen = HashSet::new();
|
||||
let input = "Hello world\nNo includes here.";
|
||||
let result = expand_includes(input, tmp.path(), 0, &mut seen);
|
||||
assert_eq!(result, input);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_simple_include() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
fs::write(tmp.path().join("other.md"), "INCLUDED_CONTENT").unwrap();
|
||||
let mut seen = HashSet::new();
|
||||
let input = "@other.md";
|
||||
let result = expand_includes(input, tmp.path(), 0, &mut seen);
|
||||
assert!(result.contains("INCLUDED_CONTENT"));
|
||||
assert!(!result.contains("@other.md"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_include_relative_dot() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
fs::write(tmp.path().join("sub.md"), "SUB_CONTENT").unwrap();
|
||||
let mut seen = HashSet::new();
|
||||
let input = "@./sub.md";
|
||||
let result = expand_includes(input, tmp.path(), 0, &mut seen);
|
||||
assert!(result.contains("SUB_CONTENT"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_include_inside_code_block_ignored() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
fs::write(tmp.path().join("skip.md"), "SHOULD_NOT_APPEAR").unwrap();
|
||||
let mut seen = HashSet::new();
|
||||
let input = "```\n@skip.md\n```";
|
||||
let result = expand_includes(input, tmp.path(), 0, &mut seen);
|
||||
assert!(!result.contains("SHOULD_NOT_APPEAR"));
|
||||
assert!(result.contains("@skip.md"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_include_missing_file_silently_skipped() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let mut seen = HashSet::new();
|
||||
let input = "before\n@nonexistent.md\nafter";
|
||||
let result = expand_includes(input, tmp.path(), 0, &mut seen);
|
||||
assert!(result.contains("before"));
|
||||
assert!(result.contains("after"));
|
||||
assert!(!result.contains("@nonexistent.md"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_include_circular_reference() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
fs::write(tmp.path().join("a.md"), "A_CONTENT\n@b.md").unwrap();
|
||||
fs::write(tmp.path().join("b.md"), "B_CONTENT\n@a.md").unwrap();
|
||||
let mut seen = HashSet::new();
|
||||
let result = expand_includes("@a.md", tmp.path(), 0, &mut seen);
|
||||
assert!(result.contains("A_CONTENT"));
|
||||
assert!(result.contains("B_CONTENT"));
|
||||
// @a.md in b.md should be skipped (circular)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_include_max_depth() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
// Chain: d0 → d1 → d2 → d3 → d4 → d5 → d6
|
||||
// With MAX_INCLUDE_DEPTH=5, expansion from the outer call:
|
||||
// outer(0) expands @d0 at depth 0 → d0 content expanded at depth 1
|
||||
// depth 1 expands @d1 → d1 at depth 2 → ... → d3 at depth 4
|
||||
// depth 4 expands @d4 → d4 at depth 5 → depth 5 >= MAX, @d5 NOT expanded
|
||||
for i in 0..7 {
|
||||
let content = if i < 6 {
|
||||
format!("DEPTH_{i}\n@d{}.md", i + 1)
|
||||
} else {
|
||||
format!("DEPTH_{i}")
|
||||
};
|
||||
fs::write(tmp.path().join(format!("d{i}.md")), content).unwrap();
|
||||
}
|
||||
let mut seen = HashSet::new();
|
||||
let result = expand_includes("@d0.md", tmp.path(), 0, &mut seen);
|
||||
assert!(result.contains("DEPTH_0"));
|
||||
assert!(result.contains("DEPTH_3"));
|
||||
assert!(result.contains("DEPTH_4"));
|
||||
// DEPTH_5 should NOT appear — depth limit reached
|
||||
assert!(!result.contains("DEPTH_5"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_include_disallowed_extension() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
fs::write(tmp.path().join("image.png"), "BINARY_DATA").unwrap();
|
||||
let mut seen = HashSet::new();
|
||||
let input = "@image.png";
|
||||
let result = expand_includes(input, tmp.path(), 0, &mut seen);
|
||||
assert!(!result.contains("BINARY_DATA"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_include_with_surrounding_text() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
fs::write(tmp.path().join("inc.md"), "MIDDLE").unwrap();
|
||||
let mut seen = HashSet::new();
|
||||
let input = "TOP\n@inc.md\nBOTTOM";
|
||||
let result = expand_includes(input, tmp.path(), 0, &mut seen);
|
||||
assert_eq!(result, "TOP\nMIDDLE\nBOTTOM");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_allowed_extension() {
|
||||
assert!(is_allowed_extension(Path::new("file.md")));
|
||||
assert!(is_allowed_extension(Path::new("file.txt")));
|
||||
assert!(is_allowed_extension(Path::new("file.yaml")));
|
||||
assert!(is_allowed_extension(Path::new("file.yml")));
|
||||
assert!(is_allowed_extension(Path::new("file.toml")));
|
||||
assert!(is_allowed_extension(Path::new("file.json")));
|
||||
assert!(!is_allowed_extension(Path::new("file.png")));
|
||||
assert!(!is_allowed_extension(Path::new("file.rs")));
|
||||
assert!(!is_allowed_extension(Path::new("file")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_inline_code_span_not_expanded() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
fs::write(tmp.path().join("x.md"), "SHOULD_NOT_APPEAR").unwrap();
|
||||
let mut seen = HashSet::new();
|
||||
let input = "Use `@x.md` for config";
|
||||
let result = expand_includes(input, tmp.path(), 0, &mut seen);
|
||||
assert!(!result.contains("SHOULD_NOT_APPEAR"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_home_path_expansion() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let mut seen = HashSet::new();
|
||||
let input = "@~/nonexistent-test-file.md";
|
||||
let result = expand_includes(input, tmp.path(), 0, &mut seen);
|
||||
assert!(!result.contains("@~/"));
|
||||
}
|
||||
|
||||
// --- Discovery tests ---
|
||||
|
||||
#[test]
|
||||
fn test_collect_no_agents_md_anywhere() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let cwd = tmp.path();
|
||||
fs::create_dir(cwd.join(".git")).unwrap();
|
||||
let files = collect_agents_md(&cwd.to_string_lossy());
|
||||
assert!(files.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_collect_cwd_only() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let cwd = tmp.path();
|
||||
fs::create_dir(cwd.join(".git")).unwrap();
|
||||
fs::write(cwd.join("AGENTS.md"), "CWD_RULES").unwrap();
|
||||
|
||||
let files = collect_agents_md(&cwd.to_string_lossy());
|
||||
assert_eq!(files.len(), 1);
|
||||
assert!(files[0].content.contains("CWD_RULES"));
|
||||
assert!(!files[0].is_global);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_collect_hierarchical_ordering() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let root = tmp.path();
|
||||
fs::create_dir(root.join(".git")).unwrap();
|
||||
fs::write(root.join("AGENTS.md"), "ROOT_RULES").unwrap();
|
||||
|
||||
let sub = root.join("packages").join("server");
|
||||
fs::create_dir_all(&sub).unwrap();
|
||||
fs::write(sub.join("AGENTS.md"), "SUB_RULES").unwrap();
|
||||
|
||||
let files = collect_agents_md(&sub.to_string_lossy());
|
||||
assert_eq!(files.len(), 2);
|
||||
assert!(files[0].content.contains("ROOT_RULES"));
|
||||
assert!(files[1].content.contains("SUB_RULES"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_collect_stops_at_git_root() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let above_git = tmp.path();
|
||||
fs::write(above_git.join("AGENTS.md"), "ABOVE_GIT_SHOULD_NOT_APPEAR").unwrap();
|
||||
|
||||
let repo = above_git.join("repo");
|
||||
fs::create_dir_all(&repo).unwrap();
|
||||
fs::create_dir(repo.join(".git")).unwrap();
|
||||
fs::write(repo.join("AGENTS.md"), "REPO_RULES").unwrap();
|
||||
|
||||
let files = collect_agents_md(&repo.to_string_lossy());
|
||||
assert_eq!(files.len(), 1);
|
||||
assert!(files[0].content.contains("REPO_RULES"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_collect_skips_empty_agents_md() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let cwd = tmp.path();
|
||||
fs::create_dir(cwd.join(".git")).unwrap();
|
||||
fs::write(cwd.join("AGENTS.md"), " \n ").unwrap();
|
||||
|
||||
let files = collect_agents_md(&cwd.to_string_lossy());
|
||||
assert!(files.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_collect_with_include_expanded() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let cwd = tmp.path();
|
||||
fs::create_dir(cwd.join(".git")).unwrap();
|
||||
fs::write(cwd.join("AGENTS.md"), "@rules.md").unwrap();
|
||||
fs::write(cwd.join("rules.md"), "INCLUDED_RULES").unwrap();
|
||||
|
||||
let files = collect_agents_md(&cwd.to_string_lossy());
|
||||
assert_eq!(files.len(), 1);
|
||||
assert!(files[0].content.contains("INCLUDED_RULES"));
|
||||
}
|
||||
|
||||
// --- Formatting tests ---
|
||||
|
||||
#[test]
|
||||
fn test_format_empty() {
|
||||
let files: Vec<AgentsMdFile> = vec![];
|
||||
let result = format_agents_md_section(&files);
|
||||
assert!(result.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_format_single_project() {
|
||||
let files = vec![AgentsMdFile {
|
||||
path: PathBuf::from("/workspace/AGENTS.md"),
|
||||
content: "My rules".to_string(),
|
||||
is_global: false,
|
||||
}];
|
||||
let result = format_agents_md_section(&files);
|
||||
assert!(result.contains("Be sure to adhere to these instructions"));
|
||||
assert!(result.contains("Contents of /workspace/AGENTS.md (project instructions):"));
|
||||
assert!(result.contains("My rules"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_format_global_and_project() {
|
||||
let files = vec![
|
||||
AgentsMdFile {
|
||||
path: PathBuf::from("/home/user/.config/nomi/AGENTS.md"),
|
||||
content: "Global rules".to_string(),
|
||||
is_global: true,
|
||||
},
|
||||
AgentsMdFile {
|
||||
path: PathBuf::from("/workspace/AGENTS.md"),
|
||||
content: "Project rules".to_string(),
|
||||
is_global: false,
|
||||
},
|
||||
];
|
||||
let result = format_agents_md_section(&files);
|
||||
let global_pos = result.find("Global rules").unwrap();
|
||||
let project_pos = result.find("Project rules").unwrap();
|
||||
assert!(global_pos < project_pos, "global before project");
|
||||
assert!(result.contains("(user's global instructions for all projects)"));
|
||||
assert!(result.contains("(project instructions)"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,921 @@
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
use nomi_config::config::Config;
|
||||
use nomi_mcp::manager::McpManager;
|
||||
use nomi_providers::LlmProvider;
|
||||
|
||||
use crate::engine::AgentEngine;
|
||||
use crate::output::OutputSink;
|
||||
use crate::session::Session;
|
||||
|
||||
/// **extract-llm: session-model adapter for `BrowserTool`'s extract seam.**
|
||||
///
|
||||
/// Wraps the session's [`LlmProvider`] + model params so `act(Extract)` can do real
|
||||
/// LLM-driven structured extraction. The browser engine itself stays LLM-free — this
|
||||
/// adapter lives at the bootstrap/facade layer (架构铁律). `complete(prompt)` issues a
|
||||
/// minimal one-shot request — the (already spotlighted, untrusted-data-wrapped) extract
|
||||
/// prompt as a single user message, no tools, no system prompt, no extended thinking
|
||||
/// (extraction is mechanical) — and collects the streamed text deltas into the full
|
||||
/// completion. Reuses the session's own model (user decision: "extract 用会话模型"), so
|
||||
/// there is no separate model/cost surface. `None` provider → seam stays unwired (the
|
||||
/// facade returns its deterministic `<data>` payload = zero-regression graceful degrade).
|
||||
#[cfg(feature = "browser-use")]
|
||||
struct SessionExtractModel {
|
||||
provider: Arc<dyn LlmProvider>,
|
||||
model: String,
|
||||
max_tokens: u32,
|
||||
}
|
||||
|
||||
#[cfg(feature = "browser-use")]
|
||||
#[async_trait::async_trait]
|
||||
impl nomi_browser::extract::ExtractModel for SessionExtractModel {
|
||||
async fn complete(&self, prompt: &str) -> Result<String, String> {
|
||||
use nomi_types::llm::{LlmEvent, LlmRequest};
|
||||
use nomi_types::message::{ContentBlock, Message, Role};
|
||||
|
||||
let request = LlmRequest {
|
||||
model: self.model.clone(),
|
||||
system: String::new(),
|
||||
messages: vec![Message::new(
|
||||
Role::User,
|
||||
vec![ContentBlock::Text { text: prompt.to_string() }],
|
||||
)],
|
||||
tools: Vec::new(),
|
||||
max_tokens: self.max_tokens,
|
||||
thinking: None,
|
||||
reasoning_effort: None,
|
||||
};
|
||||
let mut rx = self
|
||||
.provider
|
||||
.stream(&request)
|
||||
.await
|
||||
.map_err(|e| format!("extract model stream failed: {e}"))?;
|
||||
let mut out = String::new();
|
||||
while let Some(event) = rx.recv().await {
|
||||
match event {
|
||||
LlmEvent::TextDelta(t) => out.push_str(&t),
|
||||
LlmEvent::Done { .. } => break,
|
||||
LlmEvent::Error(e) => return Err(format!("extract model error: {e}")),
|
||||
// No tools / no thinking requested → other events aren't expected; ignore.
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
/// **P7B: session-model adapter for `BrowserTool`'s visual-fallback locator seam.**
|
||||
///
|
||||
/// Wraps the session's [`LlmProvider`] + model params so the facade can do vision-based
|
||||
/// element location when DOM/aria anchoring fails (a `ref` went stale/detached). The
|
||||
/// browser engine stays LLM-free — like [`SessionExtractModel`], this adapter lives at the
|
||||
/// bootstrap/facade layer (架构铁律). `locate` sends one multimodal user message — a strict
|
||||
/// "return JSON bounding box" instruction plus the (already engine-redacted) screenshot as a
|
||||
/// [`ContentBlock::Image`] — and parses the model's pixel-box reply. No system prompt, no
|
||||
/// tools, no extended thinking. Reuses the session's own model (same decision as Extract: no
|
||||
/// separate model/cost surface). Gated behind `agent.browserUse.visualFallback` (host_default
|
||||
/// OFF) because every fallback round-trips the vision model — a real token cost.
|
||||
#[cfg(feature = "browser-use")]
|
||||
struct SessionVisualLocator {
|
||||
provider: Arc<dyn LlmProvider>,
|
||||
model: String,
|
||||
max_tokens: u32,
|
||||
}
|
||||
|
||||
#[cfg(feature = "browser-use")]
|
||||
impl SessionVisualLocator {
|
||||
/// Shared multimodal round-trip: one user message (`prompt` text + a base64 PNG) → stream →
|
||||
/// collected text. No system prompt, no tools, no thinking (vision locating is mechanical).
|
||||
/// Both [`VisualLocator::locate`] (bbox) and [`VisualLocator::locate_labeled`] (SoM label)
|
||||
/// share this; only the prompt + reply parser differ.
|
||||
async fn run_vision(&self, prompt: String, png: &[u8]) -> Result<String, String> {
|
||||
use base64::Engine as _;
|
||||
use nomi_types::llm::{LlmEvent, LlmRequest};
|
||||
use nomi_types::message::{ContentBlock, Message, Role};
|
||||
|
||||
let data = base64::engine::general_purpose::STANDARD.encode(png);
|
||||
let request = LlmRequest {
|
||||
model: self.model.clone(),
|
||||
system: String::new(),
|
||||
messages: vec![Message::new(
|
||||
Role::User,
|
||||
vec![
|
||||
ContentBlock::Text { text: prompt },
|
||||
ContentBlock::Image { media_type: "image/png".to_string(), data },
|
||||
],
|
||||
)],
|
||||
tools: Vec::new(),
|
||||
max_tokens: self.max_tokens,
|
||||
thinking: None,
|
||||
reasoning_effort: None,
|
||||
};
|
||||
|
||||
let mut rx = self
|
||||
.provider
|
||||
.stream(&request)
|
||||
.await
|
||||
.map_err(|e| format!("visual locator stream failed: {e}"))?;
|
||||
let mut out = String::new();
|
||||
while let Some(event) = rx.recv().await {
|
||||
match event {
|
||||
LlmEvent::TextDelta(t) => out.push_str(&t),
|
||||
LlmEvent::Done { .. } => break,
|
||||
LlmEvent::Error(e) => return Err(format!("visual locator error: {e}")),
|
||||
// No tools / no thinking requested → other events aren't expected; ignore.
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "browser-use")]
|
||||
#[async_trait::async_trait]
|
||||
impl nomi_browser::visual_fallback::VisualLocator for SessionVisualLocator {
|
||||
async fn locate(
|
||||
&self,
|
||||
screenshot: &[u8],
|
||||
instruction: &str,
|
||||
) -> Result<nomi_browser::visual_fallback::VisualLocateResult, String> {
|
||||
// Strict-JSON instruction. The screenshot is at device-pixel resolution; the model
|
||||
// returns the box in that same image-pixel space (the facade divides by the live DPR
|
||||
// before dispatching the click). A not-found answer is `{"confidence": 0}`.
|
||||
let prompt = format!(
|
||||
"You are a precise UI element locator. The attached PNG is a screenshot of a web \
|
||||
page rendered at device-pixel resolution; the origin (0,0) is its top-left corner. \
|
||||
Find the single element described below and return its bounding box in IMAGE PIXEL \
|
||||
coordinates.\n\n\
|
||||
Target element: {instruction}\n\n\
|
||||
Respond with ONLY a JSON object — no markdown, no code fence, no prose:\n\
|
||||
{{\"x\": <left px>, \"y\": <top px>, \"width\": <px>, \"height\": <px>, \"confidence\": <0..1>}}\n\
|
||||
If you cannot confidently find the element, respond with {{\"confidence\": 0}}."
|
||||
);
|
||||
let out = self.run_vision(prompt, screenshot).await?;
|
||||
parse_visual_locate_result(&out)
|
||||
}
|
||||
|
||||
async fn locate_labeled(
|
||||
&self,
|
||||
annotated_screenshot: &[u8],
|
||||
instruction: &str,
|
||||
n_labels: usize,
|
||||
) -> Result<nomi_browser::visual_fallback::SomLabelResult, String> {
|
||||
// SoM mode: the screenshot has numbered labels (1..=n_labels) drawn on its clickable
|
||||
// elements. The model picks the label that matches the target — a finite choice, far
|
||||
// more reliable than free-form pixel regression. A not-found answer is `{"label": 0}`.
|
||||
let prompt = format!(
|
||||
"You are a precise UI element selector. The attached PNG is a screenshot of a web \
|
||||
page with numbered labels 1 to {n_labels} drawn on its interactive elements (each \
|
||||
label sits at the top-left of its element's box). Identify which single label \
|
||||
number marks the element described below.\n\n\
|
||||
Target element: {instruction}\n\n\
|
||||
Respond with ONLY a JSON object — no markdown, no code fence, no prose:\n\
|
||||
{{\"label\": <integer 1..{n_labels}>, \"confidence\": <0..1>}}\n\
|
||||
If none of the labeled elements matches, respond with {{\"label\": 0, \"confidence\": 0}}."
|
||||
);
|
||||
let out = self.run_vision(prompt, annotated_screenshot).await?;
|
||||
parse_som_locate_result(&out, n_labels)
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse the vision model's reply into a [`VisualLocateResult`](nomi_browser::visual_fallback::VisualLocateResult).
|
||||
///
|
||||
/// Tolerates models that wrap the JSON in prose or a ```` ```json ```` fence by extracting the
|
||||
/// outermost `{...}` span. A reply with `confidence == 0`, a missing/zero box, or any
|
||||
/// non-numeric field is treated as "element not found" (an `Err`) — the facade then surfaces
|
||||
/// the original anchor error rather than clicking a bogus coordinate.
|
||||
#[cfg(feature = "browser-use")]
|
||||
fn parse_visual_locate_result(
|
||||
raw: &str,
|
||||
) -> Result<nomi_browser::visual_fallback::VisualLocateResult, String> {
|
||||
use nomi_browser::visual_fallback::{PixelBox, VisualLocateResult};
|
||||
|
||||
let trimmed = raw.trim();
|
||||
let start = trimmed
|
||||
.find('{')
|
||||
.ok_or_else(|| format!("visual locator returned no JSON object: {trimmed:.200}"))?;
|
||||
let end = trimmed
|
||||
.rfind('}')
|
||||
.filter(|e| *e >= start)
|
||||
.ok_or_else(|| format!("visual locator JSON span malformed: {trimmed:.200}"))?;
|
||||
let json_str = &trimmed[start..=end];
|
||||
let v: serde_json::Value = serde_json::from_str(json_str)
|
||||
.map_err(|e| format!("visual locator JSON parse failed ({e}): {json_str:.200}"))?;
|
||||
|
||||
let confidence = v.get("confidence").and_then(serde_json::Value::as_f64).unwrap_or(0.0);
|
||||
let num = |k: &str| v.get(k).and_then(serde_json::Value::as_f64);
|
||||
match (num("x"), num("y"), num("width"), num("height")) {
|
||||
(Some(x), Some(y), Some(width), Some(height))
|
||||
if confidence > 0.0 && width > 0.0 && height > 0.0 =>
|
||||
{
|
||||
Ok(VisualLocateResult {
|
||||
pixel_box: PixelBox { x, y, width, height },
|
||||
confidence: confidence.clamp(0.0, 1.0),
|
||||
})
|
||||
}
|
||||
_ => Err(format!(
|
||||
"visual locator could not locate the element (confidence={confidence})"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse the vision model's SoM reply into a [`SomLabelResult`](nomi_browser::visual_fallback::SomLabelResult).
|
||||
///
|
||||
/// Same JSON-extraction tolerance as [`parse_visual_locate_result`] (handles prose / ```` ```json ````
|
||||
/// fences). The label MUST be an integer in `1..=n_labels` with `confidence > 0`; `label == 0`,
|
||||
/// out-of-range, missing, or zero-confidence all map to `Err` ("no label matched") so the facade
|
||||
/// surfaces the original anchor error rather than indexing a bogus / out-of-bounds label.
|
||||
#[cfg(feature = "browser-use")]
|
||||
fn parse_som_locate_result(
|
||||
raw: &str,
|
||||
n_labels: usize,
|
||||
) -> Result<nomi_browser::visual_fallback::SomLabelResult, String> {
|
||||
use nomi_browser::visual_fallback::SomLabelResult;
|
||||
|
||||
let trimmed = raw.trim();
|
||||
let start = trimmed
|
||||
.find('{')
|
||||
.ok_or_else(|| format!("SoM locator returned no JSON object: {trimmed:.200}"))?;
|
||||
let end = trimmed
|
||||
.rfind('}')
|
||||
.filter(|e| *e >= start)
|
||||
.ok_or_else(|| format!("SoM locator JSON span malformed: {trimmed:.200}"))?;
|
||||
let json_str = &trimmed[start..=end];
|
||||
let v: serde_json::Value = serde_json::from_str(json_str)
|
||||
.map_err(|e| format!("SoM locator JSON parse failed ({e}): {json_str:.200}"))?;
|
||||
|
||||
let confidence = v.get("confidence").and_then(serde_json::Value::as_f64).unwrap_or(0.0);
|
||||
// Accept integer or float-encoded label; reject anything not in 1..=n_labels.
|
||||
let label = v.get("label").and_then(serde_json::Value::as_f64);
|
||||
match label {
|
||||
Some(l) if l.fract() == 0.0 && l >= 1.0 && (l as usize) <= n_labels && confidence > 0.0 => {
|
||||
Ok(SomLabelResult {
|
||||
label: l as usize,
|
||||
confidence: confidence.clamp(0.0, 1.0),
|
||||
})
|
||||
}
|
||||
_ => Err(format!(
|
||||
"SoM locator picked no valid label (label={label:?}, n_labels={n_labels}, confidence={confidence})"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of bootstrapping an agent engine with all features initialized.
|
||||
pub struct BootstrapResult {
|
||||
pub engine: AgentEngine,
|
||||
pub provider: Arc<dyn LlmProvider>,
|
||||
pub mcp_managers: Vec<Arc<McpManager>>,
|
||||
pub has_mcp: bool,
|
||||
}
|
||||
|
||||
/// Builder for creating a fully-initialized `AgentEngine`.
|
||||
///
|
||||
/// Encapsulates the complete initialization pipeline so all consumers
|
||||
/// (CLI, backend, sub-agents) get consistent behavior:
|
||||
///
|
||||
/// - System prompt always includes model identity, working directory, date
|
||||
/// - Tool usage guidance is always injected
|
||||
/// - AGENTS.md is loaded from the workspace hierarchy
|
||||
/// - Skills, MCP, plan mode, spawn are enabled based on `Config` fields
|
||||
pub struct AgentBootstrap {
|
||||
config: Config,
|
||||
workspace: String,
|
||||
output: Arc<dyn OutputSink>,
|
||||
provider: Option<Arc<dyn LlmProvider>>,
|
||||
resume_session: Option<Session>,
|
||||
extra_skill_dirs: Vec<PathBuf>,
|
||||
goal: Option<crate::goal::runtime::GoalSpec>,
|
||||
/// **P3-X1: the session's shared runtime approval-mode handle** (the same
|
||||
/// `Arc<ToolApprovalManager>` the host later installs on the engine via
|
||||
/// `set_approval_manager`). When present it is threaded into the native
|
||||
/// `BrowserTool` so its fail-closed redline gate reads the *runtime* session mode
|
||||
/// LIVE — a mid-session `set_mode` to yolo arms the gate immediately, instead of
|
||||
/// being pinned to the construction-time `auto_approve` snapshot. Hosts that have
|
||||
/// no protocol approval manager (e.g. the interactive REPL) leave it `None` and the
|
||||
/// facade falls back to the construction-time snapshot (unchanged behavior).
|
||||
approval_manager: Option<Arc<nomi_protocol::ToolApprovalManager>>,
|
||||
/// **P3-X2: the session's per-pet browser secret vault source** (vault file path +
|
||||
/// machine-bound 32-byte key). Threaded into the native `BrowserTool` so it can lazily
|
||||
/// load the registered credentials (`secret:NAME` resolves, origin-gated) and derive the
|
||||
/// firewall domain allowlist from the same per-pet `allowed_origins` (裁决⑤). Stored as
|
||||
/// the raw pieces (NOT the `nomi_browser` type) so the field exists regardless of the
|
||||
/// `browser-use` feature; the `BrowserSecretSource` is constructed only at the
|
||||
/// feature-gated `with_policy` call site. `None` → empty store + unrestricted egress.
|
||||
#[cfg_attr(not(feature = "browser-use"), allow(dead_code))]
|
||||
browser_secret_source: Option<(PathBuf, [u8; 32])>,
|
||||
/// **Phase D: the session's browser approval gate** (human takeover + SD-5 cross-origin
|
||||
/// egress). Threaded into the native `BrowserTool` so an irreversible action in a bypass
|
||||
/// session — and a gated cross-origin POST — is surfaced to the user and awaited. `None`
|
||||
/// (default) → fail-closed (current behavior). Feature-gated: the trait is in `nomi_browser`.
|
||||
#[cfg(feature = "browser-use")]
|
||||
approval_gate: Option<Arc<dyn nomi_browser::BrowserApprovalGate>>,
|
||||
}
|
||||
|
||||
impl AgentBootstrap {
|
||||
pub fn new(config: Config, workspace: impl Into<String>, output: Arc<dyn OutputSink>) -> Self {
|
||||
Self {
|
||||
config,
|
||||
workspace: workspace.into(),
|
||||
output,
|
||||
provider: None,
|
||||
resume_session: None,
|
||||
extra_skill_dirs: Vec::new(),
|
||||
goal: None,
|
||||
approval_manager: None,
|
||||
browser_secret_source: None,
|
||||
#[cfg(feature = "browser-use")]
|
||||
approval_gate: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Use a pre-created provider instead of creating one from config.
|
||||
pub fn provider(mut self, provider: Arc<dyn LlmProvider>) -> Self {
|
||||
self.provider = Some(provider);
|
||||
self
|
||||
}
|
||||
|
||||
/// Enable goal-driven continuation for this session (opt-in). Omit it (the
|
||||
/// default) and the engine behaves exactly as before.
|
||||
pub fn goal(mut self, goal: Option<crate::goal::runtime::GoalSpec>) -> Self {
|
||||
self.goal = goal;
|
||||
self
|
||||
}
|
||||
|
||||
/// **P3-X1: provide the session's shared `Arc<ToolApprovalManager>`** so the native
|
||||
/// `BrowserTool`'s redline gate reads the *runtime* approval mode LIVE (a mid-session
|
||||
/// `set_mode` to yolo arms the gate immediately). Pass the *same* Arc that is later
|
||||
/// installed on the engine via `set_approval_manager`, so the facade and orchestration
|
||||
/// observe one mode cell with zero drift. Omit it (the default) to keep the
|
||||
/// construction-time `auto_approve` snapshot as the (fail-closed) source of truth.
|
||||
pub fn approval_manager(mut self, mgr: Arc<nomi_protocol::ToolApprovalManager>) -> Self {
|
||||
self.approval_manager = Some(mgr);
|
||||
self
|
||||
}
|
||||
|
||||
/// **P3-X2: provide the session's per-pet browser secret vault source** so the native
|
||||
/// `BrowserTool` can load the user-registered credentials (`secret:NAME`, origin-gated)
|
||||
/// and derive the firewall domain allowlist from the same per-pet `allowed_origins`
|
||||
/// (裁决⑤). Takes the raw pieces (vault file path + machine-bound 32-byte key) so backend
|
||||
/// callers (`nomifun-ai-agent`) need not depend on `nomi-browser` to wire it. Omit it
|
||||
/// (the default) to keep an empty store + unrestricted egress (current behavior).
|
||||
pub fn browser_secret_source(mut self, vault_path: PathBuf, key: [u8; 32]) -> Self {
|
||||
self.browser_secret_source = Some((vault_path, key));
|
||||
self
|
||||
}
|
||||
|
||||
/// **Phase D: provide the browser approval gate** (host impl raises a `Confirmation`
|
||||
/// and awaits the shared `ToolApprovalManager`). Threaded into the native `BrowserTool`,
|
||||
/// enabling human takeover of irreversible actions + SD-5 cross-origin egress approval.
|
||||
/// Omit it (the default) → fail-closed (irreversible stays Blocked, gated egress fails).
|
||||
#[cfg(feature = "browser-use")]
|
||||
pub fn approval_gate(mut self, gate: Arc<dyn nomi_browser::BrowserApprovalGate>) -> Self {
|
||||
self.approval_gate = Some(gate);
|
||||
self
|
||||
}
|
||||
|
||||
/// Resume from a previously saved session.
|
||||
pub fn resume(mut self, session: Session) -> Self {
|
||||
self.resume_session = Some(session);
|
||||
self
|
||||
}
|
||||
|
||||
/// Add extra directories to scan for skills.
|
||||
pub fn extra_skill_dirs(mut self, dirs: Vec<PathBuf>) -> Self {
|
||||
self.extra_skill_dirs = dirs;
|
||||
self
|
||||
}
|
||||
|
||||
/// Read-only access to the config (for session management before build).
|
||||
pub fn config(&self) -> &Config {
|
||||
&self.config
|
||||
}
|
||||
|
||||
/// Build the fully-initialized engine.
|
||||
pub async fn build(mut self) -> anyhow::Result<BootstrapResult> {
|
||||
let cwd = &self.workspace;
|
||||
let cwd_path = std::path::Path::new(cwd);
|
||||
|
||||
tracing::info!(target: "nomi_agent", workspace = %cwd, "agent bootstrap: workspace cwd resolved");
|
||||
|
||||
let provider = self
|
||||
.provider
|
||||
.unwrap_or_else(|| nomi_providers::create_provider(&self.config));
|
||||
|
||||
let memory_dir = nomi_memory::paths::auto_memory_dir(cwd_path);
|
||||
|
||||
let file_cache = if self.config.file_cache.enabled {
|
||||
Some(Arc::new(std::sync::RwLock::new(
|
||||
nomi_tools::file_cache::FileStateCache::new(&self.config.file_cache),
|
||||
)))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let mut registry = nomi_tools::registry::ToolRegistry::new();
|
||||
// Opt-in write-root containment (§3.6): when `tools.write_root` is set,
|
||||
// resolve it to an absolute path the write tools enforce. Empty = off.
|
||||
let write_root: Option<std::path::PathBuf> = {
|
||||
let wr = self.config.tools.write_root.trim();
|
||||
if wr.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(std::path::PathBuf::from(wr))
|
||||
}
|
||||
};
|
||||
registry.register(Box::new(nomi_tools::read::ReadTool::new(
|
||||
file_cache.clone(),
|
||||
Some(cwd_path.to_path_buf()),
|
||||
)));
|
||||
registry.register(Box::new(
|
||||
nomi_tools::write::WriteTool::new(file_cache.clone())
|
||||
.with_write_root(write_root.clone())
|
||||
.with_cwd(Some(cwd_path.to_path_buf())),
|
||||
));
|
||||
registry.register(Box::new(
|
||||
nomi_tools::edit::EditTool::new(file_cache.clone())
|
||||
.with_write_root(write_root.clone())
|
||||
.with_cwd(Some(cwd_path.to_path_buf())),
|
||||
));
|
||||
registry.register(Box::new(
|
||||
nomi_tools::apply_patch::ApplyPatchTool::new(file_cache)
|
||||
.with_write_root(write_root)
|
||||
.with_cwd(Some(cwd_path.to_path_buf())),
|
||||
));
|
||||
// Experimental `Lsp` code-navigation tool: registered only when at least
|
||||
// one language server is configured (default off → no behaviour change).
|
||||
{
|
||||
let mut lsp_map: std::collections::HashMap<String, Vec<String>> =
|
||||
std::collections::HashMap::new();
|
||||
for entry in &self.config.tools.lsp_servers {
|
||||
if entry.command.is_empty() {
|
||||
continue;
|
||||
}
|
||||
for ext in &entry.extensions {
|
||||
lsp_map.insert(ext.trim_start_matches('.').to_ascii_lowercase(), entry.command.clone());
|
||||
}
|
||||
}
|
||||
if !lsp_map.is_empty() {
|
||||
registry.register(Box::new(nomi_tools::lsp::LspTool::new(
|
||||
lsp_map,
|
||||
cwd_path.to_path_buf(),
|
||||
)));
|
||||
}
|
||||
}
|
||||
// Native `remember` tool: persist durable project/user memories mid-session
|
||||
// to the file-based long-term memory (injected into future sessions).
|
||||
if let Some(mem_dir) = memory_dir.clone() {
|
||||
registry.register(Box::new(crate::memory_tools::RememberTool::new(mem_dir)));
|
||||
}
|
||||
// Bash registration, precedence: Seatbelt sandbox (macOS, opt-in) >
|
||||
// persistent shell (Unix, opt-in) > stateless one-shot. The sandbox wins
|
||||
// so no unconfined path coexists with it.
|
||||
#[cfg(target_os = "macos")]
|
||||
let bash_sandbox_on = self.config.tools.bash_sandbox;
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
let bash_sandbox_on = false;
|
||||
|
||||
if bash_sandbox_on {
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
if self.config.tools.persistent_shell {
|
||||
tracing::warn!(
|
||||
target: "nomi_agent",
|
||||
"bash_sandbox enabled — persistent_shell is disabled under the sandbox so no unconfined shell path coexists"
|
||||
);
|
||||
}
|
||||
registry.register(Box::new(
|
||||
nomi_tools::bash::BashTool::new(cwd_path.to_path_buf())
|
||||
.with_sandbox(Some(vec![cwd_path.to_path_buf()])),
|
||||
));
|
||||
}
|
||||
} else {
|
||||
#[cfg(unix)]
|
||||
if self.config.tools.persistent_shell {
|
||||
let shell = std::sync::Arc::new(nomi_tools::persistent_shell::PersistentShell::new(
|
||||
cwd_path.to_string_lossy().into_owned(),
|
||||
));
|
||||
registry.register(Box::new(nomi_tools::bash::BashTool::with_persistent_shell(
|
||||
cwd_path.to_path_buf(),
|
||||
shell,
|
||||
)));
|
||||
} else {
|
||||
registry.register(Box::new(nomi_tools::bash::BashTool::new(cwd_path.to_path_buf())));
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
registry.register(Box::new(nomi_tools::bash::BashTool::new(
|
||||
cwd_path.to_path_buf(),
|
||||
)));
|
||||
}
|
||||
registry.register(Box::new(nomi_tools::grep::GrepTool::new(
|
||||
cwd_path.to_path_buf(),
|
||||
)));
|
||||
registry.register(Box::new(nomi_tools::glob::GlobTool::new(
|
||||
cwd_path.to_path_buf(),
|
||||
)));
|
||||
|
||||
let builtin_names: Vec<String> = registry.tool_names();
|
||||
|
||||
let mut mcp_managers: Vec<Arc<McpManager>> = Vec::new();
|
||||
let mcp_manager = if !self.config.mcp.servers.is_empty() {
|
||||
match McpManager::connect_all(&self.config.mcp.servers).await {
|
||||
Ok(mgr) => {
|
||||
let mgr = Arc::new(mgr);
|
||||
nomi_mcp::tool_proxy::register_mcp_tools(
|
||||
&mut registry,
|
||||
&mgr,
|
||||
&builtin_names,
|
||||
&self.config.mcp.servers,
|
||||
);
|
||||
mcp_managers.push(mgr.clone());
|
||||
Some(mgr)
|
||||
}
|
||||
Err(e) => {
|
||||
self.output
|
||||
.emit_warning(&format!("MCP initialization error: {e}"));
|
||||
None
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let has_mcp = mcp_manager.is_some();
|
||||
|
||||
let skills = nomi_skills::loader::load_all_skills(
|
||||
cwd_path,
|
||||
&self.extra_skill_dirs,
|
||||
false,
|
||||
mcp_manager.as_deref(),
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut prompt_cache = crate::context::SystemPromptCache::new();
|
||||
let system_prompt = crate::context::build_system_prompt(
|
||||
&mut prompt_cache,
|
||||
self.config.system_prompt.as_deref(),
|
||||
cwd,
|
||||
&self.config.model,
|
||||
&skills,
|
||||
None,
|
||||
memory_dir.as_deref(),
|
||||
false,
|
||||
self.config.compact.toon,
|
||||
self.config.tools.browser.enabled,
|
||||
);
|
||||
self.config.system_prompt = Some(system_prompt);
|
||||
|
||||
let skills_arc = Arc::new(skills);
|
||||
let skill_checker = nomi_skills::permissions::SkillPermissionChecker::new(
|
||||
self.config.tools.skills.deny.clone(),
|
||||
self.config.tools.skills.allow.clone(),
|
||||
self.config.tools.auto_approve,
|
||||
);
|
||||
registry.register(Box::new(crate::skill_tool::SkillTool::new(
|
||||
skills_arc,
|
||||
cwd.to_string(),
|
||||
skill_checker,
|
||||
)));
|
||||
|
||||
let spawner = Arc::new(
|
||||
crate::spawner::AgentSpawner::new(
|
||||
provider.clone(),
|
||||
self.config.clone(),
|
||||
cwd_path.to_path_buf(),
|
||||
)
|
||||
.with_token_budget(
|
||||
self.config
|
||||
.tools
|
||||
.subagent_token_budget
|
||||
.map(|limit| Arc::new(crate::spawner::TokenBudget::new(limit))),
|
||||
),
|
||||
);
|
||||
registry.register(Box::new(crate::spawn_tool::SpawnTool::new(spawner)));
|
||||
|
||||
let plan_active_flag = Arc::new(AtomicBool::new(false));
|
||||
if self.config.plan.enabled {
|
||||
registry.register(Box::new(crate::plan::tools::EnterPlanModeTool::new(
|
||||
Arc::clone(&plan_active_flag),
|
||||
)));
|
||||
registry.register(Box::new(crate::plan::tools::ExitPlanModeTool::new(
|
||||
Arc::clone(&plan_active_flag),
|
||||
)));
|
||||
}
|
||||
|
||||
#[cfg(feature = "computer-use")]
|
||||
if self.config.tools.computer.enabled {
|
||||
tracing::info!(
|
||||
target: "nomi_agent",
|
||||
"computer-use ENABLED: registering the Computer tool (observe / click_element / \
|
||||
launch / type / scroll). Desktop control is available to this session."
|
||||
);
|
||||
registry.register(Box::new(nomi_computer::ComputerTool::new(
|
||||
&self.config.tools.computer,
|
||||
)));
|
||||
}
|
||||
#[cfg(feature = "computer-use")]
|
||||
if !self.config.tools.computer.enabled {
|
||||
tracing::info!(
|
||||
target: "nomi_agent",
|
||||
"computer-use DISABLED for this session (config.tools.computer.enabled = false); \
|
||||
the Computer tool is NOT registered — the agent falls back to the shell."
|
||||
);
|
||||
}
|
||||
#[cfg(not(feature = "computer-use"))]
|
||||
if self.config.tools.computer.enabled {
|
||||
tracing::warn!(
|
||||
target: "nomi_agent",
|
||||
"computer use enabled in config but this build lacks the computer-use feature"
|
||||
);
|
||||
}
|
||||
|
||||
// Native browser-use (in-process self-hosted CDP engine). The native
|
||||
// BrowserTool registers under the name "Browser" (actions: navigate /
|
||||
// observe / screenshot / capabilities) and is the sole browser path. The
|
||||
// engine launches lazily on the first action — registering it never starts
|
||||
// a browser.
|
||||
#[cfg(feature = "browser-use")]
|
||||
if self.config.tools.browser.enabled {
|
||||
tracing::info!(
|
||||
target: "nomi_agent",
|
||||
"browser-use ENABLED: registering the native Browser tool (navigate / observe / \
|
||||
screenshot / capabilities). The managed Chromium launches lazily on first use."
|
||||
);
|
||||
// F1-sec: thread the session-bypass policy + evaluate full-power into the
|
||||
// facade so its independent fail-closed redline gate (裁决⑧) actually fires
|
||||
// and the evaluate gate (裁决⑨) reflects the user's opt-in. `auto_approve`
|
||||
// is `true` iff orchestration approval is bypassed (yolo / companion-forced-yolo
|
||||
// / --auto-approve — see BrowserTool::session_bypasses_approval doc); the
|
||||
// `browser.full_power` flag carries the LIVE `agent.browserUse.fullPower` pref
|
||||
// (set by the backend factory per session). Constructed here where the full
|
||||
// Config is in scope.
|
||||
//
|
||||
// P3-G2: pass the session working directory `cwd` (= self.workspace) as the
|
||||
// per-session/per-pet workspace. It's the natural isolation point — for a
|
||||
// companion session it is `{companion_id}/workspace` (companion.rs sets
|
||||
// extra.workspace, which the manager resolves to this cwd); for a non-companion
|
||||
// session it's the conversation's own working dir. Downloads (E4) land in its
|
||||
// `downloads/` subdir instead of a temp dir. The non-companion
|
||||
// `{data_dir}/browser-profiles/{conversation_id}` subdivision (默认④) needs the
|
||||
// data_dir + conversation_id which the bootstrap does not hold (they live in the
|
||||
// upper manager/factory) — the cwd is already per-conversation isolated, so we
|
||||
// pass it directly (simplest correct) and leave the finer browser-profiles
|
||||
// layout to W4/deployment wiring (the field signature already supports it).
|
||||
// P3-X1: thread the session's shared runtime approval-mode handle (the same
|
||||
// Arc<ToolApprovalManager> the host installs via set_approval_manager) so the
|
||||
// facade's redline gate reads the LIVE session mode — a mid-session set_mode to
|
||||
// yolo arms it immediately, instead of being pinned to the auto_approve snapshot
|
||||
// above. `None` (e.g. the interactive REPL, which has no protocol approval
|
||||
// manager) → the facade falls back to the construction-time snapshot (unchanged).
|
||||
let mut browser_tool = nomi_browser::BrowserTool::with_policy(
|
||||
&self.config.tools.browser,
|
||||
self.config.tools.auto_approve,
|
||||
self.config.tools.browser.full_power,
|
||||
self.config.tools.browser.persistent_login,
|
||||
Some(PathBuf::from(cwd)),
|
||||
self.approval_manager.clone(),
|
||||
// P3-X2: per-pet secret vault source (vault path + machine-bound key) so the
|
||||
// facade lazily loads registered credentials + derives the firewall domain
|
||||
// allowlist from their allowed_origins (裁决⑤). None → empty store + unrestricted.
|
||||
self.browser_secret_source
|
||||
.clone()
|
||||
.map(|(vault_path, key)| nomi_browser::BrowserSecretSource { vault_path, key }),
|
||||
);
|
||||
// P7A: site-memory opt-in (LIVE pref `agent.browserUse.siteMemory`, default OFF). When
|
||||
// ON, inject a file-backed sink so the agent remembers site structure across sessions
|
||||
// (entries injected into observe as untrusted hints; secret-sourced entries dropped by
|
||||
// the store). Root is GLOBAL (browser identity is globally shared, NOT per-session) —
|
||||
// same `browser-data` root the gateway/engine use. OFF → no sink (zero behavior change).
|
||||
if self.config.tools.browser.site_memory {
|
||||
let sm_root = nomi_config::config::app_config_dir()
|
||||
.map(|d| d.join("browser-data").join("site-memory"))
|
||||
.unwrap_or_else(|| std::env::temp_dir().join("nomi-browser-data").join("site-memory"));
|
||||
let sink = nomi_browser::site_memory::FileSiteMemorySink::new(sm_root);
|
||||
let store = std::sync::Arc::new(nomi_browser::site_memory::SiteMemoryStore::new(
|
||||
Box::new(sink),
|
||||
));
|
||||
browser_tool = browser_tool.with_site_memory(store);
|
||||
}
|
||||
// extract-llm: reuse the session provider (the same `provider` driving this engine,
|
||||
// created above if none was injected) so act(Extract) does real LLM extraction. No
|
||||
// pref — extract is opt-in per-call by the agent; this only enables the capability.
|
||||
let extract_model = Arc::new(SessionExtractModel {
|
||||
provider: provider.clone(),
|
||||
model: self.config.model.clone(),
|
||||
max_tokens: self.config.max_tokens,
|
||||
});
|
||||
browser_tool = browser_tool.with_extract_model(extract_model);
|
||||
// P7B: visual fallback opt-in (LIVE pref `agent.browserUse.visualFallback`, default
|
||||
// OFF). When ON, inject a session-model `VisualLocator`: if DOM/aria anchoring fails
|
||||
// (a `ref` went stale/detached), the facade screenshots the page and asks the vision
|
||||
// model to locate the target by description, then clicks the DPR-mapped CSS point.
|
||||
// Reuses the session provider/model (no separate cost surface). OFF → no locator
|
||||
// injected, so the facade's fallback stays Unavailable (zero behavior change).
|
||||
if self.config.tools.browser.visual_fallback {
|
||||
let locator = Arc::new(SessionVisualLocator {
|
||||
provider: provider.clone(),
|
||||
model: self.config.model.clone(),
|
||||
max_tokens: self.config.max_tokens,
|
||||
});
|
||||
browser_tool = browser_tool
|
||||
.with_visual_locator(locator)
|
||||
.with_visual_fallback_enabled(true);
|
||||
}
|
||||
// Phase D: thread the host approval gate (human takeover + SD-5 egress). When the
|
||||
// gate is present it surfaces irreversible actions / gated cross-origin POSTs to the
|
||||
// user and awaits a decision; absent → fail-closed (current behavior).
|
||||
if let Some(gate) = self.approval_gate.clone() {
|
||||
browser_tool = browser_tool.with_approval_gate(gate);
|
||||
}
|
||||
registry.register(Box::new(browser_tool));
|
||||
}
|
||||
#[cfg(not(feature = "browser-use"))]
|
||||
if self.config.tools.browser.enabled {
|
||||
tracing::debug!(
|
||||
target: "nomi_agent",
|
||||
"browser-use enabled in config but this build lacks the browser-use feature; the \
|
||||
native Browser tool is not registered."
|
||||
);
|
||||
}
|
||||
|
||||
// Interactive PTY tools: exec_command + write_stdin share one
|
||||
// ProcessStore (session-level, alive for the engine's lifetime via the
|
||||
// tools held in the ToolRegistry). Same stateful-tool pattern as
|
||||
// SpawnTool/BrowserTool. The store's Drop SIGKILLs any lingering PTY
|
||||
// process groups when the engine (and its registry) is torn down.
|
||||
let process_store = Arc::new(nomi_tools::process_store::ProcessStore::new());
|
||||
registry.register(Box::new(nomi_tools::exec_command::ExecCommandTool::new(
|
||||
Arc::clone(&process_store),
|
||||
cwd_path.to_path_buf(),
|
||||
)));
|
||||
registry.register(Box::new(nomi_tools::write_stdin::WriteStdinTool::new(
|
||||
Arc::clone(&process_store),
|
||||
)));
|
||||
|
||||
// codex-style stateless todo checklist tool. Always registered (not
|
||||
// deferred), surfaced to the frontend via the Plan event bridge.
|
||||
registry.register(Box::new(nomi_tools::update_plan::UpdatePlanTool::new()));
|
||||
|
||||
let tool_defs_snapshot = registry.to_tool_defs();
|
||||
registry.register(Box::new(nomi_tools::tool_search::ToolSearchTool::new(
|
||||
tool_defs_snapshot,
|
||||
)));
|
||||
|
||||
let mut engine = if let Some(session) = self.resume_session {
|
||||
AgentEngine::resume_with_provider(
|
||||
provider.clone(),
|
||||
self.config,
|
||||
registry,
|
||||
self.output,
|
||||
session,
|
||||
cwd_path.to_path_buf(),
|
||||
)
|
||||
} else {
|
||||
AgentEngine::new_with_provider(
|
||||
provider.clone(),
|
||||
self.config,
|
||||
registry,
|
||||
self.output,
|
||||
cwd_path.to_path_buf(),
|
||||
)
|
||||
};
|
||||
engine.set_plan_active_flag(plan_active_flag);
|
||||
if let Some(spec) = self.goal {
|
||||
engine.set_goal(spec.objective, spec.max_auto_continuations);
|
||||
}
|
||||
|
||||
Ok(BootstrapResult {
|
||||
engine,
|
||||
provider,
|
||||
mcp_managers,
|
||||
has_mcp,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "browser-use"))]
|
||||
mod visual_locator_tests {
|
||||
use super::{parse_som_locate_result, parse_visual_locate_result};
|
||||
|
||||
#[test]
|
||||
fn parses_clean_json_box() {
|
||||
let r = parse_visual_locate_result(
|
||||
r#"{"x": 100, "y": 200, "width": 40, "height": 20, "confidence": 0.9}"#,
|
||||
)
|
||||
.expect("clean JSON should parse");
|
||||
assert_eq!(r.pixel_box.x, 100.0);
|
||||
assert_eq!(r.pixel_box.y, 200.0);
|
||||
assert_eq!(r.pixel_box.width, 40.0);
|
||||
assert_eq!(r.pixel_box.height, 20.0);
|
||||
assert!((r.confidence - 0.9).abs() < 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_json_from_code_fence() {
|
||||
// Vision models love wrapping JSON in a ```json fence — tolerate it.
|
||||
let raw = "```json\n{\"x\": 1, \"y\": 2, \"width\": 3, \"height\": 4, \"confidence\": 0.5}\n```";
|
||||
let r = parse_visual_locate_result(raw).expect("fenced JSON should parse");
|
||||
assert_eq!(r.pixel_box.width, 3.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_json_from_surrounding_prose() {
|
||||
let raw = "Sure! Here is the box: {\"x\": 5, \"y\": 6, \"width\": 7, \"height\": 8, \"confidence\": 0.8} — hope that helps.";
|
||||
let r = parse_visual_locate_result(raw).expect("prose-wrapped JSON should parse");
|
||||
assert_eq!(r.pixel_box.x, 5.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn confidence_clamped_to_unit_interval() {
|
||||
let r = parse_visual_locate_result(
|
||||
r#"{"x": 1, "y": 1, "width": 1, "height": 1, "confidence": 1.7}"#,
|
||||
)
|
||||
.expect("over-unit confidence should still parse");
|
||||
assert_eq!(r.confidence, 1.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn not_found_confidence_zero_is_err() {
|
||||
assert!(parse_visual_locate_result(r#"{"confidence": 0}"#).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_sized_box_is_err() {
|
||||
// A box with no area can't be a click target → not-found.
|
||||
assert!(
|
||||
parse_visual_locate_result(
|
||||
r#"{"x": 1, "y": 1, "width": 0, "height": 10, "confidence": 0.9}"#
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_box_fields_is_err() {
|
||||
// Confidence present but no coordinates → can't act.
|
||||
assert!(parse_visual_locate_result(r#"{"confidence": 0.9}"#).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_json_reply_is_err() {
|
||||
assert!(parse_visual_locate_result("I could not find that element.").is_err());
|
||||
}
|
||||
|
||||
// ── SoM label parser (parse_som_locate_result) ──
|
||||
|
||||
#[test]
|
||||
fn som_parses_valid_label() {
|
||||
let r = parse_som_locate_result(r#"{"label": 3, "confidence": 0.88}"#, 10)
|
||||
.expect("valid label should parse");
|
||||
assert_eq!(r.label, 3);
|
||||
assert!((r.confidence - 0.88).abs() < 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn som_extracts_label_from_code_fence() {
|
||||
let raw = "```json\n{\"label\": 7, \"confidence\": 0.6}\n```";
|
||||
let r = parse_som_locate_result(raw, 12).expect("fenced JSON should parse");
|
||||
assert_eq!(r.label, 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn som_label_zero_is_err() {
|
||||
// {"label": 0} is the model's "none matched" sentinel.
|
||||
assert!(parse_som_locate_result(r#"{"label": 0, "confidence": 0}"#, 10).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn som_label_out_of_range_is_err() {
|
||||
// Hallucinated label beyond n_labels must NOT index the label_map (would be OOB).
|
||||
assert!(parse_som_locate_result(r#"{"label": 99, "confidence": 0.9}"#, 12).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn som_label_below_one_is_err() {
|
||||
assert!(parse_som_locate_result(r#"{"label": -1, "confidence": 0.9}"#, 12).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn som_non_integer_label_is_err() {
|
||||
// A fractional label is not a valid 1-based index.
|
||||
assert!(parse_som_locate_result(r#"{"label": 2.5, "confidence": 0.9}"#, 12).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn som_zero_confidence_is_err() {
|
||||
assert!(parse_som_locate_result(r#"{"label": 3, "confidence": 0}"#, 10).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn som_missing_label_is_err() {
|
||||
assert!(parse_som_locate_result(r#"{"confidence": 0.9}"#, 10).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn som_boundary_label_n_is_ok() {
|
||||
// label == n_labels is in range (inclusive upper bound).
|
||||
let r = parse_som_locate_result(r#"{"label": 5, "confidence": 0.7}"#, 5)
|
||||
.expect("label == n_labels is valid");
|
||||
assert_eq!(r.label, 5);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,403 @@
|
||||
//! Prompt cache break detection.
|
||||
//!
|
||||
//! Pairs request-side prompt state (hashes) with response-side cache tokens
|
||||
//! to detect and diagnose prompt cache breaks across turns.
|
||||
|
||||
use std::hash::{DefaultHasher, Hash, Hasher};
|
||||
|
||||
use nomi_types::tool::ToolDef;
|
||||
|
||||
/// Snapshot of prompt state taken before each API call.
|
||||
#[derive(Debug, Clone)]
|
||||
struct PromptSnapshot {
|
||||
system_hash: u64,
|
||||
tools_hash: u64,
|
||||
}
|
||||
|
||||
/// Cache token statistics from a single API response.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CacheStats {
|
||||
pub input_tokens: u64,
|
||||
pub cache_read_tokens: u64,
|
||||
pub cache_creation_tokens: u64,
|
||||
}
|
||||
|
||||
/// Diagnostic result after comparing two consecutive turns.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum CacheDiagnostic {
|
||||
Healthy {
|
||||
hit_rate: f64,
|
||||
},
|
||||
PartialMiss {
|
||||
hit_rate: f64,
|
||||
cause: CacheBreakCause,
|
||||
},
|
||||
FullMiss {
|
||||
cause: CacheBreakCause,
|
||||
},
|
||||
}
|
||||
|
||||
/// What caused a cache break.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum CacheBreakCause {
|
||||
SystemPromptChanged,
|
||||
ToolsChanged,
|
||||
TtlExpiry,
|
||||
FirstRequest,
|
||||
}
|
||||
|
||||
/// Detects prompt cache breaks by comparing consecutive turns.
|
||||
pub struct CacheBreakDetector {
|
||||
/// Snapshot from the PREVIOUS turn (used for attribution on cache break).
|
||||
prev_snapshot: Option<PromptSnapshot>,
|
||||
/// Snapshot from the CURRENT turn (just recorded by record_request).
|
||||
current_snapshot: Option<PromptSnapshot>,
|
||||
/// Cache stats from the previous API response.
|
||||
prev_stats: Option<CacheStats>,
|
||||
}
|
||||
|
||||
impl CacheBreakDetector {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
prev_snapshot: None,
|
||||
current_snapshot: None,
|
||||
prev_stats: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Record the prompt state before an API call.
|
||||
pub fn record_request(&mut self, system: &str, tools: &[ToolDef]) {
|
||||
let mut system_hasher = DefaultHasher::new();
|
||||
system.hash(&mut system_hasher);
|
||||
let system_hash = system_hasher.finish();
|
||||
|
||||
let mut tools_hasher = DefaultHasher::new();
|
||||
for t in tools {
|
||||
t.name.hash(&mut tools_hasher);
|
||||
t.description.hash(&mut tools_hasher);
|
||||
let schema_str = serde_json::to_string(&t.input_schema).unwrap_or_default();
|
||||
schema_str.hash(&mut tools_hasher);
|
||||
t.deferred.hash(&mut tools_hasher);
|
||||
}
|
||||
let tools_hash = tools_hasher.finish();
|
||||
|
||||
// Rotate: current becomes prev, new snapshot becomes current
|
||||
self.prev_snapshot = self.current_snapshot.take();
|
||||
self.current_snapshot = Some(PromptSnapshot {
|
||||
system_hash,
|
||||
tools_hash,
|
||||
});
|
||||
}
|
||||
|
||||
/// Check the response cache tokens against the previous turn.
|
||||
///
|
||||
/// Returns `None` if no snapshot was recorded before the call.
|
||||
pub fn check_response(&mut self, stats: CacheStats) -> Option<CacheDiagnostic> {
|
||||
let current = self.current_snapshot.as_ref()?;
|
||||
let diagnostic = self.compute_diagnostic(current, &stats);
|
||||
self.prev_stats = Some(stats);
|
||||
Some(diagnostic)
|
||||
}
|
||||
|
||||
fn compute_diagnostic(&self, current: &PromptSnapshot, stats: &CacheStats) -> CacheDiagnostic {
|
||||
let Some(prev) = &self.prev_stats else {
|
||||
// First request — no previous data to compare
|
||||
return CacheDiagnostic::Healthy { hit_rate: 0.0 };
|
||||
};
|
||||
|
||||
// If provider doesn't support caching (both turns have 0 cache tokens),
|
||||
// report healthy to avoid false alarms (e.g., OpenAI).
|
||||
if prev.cache_read_tokens == 0
|
||||
&& prev.cache_creation_tokens == 0
|
||||
&& stats.cache_read_tokens == 0
|
||||
&& stats.cache_creation_tokens == 0
|
||||
{
|
||||
return CacheDiagnostic::Healthy { hit_rate: 0.0 };
|
||||
}
|
||||
|
||||
let prev_had_cache = prev.cache_read_tokens > 0 || prev.cache_creation_tokens > 0;
|
||||
|
||||
// Full miss: had cache before, now read tokens dropped to 0
|
||||
if prev_had_cache && stats.cache_read_tokens == 0 {
|
||||
let cause = self.attribute_cause(current);
|
||||
return CacheDiagnostic::FullMiss { cause };
|
||||
}
|
||||
|
||||
// Calculate hit rate
|
||||
let hit_rate = if stats.input_tokens > 0 {
|
||||
stats.cache_read_tokens as f64 / stats.input_tokens as f64
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
// Partial miss: cache_read dropped >5% compared to previous
|
||||
if prev.cache_read_tokens > 0 {
|
||||
let drop_pct = 1.0 - (stats.cache_read_tokens as f64 / prev.cache_read_tokens as f64);
|
||||
if drop_pct > 0.05 {
|
||||
let cause = self.attribute_cause(current);
|
||||
return CacheDiagnostic::PartialMiss { hit_rate, cause };
|
||||
}
|
||||
}
|
||||
|
||||
CacheDiagnostic::Healthy { hit_rate }
|
||||
}
|
||||
|
||||
/// Determine what caused the cache break by comparing prev vs current snapshots.
|
||||
fn attribute_cause(&self, current: &PromptSnapshot) -> CacheBreakCause {
|
||||
let Some(prev) = &self.prev_snapshot else {
|
||||
return CacheBreakCause::FirstRequest;
|
||||
};
|
||||
|
||||
if prev.system_hash != current.system_hash {
|
||||
return CacheBreakCause::SystemPromptChanged;
|
||||
}
|
||||
if prev.tools_hash != current.tools_hash {
|
||||
return CacheBreakCause::ToolsChanged;
|
||||
}
|
||||
|
||||
// Hashes match but cache was lost — server-side TTL expiry
|
||||
CacheBreakCause::TtlExpiry
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for CacheBreakDetector {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
fn make_tools() -> Vec<ToolDef> {
|
||||
vec![ToolDef {
|
||||
name: "Read".into(),
|
||||
description: "Read a file".into(),
|
||||
input_schema: json!({"type": "object"}),
|
||||
deferred: false,
|
||||
}]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_request_returns_healthy() {
|
||||
let mut detector = CacheBreakDetector::new();
|
||||
detector.record_request("system prompt", &make_tools());
|
||||
let diag = detector
|
||||
.check_response(CacheStats {
|
||||
input_tokens: 10000,
|
||||
cache_read_tokens: 0,
|
||||
cache_creation_tokens: 5000,
|
||||
})
|
||||
.unwrap();
|
||||
assert!(matches!(diag, CacheDiagnostic::Healthy { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn healthy_when_cache_read_stable() {
|
||||
let mut detector = CacheBreakDetector::new();
|
||||
|
||||
// Turn 1
|
||||
detector.record_request("prompt", &make_tools());
|
||||
detector.check_response(CacheStats {
|
||||
input_tokens: 10000,
|
||||
cache_read_tokens: 8000,
|
||||
cache_creation_tokens: 2000,
|
||||
});
|
||||
|
||||
// Turn 2 — similar cache_read
|
||||
detector.record_request("prompt", &make_tools());
|
||||
let diag = detector
|
||||
.check_response(CacheStats {
|
||||
input_tokens: 11000,
|
||||
cache_read_tokens: 8000,
|
||||
cache_creation_tokens: 0,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert!(matches!(diag, CacheDiagnostic::Healthy { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_miss_when_cache_read_drops_to_zero() {
|
||||
let mut detector = CacheBreakDetector::new();
|
||||
|
||||
// Turn 1 — cache established
|
||||
detector.record_request("prompt", &make_tools());
|
||||
detector.check_response(CacheStats {
|
||||
input_tokens: 10000,
|
||||
cache_read_tokens: 8000,
|
||||
cache_creation_tokens: 2000,
|
||||
});
|
||||
|
||||
// Turn 2 — cache_read drops to 0
|
||||
detector.record_request("prompt", &make_tools());
|
||||
let diag = detector
|
||||
.check_response(CacheStats {
|
||||
input_tokens: 10000,
|
||||
cache_read_tokens: 0,
|
||||
cache_creation_tokens: 10000,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert!(matches!(diag, CacheDiagnostic::FullMiss { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_miss_system_prompt_changed() {
|
||||
let mut detector = CacheBreakDetector::new();
|
||||
|
||||
// Turn 1
|
||||
detector.record_request("prompt v1", &make_tools());
|
||||
detector.check_response(CacheStats {
|
||||
input_tokens: 10000,
|
||||
cache_read_tokens: 8000,
|
||||
cache_creation_tokens: 2000,
|
||||
});
|
||||
|
||||
// Turn 2 — different system prompt
|
||||
detector.record_request("prompt v2", &make_tools());
|
||||
let diag = detector
|
||||
.check_response(CacheStats {
|
||||
input_tokens: 10000,
|
||||
cache_read_tokens: 0,
|
||||
cache_creation_tokens: 10000,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
match diag {
|
||||
CacheDiagnostic::FullMiss { cause } => {
|
||||
assert_eq!(cause, CacheBreakCause::SystemPromptChanged);
|
||||
}
|
||||
_ => panic!("expected FullMiss"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_miss_tools_changed() {
|
||||
let mut detector = CacheBreakDetector::new();
|
||||
|
||||
// Turn 1
|
||||
detector.record_request("prompt", &make_tools());
|
||||
detector.check_response(CacheStats {
|
||||
input_tokens: 10000,
|
||||
cache_read_tokens: 8000,
|
||||
cache_creation_tokens: 2000,
|
||||
});
|
||||
|
||||
// Turn 2 — different tools
|
||||
let new_tools = vec![ToolDef {
|
||||
name: "Write".into(),
|
||||
description: "Write a file".into(),
|
||||
input_schema: json!({"type": "object"}),
|
||||
deferred: false,
|
||||
}];
|
||||
detector.record_request("prompt", &new_tools);
|
||||
let diag = detector
|
||||
.check_response(CacheStats {
|
||||
input_tokens: 10000,
|
||||
cache_read_tokens: 0,
|
||||
cache_creation_tokens: 10000,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
match diag {
|
||||
CacheDiagnostic::FullMiss { cause } => {
|
||||
assert_eq!(cause, CacheBreakCause::ToolsChanged);
|
||||
}
|
||||
_ => panic!("expected FullMiss"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_miss_ttl_expiry() {
|
||||
let mut detector = CacheBreakDetector::new();
|
||||
|
||||
// Turn 1
|
||||
detector.record_request("prompt", &make_tools());
|
||||
detector.check_response(CacheStats {
|
||||
input_tokens: 10000,
|
||||
cache_read_tokens: 8000,
|
||||
cache_creation_tokens: 2000,
|
||||
});
|
||||
|
||||
// Turn 2 — same prompt and tools but cache lost (TTL expired server-side)
|
||||
detector.record_request("prompt", &make_tools());
|
||||
let diag = detector
|
||||
.check_response(CacheStats {
|
||||
input_tokens: 10000,
|
||||
cache_read_tokens: 0,
|
||||
cache_creation_tokens: 10000,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
match diag {
|
||||
CacheDiagnostic::FullMiss { cause } => {
|
||||
assert_eq!(cause, CacheBreakCause::TtlExpiry);
|
||||
}
|
||||
_ => panic!("expected FullMiss"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_miss_when_cache_read_drops_significantly() {
|
||||
let mut detector = CacheBreakDetector::new();
|
||||
|
||||
// Turn 1
|
||||
detector.record_request("prompt", &make_tools());
|
||||
detector.check_response(CacheStats {
|
||||
input_tokens: 10000,
|
||||
cache_read_tokens: 8000,
|
||||
cache_creation_tokens: 2000,
|
||||
});
|
||||
|
||||
// Turn 2 — 50% drop in cache_read
|
||||
detector.record_request("prompt", &make_tools());
|
||||
let diag = detector
|
||||
.check_response(CacheStats {
|
||||
input_tokens: 10000,
|
||||
cache_read_tokens: 4000,
|
||||
cache_creation_tokens: 6000,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert!(matches!(diag, CacheDiagnostic::PartialMiss { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_no_false_alarm() {
|
||||
// OpenAI never returns cache tokens — both turns have all zeros
|
||||
let mut detector = CacheBreakDetector::new();
|
||||
|
||||
detector.record_request("prompt", &make_tools());
|
||||
detector.check_response(CacheStats {
|
||||
input_tokens: 10000,
|
||||
cache_read_tokens: 0,
|
||||
cache_creation_tokens: 0,
|
||||
});
|
||||
|
||||
detector.record_request("prompt", &make_tools());
|
||||
let diag = detector
|
||||
.check_response(CacheStats {
|
||||
input_tokens: 10000,
|
||||
cache_read_tokens: 0,
|
||||
cache_creation_tokens: 0,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
// Should be Healthy, not FullMiss
|
||||
assert!(matches!(diag, CacheDiagnostic::Healthy { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_diagnostic_without_record_request() {
|
||||
let mut detector = CacheBreakDetector::new();
|
||||
let diag = detector.check_response(CacheStats {
|
||||
input_tokens: 10000,
|
||||
cache_read_tokens: 0,
|
||||
cache_creation_tokens: 0,
|
||||
});
|
||||
assert!(diag.is_none());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
use async_trait::async_trait;
|
||||
|
||||
use super::{CommandContext, CommandResult, SlashCommand};
|
||||
use crate::compact::state::CompactState;
|
||||
|
||||
pub struct ClearCommand;
|
||||
|
||||
#[async_trait]
|
||||
impl SlashCommand for ClearCommand {
|
||||
fn name(&self) -> &str {
|
||||
"clear"
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"Clear conversation history"
|
||||
}
|
||||
|
||||
async fn execute(
|
||||
&self,
|
||||
ctx: &mut CommandContext<'_>,
|
||||
_args: &str,
|
||||
) -> anyhow::Result<CommandResult> {
|
||||
ctx.messages.clear();
|
||||
*ctx.compact_state = CompactState::new();
|
||||
ctx.output.emit_info("Conversation cleared");
|
||||
Ok(CommandResult::Continue)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use nomi_providers::{LlmProvider, ProviderError};
|
||||
use nomi_types::llm::{LlmEvent, LlmRequest};
|
||||
use nomi_types::message::{ContentBlock, Message, Role};
|
||||
|
||||
use super::*;
|
||||
use crate::commands::{CommandContext, CommandRegistry};
|
||||
use crate::output::null_sink::NullSink;
|
||||
|
||||
struct NullProvider;
|
||||
#[async_trait::async_trait]
|
||||
impl LlmProvider for NullProvider {
|
||||
async fn stream(
|
||||
&self,
|
||||
_: &LlmRequest,
|
||||
) -> Result<tokio::sync::mpsc::Receiver<LlmEvent>, ProviderError> {
|
||||
let (_tx, rx) = tokio::sync::mpsc::channel(1);
|
||||
Ok(rx)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn clear_empties_messages() {
|
||||
let provider: Arc<dyn LlmProvider> = Arc::new(NullProvider);
|
||||
let registry = CommandRegistry::new();
|
||||
let output = NullSink;
|
||||
let mut messages = vec![
|
||||
Message::new(
|
||||
Role::User,
|
||||
vec![ContentBlock::Text {
|
||||
text: "hello".into(),
|
||||
}],
|
||||
),
|
||||
Message::new(
|
||||
Role::Assistant,
|
||||
vec![ContentBlock::Text { text: "hi".into() }],
|
||||
),
|
||||
];
|
||||
let mut state = CompactState::new();
|
||||
state.last_input_tokens = 5000;
|
||||
state.consecutive_failures = 2;
|
||||
let config = nomi_config::compact::CompactConfig::default();
|
||||
|
||||
let mut ctx = CommandContext {
|
||||
messages: &mut messages,
|
||||
compact_state: &mut state,
|
||||
compact_config: &config,
|
||||
provider,
|
||||
model: "test",
|
||||
output: &output,
|
||||
registry: ®istry,
|
||||
};
|
||||
|
||||
let cmd = ClearCommand;
|
||||
let result = cmd.execute(&mut ctx, "").await.unwrap();
|
||||
|
||||
assert_eq!(result, CommandResult::Continue);
|
||||
assert!(ctx.messages.is_empty());
|
||||
assert_eq!(ctx.compact_state.last_input_tokens, 0);
|
||||
assert_eq!(ctx.compact_state.consecutive_failures, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn clear_on_empty_messages() {
|
||||
let provider: Arc<dyn LlmProvider> = Arc::new(NullProvider);
|
||||
let registry = CommandRegistry::new();
|
||||
let output = NullSink;
|
||||
let mut messages: Vec<Message> = vec![];
|
||||
let mut state = CompactState::new();
|
||||
let config = nomi_config::compact::CompactConfig::default();
|
||||
|
||||
let mut ctx = CommandContext {
|
||||
messages: &mut messages,
|
||||
compact_state: &mut state,
|
||||
compact_config: &config,
|
||||
provider,
|
||||
model: "test",
|
||||
output: &output,
|
||||
registry: ®istry,
|
||||
};
|
||||
|
||||
let cmd = ClearCommand;
|
||||
let result = cmd.execute(&mut ctx, "").await.unwrap();
|
||||
assert_eq!(result, CommandResult::Continue);
|
||||
assert!(ctx.messages.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
use async_trait::async_trait;
|
||||
|
||||
use super::{CommandContext, CommandResult, SlashCommand};
|
||||
use crate::compact::auto;
|
||||
use nomi_types::compact::CompactTrigger;
|
||||
|
||||
pub struct CompactCommand;
|
||||
|
||||
#[async_trait]
|
||||
impl SlashCommand for CompactCommand {
|
||||
fn name(&self) -> &str {
|
||||
"compact"
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"Compress conversation context"
|
||||
}
|
||||
|
||||
async fn execute(
|
||||
&self,
|
||||
ctx: &mut CommandContext<'_>,
|
||||
_args: &str,
|
||||
) -> anyhow::Result<CommandResult> {
|
||||
if ctx.messages.len() <= 2 {
|
||||
ctx.output.emit_info("Context is already compact");
|
||||
return Ok(CommandResult::Continue);
|
||||
}
|
||||
|
||||
// Reset circuit breaker — manual intent overrides protection
|
||||
ctx.compact_state.consecutive_failures = 0;
|
||||
|
||||
let pre_tokens = ctx.compact_state.last_input_tokens;
|
||||
|
||||
match auto::autocompact(
|
||||
ctx.provider.as_ref(),
|
||||
ctx.messages,
|
||||
ctx.model,
|
||||
ctx.compact_config,
|
||||
ctx.compact_state,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => {
|
||||
let msgs_summarized = result.messages_summarized;
|
||||
*ctx.messages = result.messages;
|
||||
|
||||
if let Some(boundary) = ctx.messages.first_mut() {
|
||||
for block in &mut boundary.content {
|
||||
if let nomi_types::message::ContentBlock::Text { text } = block
|
||||
&& text.starts_with(auto::BOUNDARY_PREFIX)
|
||||
{
|
||||
let metadata = nomi_types::compact::CompactMetadata {
|
||||
trigger: CompactTrigger::Manual,
|
||||
pre_compact_tokens: pre_tokens,
|
||||
messages_summarized: msgs_summarized,
|
||||
};
|
||||
*text = format!(
|
||||
"{}\n{}",
|
||||
auto::BOUNDARY_PREFIX,
|
||||
serde_json::to_string(&metadata)
|
||||
.expect("metadata serialization cannot fail")
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ctx.output.emit_info(&format!(
|
||||
"Context compacted: {}k → compact ({} messages summarized)",
|
||||
pre_tokens / 1000,
|
||||
msgs_summarized
|
||||
));
|
||||
}
|
||||
Err(e) => {
|
||||
ctx.output.emit_warning(&format!("Compact failed: {}", e));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(CommandResult::Continue)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use nomi_providers::{LlmProvider, ProviderError};
|
||||
use nomi_types::llm::{LlmEvent, LlmRequest};
|
||||
use nomi_types::message::{ContentBlock, Message, Role};
|
||||
|
||||
use super::*;
|
||||
use crate::commands::{CommandContext, CommandRegistry};
|
||||
use crate::compact::state::CompactState;
|
||||
use crate::output::null_sink::NullSink;
|
||||
|
||||
struct NullProvider;
|
||||
#[async_trait::async_trait]
|
||||
impl LlmProvider for NullProvider {
|
||||
async fn stream(
|
||||
&self,
|
||||
_: &LlmRequest,
|
||||
) -> Result<tokio::sync::mpsc::Receiver<LlmEvent>, ProviderError> {
|
||||
let (_tx, rx) = tokio::sync::mpsc::channel(1);
|
||||
Ok(rx)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compact_already_compact_guard() {
|
||||
let provider: Arc<dyn LlmProvider> = Arc::new(NullProvider);
|
||||
let registry = CommandRegistry::new();
|
||||
let output = NullSink;
|
||||
let mut messages = vec![Message::new(
|
||||
Role::User,
|
||||
vec![ContentBlock::Text { text: "hi".into() }],
|
||||
)];
|
||||
let mut state = CompactState::new();
|
||||
let config = nomi_config::compact::CompactConfig::default();
|
||||
|
||||
let mut ctx = CommandContext {
|
||||
messages: &mut messages,
|
||||
compact_state: &mut state,
|
||||
compact_config: &config,
|
||||
provider,
|
||||
model: "test-model",
|
||||
output: &output,
|
||||
registry: ®istry,
|
||||
};
|
||||
|
||||
let cmd = CompactCommand;
|
||||
let result = cmd.execute(&mut ctx, "").await.unwrap();
|
||||
assert_eq!(result, CommandResult::Continue);
|
||||
assert_eq!(ctx.messages.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compact_resets_circuit_breaker() {
|
||||
let provider: Arc<dyn LlmProvider> = Arc::new(NullProvider);
|
||||
let registry = CommandRegistry::new();
|
||||
let output = NullSink;
|
||||
let mut messages: Vec<Message> = (0..10)
|
||||
.map(|i| {
|
||||
let role = if i % 2 == 0 {
|
||||
Role::User
|
||||
} else {
|
||||
Role::Assistant
|
||||
};
|
||||
Message::new(
|
||||
role,
|
||||
vec![ContentBlock::Text {
|
||||
text: format!("msg-{i}"),
|
||||
}],
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let mut state = CompactState::new();
|
||||
state.consecutive_failures = 5;
|
||||
let config = nomi_config::compact::CompactConfig::default();
|
||||
|
||||
let mut ctx = CommandContext {
|
||||
messages: &mut messages,
|
||||
compact_state: &mut state,
|
||||
compact_config: &config,
|
||||
provider,
|
||||
model: "test-model",
|
||||
output: &output,
|
||||
registry: ®istry,
|
||||
};
|
||||
|
||||
let cmd = CompactCommand;
|
||||
let _ = cmd.execute(&mut ctx, "").await;
|
||||
// Circuit breaker was reset to 0 before the call, then failure increments it
|
||||
assert!(ctx.compact_state.consecutive_failures <= 1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
use async_trait::async_trait;
|
||||
|
||||
use super::{CommandContext, CommandResult, SlashCommand};
|
||||
|
||||
pub struct HelpCommand;
|
||||
|
||||
#[async_trait]
|
||||
impl SlashCommand for HelpCommand {
|
||||
fn name(&self) -> &str {
|
||||
"help"
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"List available commands"
|
||||
}
|
||||
|
||||
async fn execute(
|
||||
&self,
|
||||
ctx: &mut CommandContext<'_>,
|
||||
_args: &str,
|
||||
) -> anyhow::Result<CommandResult> {
|
||||
let mut entries: Vec<(&str, &str)> = ctx
|
||||
.registry
|
||||
.all()
|
||||
.iter()
|
||||
.map(|cmd| (cmd.name(), cmd.description()))
|
||||
.collect();
|
||||
entries.sort_by_key(|(name, _)| *name);
|
||||
|
||||
let mut output = String::from("Available commands:\n");
|
||||
for (name, desc) in entries {
|
||||
output.push_str(&format!(" /{} — {}\n", name, desc));
|
||||
}
|
||||
ctx.output.emit_info(output.trim_end());
|
||||
Ok(CommandResult::Continue)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use nomi_providers::{LlmProvider, ProviderError};
|
||||
use nomi_types::llm::{LlmEvent, LlmRequest};
|
||||
use nomi_types::message::Message;
|
||||
|
||||
use super::*;
|
||||
use crate::commands::{CommandContext, default_registry};
|
||||
use crate::compact::state::CompactState;
|
||||
use crate::output::OutputSink;
|
||||
|
||||
struct CaptureSink {
|
||||
messages: Mutex<Vec<String>>,
|
||||
}
|
||||
|
||||
impl CaptureSink {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
messages: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
fn captured(&self) -> Vec<String> {
|
||||
self.messages.lock().unwrap().clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl OutputSink for CaptureSink {
|
||||
fn emit_text_delta(&self, _: &str, _: &str) {}
|
||||
fn emit_thinking(&self, _: &str, _: &str) {}
|
||||
fn emit_tool_call(&self, _: &str, _: &str, _: &str) {}
|
||||
fn emit_tool_result(&self, _: &str, _: &str, _: bool, _: &str) {}
|
||||
fn emit_stream_start(&self, _: &str) {}
|
||||
fn emit_stream_end(&self, _: &str, _: usize, _: u64, _: u64, _: u64, _: u64) {}
|
||||
fn emit_error(&self, _: &str) {}
|
||||
fn emit_info(&self, msg: &str) {
|
||||
self.messages.lock().unwrap().push(msg.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
struct NullProvider;
|
||||
#[async_trait::async_trait]
|
||||
impl LlmProvider for NullProvider {
|
||||
async fn stream(
|
||||
&self,
|
||||
_: &LlmRequest,
|
||||
) -> Result<tokio::sync::mpsc::Receiver<LlmEvent>, ProviderError> {
|
||||
let (_tx, rx) = tokio::sync::mpsc::channel(1);
|
||||
Ok(rx)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn help_lists_all_commands() {
|
||||
let provider: Arc<dyn LlmProvider> = Arc::new(NullProvider);
|
||||
let registry = default_registry();
|
||||
let output = CaptureSink::new();
|
||||
let mut messages: Vec<Message> = Vec::new();
|
||||
let mut state = CompactState::new();
|
||||
let config = nomi_config::compact::CompactConfig::default();
|
||||
|
||||
let mut ctx = CommandContext {
|
||||
messages: &mut messages,
|
||||
compact_state: &mut state,
|
||||
compact_config: &config,
|
||||
provider,
|
||||
model: "test",
|
||||
output: &output,
|
||||
registry: ®istry,
|
||||
};
|
||||
|
||||
let cmd = HelpCommand;
|
||||
let result = cmd.execute(&mut ctx, "").await.unwrap();
|
||||
assert_eq!(result, CommandResult::Continue);
|
||||
|
||||
let captured = output.captured();
|
||||
assert_eq!(captured.len(), 1);
|
||||
let help_text = &captured[0];
|
||||
assert!(help_text.contains("/clear"));
|
||||
assert!(help_text.contains("/compact"));
|
||||
assert!(help_text.contains("/help"));
|
||||
assert!(help_text.contains("/quit"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn help_output_is_sorted() {
|
||||
let provider: Arc<dyn LlmProvider> = Arc::new(NullProvider);
|
||||
let registry = default_registry();
|
||||
let output = CaptureSink::new();
|
||||
let mut messages: Vec<Message> = Vec::new();
|
||||
let mut state = CompactState::new();
|
||||
let config = nomi_config::compact::CompactConfig::default();
|
||||
|
||||
let mut ctx = CommandContext {
|
||||
messages: &mut messages,
|
||||
compact_state: &mut state,
|
||||
compact_config: &config,
|
||||
provider,
|
||||
model: "test",
|
||||
output: &output,
|
||||
registry: ®istry,
|
||||
};
|
||||
|
||||
let cmd = HelpCommand;
|
||||
cmd.execute(&mut ctx, "").await.unwrap();
|
||||
|
||||
let help_text = &output.captured()[0];
|
||||
let clear_pos = help_text.find("/clear").unwrap();
|
||||
let compact_pos = help_text.find("/compact").unwrap();
|
||||
let help_pos = help_text.find("/help").unwrap();
|
||||
let quit_pos = help_text.find("/quit").unwrap();
|
||||
|
||||
assert!(clear_pos < compact_pos);
|
||||
assert!(compact_pos < help_pos);
|
||||
assert!(help_pos < quit_pos);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
pub mod clear;
|
||||
pub mod compact;
|
||||
pub mod help;
|
||||
pub mod quit;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
use crate::compact::state::CompactState;
|
||||
use crate::output::OutputSink;
|
||||
use nomi_config::compact::CompactConfig;
|
||||
use nomi_providers::LlmProvider;
|
||||
use nomi_types::message::Message;
|
||||
|
||||
/// Result of executing a slash command.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum CommandResult {
|
||||
/// Command handled, continue the REPL loop.
|
||||
Continue,
|
||||
/// Exit the REPL.
|
||||
Exit,
|
||||
}
|
||||
|
||||
/// Context passed to slash commands during execution.
|
||||
pub struct CommandContext<'a> {
|
||||
pub messages: &'a mut Vec<Message>,
|
||||
pub compact_state: &'a mut CompactState,
|
||||
pub compact_config: &'a CompactConfig,
|
||||
pub provider: Arc<dyn LlmProvider>,
|
||||
pub model: &'a str,
|
||||
pub output: &'a dyn OutputSink,
|
||||
pub registry: &'a CommandRegistry,
|
||||
}
|
||||
|
||||
/// A slash command that can be executed in the REPL.
|
||||
#[async_trait]
|
||||
pub trait SlashCommand: Send + Sync {
|
||||
fn name(&self) -> &str;
|
||||
fn aliases(&self) -> &[&str] {
|
||||
&[]
|
||||
}
|
||||
fn description(&self) -> &str;
|
||||
async fn execute(
|
||||
&self,
|
||||
ctx: &mut CommandContext<'_>,
|
||||
args: &str,
|
||||
) -> anyhow::Result<CommandResult>;
|
||||
}
|
||||
|
||||
/// Registry of all available slash commands.
|
||||
pub struct CommandRegistry {
|
||||
commands: Vec<Box<dyn SlashCommand>>,
|
||||
}
|
||||
|
||||
impl CommandRegistry {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
commands: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn register(&mut self, cmd: Box<dyn SlashCommand>) {
|
||||
self.commands.push(cmd);
|
||||
}
|
||||
|
||||
pub fn find(&self, name: &str) -> Option<&dyn SlashCommand> {
|
||||
self.commands.iter().find_map(|cmd| {
|
||||
if cmd.name() == name || cmd.aliases().contains(&name) {
|
||||
Some(cmd.as_ref())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub fn all(&self) -> &[Box<dyn SlashCommand>] {
|
||||
&self.commands
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for CommandRegistry {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the default registry with all built-in commands.
|
||||
pub fn default_registry() -> CommandRegistry {
|
||||
let mut registry = CommandRegistry::new();
|
||||
registry.register(Box::new(compact::CompactCommand));
|
||||
registry.register(Box::new(clear::ClearCommand));
|
||||
registry.register(Box::new(help::HelpCommand));
|
||||
registry.register(Box::new(quit::QuitCommand));
|
||||
registry
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn registry_find_by_name() {
|
||||
let registry = default_registry();
|
||||
assert!(registry.find("compact").is_some());
|
||||
assert!(registry.find("clear").is_some());
|
||||
assert!(registry.find("help").is_some());
|
||||
assert!(registry.find("quit").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_find_by_alias() {
|
||||
let registry = default_registry();
|
||||
assert!(registry.find("exit").is_some());
|
||||
let cmd = registry.find("exit").unwrap();
|
||||
assert_eq!(cmd.name(), "quit");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_find_unknown_returns_none() {
|
||||
let registry = default_registry();
|
||||
assert!(registry.find("nonexistent").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_all_returns_all_commands() {
|
||||
let registry = default_registry();
|
||||
assert_eq!(registry.all().len(), 4);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
use async_trait::async_trait;
|
||||
|
||||
use super::{CommandContext, CommandResult, SlashCommand};
|
||||
|
||||
pub struct QuitCommand;
|
||||
|
||||
#[async_trait]
|
||||
impl SlashCommand for QuitCommand {
|
||||
fn name(&self) -> &str {
|
||||
"quit"
|
||||
}
|
||||
|
||||
fn aliases(&self) -> &[&str] {
|
||||
&["exit"]
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"Exit the REPL"
|
||||
}
|
||||
|
||||
async fn execute(
|
||||
&self,
|
||||
_ctx: &mut CommandContext<'_>,
|
||||
_args: &str,
|
||||
) -> anyhow::Result<CommandResult> {
|
||||
Ok(CommandResult::Exit)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,533 @@
|
||||
//! Autocompact: watermark-triggered LLM summarization.
|
||||
//!
|
||||
//! When the token watermark exceeds the configured threshold, this module
|
||||
//! calls the LLM to produce a structured summary of the conversation,
|
||||
//! then replaces the full history with a compact boundary marker and the
|
||||
//! summary. A circuit breaker prevents runaway retries.
|
||||
|
||||
use nomi_config::compact::CompactConfig;
|
||||
use nomi_providers::{LlmProvider, ProviderError};
|
||||
use nomi_types::compact::{CompactMetadata, CompactTrigger};
|
||||
use nomi_types::llm::{LlmEvent, LlmRequest, ThinkingConfig};
|
||||
use nomi_types::message::{ContentBlock, Message, Role, TokenUsage};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use super::prompt::{
|
||||
COMPACT_MAX_OUTPUT_TOKENS, COMPACT_SYSTEM_PROMPT, build_compact_prompt, build_summary_content,
|
||||
format_compact_summary,
|
||||
};
|
||||
use super::state::CompactState;
|
||||
|
||||
/// Maximum number of prompt-too-long retries.
|
||||
const MAX_PTL_RETRIES: u32 = 2;
|
||||
|
||||
/// Content prefix for the compact boundary marker message.
|
||||
pub const BOUNDARY_PREFIX: &str = "[Conversation compacted]";
|
||||
|
||||
// ── Public types ────────────────────────────────────────────────────────────
|
||||
|
||||
/// Result of a successful autocompact operation.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CompactResult {
|
||||
/// Post-compact messages that replace the original conversation.
|
||||
/// Contains a boundary marker and a summary message.
|
||||
pub messages: Vec<Message>,
|
||||
/// How many original messages were summarized.
|
||||
pub messages_summarized: usize,
|
||||
/// Input token count before compaction (from the last API call).
|
||||
pub pre_compact_tokens: u64,
|
||||
}
|
||||
|
||||
/// Errors specific to autocompact.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum CompactError {
|
||||
#[error("LLM provider error: {0}")]
|
||||
Provider(#[from] ProviderError),
|
||||
#[error("Prompt too long after {attempts} retries")]
|
||||
PromptTooLong { attempts: u32 },
|
||||
#[error("Empty response from LLM")]
|
||||
EmptyResponse,
|
||||
#[error("Stream error: {0}")]
|
||||
StreamError(String),
|
||||
#[error("Circuit breaker tripped after {failures} consecutive failures")]
|
||||
CircuitBroken { failures: u32 },
|
||||
}
|
||||
|
||||
// ── Trigger check ───────────────────────────────────────────────────────────
|
||||
|
||||
/// Check if autocompact should trigger based on the token watermark.
|
||||
///
|
||||
/// When `autocompact_threshold_pct` is set, threshold = context_window * pct / 100.
|
||||
/// Otherwise falls back to: `threshold = context_window - output_reserve - autocompact_buffer`
|
||||
pub fn should_autocompact(last_input_tokens: u64, config: &CompactConfig) -> bool {
|
||||
if !config.enabled {
|
||||
return false;
|
||||
}
|
||||
let threshold = if let Some(pct) = config.autocompact_threshold_pct {
|
||||
config.context_window * pct as usize / 100
|
||||
} else {
|
||||
let effective_window = config.context_window.saturating_sub(config.output_reserve);
|
||||
effective_window.saturating_sub(config.autocompact_buffer)
|
||||
};
|
||||
last_input_tokens as usize >= threshold
|
||||
}
|
||||
|
||||
// ── Core autocompact ────────────────────────────────────────────────────────
|
||||
|
||||
/// Execute autocompact: call LLM to summarize the conversation.
|
||||
///
|
||||
/// 1. Build a summary prompt and send conversation + prompt to the LLM.
|
||||
/// 2. If the prompt is too long, truncate oldest 20% messages and retry
|
||||
/// (up to [`MAX_PTL_RETRIES`] times).
|
||||
/// 3. Parse the `<summary>` from the response.
|
||||
/// 4. Return a [`CompactResult`] with boundary marker + summary messages.
|
||||
///
|
||||
/// On failure, increments `state.consecutive_failures`.
|
||||
/// On success, resets the failure counter.
|
||||
pub async fn autocompact(
|
||||
provider: &dyn LlmProvider,
|
||||
messages: &[Message],
|
||||
model: &str,
|
||||
config: &CompactConfig,
|
||||
state: &mut CompactState,
|
||||
) -> Result<CompactResult, CompactError> {
|
||||
// Circuit breaker check
|
||||
if state.is_circuit_broken(config) {
|
||||
return Err(CompactError::CircuitBroken {
|
||||
failures: state.consecutive_failures,
|
||||
});
|
||||
}
|
||||
|
||||
let pre_compact_tokens = state.last_input_tokens;
|
||||
let messages_summarized = messages.len();
|
||||
|
||||
// Build messages for the compact LLM call: conversation + summary prompt
|
||||
let prompt = build_compact_prompt();
|
||||
let mut conv_messages = messages.to_vec();
|
||||
conv_messages.push(Message::new(
|
||||
Role::User,
|
||||
vec![ContentBlock::Text { text: prompt }],
|
||||
));
|
||||
|
||||
let mut ptl_attempts = 0u32;
|
||||
|
||||
let summary_text = loop {
|
||||
let request = LlmRequest {
|
||||
model: model.to_string(),
|
||||
system: COMPACT_SYSTEM_PROMPT.to_string(),
|
||||
messages: conv_messages.clone(),
|
||||
tools: vec![],
|
||||
max_tokens: COMPACT_MAX_OUTPUT_TOKENS,
|
||||
thinking: Some(ThinkingConfig::Disabled),
|
||||
reasoning_effort: None,
|
||||
};
|
||||
|
||||
match provider.stream(&request).await {
|
||||
Ok(rx) => match collect_stream_text(rx).await {
|
||||
Ok((text, _usage)) => break text,
|
||||
Err(e) => {
|
||||
state.record_failure();
|
||||
return Err(e);
|
||||
}
|
||||
},
|
||||
Err(ProviderError::PromptTooLong(_)) if ptl_attempts < MAX_PTL_RETRIES => {
|
||||
ptl_attempts += 1;
|
||||
// Remove the summary prompt (last msg), truncate, re-add prompt
|
||||
let conversation_part = &conv_messages[..conv_messages.len() - 1];
|
||||
match truncate_for_retry(conversation_part) {
|
||||
Some(mut truncated) => {
|
||||
truncated.push(Message::new(
|
||||
Role::User,
|
||||
vec![ContentBlock::Text {
|
||||
text: build_compact_prompt(),
|
||||
}],
|
||||
));
|
||||
conv_messages = truncated;
|
||||
}
|
||||
None => {
|
||||
state.record_failure();
|
||||
return Err(CompactError::PromptTooLong {
|
||||
attempts: ptl_attempts,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(ProviderError::PromptTooLong(_)) => {
|
||||
state.record_failure();
|
||||
return Err(CompactError::PromptTooLong {
|
||||
attempts: ptl_attempts,
|
||||
});
|
||||
}
|
||||
Err(e) => {
|
||||
state.record_failure();
|
||||
return Err(CompactError::Provider(e));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if summary_text.trim().is_empty() {
|
||||
state.record_failure();
|
||||
return Err(CompactError::EmptyResponse);
|
||||
}
|
||||
|
||||
// Format and build post-compact messages
|
||||
let formatted = format_compact_summary(&summary_text);
|
||||
let summary_content = build_summary_content(&formatted, true);
|
||||
|
||||
let metadata = CompactMetadata {
|
||||
trigger: CompactTrigger::Auto,
|
||||
pre_compact_tokens,
|
||||
messages_summarized,
|
||||
};
|
||||
|
||||
let boundary_text = format!(
|
||||
"{BOUNDARY_PREFIX}\n{}",
|
||||
serde_json::to_string(&metadata).expect("CompactMetadata serialization cannot fail")
|
||||
);
|
||||
|
||||
let boundary_msg = Message::new(
|
||||
Role::User,
|
||||
vec![ContentBlock::Text {
|
||||
text: boundary_text,
|
||||
}],
|
||||
);
|
||||
|
||||
let summary_msg = Message::new(
|
||||
Role::User,
|
||||
vec![ContentBlock::Text {
|
||||
text: summary_content,
|
||||
}],
|
||||
);
|
||||
|
||||
state.record_success();
|
||||
|
||||
Ok(CompactResult {
|
||||
messages: vec![boundary_msg, summary_msg],
|
||||
messages_summarized,
|
||||
pre_compact_tokens,
|
||||
})
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Collect all text from a streaming LLM response.
|
||||
async fn collect_stream_text(
|
||||
mut rx: mpsc::Receiver<LlmEvent>,
|
||||
) -> Result<(String, TokenUsage), CompactError> {
|
||||
let mut text = String::new();
|
||||
|
||||
while let Some(event) = rx.recv().await {
|
||||
match event {
|
||||
LlmEvent::TextDelta(delta) => text.push_str(&delta),
|
||||
LlmEvent::Done { usage, .. } => return Ok((text, usage)),
|
||||
LlmEvent::Error(e) => return Err(CompactError::StreamError(e)),
|
||||
// Ignore thinking deltas and tool calls (shouldn't happen in compact)
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
// Channel closed without a Done event
|
||||
Err(CompactError::EmptyResponse)
|
||||
}
|
||||
|
||||
/// Truncate the oldest ~20% of messages for PTL retry.
|
||||
///
|
||||
/// Returns `None` if there are too few messages to truncate meaningfully.
|
||||
fn truncate_for_retry(messages: &[Message]) -> Option<Vec<Message>> {
|
||||
if messages.len() < 2 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let drop_count = (messages.len() / 5).max(1);
|
||||
if drop_count >= messages.len() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let remaining = &messages[drop_count..];
|
||||
let mut result = Vec::with_capacity(remaining.len() + 1);
|
||||
|
||||
// Ensure the first message is User role for API compatibility
|
||||
if remaining.first().map(|m| m.role) != Some(Role::User) {
|
||||
result.push(Message::new(
|
||||
Role::User,
|
||||
vec![ContentBlock::Text {
|
||||
text: "[earlier conversation truncated for compaction retry]".to_string(),
|
||||
}],
|
||||
));
|
||||
}
|
||||
|
||||
result.extend_from_slice(remaining);
|
||||
Some(result)
|
||||
}
|
||||
|
||||
/// Check if a message is a compact boundary marker.
|
||||
pub fn is_compact_boundary(message: &Message) -> bool {
|
||||
message.content.iter().any(|block| {
|
||||
if let ContentBlock::Text { text } = block {
|
||||
text.starts_with(BOUNDARY_PREFIX)
|
||||
} else {
|
||||
false
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Extract [`CompactMetadata`] from a boundary marker message.
|
||||
pub fn extract_compact_metadata(message: &Message) -> Option<CompactMetadata> {
|
||||
for block in &message.content {
|
||||
if let ContentBlock::Text { text } = block
|
||||
&& let Some(json_str) = text.strip_prefix(BOUNDARY_PREFIX)
|
||||
{
|
||||
let json_str = json_str.trim_start_matches('\n');
|
||||
return serde_json::from_str(json_str).ok();
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use nomi_types::compact::CompactTrigger;
|
||||
|
||||
fn default_config() -> CompactConfig {
|
||||
CompactConfig::default()
|
||||
}
|
||||
|
||||
// ── should_autocompact (TC-2.4-01..03, TC-2.4-14) ──────────────────
|
||||
|
||||
#[test]
|
||||
fn above_threshold_triggers() {
|
||||
// threshold = 200k - 20k - 13k = 167k
|
||||
let config = default_config();
|
||||
assert!(should_autocompact(170_000, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn below_threshold_does_not_trigger() {
|
||||
let config = default_config();
|
||||
assert!(!should_autocompact(160_000, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn at_exact_threshold_triggers() {
|
||||
let config = default_config();
|
||||
assert!(should_autocompact(167_000, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_config_never_triggers() {
|
||||
let config = CompactConfig {
|
||||
enabled: false,
|
||||
..default_config()
|
||||
};
|
||||
assert!(!should_autocompact(999_999, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_config_threshold() {
|
||||
let config = CompactConfig {
|
||||
context_window: 100_000,
|
||||
output_reserve: 10_000,
|
||||
autocompact_buffer: 5_000,
|
||||
..default_config()
|
||||
};
|
||||
// threshold = 100k - 10k - 5k = 85k
|
||||
assert!(!should_autocompact(80_000, &config));
|
||||
assert!(should_autocompact(85_000, &config));
|
||||
assert!(should_autocompact(90_000, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_tokens_does_not_trigger() {
|
||||
let config = default_config();
|
||||
assert!(!should_autocompact(0, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn threshold_pct_overrides_default_calculation() {
|
||||
let config = CompactConfig {
|
||||
context_window: 200_000,
|
||||
autocompact_threshold_pct: Some(50),
|
||||
..default_config()
|
||||
};
|
||||
// threshold = 200k * 50 / 100 = 100k
|
||||
assert!(!should_autocompact(99_999, &config));
|
||||
assert!(should_autocompact(100_000, &config));
|
||||
assert!(should_autocompact(150_000, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn threshold_pct_zero_triggers_immediately() {
|
||||
let config = CompactConfig {
|
||||
autocompact_threshold_pct: Some(0),
|
||||
..default_config()
|
||||
};
|
||||
// threshold = 0, any non-negative triggers
|
||||
assert!(should_autocompact(0, &config));
|
||||
assert!(should_autocompact(1, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn threshold_pct_100_never_triggers() {
|
||||
let config = CompactConfig {
|
||||
context_window: 200_000,
|
||||
autocompact_threshold_pct: Some(100),
|
||||
..default_config()
|
||||
};
|
||||
// threshold = 200k, provider never reports 200k input_tokens
|
||||
assert!(!should_autocompact(199_999, &config));
|
||||
assert!(should_autocompact(200_000, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn threshold_pct_none_uses_default_logic() {
|
||||
let config = CompactConfig {
|
||||
autocompact_threshold_pct: None,
|
||||
..default_config()
|
||||
};
|
||||
// Same as default: threshold = 200k - 20k - 13k = 167k
|
||||
assert!(!should_autocompact(166_999, &config));
|
||||
assert!(should_autocompact(167_000, &config));
|
||||
}
|
||||
|
||||
// ── truncate_for_retry ──────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn truncate_drops_20_percent() {
|
||||
let msgs: Vec<Message> = (0..10)
|
||||
.map(|i| {
|
||||
let role = if i % 2 == 0 {
|
||||
Role::User
|
||||
} else {
|
||||
Role::Assistant
|
||||
};
|
||||
Message::new(
|
||||
role,
|
||||
vec![ContentBlock::Text {
|
||||
text: format!("msg-{i}"),
|
||||
}],
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
|
||||
let result = truncate_for_retry(&msgs).unwrap();
|
||||
// Drop 20% of 10 = 2 messages, remaining 8
|
||||
assert_eq!(result.len(), 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncate_ensures_user_first() {
|
||||
let msgs: Vec<Message> = (0..5)
|
||||
.map(|i| {
|
||||
Message::new(
|
||||
Role::Assistant,
|
||||
vec![ContentBlock::Text {
|
||||
text: format!("msg-{i}"),
|
||||
}],
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
|
||||
let result = truncate_for_retry(&msgs).unwrap();
|
||||
assert_eq!(result[0].role, Role::User);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncate_too_few_returns_none() {
|
||||
let msgs = vec![Message::new(
|
||||
Role::User,
|
||||
vec![ContentBlock::Text {
|
||||
text: "only one".to_string(),
|
||||
}],
|
||||
)];
|
||||
assert!(truncate_for_retry(&msgs).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncate_empty_returns_none() {
|
||||
assert!(truncate_for_retry(&[]).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncate_preserves_user_first_without_placeholder() {
|
||||
// First remaining message is already User — no placeholder needed
|
||||
let msgs: Vec<Message> = (0..10)
|
||||
.map(|i| {
|
||||
let role = if i % 2 == 0 {
|
||||
Role::User
|
||||
} else {
|
||||
Role::Assistant
|
||||
};
|
||||
Message::new(
|
||||
role,
|
||||
vec![ContentBlock::Text {
|
||||
text: format!("msg-{i}"),
|
||||
}],
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
|
||||
let result = truncate_for_retry(&msgs).unwrap();
|
||||
// msgs[2] (User) should be first; no placeholder prepended
|
||||
assert_eq!(result.len(), 8);
|
||||
match &result[0].content[0] {
|
||||
ContentBlock::Text { text } => assert_eq!(text, "msg-2"),
|
||||
_ => panic!("expected Text"),
|
||||
}
|
||||
}
|
||||
|
||||
// ── boundary detection / extraction ─────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn detect_boundary_message() {
|
||||
let metadata = CompactMetadata {
|
||||
trigger: CompactTrigger::Auto,
|
||||
pre_compact_tokens: 150_000,
|
||||
messages_summarized: 42,
|
||||
};
|
||||
let text = format!(
|
||||
"{BOUNDARY_PREFIX}\n{}",
|
||||
serde_json::to_string(&metadata).unwrap()
|
||||
);
|
||||
let msg = Message::new(Role::User, vec![ContentBlock::Text { text }]);
|
||||
assert!(is_compact_boundary(&msg));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_boundary_message() {
|
||||
let msg = Message::new(
|
||||
Role::User,
|
||||
vec![ContentBlock::Text {
|
||||
text: "hello".to_string(),
|
||||
}],
|
||||
);
|
||||
assert!(!is_compact_boundary(&msg));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_metadata_from_boundary() {
|
||||
let metadata = CompactMetadata {
|
||||
trigger: CompactTrigger::Auto,
|
||||
pre_compact_tokens: 150_000,
|
||||
messages_summarized: 42,
|
||||
};
|
||||
let text = format!(
|
||||
"{BOUNDARY_PREFIX}\n{}",
|
||||
serde_json::to_string(&metadata).unwrap()
|
||||
);
|
||||
let msg = Message::new(Role::User, vec![ContentBlock::Text { text }]);
|
||||
let extracted = extract_compact_metadata(&msg).unwrap();
|
||||
assert_eq!(extracted, metadata);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_metadata_from_non_boundary_returns_none() {
|
||||
let msg = Message::new(
|
||||
Role::User,
|
||||
vec![ContentBlock::Text {
|
||||
text: "not a boundary".to_string(),
|
||||
}],
|
||||
);
|
||||
assert!(extract_compact_metadata(&msg).is_none());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
//! Emergency truncation: the last safety net before a context overflow.
|
||||
//!
|
||||
//! When `last_input_tokens` is within `emergency_buffer` of the full
|
||||
//! `context_window`, the engine should block the next API call and ask
|
||||
//! the user to compact or start a new conversation.
|
||||
//!
|
||||
//! Unlike autocompact, the emergency check always applies — even when
|
||||
//! the compaction system is disabled via `CompactConfig.enabled`.
|
||||
|
||||
use nomi_config::compact::CompactConfig;
|
||||
|
||||
/// User-facing message shown when the emergency limit is hit.
|
||||
pub const EMERGENCY_USER_MESSAGE: &str =
|
||||
"Context window nearly full. Please use /compact or start a new conversation.";
|
||||
|
||||
/// Check whether the last observed input token count has reached the
|
||||
/// emergency blocking limit.
|
||||
///
|
||||
/// The limit is `context_window - emergency_buffer`. When
|
||||
/// `last_input_tokens >= limit`, the engine must not send another API
|
||||
/// request — doing so would almost certainly fail with a prompt-too-long
|
||||
/// error from the provider.
|
||||
///
|
||||
/// This check is independent of `CompactConfig.enabled`; the emergency
|
||||
/// safety net is always active.
|
||||
pub fn is_at_emergency_limit(last_input_tokens: u64, config: &CompactConfig) -> bool {
|
||||
let limit = config
|
||||
.context_window
|
||||
.saturating_sub(config.emergency_buffer);
|
||||
last_input_tokens as usize >= limit
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn default_config() -> CompactConfig {
|
||||
CompactConfig::default()
|
||||
}
|
||||
|
||||
// ── is_at_emergency_limit ──────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn below_limit_returns_false() {
|
||||
// limit = 200k - 3k = 197k; 190k < 197k
|
||||
let config = default_config();
|
||||
assert!(!is_at_emergency_limit(190_000, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn above_limit_returns_true() {
|
||||
// 198k >= 197k
|
||||
let config = default_config();
|
||||
assert!(is_at_emergency_limit(198_000, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn at_exact_limit_returns_true() {
|
||||
// 197k >= 197k
|
||||
let config = default_config();
|
||||
assert!(is_at_emergency_limit(197_000, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn small_context_window() {
|
||||
let config = CompactConfig {
|
||||
context_window: 8_000,
|
||||
emergency_buffer: 3_000,
|
||||
..default_config()
|
||||
};
|
||||
// limit = 8k - 3k = 5k; 6k >= 5k
|
||||
assert!(is_at_emergency_limit(6_000, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_tokens_below_limit() {
|
||||
let config = default_config();
|
||||
assert!(!is_at_emergency_limit(0, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_emergency_buffer() {
|
||||
let config = CompactConfig {
|
||||
context_window: 100_000,
|
||||
emergency_buffer: 10_000,
|
||||
..default_config()
|
||||
};
|
||||
// limit = 100k - 10k = 90k
|
||||
assert!(!is_at_emergency_limit(89_999, &config));
|
||||
assert!(is_at_emergency_limit(90_000, &config));
|
||||
assert!(is_at_emergency_limit(95_000, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn works_regardless_of_enabled_flag() {
|
||||
let config = CompactConfig {
|
||||
enabled: false,
|
||||
..default_config()
|
||||
};
|
||||
// Emergency check ignores the enabled flag
|
||||
assert!(is_at_emergency_limit(198_000, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn emergency_buffer_larger_than_context_window_saturates() {
|
||||
let config = CompactConfig {
|
||||
context_window: 1_000,
|
||||
emergency_buffer: 5_000,
|
||||
..default_config()
|
||||
};
|
||||
// saturating_sub: limit = 0; any positive token count triggers
|
||||
assert!(is_at_emergency_limit(1, &config));
|
||||
// 0 tokens = 0 >= 0 → true (degenerate but safe)
|
||||
assert!(is_at_emergency_limit(0, &config));
|
||||
}
|
||||
|
||||
// ── EMERGENCY_USER_MESSAGE ─────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn user_message_mentions_compact() {
|
||||
assert!(EMERGENCY_USER_MESSAGE.contains("/compact"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn user_message_mentions_new_conversation() {
|
||||
assert!(EMERGENCY_USER_MESSAGE.contains("new conversation"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
use nomi_types::message::{ContentBlock, Message};
|
||||
|
||||
const CHARS_PER_TOKEN_TEXT: usize = 4;
|
||||
|
||||
const CHARS_PER_TOKEN_JSON: usize = 3;
|
||||
|
||||
/// Flat per-image token estimate. A 1568px-edge screenshot costs roughly
|
||||
/// 1100-1600 tokens on Anthropic's vision pricing; over-estimating keeps
|
||||
/// compaction triggering early rather than late.
|
||||
const TOKENS_PER_IMAGE: usize = 1600;
|
||||
|
||||
/// Estimate the total token count for a slice of messages.
|
||||
///
|
||||
/// Intentionally conservative (slightly over-estimates) to ensure
|
||||
/// compaction triggers rather than being skipped.
|
||||
pub fn estimate_tokens_from_messages(messages: &[Message]) -> u64 {
|
||||
let mut total_chars: usize = 0;
|
||||
let mut json_chars: usize = 0;
|
||||
let mut image_tokens: usize = 0;
|
||||
|
||||
for msg in messages {
|
||||
for block in &msg.content {
|
||||
match block {
|
||||
ContentBlock::Text { text } => {
|
||||
total_chars += text.len();
|
||||
}
|
||||
ContentBlock::Thinking { thinking, .. } => {
|
||||
total_chars += thinking.len();
|
||||
}
|
||||
ContentBlock::ToolUse { name, input, .. } => {
|
||||
let input_str = input.to_string();
|
||||
json_chars += name.len() + input_str.len();
|
||||
}
|
||||
ContentBlock::ToolResult { content, images, .. } => {
|
||||
total_chars += content.len();
|
||||
image_tokens += images.len() * TOKENS_PER_IMAGE;
|
||||
}
|
||||
ContentBlock::Image { .. } => {
|
||||
image_tokens += TOKENS_PER_IMAGE;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let text_tokens = total_chars / CHARS_PER_TOKEN_TEXT;
|
||||
let json_tokens = json_chars / CHARS_PER_TOKEN_JSON;
|
||||
|
||||
(text_tokens + json_tokens + image_tokens) as u64
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use nomi_types::message::{Message, Role};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn empty_messages_returns_zero() {
|
||||
assert_eq!(estimate_tokens_from_messages(&[]), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_only_message() {
|
||||
let text = "a".repeat(400);
|
||||
let msg = Message::new(Role::User, vec![ContentBlock::Text { text }]);
|
||||
assert_eq!(estimate_tokens_from_messages(&[msg]), 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_use_message_uses_json_ratio() {
|
||||
let input = json!({"cmd": "ls -la"});
|
||||
let input_len = "Bash".len() + input.to_string().len();
|
||||
let msg = Message::new(
|
||||
Role::Assistant,
|
||||
vec![ContentBlock::ToolUse {
|
||||
id: "call_1".into(),
|
||||
name: "Bash".into(),
|
||||
input,
|
||||
extra: None,
|
||||
}],
|
||||
);
|
||||
let result = estimate_tokens_from_messages(&[msg]);
|
||||
assert_eq!(result, (input_len / CHARS_PER_TOKEN_JSON) as u64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_result_uses_text_ratio() {
|
||||
let content = "x".repeat(800);
|
||||
let msg = Message::new(
|
||||
Role::User,
|
||||
vec![ContentBlock::ToolResult {
|
||||
tool_use_id: "call_1".into(),
|
||||
content,
|
||||
is_error: false,
|
||||
images: Vec::new(),
|
||||
}],
|
||||
);
|
||||
assert_eq!(estimate_tokens_from_messages(&[msg]), 200);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_conversation_accumulates() {
|
||||
let messages = vec![
|
||||
Message::new(
|
||||
Role::User,
|
||||
vec![ContentBlock::Text {
|
||||
text: "a".repeat(400),
|
||||
}],
|
||||
),
|
||||
Message::new(
|
||||
Role::Assistant,
|
||||
vec![
|
||||
ContentBlock::Text {
|
||||
text: "b".repeat(200),
|
||||
},
|
||||
ContentBlock::ToolUse {
|
||||
id: "c1".into(),
|
||||
name: "Read".into(),
|
||||
input: json!({"path": "/foo/bar.rs"}),
|
||||
extra: None,
|
||||
},
|
||||
],
|
||||
),
|
||||
Message::new(
|
||||
Role::User,
|
||||
vec![ContentBlock::ToolResult {
|
||||
tool_use_id: "c1".into(),
|
||||
content: "c".repeat(1200),
|
||||
is_error: false,
|
||||
images: Vec::new(),
|
||||
}],
|
||||
),
|
||||
];
|
||||
let estimate = estimate_tokens_from_messages(&messages);
|
||||
// text_tokens = (400 + 200 + 1200) / 4 = 450
|
||||
// json_tokens = ("Read".len() + json_string.len()) / 3
|
||||
assert!(estimate > 450);
|
||||
assert!(estimate < 600);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thinking_block_counted() {
|
||||
let thinking = "t".repeat(4000);
|
||||
let msg = Message::new(
|
||||
Role::Assistant,
|
||||
vec![ContentBlock::Thinking {
|
||||
thinking,
|
||||
signature: None,
|
||||
}],
|
||||
);
|
||||
assert_eq!(estimate_tokens_from_messages(&[msg]), 1000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn large_conversation_realistic_estimate() {
|
||||
let big_result = "x".repeat(400_000);
|
||||
let messages = vec![Message::new(
|
||||
Role::User,
|
||||
vec![ContentBlock::ToolResult {
|
||||
tool_use_id: "c1".into(),
|
||||
content: big_result,
|
||||
is_error: false,
|
||||
images: Vec::new(),
|
||||
}],
|
||||
)];
|
||||
let estimate = estimate_tokens_from_messages(&messages);
|
||||
assert_eq!(estimate, 100_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn effective_watermark_uses_max() {
|
||||
let provider_reported: u64 = 500;
|
||||
let messages = vec![Message::new(
|
||||
Role::User,
|
||||
vec![ContentBlock::ToolResult {
|
||||
tool_use_id: "c1".into(),
|
||||
content: "x".repeat(400_000),
|
||||
is_error: false,
|
||||
images: Vec::new(),
|
||||
}],
|
||||
)];
|
||||
let local_estimate = estimate_tokens_from_messages(&messages);
|
||||
let effective = provider_reported.max(local_estimate);
|
||||
|
||||
assert_eq!(effective, 100_000);
|
||||
assert!(effective > provider_reported);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,586 @@
|
||||
//! Microcompact: clear old tool result content without any LLM call.
|
||||
//!
|
||||
//! This is the lightest compaction level. It walks the conversation,
|
||||
//! identifies tool results from compactable tools, and replaces the
|
||||
//! content of all but the N most recent with a short placeholder.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use chrono::Utc;
|
||||
use nomi_config::compact::CompactConfig;
|
||||
use nomi_types::message::{ContentBlock, Message, Role};
|
||||
|
||||
/// Placeholder that replaces cleared tool result content.
|
||||
pub const CLEARED_TOOL_RESULT: &str = "[Tool result cleared]";
|
||||
|
||||
/// Statistics returned after a microcompact pass.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct MicrocompactResult {
|
||||
/// Number of tool results whose content was cleared.
|
||||
pub cleared_count: usize,
|
||||
/// Rough estimate of tokens freed (content bytes / 4).
|
||||
pub estimated_tokens_freed: usize,
|
||||
}
|
||||
|
||||
// ── Trigger checks ──────────────────────────────────────────────────────────
|
||||
|
||||
/// Decide whether microcompact should run.
|
||||
///
|
||||
/// Returns `true` if **either** trigger fires:
|
||||
/// - **Time**: the most recent assistant message is older than
|
||||
/// `config.micro_gap_seconds`.
|
||||
/// - **Count**: total compactable (non-cleared) tool results exceed
|
||||
/// `config.micro_keep_recent * 2`.
|
||||
pub fn should_microcompact(messages: &[Message], config: &CompactConfig) -> bool {
|
||||
if !config.enabled {
|
||||
return false;
|
||||
}
|
||||
time_trigger(messages, config) || count_trigger(messages, config)
|
||||
}
|
||||
|
||||
/// Time-based trigger: last assistant timestamp older than gap threshold.
|
||||
fn time_trigger(messages: &[Message], config: &CompactConfig) -> bool {
|
||||
let last_assistant_ts = messages
|
||||
.iter()
|
||||
.rev()
|
||||
.filter(|m| m.role == Role::Assistant)
|
||||
.find_map(|m| m.timestamp);
|
||||
|
||||
let Some(ts) = last_assistant_ts else {
|
||||
return false;
|
||||
};
|
||||
|
||||
let gap = Utc::now().signed_duration_since(ts);
|
||||
gap.num_seconds() >= config.micro_gap_seconds as i64
|
||||
}
|
||||
|
||||
/// Count-based trigger: compactable tool results > keep_recent * 2.
|
||||
fn count_trigger(messages: &[Message], config: &CompactConfig) -> bool {
|
||||
let tool_names = build_tool_name_map(messages);
|
||||
let compactable_set: HashSet<&str> = config
|
||||
.compactable_tools
|
||||
.iter()
|
||||
.map(String::as_str)
|
||||
.collect();
|
||||
|
||||
let count = count_compactable_results(messages, &tool_names, &compactable_set);
|
||||
count > config.micro_keep_recent * 2
|
||||
}
|
||||
|
||||
// ── Core compaction ─────────────────────────────────────────────────────────
|
||||
|
||||
/// Clear old tool result content in-place.
|
||||
///
|
||||
/// Keeps the `config.micro_keep_recent` most recent compactable results
|
||||
/// (minimum 1) and replaces older ones with [`CLEARED_TOOL_RESULT`].
|
||||
/// Already-cleared results are left untouched and do not count toward
|
||||
/// the keep budget.
|
||||
pub fn microcompact(messages: &mut [Message], config: &CompactConfig) -> MicrocompactResult {
|
||||
let tool_names = build_tool_name_map(messages);
|
||||
let compactable_set: HashSet<&str> = config
|
||||
.compactable_tools
|
||||
.iter()
|
||||
.map(String::as_str)
|
||||
.collect();
|
||||
|
||||
// Collect (message_index, block_index) of all compactable, non-cleared
|
||||
// tool results, in conversation order.
|
||||
let targets = collect_compactable_locations(messages, &tool_names, &compactable_set);
|
||||
|
||||
let keep = config.micro_keep_recent.max(1);
|
||||
if targets.len() <= keep {
|
||||
return MicrocompactResult {
|
||||
cleared_count: 0,
|
||||
estimated_tokens_freed: 0,
|
||||
};
|
||||
}
|
||||
|
||||
let to_clear = &targets[..targets.len() - keep];
|
||||
|
||||
let mut cleared_count = 0usize;
|
||||
let mut tokens_freed = 0usize;
|
||||
|
||||
for &(mi, bi) in to_clear {
|
||||
if let ContentBlock::ToolResult { content, images, .. } = &mut messages[mi].content[bi] {
|
||||
// Rough token estimate: ~4 chars per token.
|
||||
tokens_freed += content.len() / 4;
|
||||
*content = CLEARED_TOOL_RESULT.to_string();
|
||||
images.clear();
|
||||
cleared_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
MicrocompactResult {
|
||||
cleared_count,
|
||||
estimated_tokens_freed: tokens_freed,
|
||||
}
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Build a map from tool_use_id → tool name by scanning ToolUse blocks
|
||||
/// across all messages.
|
||||
fn build_tool_name_map(messages: &[Message]) -> HashMap<String, String> {
|
||||
let mut map = HashMap::new();
|
||||
for msg in messages {
|
||||
for block in &msg.content {
|
||||
if let ContentBlock::ToolUse { id, name, .. } = block {
|
||||
map.insert(id.clone(), name.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
map
|
||||
}
|
||||
|
||||
/// Count compactable, non-cleared tool results.
|
||||
fn count_compactable_results(
|
||||
messages: &[Message],
|
||||
tool_names: &HashMap<String, String>,
|
||||
compactable_set: &HashSet<&str>,
|
||||
) -> usize {
|
||||
messages
|
||||
.iter()
|
||||
.flat_map(|m| &m.content)
|
||||
.filter(|b| is_compactable_and_live(b, tool_names, compactable_set))
|
||||
.count()
|
||||
}
|
||||
|
||||
/// Collect `(message_index, block_index)` of every compactable, non-cleared
|
||||
/// tool result in conversation order.
|
||||
fn collect_compactable_locations(
|
||||
messages: &[Message],
|
||||
tool_names: &HashMap<String, String>,
|
||||
compactable_set: &HashSet<&str>,
|
||||
) -> Vec<(usize, usize)> {
|
||||
let mut locations = Vec::new();
|
||||
for (mi, msg) in messages.iter().enumerate() {
|
||||
for (bi, block) in msg.content.iter().enumerate() {
|
||||
if is_compactable_and_live(block, tool_names, compactable_set) {
|
||||
locations.push((mi, bi));
|
||||
}
|
||||
}
|
||||
}
|
||||
locations
|
||||
}
|
||||
|
||||
/// A tool result is "compactable and live" when:
|
||||
/// 1. It is a `ToolResult` variant.
|
||||
/// 2. Its corresponding tool name is in the compactable set.
|
||||
/// 3. Its content has not already been cleared.
|
||||
fn is_compactable_and_live(
|
||||
block: &ContentBlock,
|
||||
tool_names: &HashMap<String, String>,
|
||||
compactable_set: &HashSet<&str>,
|
||||
) -> bool {
|
||||
if let ContentBlock::ToolResult {
|
||||
tool_use_id,
|
||||
content,
|
||||
..
|
||||
} = block
|
||||
{
|
||||
if content == CLEARED_TOOL_RESULT {
|
||||
return false;
|
||||
}
|
||||
if let Some(name) = tool_names.get(tool_use_id) {
|
||||
return compactable_set.contains(name.as_str());
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use chrono::Duration;
|
||||
use serde_json::json;
|
||||
|
||||
// ── Test helpers ────────────────────────────────────────────────────
|
||||
|
||||
fn tool_use_block(id: &str, name: &str) -> ContentBlock {
|
||||
ContentBlock::ToolUse {
|
||||
id: id.to_string(),
|
||||
name: name.to_string(),
|
||||
input: json!({}),
|
||||
extra: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn tool_result_block(id: &str, content: &str) -> ContentBlock {
|
||||
ContentBlock::ToolResult {
|
||||
tool_use_id: id.to_string(),
|
||||
content: content.to_string(),
|
||||
is_error: false,
|
||||
images: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn text_block(text: &str) -> ContentBlock {
|
||||
ContentBlock::Text {
|
||||
text: text.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn assistant_msg(blocks: Vec<ContentBlock>) -> Message {
|
||||
Message::new(Role::Assistant, blocks)
|
||||
}
|
||||
|
||||
fn user_msg(blocks: Vec<ContentBlock>) -> Message {
|
||||
Message::new(Role::User, blocks)
|
||||
}
|
||||
|
||||
fn assistant_msg_at(blocks: Vec<ContentBlock>, ts: chrono::DateTime<Utc>) -> Message {
|
||||
Message {
|
||||
role: Role::Assistant,
|
||||
content: blocks,
|
||||
timestamp: Some(ts),
|
||||
}
|
||||
}
|
||||
|
||||
fn default_config() -> CompactConfig {
|
||||
CompactConfig::default()
|
||||
}
|
||||
|
||||
// ── build_tool_name_map ─────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tool_name_map_from_single_assistant() {
|
||||
let msgs = vec![assistant_msg(vec![
|
||||
tool_use_block("t1", "Read"),
|
||||
tool_use_block("t2", "Bash"),
|
||||
])];
|
||||
let map = build_tool_name_map(&msgs);
|
||||
assert_eq!(map.get("t1").unwrap(), "Read");
|
||||
assert_eq!(map.get("t2").unwrap(), "Bash");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_name_map_ignores_non_tool_use() {
|
||||
let msgs = vec![
|
||||
user_msg(vec![text_block("hello")]),
|
||||
user_msg(vec![tool_result_block("t1", "output")]),
|
||||
];
|
||||
let map = build_tool_name_map(&msgs);
|
||||
assert!(map.is_empty());
|
||||
}
|
||||
|
||||
// ── is_compactable_and_live ─────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn live_compactable_result_returns_true() {
|
||||
let tool_names: HashMap<String, String> =
|
||||
[("t1".into(), "Read".into())].into_iter().collect();
|
||||
let set: HashSet<&str> = ["Read"].into_iter().collect();
|
||||
let block = tool_result_block("t1", "file content here");
|
||||
assert!(is_compactable_and_live(&block, &tool_names, &set));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn already_cleared_result_returns_false() {
|
||||
let tool_names: HashMap<String, String> =
|
||||
[("t1".into(), "Read".into())].into_iter().collect();
|
||||
let set: HashSet<&str> = ["Read"].into_iter().collect();
|
||||
let block = tool_result_block("t1", CLEARED_TOOL_RESULT);
|
||||
assert!(!is_compactable_and_live(&block, &tool_names, &set));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_compactable_tool_returns_false() {
|
||||
let tool_names: HashMap<String, String> =
|
||||
[("t1".into(), "Skill".into())].into_iter().collect();
|
||||
let set: HashSet<&str> = ["Read", "Bash"].into_iter().collect();
|
||||
let block = tool_result_block("t1", "result");
|
||||
assert!(!is_compactable_and_live(&block, &tool_names, &set));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_block_returns_false() {
|
||||
let tool_names = HashMap::new();
|
||||
let set: HashSet<&str> = ["Read"].into_iter().collect();
|
||||
let block = text_block("hello");
|
||||
assert!(!is_compactable_and_live(&block, &tool_names, &set));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_tool_use_id_returns_false() {
|
||||
let tool_names = HashMap::new(); // no ToolUse registered
|
||||
let set: HashSet<&str> = ["Read"].into_iter().collect();
|
||||
let block = tool_result_block("orphan", "data");
|
||||
assert!(!is_compactable_and_live(&block, &tool_names, &set));
|
||||
}
|
||||
|
||||
// ── time_trigger ────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn time_trigger_fires_when_gap_exceeded() {
|
||||
let old_ts = Utc::now() - Duration::seconds(3700);
|
||||
let msgs = vec![assistant_msg_at(vec![text_block("hi")], old_ts)];
|
||||
let config = CompactConfig {
|
||||
micro_gap_seconds: 3600,
|
||||
..default_config()
|
||||
};
|
||||
assert!(time_trigger(&msgs, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn time_trigger_silent_when_within_gap() {
|
||||
let recent_ts = Utc::now() - Duration::seconds(1800);
|
||||
let msgs = vec![assistant_msg_at(vec![text_block("hi")], recent_ts)];
|
||||
let config = CompactConfig {
|
||||
micro_gap_seconds: 3600,
|
||||
..default_config()
|
||||
};
|
||||
assert!(!time_trigger(&msgs, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn time_trigger_silent_when_no_timestamp() {
|
||||
let msgs = vec![assistant_msg(vec![text_block("hi")])];
|
||||
let config = default_config();
|
||||
assert!(!time_trigger(&msgs, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn time_trigger_uses_latest_assistant() {
|
||||
let old_ts = Utc::now() - Duration::seconds(7200);
|
||||
let recent_ts = Utc::now() - Duration::seconds(100);
|
||||
let msgs = vec![
|
||||
assistant_msg_at(vec![text_block("first")], old_ts),
|
||||
assistant_msg_at(vec![text_block("second")], recent_ts),
|
||||
];
|
||||
let config = CompactConfig {
|
||||
micro_gap_seconds: 3600,
|
||||
..default_config()
|
||||
};
|
||||
// The most recent assistant (100s ago) is within the gap.
|
||||
assert!(!time_trigger(&msgs, &config));
|
||||
}
|
||||
|
||||
// ── count_trigger ───────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn count_trigger_fires_above_threshold() {
|
||||
// keep_recent=3, threshold=6. Create 7 compactable results.
|
||||
let mut msgs = Vec::new();
|
||||
for i in 0..7 {
|
||||
let id = format!("t{i}");
|
||||
msgs.push(assistant_msg(vec![tool_use_block(&id, "Read")]));
|
||||
msgs.push(user_msg(vec![tool_result_block(&id, "data")]));
|
||||
}
|
||||
let config = CompactConfig {
|
||||
micro_keep_recent: 3,
|
||||
..default_config()
|
||||
};
|
||||
assert!(count_trigger(&msgs, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn count_trigger_silent_at_threshold() {
|
||||
// keep_recent=3, threshold=6. Create exactly 6 results.
|
||||
let mut msgs = Vec::new();
|
||||
for i in 0..6 {
|
||||
let id = format!("t{i}");
|
||||
msgs.push(assistant_msg(vec![tool_use_block(&id, "Read")]));
|
||||
msgs.push(user_msg(vec![tool_result_block(&id, "data")]));
|
||||
}
|
||||
let config = CompactConfig {
|
||||
micro_keep_recent: 3,
|
||||
..default_config()
|
||||
};
|
||||
assert!(!count_trigger(&msgs, &config));
|
||||
}
|
||||
|
||||
// ── microcompact ────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn clears_oldest_keeps_recent() {
|
||||
// 5 tool results, keep_recent=2 → clear 3.
|
||||
let mut msgs = Vec::new();
|
||||
for i in 0..5 {
|
||||
let id = format!("t{i}");
|
||||
msgs.push(assistant_msg(vec![tool_use_block(&id, "Read")]));
|
||||
msgs.push(user_msg(vec![tool_result_block(&id, &format!("data-{i}"))]));
|
||||
}
|
||||
let config = CompactConfig {
|
||||
micro_keep_recent: 2,
|
||||
..default_config()
|
||||
};
|
||||
|
||||
let result = microcompact(&mut msgs, &config);
|
||||
assert_eq!(result.cleared_count, 3);
|
||||
assert!(result.estimated_tokens_freed > 0);
|
||||
|
||||
// First 3 user msgs (indices 1,3,5) should be cleared.
|
||||
for idx in [1, 3, 5] {
|
||||
let content = match &msgs[idx].content[0] {
|
||||
ContentBlock::ToolResult { content, .. } => content.as_str(),
|
||||
_ => panic!("expected ToolResult"),
|
||||
};
|
||||
assert_eq!(content, CLEARED_TOOL_RESULT);
|
||||
}
|
||||
// Last 2 user msgs (indices 7,9) should retain original content.
|
||||
for (idx, expected) in [(7, "data-3"), (9, "data-4")] {
|
||||
let content = match &msgs[idx].content[0] {
|
||||
ContentBlock::ToolResult { content, .. } => content.as_str(),
|
||||
_ => panic!("expected ToolResult"),
|
||||
};
|
||||
assert_eq!(content, expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_clear_when_below_keep_recent() {
|
||||
let mut msgs = vec![
|
||||
assistant_msg(vec![tool_use_block("t1", "Read")]),
|
||||
user_msg(vec![tool_result_block("t1", "data")]),
|
||||
];
|
||||
let config = CompactConfig {
|
||||
micro_keep_recent: 5,
|
||||
..default_config()
|
||||
};
|
||||
let result = microcompact(&mut msgs, &config);
|
||||
assert_eq!(result.cleared_count, 0);
|
||||
assert_eq!(result.estimated_tokens_freed, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skips_non_compactable_tools() {
|
||||
let mut msgs = vec![
|
||||
assistant_msg(vec![tool_use_block("t1", "Read")]),
|
||||
user_msg(vec![tool_result_block("t1", "file-data")]),
|
||||
assistant_msg(vec![tool_use_block("t2", "Skill")]),
|
||||
user_msg(vec![tool_result_block("t2", "skill-output")]),
|
||||
assistant_msg(vec![tool_use_block("t3", "Bash")]),
|
||||
user_msg(vec![tool_result_block("t3", "bash-output")]),
|
||||
];
|
||||
// compactable_tools does NOT include Skill.
|
||||
let config = CompactConfig {
|
||||
micro_keep_recent: 1,
|
||||
compactable_tools: vec!["Read".into(), "Bash".into()],
|
||||
..default_config()
|
||||
};
|
||||
|
||||
let result = microcompact(&mut msgs, &config);
|
||||
// Only Read(t1) should be cleared; Bash(t3) kept as most recent.
|
||||
assert_eq!(result.cleared_count, 1);
|
||||
|
||||
// Skill result untouched.
|
||||
match &msgs[3].content[0] {
|
||||
ContentBlock::ToolResult { content, .. } => {
|
||||
assert_eq!(content, "skill-output");
|
||||
}
|
||||
_ => panic!("expected ToolResult"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn does_not_recleared_already_cleared() {
|
||||
let mut msgs = vec![
|
||||
assistant_msg(vec![tool_use_block("t1", "Read")]),
|
||||
user_msg(vec![tool_result_block("t1", CLEARED_TOOL_RESULT)]),
|
||||
assistant_msg(vec![tool_use_block("t2", "Read")]),
|
||||
user_msg(vec![tool_result_block("t2", "live-data")]),
|
||||
];
|
||||
let config = CompactConfig {
|
||||
micro_keep_recent: 1,
|
||||
..default_config()
|
||||
};
|
||||
let result = microcompact(&mut msgs, &config);
|
||||
// t1 already cleared → not in compactable list.
|
||||
// Only t2 is compactable, and it's the most recent → keep it.
|
||||
assert_eq!(result.cleared_count, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_messages_returns_zero() {
|
||||
let mut msgs: Vec<Message> = Vec::new();
|
||||
let result = microcompact(&mut msgs, &default_config());
|
||||
assert_eq!(result.cleared_count, 0);
|
||||
assert_eq!(result.estimated_tokens_freed, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn message_count_and_order_preserved() {
|
||||
let mut msgs = vec![
|
||||
assistant_msg(vec![tool_use_block("t1", "Read")]),
|
||||
user_msg(vec![tool_result_block("t1", &"a".repeat(100))]),
|
||||
assistant_msg(vec![tool_use_block("t2", "Read")]),
|
||||
user_msg(vec![tool_result_block("t2", &"b".repeat(100))]),
|
||||
assistant_msg(vec![tool_use_block("t3", "Read")]),
|
||||
user_msg(vec![tool_result_block("t3", &"c".repeat(100))]),
|
||||
];
|
||||
let original_len = msgs.len();
|
||||
let config = CompactConfig {
|
||||
micro_keep_recent: 1,
|
||||
..default_config()
|
||||
};
|
||||
microcompact(&mut msgs, &config);
|
||||
|
||||
assert_eq!(msgs.len(), original_len);
|
||||
// Roles alternate: Assistant, User, Assistant, User, ...
|
||||
for (i, msg) in msgs.iter().enumerate() {
|
||||
let expected = if i % 2 == 0 {
|
||||
Role::Assistant
|
||||
} else {
|
||||
Role::User
|
||||
};
|
||||
assert_eq!(msg.role, expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_estimate_proportional_to_content() {
|
||||
let long_content = "x".repeat(400); // ~100 tokens
|
||||
let mut msgs = vec![
|
||||
assistant_msg(vec![tool_use_block("t1", "Read")]),
|
||||
user_msg(vec![tool_result_block("t1", &long_content)]),
|
||||
assistant_msg(vec![tool_use_block("t2", "Read")]),
|
||||
user_msg(vec![tool_result_block("t2", "keep")]),
|
||||
];
|
||||
let config = CompactConfig {
|
||||
micro_keep_recent: 1,
|
||||
..default_config()
|
||||
};
|
||||
let result = microcompact(&mut msgs, &config);
|
||||
assert_eq!(result.cleared_count, 1);
|
||||
assert_eq!(result.estimated_tokens_freed, 100); // 400 / 4
|
||||
}
|
||||
|
||||
// ── should_microcompact ─────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn should_returns_false_when_disabled() {
|
||||
let old_ts = Utc::now() - Duration::seconds(7200);
|
||||
let msgs = vec![assistant_msg_at(vec![text_block("hi")], old_ts)];
|
||||
let config = CompactConfig {
|
||||
enabled: false,
|
||||
micro_gap_seconds: 3600,
|
||||
..default_config()
|
||||
};
|
||||
assert!(!should_microcompact(&msgs, &config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keep_recent_floored_at_one() {
|
||||
// Even with keep_recent=0, we never clear everything.
|
||||
let mut msgs = vec![
|
||||
assistant_msg(vec![tool_use_block("t1", "Read")]),
|
||||
user_msg(vec![tool_result_block("t1", "data-1")]),
|
||||
assistant_msg(vec![tool_use_block("t2", "Read")]),
|
||||
user_msg(vec![tool_result_block("t2", "data-2")]),
|
||||
];
|
||||
let config = CompactConfig {
|
||||
micro_keep_recent: 0,
|
||||
..default_config()
|
||||
};
|
||||
let result = microcompact(&mut msgs, &config);
|
||||
// 2 compactable, keep at least 1 → clear 1.
|
||||
assert_eq!(result.cleared_count, 1);
|
||||
// The most recent (t2) must survive.
|
||||
match &msgs[3].content[0] {
|
||||
ContentBlock::ToolResult { content, .. } => {
|
||||
assert_eq!(content, "data-2");
|
||||
}
|
||||
_ => panic!("expected ToolResult"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
//! Multi-level context compaction for long conversations.
|
||||
//!
|
||||
//! Three levels, from lightest to heaviest:
|
||||
//! - **Microcompact**: clears old tool result content (no LLM call)
|
||||
//! - **Autocompact**: watermark-triggered LLM summarization
|
||||
//! - **Emergency**: blocks API calls when near the context window limit
|
||||
|
||||
pub mod auto;
|
||||
pub mod emergency;
|
||||
pub mod estimate;
|
||||
pub mod micro;
|
||||
pub mod prompt;
|
||||
pub mod state;
|
||||
@@ -0,0 +1,314 @@
|
||||
//! Compact prompt templates for LLM-based conversation summarization.
|
||||
//!
|
||||
//! Provides the 9-section summary prompt, response parsing, and
|
||||
//! post-compact message construction.
|
||||
|
||||
/// System prompt used for the compact LLM call.
|
||||
pub const COMPACT_SYSTEM_PROMPT: &str =
|
||||
"You are a helpful AI assistant tasked with summarizing conversations.";
|
||||
|
||||
/// Maximum output tokens for the compact LLM call.
|
||||
pub const COMPACT_MAX_OUTPUT_TOKENS: u32 = 20_000;
|
||||
|
||||
// ── Prompt construction ─────────────────────────────────────────────────────
|
||||
|
||||
/// Build the 9-section compact prompt that asks the LLM to summarize.
|
||||
pub fn build_compact_prompt() -> String {
|
||||
format!("{PREAMBLE}\n\n{BODY}\n\n{FORMAT_INSTRUCTIONS}\n\n{REMINDER}")
|
||||
}
|
||||
|
||||
const PREAMBLE: &str = "\
|
||||
CRITICAL: Respond with TEXT ONLY. Do NOT call any tools.
|
||||
- Do NOT use Read, Bash, Grep, Glob, Edit, Write, or ANY other tool.
|
||||
- You already have all the context you need in the conversation above.
|
||||
- Tool calls will be REJECTED and will waste your only turn — you will fail the task.
|
||||
- Your entire response must be plain text: an <analysis> block followed by a <summary> block.";
|
||||
|
||||
const BODY: &str = "\
|
||||
Your task is to create a detailed summary of the conversation so far, paying close attention \
|
||||
to the user's explicit requests and your previous actions. This summary should be thorough in \
|
||||
capturing technical details, code patterns, and architectural decisions that would be essential \
|
||||
for continuing development work.
|
||||
|
||||
Before providing your final summary, wrap your analysis in <analysis> tags to organize your \
|
||||
thoughts and ensure completeness.
|
||||
|
||||
Your summary should include the following sections:
|
||||
|
||||
1. **Primary Request and Intent**: What has the user asked for? Include ALL explicit requests \
|
||||
made during the conversation.
|
||||
2. **Key Technical Concepts**: Important technical details, patterns, or architectural decisions discussed.
|
||||
3. **Files and Code Sections**: All files that have been viewed or modified, with brief descriptions of changes.
|
||||
4. **Errors and Fixes**: Any errors encountered and how they were resolved.
|
||||
5. **Problem Solving Progress**: Current state of each problem — what's solved and what remains.
|
||||
6. **All User Messages**: A summary of every non-tool user message, preserving intent and context.
|
||||
7. **Pending Tasks**: Any tasks that are not yet complete.
|
||||
8. **Current Work**: What was being worked on immediately before this summary.
|
||||
9. **Suggested Next Step**: The single most logical next action, which MUST be directly in line \
|
||||
with the most recent explicit user request. Quote the user's request verbatim to prevent drift.";
|
||||
|
||||
const FORMAT_INSTRUCTIONS: &str = "\
|
||||
Format your response exactly as follows:
|
||||
|
||||
<analysis>
|
||||
Your reasoning about what information is most important to preserve
|
||||
</analysis>
|
||||
|
||||
<summary>
|
||||
Your detailed, structured summary following the 9 sections above
|
||||
</summary>";
|
||||
|
||||
const REMINDER: &str = "\
|
||||
REMINDER: Do NOT call any tools. Respond with plain text only — an <analysis> block followed \
|
||||
by a <summary> block. Tool calls will be rejected and you will fail the task.";
|
||||
|
||||
// ── Response parsing ────────────────────────────────────────────────────────
|
||||
|
||||
/// Parse the raw LLM response: strip `<analysis>`, extract `<summary>` content.
|
||||
///
|
||||
/// If no `<summary>` tags are found, returns the raw text as-is (graceful degradation).
|
||||
pub fn format_compact_summary(raw: &str) -> String {
|
||||
// Step 1: remove <analysis>...</analysis>
|
||||
let without_analysis = strip_tag(raw, "analysis");
|
||||
|
||||
// Step 2: extract <summary>...</summary> content
|
||||
if let Some(summary_content) = extract_tag_content(&without_analysis, "summary") {
|
||||
let trimmed = summary_content.trim();
|
||||
if trimmed.is_empty() {
|
||||
return collapse_blank_lines(&without_analysis).trim().to_string();
|
||||
}
|
||||
format!("Summary:\n{trimmed}")
|
||||
} else {
|
||||
// Graceful degradation: use the text with analysis stripped
|
||||
collapse_blank_lines(&without_analysis).trim().to_string()
|
||||
}
|
||||
}
|
||||
|
||||
// ── Post-compact message content ────────────────────────────────────────────
|
||||
|
||||
/// Build the user message content for the post-compact summary.
|
||||
///
|
||||
/// For autocompact (`is_auto = true`), appends an instruction telling the
|
||||
/// model to continue seamlessly without acknowledging the compaction.
|
||||
pub fn build_summary_content(formatted_summary: &str, is_auto: bool) -> String {
|
||||
let mut content = String::from(
|
||||
"This session is being continued from a previous conversation that ran out of context. \
|
||||
The summary below covers the earlier portion of the conversation.\n\n",
|
||||
);
|
||||
content.push_str(formatted_summary);
|
||||
|
||||
if is_auto {
|
||||
content.push_str(
|
||||
"\n\nContinue the conversation from where it left off without asking the user \
|
||||
any further questions. Resume directly — do not acknowledge the summary, \
|
||||
do not recap what was happening, do not preface with \"I'll continue\" or similar. \
|
||||
Pick up the last task as if the break never happened.",
|
||||
);
|
||||
}
|
||||
|
||||
content
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Remove `<tag>...</tag>` (first occurrence) from text.
|
||||
///
|
||||
/// If the closing tag appears before the opening tag (reversed order),
|
||||
/// the text is returned unchanged to avoid producing duplicate content.
|
||||
fn strip_tag(text: &str, tag: &str) -> String {
|
||||
let open = format!("<{tag}>");
|
||||
let close = format!("</{tag}>");
|
||||
|
||||
let Some(start) = text.find(&open) else {
|
||||
return text.to_string();
|
||||
};
|
||||
let Some(end) = text.find(&close) else {
|
||||
return text.to_string();
|
||||
};
|
||||
|
||||
// Guard: closing tag before opening tag → no-op
|
||||
if end < start {
|
||||
return text.to_string();
|
||||
}
|
||||
|
||||
let mut result = String::with_capacity(text.len());
|
||||
result.push_str(&text[..start]);
|
||||
result.push_str(&text[end + close.len()..]);
|
||||
collapse_blank_lines(&result)
|
||||
}
|
||||
|
||||
/// Extract the content between `<tag>` and `</tag>` (first occurrence).
|
||||
fn extract_tag_content<'a>(text: &'a str, tag: &str) -> Option<&'a str> {
|
||||
let open = format!("<{tag}>");
|
||||
let close = format!("</{tag}>");
|
||||
|
||||
let start = text.find(&open)? + open.len();
|
||||
let end = text.find(&close)?;
|
||||
|
||||
if start <= end {
|
||||
Some(&text[start..end])
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Collapse consecutive blank lines into a single blank line.
|
||||
fn collapse_blank_lines(text: &str) -> String {
|
||||
let mut result = String::with_capacity(text.len());
|
||||
let mut prev_was_blank = false;
|
||||
|
||||
for line in text.lines() {
|
||||
let is_blank = line.trim().is_empty();
|
||||
if is_blank && prev_was_blank {
|
||||
continue;
|
||||
}
|
||||
if !result.is_empty() {
|
||||
result.push('\n');
|
||||
}
|
||||
result.push_str(line);
|
||||
prev_was_blank = is_blank;
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// ── build_compact_prompt ────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn prompt_contains_all_nine_sections() {
|
||||
let prompt = build_compact_prompt();
|
||||
for i in 1..=9 {
|
||||
assert!(prompt.contains(&format!("{i}.")), "Missing section {i}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prompt_forbids_tool_calls() {
|
||||
let prompt = build_compact_prompt();
|
||||
assert!(prompt.contains("Do NOT call any tools"));
|
||||
assert!(prompt.contains("CRITICAL"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prompt_requires_analysis_and_summary_tags() {
|
||||
let prompt = build_compact_prompt();
|
||||
assert!(prompt.contains("<analysis>"));
|
||||
assert!(prompt.contains("<summary>"));
|
||||
}
|
||||
|
||||
// ── format_compact_summary ──────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn strips_analysis_extracts_summary() {
|
||||
let raw =
|
||||
"<analysis>thinking about things</analysis>\n<summary>the actual result</summary>";
|
||||
assert_eq!(format_compact_summary(raw), "Summary:\nthe actual result");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_summary_without_analysis() {
|
||||
let raw = "<summary>result only</summary>";
|
||||
assert_eq!(format_compact_summary(raw), "Summary:\nresult only");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn graceful_degradation_without_tags() {
|
||||
let raw = "plain text without any tags";
|
||||
assert_eq!(format_compact_summary(raw), "plain text without any tags");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn handles_multiline_summary() {
|
||||
let raw =
|
||||
"<analysis>analysis\nwith lines</analysis>\n<summary>\nLine 1\nLine 2\n</summary>";
|
||||
let result = format_compact_summary(raw);
|
||||
assert!(result.starts_with("Summary:\n"));
|
||||
assert!(result.contains("Line 1"));
|
||||
assert!(result.contains("Line 2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_summary_tags_falls_back() {
|
||||
let raw = "<analysis>thinking</analysis>\n<summary></summary>";
|
||||
let result = format_compact_summary(raw);
|
||||
// Falls back since summary content is empty
|
||||
assert!(!result.is_empty());
|
||||
}
|
||||
|
||||
// ── build_summary_content ───────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn auto_summary_includes_continuation_instruction() {
|
||||
let content = build_summary_content("Summary:\ntest", true);
|
||||
assert!(content.contains("Continue the conversation"));
|
||||
assert!(content.contains("as if the break never happened"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_summary_no_continuation_instruction() {
|
||||
let content = build_summary_content("Summary:\ntest", false);
|
||||
assert!(!content.contains("Continue the conversation"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summary_content_includes_session_header() {
|
||||
let content = build_summary_content("Summary:\ntest", false);
|
||||
assert!(content.contains("This session is being continued"));
|
||||
}
|
||||
|
||||
// ── strip_tag ───────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn strip_tag_removes_complete_tag() {
|
||||
let text = "before<foo>inside</foo>after";
|
||||
assert_eq!(strip_tag(text, "foo"), "beforeafter");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strip_tag_noop_when_tag_missing() {
|
||||
let text = "no tags here";
|
||||
assert_eq!(strip_tag(text, "foo"), "no tags here");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strip_tag_noop_when_reversed_order() {
|
||||
// Closing tag before opening tag should be treated as no-op
|
||||
let text = "before</foo>middle<foo>inside</foo>after";
|
||||
// The first </foo> is at position 6, first <foo> is at position 17
|
||||
// Since end < start, the text should be returned unchanged
|
||||
assert_eq!(strip_tag(text, "foo"), text);
|
||||
}
|
||||
|
||||
// ── extract_tag_content ─────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn extract_existing_tag() {
|
||||
let text = "<summary>hello world</summary>";
|
||||
assert_eq!(extract_tag_content(text, "summary"), Some("hello world"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_missing_tag() {
|
||||
let text = "no summary here";
|
||||
assert_eq!(extract_tag_content(text, "summary"), None);
|
||||
}
|
||||
|
||||
// ── collapse_blank_lines ────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn collapses_multiple_blank_lines() {
|
||||
let text = "a\n\n\n\nb";
|
||||
let result = collapse_blank_lines(text);
|
||||
assert_eq!(result, "a\n\nb");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_single_blank_line() {
|
||||
let text = "a\n\nb";
|
||||
assert_eq!(collapse_blank_lines(text), "a\n\nb");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
use nomi_config::compact::CompactConfig;
|
||||
|
||||
/// Runtime state for the compaction circuit breaker.
|
||||
///
|
||||
/// Tracks consecutive autocompact failures so we can stop retrying
|
||||
/// after `config.max_failures` consecutive failures.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CompactState {
|
||||
/// Number of consecutive autocompact failures.
|
||||
pub consecutive_failures: u32,
|
||||
/// Input token count from the last API call (used as the watermark).
|
||||
pub last_input_tokens: u64,
|
||||
}
|
||||
|
||||
impl CompactState {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
consecutive_failures: 0,
|
||||
last_input_tokens: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Check whether the circuit breaker has tripped.
|
||||
pub fn is_circuit_broken(&self, config: &CompactConfig) -> bool {
|
||||
self.consecutive_failures >= config.max_failures
|
||||
}
|
||||
|
||||
/// Record a successful autocompact — resets the failure counter.
|
||||
pub fn record_success(&mut self) {
|
||||
self.consecutive_failures = 0;
|
||||
}
|
||||
|
||||
/// Record a failed autocompact — increments the failure counter.
|
||||
pub fn record_failure(&mut self) {
|
||||
self.consecutive_failures += 1;
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for CompactState {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn test_config() -> CompactConfig {
|
||||
CompactConfig {
|
||||
max_failures: 3,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_state_not_circuit_broken() {
|
||||
let state = CompactState::new();
|
||||
assert_eq!(state.consecutive_failures, 0);
|
||||
assert_eq!(state.last_input_tokens, 0);
|
||||
assert!(!state.is_circuit_broken(&test_config()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn circuit_breaker_trips_at_max_failures() {
|
||||
let config = test_config();
|
||||
let mut state = CompactState::new();
|
||||
|
||||
state.record_failure();
|
||||
assert!(!state.is_circuit_broken(&config));
|
||||
state.record_failure();
|
||||
assert!(!state.is_circuit_broken(&config));
|
||||
state.record_failure();
|
||||
assert!(state.is_circuit_broken(&config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn success_resets_failure_counter() {
|
||||
let config = test_config();
|
||||
let mut state = CompactState::new();
|
||||
|
||||
state.record_failure();
|
||||
state.record_failure();
|
||||
assert_eq!(state.consecutive_failures, 2);
|
||||
|
||||
state.record_success();
|
||||
assert_eq!(state.consecutive_failures, 0);
|
||||
assert!(!state.is_circuit_broken(&config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn circuit_breaker_with_max_failures_one() {
|
||||
let config = CompactConfig {
|
||||
max_failures: 1,
|
||||
..Default::default()
|
||||
};
|
||||
let mut state = CompactState::new();
|
||||
|
||||
assert!(!state.is_circuit_broken(&config));
|
||||
state.record_failure();
|
||||
assert!(state.is_circuit_broken(&config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_impl_matches_new() {
|
||||
let a = CompactState::new();
|
||||
let b = CompactState::default();
|
||||
assert_eq!(a.consecutive_failures, b.consecutive_failures);
|
||||
assert_eq!(a.last_input_tokens, b.last_input_tokens);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,498 @@
|
||||
//! Native tools that give a companion-companion agent access to its memory store
|
||||
//! through a `CompanionMemorySink` trait object. The backend (nomifun-companion)
|
||||
//! injects a concrete sink; other hosts pass `None` and these are not
|
||||
//! registered. Mirrors `requirement_tools.rs`.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use nomi_protocol::events::ToolCategory;
|
||||
use nomi_tools::Tool;
|
||||
use nomi_types::tool::{JsonSchema, ToolResult};
|
||||
|
||||
/// Memory kinds shared with the companion store taxonomy.
|
||||
pub const COMPANION_MEMORY_KINDS: [&str; 6] = ["profile", "preference", "knowledge", "episode", "task", "affective"];
|
||||
|
||||
/// Backend seam for the companion's long-term memory + activity feed. Implemented
|
||||
/// by `nomifun-companion` over its `CompanionStore`; `nomi-agent` only depends on this.
|
||||
#[async_trait]
|
||||
pub trait CompanionMemorySink: Send + Sync {
|
||||
/// Search memories by keyword (optionally by kind / incl. archived).
|
||||
/// `conversation_id` scopes the search to the owning companion (shared +
|
||||
/// its own private memories), so one companion never recalls another's
|
||||
/// private memories. Returns a human-readable digest the model can quote.
|
||||
async fn recall(&self, conversation_id: &str, query: &str, kind: Option<&str>, include_archived: bool) -> Result<String, String>;
|
||||
|
||||
/// Persist one memory; implementations dedup. Returns a confirmation line.
|
||||
/// `conversation_id` identifies the session the save came from, so the
|
||||
/// backend can attribute per-companion rewards (XP) to the owning companion.
|
||||
async fn save(&self, conversation_id: &str, kind: &str, content: &str, tags: &[String]) -> Result<String, String>;
|
||||
|
||||
/// Newest collected work events (already sanitized), newest-last digest.
|
||||
async fn recent_events(&self, limit: usize) -> Result<String, String>;
|
||||
}
|
||||
|
||||
/// `recall_memories` — search the companion's full memory store.
|
||||
pub struct RecallMemoriesTool {
|
||||
sink: Arc<dyn CompanionMemorySink>,
|
||||
/// The conversation this tool instance serves — passed to the sink so the
|
||||
/// backend can scope recall to the owning companion (shared + own private).
|
||||
conversation_id: String,
|
||||
}
|
||||
|
||||
impl RecallMemoriesTool {
|
||||
pub fn new(sink: Arc<dyn CompanionMemorySink>, conversation_id: impl Into<String>) -> Self {
|
||||
Self {
|
||||
sink,
|
||||
conversation_id: conversation_id.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for RecallMemoriesTool {
|
||||
fn name(&self) -> &str {
|
||||
"recall_memories"
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"搜索你对主人的全部长期记忆（包含未注入上下文的与已归档的）。当主人问起过去的事、\
|
||||
或你需要确认自己是否记得某事时使用。返回匹配的记忆列表。"
|
||||
}
|
||||
|
||||
fn input_schema(&self) -> JsonSchema {
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string", "description": "关键词（按内容模糊匹配）"},
|
||||
"kind": {"type": "string", "enum": COMPANION_MEMORY_KINDS, "description": "可选：限定记忆类型"},
|
||||
"include_archived": {"type": "boolean", "description": "是否包含已归档记忆，默认 false"}
|
||||
},
|
||||
"required": ["query"]
|
||||
})
|
||||
}
|
||||
|
||||
fn is_concurrency_safe(&self, _input: &Value) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
async fn execute(&self, input: Value) -> ToolResult {
|
||||
let query = input.get("query").and_then(|v| v.as_str()).unwrap_or("").trim();
|
||||
if query.is_empty() {
|
||||
return ToolResult {
|
||||
content: "query 不能为空".into(),
|
||||
is_error: true,
|
||||
images: Vec::new(),
|
||||
};
|
||||
}
|
||||
let kind = input
|
||||
.get("kind")
|
||||
.and_then(|v| v.as_str())
|
||||
.filter(|k| COMPANION_MEMORY_KINDS.contains(k));
|
||||
let include_archived = input.get("include_archived").and_then(|v| v.as_bool()).unwrap_or(false);
|
||||
match self.sink.recall(&self.conversation_id, query, kind, include_archived).await {
|
||||
Ok(out) => ToolResult {
|
||||
content: out,
|
||||
is_error: false,
|
||||
images: Vec::new(),
|
||||
},
|
||||
Err(e) => ToolResult {
|
||||
content: e,
|
||||
is_error: true,
|
||||
images: Vec::new(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn category(&self) -> ToolCategory {
|
||||
ToolCategory::Info
|
||||
}
|
||||
}
|
||||
|
||||
/// `save_memory` — persist a long-term memory about the user.
|
||||
pub struct SaveMemoryTool {
|
||||
sink: Arc<dyn CompanionMemorySink>,
|
||||
/// The conversation this tool instance serves — passed to the sink so the
|
||||
/// backend can attribute the save to the owning companion.
|
||||
conversation_id: String,
|
||||
}
|
||||
|
||||
impl SaveMemoryTool {
|
||||
pub fn new(sink: Arc<dyn CompanionMemorySink>, conversation_id: impl Into<String>) -> Self {
|
||||
Self {
|
||||
sink,
|
||||
conversation_id: conversation_id.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for SaveMemoryTool {
|
||||
fn name(&self) -> &str {
|
||||
"save_memory"
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"立即保存一条关于主人的长期记忆。当主人告诉你值得记住的事（偏好、约定、计划、\
|
||||
纠正你的认知）时使用；一句话自包含，宁缺毋滥。kind 取值：profile(稳定画像)/\
|
||||
preference(偏好)/knowledge(可复用结论)/episode(带时间的经历)/task(待办线索)/affective(情感)。"
|
||||
}
|
||||
|
||||
fn input_schema(&self) -> JsonSchema {
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kind": {"type": "string", "enum": COMPANION_MEMORY_KINDS},
|
||||
"content": {"type": "string", "description": "一句话记忆内容（中文，自包含）"},
|
||||
"tags": {"type": "array", "items": {"type": "string"}}
|
||||
},
|
||||
"required": ["kind", "content"]
|
||||
})
|
||||
}
|
||||
|
||||
fn is_concurrency_safe(&self, _input: &Value) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
async fn execute(&self, input: Value) -> ToolResult {
|
||||
let kind = input.get("kind").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let content = input.get("content").and_then(|v| v.as_str()).unwrap_or("").trim();
|
||||
if !COMPANION_MEMORY_KINDS.contains(&kind) {
|
||||
return ToolResult {
|
||||
content: format!("kind 必须是 {COMPANION_MEMORY_KINDS:?} 之一"),
|
||||
is_error: true,
|
||||
images: Vec::new(),
|
||||
};
|
||||
}
|
||||
if content.is_empty() {
|
||||
return ToolResult {
|
||||
content: "content 不能为空".into(),
|
||||
is_error: true,
|
||||
images: Vec::new(),
|
||||
};
|
||||
}
|
||||
let tags: Vec<String> = input
|
||||
.get("tags")
|
||||
.and_then(|v| v.as_array())
|
||||
.map(|a| a.iter().filter_map(|t| t.as_str().map(str::to_owned)).collect())
|
||||
.unwrap_or_default();
|
||||
match self.sink.save(&self.conversation_id, kind, content, &tags).await {
|
||||
Ok(out) => ToolResult {
|
||||
content: out,
|
||||
is_error: false,
|
||||
images: Vec::new(),
|
||||
},
|
||||
Err(e) => ToolResult {
|
||||
content: e,
|
||||
is_error: true,
|
||||
images: Vec::new(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn category(&self) -> ToolCategory {
|
||||
// Writes only to the companion's own memory.db (never user files) — treat
|
||||
// as Info so default session mode doesn't gate it behind approval.
|
||||
ToolCategory::Info
|
||||
}
|
||||
}
|
||||
|
||||
/// `list_recent_events` — peek at the user's recent collected work activity.
|
||||
pub struct ListRecentEventsTool {
|
||||
sink: Arc<dyn CompanionMemorySink>,
|
||||
}
|
||||
|
||||
impl ListRecentEventsTool {
|
||||
pub fn new(sink: Arc<dyn CompanionMemorySink>) -> Self {
|
||||
Self { sink }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for ListRecentEventsTool {
|
||||
fn name(&self) -> &str {
|
||||
"list_recent_events"
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"查看最近采集到的主人工作事件（已脱敏摘要）。当主人问「我今天/最近都干了啥」\
|
||||
或你想结合实际活动给建议时使用。"
|
||||
}
|
||||
|
||||
fn input_schema(&self) -> JsonSchema {
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"limit": {"type": "integer", "description": "最多返回多少条，默认 20，上限 50"}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn is_concurrency_safe(&self, _input: &Value) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
async fn execute(&self, input: Value) -> ToolResult {
|
||||
let limit = input
|
||||
.get("limit")
|
||||
.and_then(|v| v.as_i64())
|
||||
.unwrap_or(20)
|
||||
.clamp(1, 50) as usize;
|
||||
match self.sink.recent_events(limit).await {
|
||||
Ok(out) => ToolResult {
|
||||
content: out,
|
||||
is_error: false,
|
||||
images: Vec::new(),
|
||||
},
|
||||
Err(e) => ToolResult {
|
||||
content: e,
|
||||
is_error: true,
|
||||
images: Vec::new(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn category(&self) -> ToolCategory {
|
||||
ToolCategory::Info
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 自进化技能：自调用（design §7）
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// 一个可调用技能的精简描述，用于每轮的 when_to_use 索引注入。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SkillListing {
|
||||
pub name: String,
|
||||
pub when_to_use: String,
|
||||
}
|
||||
|
||||
/// Backend seam for the companion's self-evolved skills. Implemented by
|
||||
/// `nomifun-companion` over its store + `skill_service`; `nomi-agent` only depends
|
||||
/// on this (engine stays host-agnostic).
|
||||
#[async_trait]
|
||||
pub trait CompanionSkillSink: Send + Sync {
|
||||
/// This companion's currently-active skills (for per-turn `when_to_use` injection).
|
||||
/// Must be cheap — called once per turn.
|
||||
async fn active_skills(&self) -> Vec<SkillListing>;
|
||||
/// The SKILL.md body of a named active skill, or `None` if unknown.
|
||||
async fn load_skill_body(&self, name: &str) -> Option<String>;
|
||||
}
|
||||
|
||||
/// `companion_skill` — invoke a learned skill by name to fetch its playbook.
|
||||
pub struct CompanionSkillTool {
|
||||
sink: Arc<dyn CompanionSkillSink>,
|
||||
}
|
||||
|
||||
impl CompanionSkillTool {
|
||||
pub fn new(sink: Arc<dyn CompanionSkillSink>) -> Self {
|
||||
Self { sink }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for CompanionSkillTool {
|
||||
fn name(&self) -> &str {
|
||||
"companion_skill"
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"调用你已学会的某个技能，获取它的操作手册（步骤），然后照着执行。\
|
||||
当当前任务匹配系统提示里列出的某个技能的适用场景时使用。"
|
||||
}
|
||||
|
||||
fn input_schema(&self) -> JsonSchema {
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"skill": {"type": "string", "description": "技能名（见系统提示里列出的可用技能）"}
|
||||
},
|
||||
"required": ["skill"]
|
||||
})
|
||||
}
|
||||
|
||||
fn is_concurrency_safe(&self, _input: &Value) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
async fn execute(&self, input: Value) -> ToolResult {
|
||||
let name = input.get("skill").and_then(|v| v.as_str()).unwrap_or("").trim();
|
||||
if name.is_empty() {
|
||||
return ToolResult {
|
||||
content: "skill 不能为空".into(),
|
||||
is_error: true,
|
||||
images: Vec::new(),
|
||||
};
|
||||
}
|
||||
match self.sink.load_skill_body(name).await {
|
||||
Some(body) => ToolResult {
|
||||
content: body,
|
||||
is_error: false,
|
||||
images: Vec::new(),
|
||||
},
|
||||
None => ToolResult {
|
||||
content: format!("未找到技能：{name}"),
|
||||
is_error: true,
|
||||
images: Vec::new(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn category(&self) -> ToolCategory {
|
||||
ToolCategory::Info
|
||||
}
|
||||
}
|
||||
|
||||
/// 每轮把该伙伴 active 技能的 `when_to_use` 索引注入系统提示（design §7）。
|
||||
/// 空技能集 → `None`（no-op 快路，引擎据此每轮零成本跳过）。
|
||||
pub struct CompanionSkillContributor {
|
||||
sink: Arc<dyn CompanionSkillSink>,
|
||||
}
|
||||
|
||||
impl CompanionSkillContributor {
|
||||
pub fn new(sink: Arc<dyn CompanionSkillSink>) -> Self {
|
||||
Self { sink }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl crate::context_contributor::ContextContributor for CompanionSkillContributor {
|
||||
async fn pre_turn_context(&self) -> Option<String> {
|
||||
let skills = self.sink.active_skills().await;
|
||||
if skills.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let mut s = String::from(
|
||||
"<system-reminder>\n你已经学会以下技能。遇到匹配场景时，用 companion_skill 工具按名调用以获取操作手册并照做：\n",
|
||||
);
|
||||
for sk in &skills {
|
||||
s.push_str(&format!("- {}: {}\n", sk.name, sk.when_to_use));
|
||||
}
|
||||
s.push_str("</system-reminder>");
|
||||
Some(s)
|
||||
}
|
||||
|
||||
fn label(&self) -> &str {
|
||||
"companion_skills"
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Mutex;
|
||||
|
||||
struct RecordingSink {
|
||||
saved: Mutex<Vec<(String, String, String)>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl CompanionMemorySink for RecordingSink {
|
||||
async fn recall(&self, _conversation_id: &str, query: &str, kind: Option<&str>, _archived: bool) -> Result<String, String> {
|
||||
Ok(format!("hits for {query} kind={kind:?}"))
|
||||
}
|
||||
async fn save(&self, conversation_id: &str, kind: &str, content: &str, _tags: &[String]) -> Result<String, String> {
|
||||
self.saved
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push((conversation_id.into(), kind.into(), content.into()));
|
||||
Ok("saved".into())
|
||||
}
|
||||
async fn recent_events(&self, limit: usize) -> Result<String, String> {
|
||||
Ok(format!("{limit} events"))
|
||||
}
|
||||
}
|
||||
|
||||
fn sink() -> Arc<RecordingSink> {
|
||||
Arc::new(RecordingSink {
|
||||
saved: Mutex::new(vec![]),
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recall_requires_query_and_filters_kind() {
|
||||
let tool = RecallMemoriesTool::new(sink(), "conv_t");
|
||||
let bad = tool.execute(json!({})).await;
|
||||
assert!(bad.is_error);
|
||||
let ok = tool.execute(json!({"query": "结论", "kind": "preference"})).await;
|
||||
assert!(!ok.is_error);
|
||||
assert!(ok.content.contains("preference"));
|
||||
// Invalid kind is dropped, not an error.
|
||||
let loose = tool.execute(json!({"query": "x", "kind": "bogus"})).await;
|
||||
assert!(!loose.is_error);
|
||||
assert!(loose.content.contains("None"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn save_validates_kind_and_content() {
|
||||
let s = sink();
|
||||
let tool = SaveMemoryTool::new(s.clone(), "conv_t");
|
||||
assert!(tool.execute(json!({"kind": "bogus", "content": "x"})).await.is_error);
|
||||
assert!(tool.execute(json!({"kind": "task", "content": " "})).await.is_error);
|
||||
let ok = tool.execute(json!({"kind": "task", "content": "明天修 bug"})).await;
|
||||
assert!(!ok.is_error);
|
||||
let saved = s.saved.lock().unwrap();
|
||||
assert_eq!(saved.len(), 1);
|
||||
// The tool stamps the conversation it serves into every save.
|
||||
assert_eq!(saved[0].0, "conv_t");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recent_events_clamps_limit() {
|
||||
let tool = ListRecentEventsTool::new(sink());
|
||||
let out = tool.execute(json!({"limit": 9999})).await;
|
||||
assert_eq!(out.content, "50 events");
|
||||
let out = tool.execute(json!({})).await;
|
||||
assert_eq!(out.content, "20 events");
|
||||
}
|
||||
|
||||
use crate::context_contributor::ContextContributor;
|
||||
|
||||
struct FakeSkillSink {
|
||||
skills: Vec<SkillListing>,
|
||||
}
|
||||
#[async_trait]
|
||||
impl CompanionSkillSink for FakeSkillSink {
|
||||
async fn active_skills(&self) -> Vec<SkillListing> {
|
||||
self.skills.clone()
|
||||
}
|
||||
async fn load_skill_body(&self, name: &str) -> Option<String> {
|
||||
self.skills.iter().find(|s| s.name == name).map(|s| format!("# {}\nbody", s.name))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn skill_contributor_is_noop_when_empty() {
|
||||
let sink = Arc::new(FakeSkillSink { skills: vec![] });
|
||||
let c = CompanionSkillContributor::new(sink);
|
||||
assert!(c.pre_turn_context().await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn skill_contributor_lists_when_to_use() {
|
||||
let sink = Arc::new(FakeSkillSink {
|
||||
skills: vec![SkillListing { name: "weekly-report".into(), when_to_use: "周五出周报".into() }],
|
||||
});
|
||||
let c = CompanionSkillContributor::new(sink);
|
||||
let out = c.pre_turn_context().await.unwrap();
|
||||
assert!(out.contains("weekly-report"));
|
||||
assert!(out.contains("周五出周报"));
|
||||
assert!(out.contains("companion_skill"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn skill_tool_returns_body_or_error() {
|
||||
let sink = Arc::new(FakeSkillSink {
|
||||
skills: vec![SkillListing { name: "fmt".into(), when_to_use: "x".into() }],
|
||||
});
|
||||
let tool = CompanionSkillTool::new(sink);
|
||||
assert!(tool.execute(json!({})).await.is_error);
|
||||
let ok = tool.execute(json!({"skill": "fmt"})).await;
|
||||
assert!(!ok.is_error);
|
||||
assert!(ok.content.contains("fmt"));
|
||||
assert!(tool.execute(json!({"skill": "nope"})).await.is_error);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
use std::collections::HashSet;
|
||||
use std::io::{self, BufRead, Write};
|
||||
|
||||
pub struct ToolConfirmer {
|
||||
auto_approve: bool,
|
||||
allow_list: HashSet<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ConfirmResult {
|
||||
Approved,
|
||||
Denied,
|
||||
Quit,
|
||||
}
|
||||
|
||||
impl ToolConfirmer {
|
||||
pub fn new(auto_approve: bool, allow_list: Vec<String>) -> Self {
|
||||
Self {
|
||||
auto_approve,
|
||||
allow_list: allow_list.into_iter().collect(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether auto-approve is enabled
|
||||
pub fn is_auto_approve(&self) -> bool {
|
||||
self.auto_approve
|
||||
}
|
||||
|
||||
/// Add a tool name to the allow list at runtime.
|
||||
/// Used by skill context modifiers to grant auto-approval for specified tools.
|
||||
pub fn add_to_allow_list(&mut self, name: &str) {
|
||||
self.allow_list.insert(name.to_string());
|
||||
}
|
||||
|
||||
/// Check if the tool needs confirmation. Returns the user's decision.
|
||||
pub fn check(&mut self, tool_name: &str, tool_input_display: &str) -> ConfirmResult {
|
||||
if self.auto_approve || self.allow_list.contains(tool_name) {
|
||||
return ConfirmResult::Approved;
|
||||
}
|
||||
|
||||
eprint!(
|
||||
"\n[tool] {}({})\nAllow? [y]es / [n]o / [a]lways / [q]uit > ",
|
||||
tool_name, tool_input_display
|
||||
);
|
||||
io::stderr().flush().unwrap();
|
||||
|
||||
let mut input = String::new();
|
||||
if io::stdin().lock().read_line(&mut input).is_err() {
|
||||
return ConfirmResult::Denied;
|
||||
}
|
||||
|
||||
match input.trim().to_lowercase().as_str() {
|
||||
"y" | "yes" | "" => ConfirmResult::Approved,
|
||||
"a" | "always" => {
|
||||
self.allow_list.insert(tool_name.to_string());
|
||||
ConfirmResult::Approved
|
||||
}
|
||||
"q" | "quit" => ConfirmResult::Quit,
|
||||
_ => ConfirmResult::Denied,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_auto_approve_always_allows() {
|
||||
let mut confirmer = ToolConfirmer::new(true, vec![]);
|
||||
assert_eq!(
|
||||
confirmer.check("Bash", "echo hello"),
|
||||
ConfirmResult::Approved
|
||||
);
|
||||
assert_eq!(
|
||||
confirmer.check("Read", "/tmp/file"),
|
||||
ConfirmResult::Approved
|
||||
);
|
||||
assert_eq!(
|
||||
confirmer.check("Write", "/tmp/out"),
|
||||
ConfirmResult::Approved
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_allowlist_contains_tool() {
|
||||
let mut confirmer = ToolConfirmer::new(false, vec!["Read".into(), "Write".into()]);
|
||||
assert_eq!(
|
||||
confirmer.check("Read", "/tmp/file"),
|
||||
ConfirmResult::Approved
|
||||
);
|
||||
assert_eq!(
|
||||
confirmer.check("Write", "/tmp/out"),
|
||||
ConfirmResult::Approved
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_allowlist_approves_even_when_auto_approve_is_false() {
|
||||
let mut confirmer = ToolConfirmer::new(false, vec!["Read".into()]);
|
||||
assert_eq!(
|
||||
confirmer.check("Read", "/some/path"),
|
||||
ConfirmResult::Approved
|
||||
);
|
||||
}
|
||||
|
||||
// Phase 6: add_to_allow_list() grants runtime approval
|
||||
#[test]
|
||||
fn test_add_to_allow_list_grants_approval() {
|
||||
let mut confirmer = ToolConfirmer::new(false, vec![]);
|
||||
// Before: tool not in list (would prompt — skip interactive check, just verify membership)
|
||||
confirmer.add_to_allow_list("Write");
|
||||
// After: auto-approved without interactive prompt
|
||||
assert_eq!(
|
||||
confirmer.check("Write", "file.txt"),
|
||||
ConfirmResult::Approved
|
||||
);
|
||||
}
|
||||
|
||||
// Phase 6: add_to_allow_list() is idempotent — adding twice has no bad effect
|
||||
#[test]
|
||||
fn test_add_to_allow_list_idempotent() {
|
||||
let mut confirmer = ToolConfirmer::new(false, vec![]);
|
||||
confirmer.add_to_allow_list("Bash");
|
||||
confirmer.add_to_allow_list("Bash"); // duplicate — HashSet, no panic
|
||||
assert_eq!(confirmer.check("Bash", "echo hi"), ConfirmResult::Approved);
|
||||
}
|
||||
|
||||
// Phase 6: add_to_allow_list() does not affect unrelated tools
|
||||
#[test]
|
||||
fn test_add_to_allow_list_does_not_affect_other_tools() {
|
||||
let mut confirmer = ToolConfirmer::new(false, vec![]);
|
||||
confirmer.add_to_allow_list("Read");
|
||||
// Write is not in the list — check returns non-Approved for non-interactive
|
||||
// (we cannot test interactive input; verify Read is approved and Write is not in list)
|
||||
assert_eq!(confirmer.check("Read", "file.txt"), ConfirmResult::Approved);
|
||||
// We can't test the Denied path without stdin, but we verify allow_list state:
|
||||
assert!(confirmer.allow_list.contains("Read"));
|
||||
assert!(!confirmer.allow_list.contains("Write"));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,97 @@
|
||||
//! `ContextContributor` — the host-agnostic seam (design §3.5) that lets the
|
||||
//! backend inject dynamic, per-turn context into the system prompt without the
|
||||
//! engine hard-coding each source. The engine holds a list of contributors
|
||||
//! (empty by default → behaviour byte-for-byte unchanged) and, at the start of
|
||||
//! each turn, appends whatever they contribute to the system prompt.
|
||||
//!
|
||||
//! This is the foundation for turning "passive" platform features into "active"
|
||||
//! injection (knowledge auto-RAG, inline memory, etc.) as registered
|
||||
//! contributors rather than bespoke call-sites. It is purely additive: with no
|
||||
//! contributors registered, `merge_pre_turn_context` returns the system prompt
|
||||
//! unchanged.
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
/// A source of dynamic per-turn context. Implementations live in the backend
|
||||
/// (host) and are registered onto the engine; the engine stays host-agnostic.
|
||||
#[async_trait]
|
||||
pub trait ContextContributor: Send + Sync {
|
||||
/// Context to add to the system prompt for the upcoming turn, or `None` to
|
||||
/// contribute nothing this turn. Called once per turn before the model call.
|
||||
async fn pre_turn_context(&self) -> Option<String>;
|
||||
|
||||
/// A short stable label for diagnostics/telemetry.
|
||||
fn label(&self) -> &str {
|
||||
"context_contributor"
|
||||
}
|
||||
}
|
||||
|
||||
/// Append non-empty contributor contributions to `system`, each under a blank
|
||||
/// line, in registration order. Empty / all-`None` → `system` returned
|
||||
/// unchanged (the zero-contributor fast path the engine relies on). Pure so the
|
||||
/// merge rule is unit-testable without an engine.
|
||||
pub fn merge_pre_turn_context(system: String, contributions: Vec<String>) -> String {
|
||||
let mut out = system;
|
||||
for c in contributions {
|
||||
let trimmed = c.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if !out.is_empty() {
|
||||
out.push_str("\n\n");
|
||||
}
|
||||
out.push_str(trimmed);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn no_contributions_returns_system_unchanged() {
|
||||
let sys = "SYSTEM PROMPT".to_string();
|
||||
assert_eq!(merge_pre_turn_context(sys.clone(), vec![]), sys);
|
||||
// All-empty contributions are also a no-op.
|
||||
assert_eq!(
|
||||
merge_pre_turn_context(sys.clone(), vec!["".into(), " ".into()]),
|
||||
sys
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn appends_non_empty_contributions_in_order() {
|
||||
let out = merge_pre_turn_context(
|
||||
"BASE".to_string(),
|
||||
vec!["[KB] hit".into(), "".into(), "[memory] fact".into()],
|
||||
);
|
||||
assert_eq!(out, "BASE\n\n[KB] hit\n\n[memory] fact");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_system_with_one_contribution_has_no_leading_blank() {
|
||||
let out = merge_pre_turn_context(String::new(), vec!["only".into()]);
|
||||
assert_eq!(out, "only");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn trait_object_contributes_through_merge() {
|
||||
struct Fixed(&'static str);
|
||||
#[async_trait]
|
||||
impl ContextContributor for Fixed {
|
||||
async fn pre_turn_context(&self) -> Option<String> {
|
||||
Some(self.0.to_string())
|
||||
}
|
||||
}
|
||||
let contributors: Vec<Box<dyn ContextContributor>> =
|
||||
vec![Box::new(Fixed("alpha")), Box::new(Fixed("beta"))];
|
||||
let mut contributions = Vec::new();
|
||||
for c in &contributors {
|
||||
if let Some(s) = c.pre_turn_context().await {
|
||||
contributions.push(s);
|
||||
}
|
||||
}
|
||||
assert_eq!(merge_pre_turn_context("S".into(), contributions), "S\n\nalpha\n\nbeta");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
//! Native tools that let an in-process agent schedule / list / delete its own
|
||||
//! recurring (cron) jobs through a `CronSink` trait object. The backend injects
|
||||
//! a concrete sink bound to the agent's conversation; standalone `nomi-cli`
|
||||
//! passes `None` and these are not registered. Mirrors `requirement_tools`.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use nomi_protocol::events::ToolCategory;
|
||||
use nomi_tools::Tool;
|
||||
use nomi_types::tool::{JsonSchema, ToolResult};
|
||||
|
||||
/// One scheduled job, as surfaced to the agent by `CronSink::list`.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CronJobSummary {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
/// Human-readable schedule summary (e.g. the cron expression).
|
||||
pub schedule: String,
|
||||
pub enabled: bool,
|
||||
}
|
||||
|
||||
/// Backend seam for the agent's own scheduled jobs. Implemented by the backend
|
||||
/// over its `CronService`, bound to the agent's conversation; `nomi-agent` only
|
||||
/// depends on this trait.
|
||||
#[async_trait]
|
||||
pub trait CronSink: Send + Sync {
|
||||
/// Schedule `prompt` to re-run on `cron_expr` (5-field cron) in the agent's
|
||||
/// own conversation. Returns the new job id.
|
||||
async fn create(&self, name: &str, cron_expr: &str, prompt: &str) -> Result<String, String>;
|
||||
|
||||
/// List the agent's scheduled jobs.
|
||||
async fn list(&self) -> Result<Vec<CronJobSummary>, String>;
|
||||
|
||||
/// Delete a scheduled job by id.
|
||||
async fn delete(&self, job_id: &str) -> Result<(), String>;
|
||||
}
|
||||
|
||||
fn ok(content: String) -> ToolResult {
|
||||
ToolResult { content, is_error: false, images: Vec::new() }
|
||||
}
|
||||
fn err(content: String) -> ToolResult {
|
||||
ToolResult { content, is_error: true, images: Vec::new() }
|
||||
}
|
||||
|
||||
/// `cron_create` — schedule a recurring prompt.
|
||||
pub struct CronCreateTool {
|
||||
sink: Arc<dyn CronSink>,
|
||||
}
|
||||
impl CronCreateTool {
|
||||
pub fn new(sink: Arc<dyn CronSink>) -> Self {
|
||||
Self { sink }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for CronCreateTool {
|
||||
fn name(&self) -> &str {
|
||||
"cron_create"
|
||||
}
|
||||
fn description(&self) -> &str {
|
||||
"Schedule a prompt to re-run automatically on a recurring schedule, in this \
|
||||
conversation. Use for 'every morning…', 'check X every 5 minutes', etc. The \
|
||||
schedule is a standard 5-field cron expression in the user's local time \
|
||||
(minute hour day-of-month month day-of-week)."
|
||||
}
|
||||
fn input_schema(&self) -> JsonSchema {
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": { "type": "string", "description": "Short human-readable name for the job" },
|
||||
"cron": { "type": "string", "description": "5-field cron expression, e.g. \"0 9 * * 1-5\" (weekdays 9am)" },
|
||||
"prompt": { "type": "string", "description": "The prompt to run each time the schedule fires" }
|
||||
},
|
||||
"required": ["name", "cron", "prompt"]
|
||||
})
|
||||
}
|
||||
fn is_concurrency_safe(&self, _input: &Value) -> bool {
|
||||
false
|
||||
}
|
||||
async fn execute(&self, input: Value) -> ToolResult {
|
||||
let (Some(name), Some(cron), Some(prompt)) = (
|
||||
input["name"].as_str(),
|
||||
input["cron"].as_str(),
|
||||
input["prompt"].as_str(),
|
||||
) else {
|
||||
return err("cron_create requires: name, cron, prompt".to_string());
|
||||
};
|
||||
match self.sink.create(name, cron, prompt).await {
|
||||
Ok(id) => ok(format!("Scheduled job '{name}' ({cron}) — id {id}")),
|
||||
Err(e) => err(format!("Failed to schedule job: {e}")),
|
||||
}
|
||||
}
|
||||
fn category(&self) -> ToolCategory {
|
||||
ToolCategory::Exec
|
||||
}
|
||||
}
|
||||
|
||||
/// `cron_list` — list this conversation's scheduled jobs.
|
||||
pub struct CronListTool {
|
||||
sink: Arc<dyn CronSink>,
|
||||
}
|
||||
impl CronListTool {
|
||||
pub fn new(sink: Arc<dyn CronSink>) -> Self {
|
||||
Self { sink }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for CronListTool {
|
||||
fn name(&self) -> &str {
|
||||
"cron_list"
|
||||
}
|
||||
fn description(&self) -> &str {
|
||||
"List the scheduled (cron) jobs for this conversation."
|
||||
}
|
||||
fn input_schema(&self) -> JsonSchema {
|
||||
json!({ "type": "object", "properties": {} })
|
||||
}
|
||||
fn is_concurrency_safe(&self, _input: &Value) -> bool {
|
||||
true
|
||||
}
|
||||
async fn execute(&self, _input: Value) -> ToolResult {
|
||||
match self.sink.list().await {
|
||||
Ok(jobs) if jobs.is_empty() => ok("No scheduled jobs.".to_string()),
|
||||
Ok(jobs) => {
|
||||
let lines: Vec<String> = jobs
|
||||
.iter()
|
||||
.map(|j| {
|
||||
format!(
|
||||
"- {} [{}] {} ({})",
|
||||
j.id,
|
||||
if j.enabled { "on" } else { "off" },
|
||||
j.name,
|
||||
j.schedule
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
ok(lines.join("\n"))
|
||||
}
|
||||
Err(e) => err(format!("Failed to list jobs: {e}")),
|
||||
}
|
||||
}
|
||||
fn category(&self) -> ToolCategory {
|
||||
ToolCategory::Info
|
||||
}
|
||||
}
|
||||
|
||||
/// `cron_delete` — delete a scheduled job by id.
|
||||
pub struct CronDeleteTool {
|
||||
sink: Arc<dyn CronSink>,
|
||||
}
|
||||
impl CronDeleteTool {
|
||||
pub fn new(sink: Arc<dyn CronSink>) -> Self {
|
||||
Self { sink }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for CronDeleteTool {
|
||||
fn name(&self) -> &str {
|
||||
"cron_delete"
|
||||
}
|
||||
fn description(&self) -> &str {
|
||||
"Delete a scheduled (cron) job by its id (from cron_list)."
|
||||
}
|
||||
fn input_schema(&self) -> JsonSchema {
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": { "job_id": { "type": "string", "description": "The job id to delete" } },
|
||||
"required": ["job_id"]
|
||||
})
|
||||
}
|
||||
fn is_concurrency_safe(&self, _input: &Value) -> bool {
|
||||
false
|
||||
}
|
||||
async fn execute(&self, input: Value) -> ToolResult {
|
||||
let Some(job_id) = input["job_id"].as_str() else {
|
||||
return err("cron_delete requires: job_id".to_string());
|
||||
};
|
||||
match self.sink.delete(job_id).await {
|
||||
Ok(()) => ok(format!("Deleted scheduled job {job_id}")),
|
||||
Err(e) => err(format!("Failed to delete job: {e}")),
|
||||
}
|
||||
}
|
||||
fn category(&self) -> ToolCategory {
|
||||
ToolCategory::Exec
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Mutex;
|
||||
|
||||
#[derive(Default)]
|
||||
struct MockCronSink {
|
||||
created: Mutex<Vec<(String, String, String)>>,
|
||||
jobs: Mutex<Vec<CronJobSummary>>,
|
||||
deleted: Mutex<Vec<String>>,
|
||||
fail: bool,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl CronSink for MockCronSink {
|
||||
async fn create(&self, name: &str, cron: &str, prompt: &str) -> Result<String, String> {
|
||||
if self.fail {
|
||||
return Err("boom".into());
|
||||
}
|
||||
self.created
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push((name.into(), cron.into(), prompt.into()));
|
||||
Ok("job-1".into())
|
||||
}
|
||||
async fn list(&self) -> Result<Vec<CronJobSummary>, String> {
|
||||
Ok(self.jobs.lock().unwrap().clone())
|
||||
}
|
||||
async fn delete(&self, job_id: &str) -> Result<(), String> {
|
||||
self.deleted.lock().unwrap().push(job_id.into());
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cron_create_calls_sink_and_reports_id() {
|
||||
let sink = Arc::new(MockCronSink::default());
|
||||
let tool = CronCreateTool::new(sink.clone());
|
||||
let r = tool
|
||||
.execute(json!({ "name": "daily", "cron": "0 9 * * *", "prompt": "do it" }))
|
||||
.await;
|
||||
assert!(!r.is_error, "{}", r.content);
|
||||
assert!(r.content.contains("job-1"));
|
||||
assert_eq!(sink.created.lock().unwrap().len(), 1);
|
||||
assert_eq!(sink.created.lock().unwrap()[0].1, "0 9 * * *");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cron_create_missing_params_is_error() {
|
||||
let tool = CronCreateTool::new(Arc::new(MockCronSink::default()));
|
||||
let r = tool.execute(json!({ "name": "x" })).await;
|
||||
assert!(r.is_error);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cron_create_surfaces_sink_error() {
|
||||
let sink = Arc::new(MockCronSink { fail: true, ..Default::default() });
|
||||
let tool = CronCreateTool::new(sink);
|
||||
let r = tool
|
||||
.execute(json!({ "name": "x", "cron": "* * * * *", "prompt": "p" }))
|
||||
.await;
|
||||
assert!(r.is_error);
|
||||
assert!(r.content.contains("boom"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cron_list_formats_jobs_and_empty() {
|
||||
let sink = Arc::new(MockCronSink::default());
|
||||
let empty = CronListTool::new(sink.clone()).execute(json!({})).await;
|
||||
assert!(empty.content.contains("No scheduled jobs"));
|
||||
|
||||
sink.jobs.lock().unwrap().push(CronJobSummary {
|
||||
id: "j1".into(),
|
||||
name: "nightly".into(),
|
||||
schedule: "0 0 * * *".into(),
|
||||
enabled: true,
|
||||
});
|
||||
let listed = CronListTool::new(sink).execute(json!({})).await;
|
||||
assert!(listed.content.contains("j1"));
|
||||
assert!(listed.content.contains("nightly"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cron_delete_calls_sink() {
|
||||
let sink = Arc::new(MockCronSink::default());
|
||||
let r = CronDeleteTool::new(sink.clone())
|
||||
.execute(json!({ "job_id": "j9" }))
|
||||
.await;
|
||||
assert!(!r.is_error);
|
||||
assert_eq!(sink.deleted.lock().unwrap()[0], "j9");
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user