Files
freedak f7a720204a Update: 将子项目从 submodule 转为完整内容
- 移除 GovAI, nomifun-tauri, 算力盒子 的 submodule 引用
- 添加所有子项目的完整源代码
- 保留原始 .git 为 .git.bak 备份
2026-07-04 19:20:46 +08:00

762 lines
36 KiB
Rust
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! **P2 F3:多步 e2efacade 端到端)+ 安全门生效证据**`#[ignore]`,本机/打包 chrome)。
//!
//! 这是 P2 收官的端到端验证:**经 `BrowserTool` facade`Tool::execute`**串起完整真实流程,证明
//! P2 的各组件(navigate settle / observe ref 表 / actionability 五检查 + 三级兜底 / verify-after-act /
//! 不可逆分类器 + facade 独立 fail-closed 门 / secret 域绑定)在真 Chrome 上**协同工作**。
//!
//! 与 engine 层集成测试(`nomi-browser-engine/tests/integration_act.rs` 的 `c1_*`/`c2_*`)的区别:
//! 那些直接驱动 `engine.act(&ActSpec, &Progress)`(引擎契约);本测试走**更高层**——经 facade 的
//! `execute(json!{...})`LLM 真正调用的入口),故同时覆盖:①facade 的 dispatch/参数解析;②facade 的
//! redline 独立门(在 dispatch 前拦审批旁路会话的不可逆动作);③facade 的 `secret:NAME` origin 门。
//!
//! ## 覆盖的 P2 DoD 验收点
//! - **多步协同**navigate → observe → type username → type password → select_option Pro → click submit
//! (普通会话 submit 真提交 → onsubmit 标记 `submitted:<user>:<plan>`,经再 observe 读回证实)。
//! - **安全门生效(红线)**:
//! 1. **yolo/审批旁路会话** click submitaccname="Submit order" → 分类 Irreversible)→ facade redline
//! 门 **hard-deny Blocked**(设计裁决⑧:不靠被旁路的 orchestration,靠 facade 独立 fail-closed 门);
//! 2. **普通会话** 同一 submit → 门**不拦**(交 orchestration),动作真执行;
//! 3. **secret 域绑定 fail-closed**`secret:NAME` 在 file:// 源(无 eTLD+1)→ Blocked,明文不入输出。
//!
//! 手动跑(本机 Windows 有系统 Chrome):
//! set NOMIFUN_CHROME_BINARY=C:\Program Files\Google\Chrome\Application\chrome.exe
//! cargo nextest run -p nomi-browser --run-ignored all -E 'test(e2e)'
//! 跑完核对任务管理器无残留 chromeengine 的 Builder kill_on_drop 应自动清;tool Drop 即释放)。
use std::path::PathBuf;
use nomi_browser::BrowserTool;
use nomi_config::config::BrowserConfig;
use nomi_tools::Tool;
use serde_json::json;
/// fixture 的 file:// URL。`CARGO_MANIFEST_DIR` 在 unix 是 `/abs`(已带前导斜杠)、在 windows
/// 是 `C:/abs`(需补一个),故仅缺失时补斜杠——避免 unix 上 `file:///{manifest}` 产生四斜杠
/// `file:////...`)触发 chrome 归一成三斜杠 → navigate redirect 误判。
fn fixture_url(name: &str) -> String {
let manifest = env!("CARGO_MANIFEST_DIR").replace('\\', "/");
let abs = if manifest.starts_with('/') {
manifest
} else {
format!("/{manifest}")
};
format!("file://{abs}/tests/fixtures/{name}")
}
/// 从 facade observe 的 aria YAML 文本里,按 `role` + accname 子串找到 `[ref=f<seq>e<n>]`。
///
/// observe 输出形如 `- textbox "Username" [ref=f0e1]` / `- button "Submit order" [ref=f0e4]`。
/// 我们找含 role 词 + accname 子串 + `[ref=` 标记的那一行,抽出 ref。facade 不暴露结构化
/// `Observation`(那是 engine 契约),故按 LLM 真正看到的文本解析(与模型同视角)。
fn find_ref(observe_text: &str, role: &str, accname: &str) -> String {
observe_text
.lines()
.find(|line| line.contains(role) && line.contains(accname) && line.contains("[ref="))
.and_then(|line| {
let start = line.find("[ref=")? + 5;
let end = line[start..].find(']')? + start;
Some(line[start..end].to_string())
})
.unwrap_or_else(|| {
panic!("observe output should expose a {role:?} with accname {accname:?}; got:\n{observe_text}")
})
}
/// headless BrowserConfig(本机集成测试默认 headless;不依赖显示)。
fn headless_config() -> BrowserConfig {
BrowserConfig { headless: true, ..Default::default() }
}
/// 本测试专属隔离 data_dir(避免与运行中的 app browser-data 争用同一 profile)。
fn isolated_data_dir(suffix: &str) -> PathBuf {
std::env::temp_dir().join(format!("nomifun-f3-e2e-{suffix}-data"))
}
/// **多步 e2e(普通会话,经 facade)+ 安全门「普通会话 submit 不被门拦」证据。**
///
/// navigate → observe → type username → type password → select_option Pro → click submit →
/// 再 observe 读回 `#form-status == submitted:e2e-user:pro`(证 type/select 真写入 + submit 真触发,
/// 且普通会话的 Irreversible submit **未被 facade 门拦**——门方向正确:只拦审批旁路会话)。
#[tokio::test]
#[ignore = "需本机/打包 chromeset NOMIFUN_CHROME_BINARY 后 --run-ignored all"]
async fn e2e_multistep_form_flow_through_facade_normal_session() {
// 普通会话(session_bypasses_approval=false):facade redline 门不拦不可逆动作(交 orchestration)。
let tool = BrowserTool::with_data_dir(isolated_data_dir("normal"), false);
// ── 1. navigate ────────────────────────────────────────────────────────────
let nav = tool
.execute(json!({"action": "navigate", "url": fixture_url("e2e-form.html")}))
.await;
eprintln!("navigate -> is_error={} content={:?}", nav.is_error, nav.content);
assert!(!nav.is_error, "navigate must succeed: {}", nav.content);
assert!(nav.content.contains("Navigated to"), "navigate message: {}", nav.content);
// ── 2. observe(填 ref 表 + 武装注入侧 elements 缓存,act 反查的前置)────────────
let obs = tool.execute(json!({"action": "observe"})).await;
eprintln!("=== observe output ===\n{}", obs.content);
assert!(!obs.is_error, "observe must succeed: {}", obs.content);
let user_ref = find_ref(&obs.content, "textbox", "Username");
let pass_ref = find_ref(&obs.content, "textbox", "Password");
let plan_ref = find_ref(&obs.content, "combobox", "Plan");
let submit_ref = find_ref(&obs.content, "button", "Submit order");
eprintln!("refs: user={user_ref} pass={pass_ref} plan={plan_ref} submit={submit_ref}");
// ── 3. type usernameliteral)→ verify changed ──────────────────────────────
let type_user = tool
.execute(json!({"action": "type", "ref": user_ref, "text": "e2e-user"}))
.await;
eprintln!("type username -> is_error={} content={:?}", type_user.is_error, type_user.content);
assert!(!type_user.is_error, "type username must succeed: {}", type_user.content);
assert!(type_user.content.contains("changed=true"), "type username should change value: {}", type_user.content);
// ── 4. type passwordliteral——secret 路径的 fail-closed 在专门用例验,见下;正向 secret
// 路径需真 http 源 + eTLD+1,离线 file:// 测不到,由 facade/engine 既有测试覆盖)─────
let type_pass = tool
.execute(json!({"action": "type", "ref": pass_ref, "text": "literal-pw-not-secret"}))
.await;
eprintln!("type password -> is_error={} content={:?}", type_pass.is_error, type_pass.content);
assert!(!type_pass.is_error, "type password must succeed: {}", type_pass.content);
assert!(type_pass.content.contains("changed=true"), "type password should change value: {}", type_pass.content);
// ── 5. select_option Pro → verify after-anchor 含 "pro"C2 修复点:读 .value 非 textContent)─
let select = tool
.execute(json!({"action": "select_option", "ref": plan_ref, "options": ["Pro"]}))
.await;
eprintln!("select_option -> is_error={} content={:?}", select.is_error, select.content);
assert!(!select.is_error, "select_option must succeed: {}", select.content);
assert!(select.content.contains("changed=true"), "select Pro should change value (free→pro): {}", select.content);
assert!(
select.content.contains("pro"),
"select_option verify after-anchor should reflect the chosen value 'pro': {}",
select.content
);
// ── 6. 普通会话 click submitaccname="Submit order" → Irreversible)→ 门不拦 + 真提交 ──
// 先确认 facade 把它分类为 Irreversiblecategory_for 据 last_snapshot 的 accname 判)。
assert_eq!(
tool.category_for(&json!({"action": "click", "ref": submit_ref})),
nomi_protocol::events::ToolCategory::Irreversible,
"submit-order click must classify as Irreversible (so orchestration prompts in a normal session)"
);
let submit = tool
.execute(json!({"action": "click", "ref": submit_ref}))
.await;
eprintln!("click submit (normal session) -> is_error={} content={:?}", submit.is_error, submit.content);
// 普通会话:facade 门**不**hard-deny(方向正确)。click 真执行(成功或良性失败,但绝不是 Blocked)。
let lower = submit.content.to_lowercase();
assert!(
!(submit.is_error && (lower.contains("blocked") || lower.contains("irreversible"))),
"normal-session irreversible submit must NOT be hard-denied by the facade gate: {}",
submit.content
);
// ── 7. 再 observe 读回 #form-statusrole=status== submitted:e2e-user:pro ─────────
let after = tool.execute(json!({"action": "observe"})).await;
eprintln!("=== observe after submit ===\n{}", after.content);
assert!(!after.is_error, "post-submit observe must succeed: {}", after.content);
assert!(
after.content.contains("submitted:e2e-user:pro"),
"form submit should fire with the typed username + selected plan (onsubmit marker); \
observe output:\n{}",
after.content
);
eprintln!(
"=== F3 E2E READBACK SUMMARY (normal session) ===\n\
navigate = ok\n\
observe refs = user={user_ref} pass={pass_ref} plan={plan_ref} submit={submit_ref}\n\
type user = changed=true\n\
type pass = changed=true\n\
select Pro = changed=true (value 'pro')\n\
submit = NOT blocked in normal session (classified Irreversible → orchestration)\n\
form-status = submitted:e2e-user:pro (onsubmit fired)"
);
}
/// **安全门生效证据(红线):审批旁路(yolo/companion)会话里的不可逆 submit → facade hard-deny Blocked。**
///
/// 这是设计裁决⑧的端到端证明:不靠被旁路的 orchestration 审批闸,靠 facade 的独立 fail-closed 门。
/// 经 `with_policy(.., session_bypasses_approval=true, ..)` 构造一个审批旁路会话的 tool= yolo / companion
/// 强制 yolo / --auto-approve 的等价 test seam),navigate + observe 真页拿到真 submit refaccname
/// "Submit order" → 分类 Irreversible),然后 `execute(click submit)` → **Blocked**(门在 dispatch 之前拦)。
#[tokio::test]
#[ignore = "需本机/打包 chromeset NOMIFUN_CHROME_BINARY 后 --run-ignored all"]
async fn e2e_security_gate_blocks_irreversible_submit_in_bypassing_session() {
// 审批旁路会话(with_policy 第二参 = config.tools.auto_approve = true)→ redline 门武装。
// 注意:with_policy 用 app_config_dir 的 browser-data;为隔离,先 with_data_dir 再... 但 with_data_dir
// 不带 policy。这里直接用 with_policyheadless)——它的 data_dir 是 app browser-data;本测试只 navigate
// 一个 file:// fixture(不落数据),且 chrome user-data-dir 由 engine 专属管理,争用风险低。
let tool = BrowserTool::with_policy(&headless_config(), /* session_bypasses_approval */ true, false, false, None, None, None);
let nav = tool
.execute(json!({"action": "navigate", "url": fixture_url("e2e-form.html")}))
.await;
assert!(!nav.is_error, "navigate must succeed: {}", nav.content);
let obs = tool.execute(json!({"action": "observe"})).await;
assert!(!obs.is_error, "observe must succeed: {}", obs.content);
let submit_ref = find_ref(&obs.content, "button", "Submit order");
eprintln!("yolo session submit ref = {submit_ref}");
// 旁路会话 + 不可逆 submit → facade redline 门 hard-denydispatch 前拦)。
let blocked = tool
.execute(json!({"action": "click", "ref": submit_ref}))
.await;
eprintln!("yolo click submit -> is_error={} content={:?}", blocked.is_error, blocked.content);
assert!(
blocked.is_error,
"irreversible submit in an approval-bypassing session MUST be hard-denied: {}",
blocked.content
);
let lower = blocked.content.to_lowercase();
assert!(
lower.contains("blocked") || lower.contains("irreversible"),
"block message should explain the redline (blocked/irreversible): {}",
blocked.content
);
eprintln!(
"=== F3 SECURITY GATE EVIDENCE ===\n\
session = approval-bypassing (yolo/companion/auto_approve)\n\
action = click submit [ref={submit_ref}] (accname 'Submit order' → Irreversible)\n\
result = HARD-DENY Blocked (facade fail-closed gate, NOT orchestration)\n\
message = {:?}",
blocked.content
);
}
/// **安全门生效证据(secret 域绑定 fail-closed):`secret:NAME` 在无 eTLD+1 的 file:// 源 → Blocked
/// 明文绝不入输出。**
///
/// secret 正向注入路径需真 http 源(eTLD+1 域绑定),离线 file:// 无 registrable domain → 域门 fail-closed。
/// 这正好验**最关键的安全方向**:源不匹配 / 无源 → 拒绝解析,且 `secret:NAME` 字面量绝不当普通文本输入、
/// 也绝不泄漏配置的值。即便 yolo 会话也拦(门是 vault 的属性,非 orchestration 审批)。
#[tokio::test]
#[ignore = "需本机/打包 chromeset NOMIFUN_CHROME_BINARY 后 --run-ignored all"]
async fn e2e_secret_origin_gate_fails_closed_on_file_origin() {
use nomifun_secret::SecretStore;
// 配一个绑定到 example.com 的 secret(其值绝不应出现在任何输出里)。
let mut store = SecretStore::ephemeral().expect("ephemeral store");
let secret_plaintext = "F3-TOP-SECRET-PLAINTEXT-must-never-leak";
store
.register("login_pw", secret_plaintext, vec!["example.com".to_string()])
.expect("register secret");
let tool = BrowserTool::with_secret_store(isolated_data_dir("secret"), false, store);
let nav = tool
.execute(json!({"action": "navigate", "url": fixture_url("e2e-form.html")}))
.await;
assert!(!nav.is_error, "navigate must succeed: {}", nav.content);
let obs = tool.execute(json!({"action": "observe"})).await;
assert!(!obs.is_error, "observe must succeed: {}", obs.content);
let pass_ref = find_ref(&obs.content, "textbox", "Password");
// current origin = file://...e2e-form.html → 无 eTLD+1 → secret 域门 fail-closed(即便 secret 存在)。
let res = tool
.execute(json!({"action": "type", "ref": pass_ref, "text": "secret:login_pw"}))
.await;
eprintln!("type secret on file:// origin -> is_error={} content={:?}", res.is_error, res.content);
assert!(
res.is_error,
"a secret bound to example.com must NOT resolve on a file:// origin (fail-closed): {}",
res.content
);
// 安全铁律:明文绝不出现在错误输出里;`secret:login_pw` 字面量也不能被当普通文本输入(值不泄漏)。
assert!(
!res.content.contains(secret_plaintext),
"SECURITY: the secret plaintext must NEVER appear in the tool output: {}",
res.content
);
eprintln!(
"=== F3 SECRET GATE EVIDENCE ===\n\
origin = file:// (no registrable eTLD+1)\n\
secret = bound to example.com (mismatch)\n\
result = fail-closed Blocked; plaintext NOT typed, NOT in output\n\
message = {:?}",
res.content
);
}
// ─── P3 Structured Extract (real Chrome + stub model) ───────────────────────
use std::sync::Arc;
use nomi_browser::extract::ExtractModel;
/// A stub model that "extracts" by returning a hardcoded JSON response.
/// In a real scenario the LLM would parse the aria snapshot; here we simulate
/// a correct extraction to verify the end-to-end facade wiring.
struct StubExtractModel;
#[async_trait::async_trait]
impl ExtractModel for StubExtractModel {
async fn complete(&self, _prompt: &str) -> Result<String, String> {
// Return structured JSON matching the schema we'll request.
Ok(r#"{"products": [{"name": "Widget A", "price": 9.99}, {"name": "Gadget B", "price": 19.50}, {"name": "Doohickey C", "price": 4.25}]}"#.into())
}
}
/// **P3 e2e: structured extract with a stub model on a real Chrome page.**
///
/// navigate fixture table → Extract{schema} with StubExtractModel injected →
/// verify the response is the model's structured JSON (not the raw deterministic payload).
///
/// Run: `NOMIFUN_CHROME_BINARY="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" cargo nextest run -p nomi-browser --run-ignored all -E 'test(e2e_structured_extract)'`
#[tokio::test]
#[ignore = "需本机/打包 chromeset NOMIFUN_CHROME_BINARY 后 --run-ignored all"]
async fn e2e_structured_extract_with_stub_model() {
let data_dir = isolated_data_dir("extract");
let tool = BrowserTool::with_data_dir(data_dir.clone(), false)
.with_extract_model(Arc::new(StubExtractModel));
// Navigate to the fixture table.
let nav = tool
.execute(json!({"action": "navigate", "url": fixture_url("extract-products.html")}))
.await;
eprintln!("navigate -> is_error={} content={:?}", nav.is_error, nav.content);
assert!(!nav.is_error, "navigate must succeed: {}", nav.content);
// Run Extract with a schema requesting products.
let extract = tool
.execute(json!({
"action": "extract",
"schema": {
"type": "object",
"required": ["products"],
"properties": {
"products": {
"type": "array"
}
}
}
}))
.await;
eprintln!("extract -> is_error={} content={:?}", extract.is_error, extract.content);
assert!(!extract.is_error, "extract must succeed: {}", extract.content);
// The output should be the model's structured JSON (pretty-printed).
let parsed: serde_json::Value = serde_json::from_str(&extract.content)
.expect("extract output must be valid JSON when model is available");
assert!(parsed.get("products").is_some(), "response must have 'products' field");
let products = parsed["products"].as_array().unwrap();
assert_eq!(products.len(), 3, "expected 3 products");
assert_eq!(products[0]["name"], "Widget A");
assert_eq!(products[1]["price"], 19.50);
eprintln!("=== P3 STRUCTURED EXTRACT EVIDENCE ===\nmodel output parsed as valid JSON with schema fields");
let _ = std::fs::remove_dir_all(&data_dir);
}
/// **P3 e2e: extract WITHOUT model returns deterministic payload (graceful degradation).**
#[tokio::test]
#[ignore = "需本机/打包 chromeset NOMIFUN_CHROME_BINARY 后 --run-ignored all"]
async fn e2e_extract_without_model_returns_deterministic_payload() {
let data_dir = isolated_data_dir("extract-no-model");
// No model injected → graceful degradation.
let tool = BrowserTool::with_data_dir(data_dir.clone(), false);
let nav = tool
.execute(json!({"action": "navigate", "url": fixture_url("extract-products.html")}))
.await;
assert!(!nav.is_error, "navigate must succeed: {}", nav.content);
let extract = tool
.execute(json!({
"action": "extract",
"schema": { "type": "object", "required": ["products"] }
}))
.await;
eprintln!("extract (no model) -> is_error={} content length={}", extract.is_error, extract.content.len());
assert!(!extract.is_error, "extract must succeed even without model");
// Without model, the output is the deterministic payload (not JSON-parseable as structured data).
assert!(
extract.content.contains("structured page representation")
|| extract.content.contains("accessibility snapshot")
|| extract.content.contains("[visible text]"),
"without model, output must be the engine's deterministic payload, got: {}",
&extract.content[..extract.content.len().min(200)]
);
let _ = std::fs::remove_dir_all(&data_dir);
}
/// **Task 6 P7C: record→replay e2e** (`#[ignore]`, needs `NOMIFUN_CHROME_BINARY`).
///
/// Records click + type on a fixture form, replays on a fresh page, asserts the
/// same end-state. Proves the full record→replay pipeline end-to-end with a real
/// browser.
///
/// Run: `NOMIFUN_CHROME_BINARY="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" \
/// cargo nextest run -p nomi-browser --run-ignored all -E 'test(record_replay_e2e_smoke)'`
#[tokio::test]
#[ignore = "需 NOMIFUN_CHROME_BINARY(真 Chrome):record→replay 端到端冒烟"]
async fn record_replay_e2e_smoke() {
use nomi_browser::recording::{RecordedStep, Recording};
use nomi_browser::replay::ReplayRunner;
let data_dir = isolated_data_dir("record-replay");
let tool = BrowserTool::with_data_dir(data_dir.clone(), false);
// 1) Navigate to the fixture form.
let nav = tool
.execute(json!({"action": "navigate", "url": fixture_url("e2e-form.html")}))
.await;
assert!(!nav.is_error, "navigate: {}", nav.content);
// 2) Observe to get refs.
let obs = tool.execute(json!({"action": "observe"})).await;
assert!(!obs.is_error, "observe: {}", obs.content);
let user_ref = find_ref(&obs.content, "textbox", "Username");
let pass_ref = find_ref(&obs.content, "textbox", "Password");
eprintln!("record: user_ref={user_ref}, pass_ref={pass_ref}");
// 3) Start recording and type into the username field.
tool.start_recording();
assert!(tool.is_recording());
let type_res = tool
.execute(json!({"action": "type", "ref": &user_ref, "text": "replay-test-user"}))
.await;
assert!(!type_res.is_error, "type: {}", type_res.content);
let type_pass = tool
.execute(json!({"action": "type", "ref": &pass_ref, "text": "replay-pass-123"}))
.await;
assert!(!type_pass.is_error, "type pass: {}", type_pass.content);
// 4) Stop recording.
let recording = tool.stop_recording().expect("should have recording");
assert_eq!(recording.steps.len(), 2, "should have 2 recorded steps");
assert_eq!(recording.steps[0].action, "type");
assert_eq!(recording.steps[1].action, "type");
eprintln!("recorded {} steps", recording.steps.len());
// 5) Navigate to a fresh instance of the same page.
let nav2 = tool
.execute(json!({"action": "navigate", "url": fixture_url("e2e-form.html")}))
.await;
assert!(!nav2.is_error, "navigate fresh: {}", nav2.content);
// 6) Observe on the fresh page to get new refs.
let obs2 = tool.execute(json!({"action": "observe"})).await;
assert!(!obs2.is_error, "observe fresh: {}", obs2.content);
let new_user_ref = find_ref(&obs2.content, "textbox", "Username");
let new_pass_ref = find_ref(&obs2.content, "textbox", "Password");
eprintln!("replay: new_user_ref={new_user_ref}, new_pass_ref={new_pass_ref}");
// 7) Build a replay recording with the fresh refs (simulating selector→ref
// re-resolution that a real replay system would do).
let replay_recording = Recording {
steps: vec![
RecordedStep {
intent: recording.steps[0].intent.clone(),
action: "type".into(),
args: json!({"ref": &new_user_ref, "text": "replay-test-user"}),
selector: recording.steps[0].selector.clone(),
url: recording.steps[0].url.clone(),
},
RecordedStep {
intent: recording.steps[1].intent.clone(),
action: "type".into(),
args: json!({"ref": &new_pass_ref, "text": "replay-pass-123"}),
selector: recording.steps[1].selector.clone(),
url: recording.steps[1].url.clone(),
},
],
created_url: recording.created_url.clone(),
};
// 8) Replay.
let replay_result = ReplayRunner::replay(&replay_recording, &tool).await;
assert_eq!(
replay_result.succeeded, 2,
"both replay steps should succeed; outcomes: {:?}",
replay_result.outcomes.iter().map(|o| (&o.action, o.success, &o.result.content)).collect::<Vec<_>>()
);
assert_eq!(replay_result.failed, 0);
// 9) Verify the page state matches: re-observe and check the inputs have values.
let final_obs = tool.execute(json!({"action": "observe"})).await;
assert!(!final_obs.is_error, "final observe: {}", final_obs.content);
eprintln!(
"=== P7C RECORD→REPLAY E2E ===\n\
recorded = 2 type actions\n\
replayed = 2 steps, all succeeded\n\
pipeline = recording → fresh page → re-resolve refs → replay via act path\n\
gates intact = replay dispatches through execute() (same path as live actions)"
);
let _ = std::fs::remove_dir_all(&data_dir);
}
///
/// Proves the full takeover flow end-to-end against a real browser:
/// 1. Opens a headful window with a bypass session + takeover enabled.
/// 2. Navigates to a form, observes to get refs.
/// 3. Clicks the "Submit order" button (irreversible).
/// 4. With force_resolution=Confirmed, the redline gate releases the action.
/// 5. The submit actually executes (verify via re-observe).
///
/// Manual run:
/// NOMIFUN_CHROME_BINARY="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" \
/// cargo nextest run -p nomi-browser --run-ignored all -E 'test(takeover_smoke)'
#[tokio::test]
#[ignore = "requires NOMIFUN_CHROME_BINARY + display (headful takeover smoke)"]
async fn takeover_smoke_confirmed_releases_irreversible_through_facade() {
use nomi_browser::takeover::TakeoverResolution;
let data_dir = isolated_data_dir("takeover-smoke");
// Bypass session (yolo) + takeover enabled with forced Confirmed.
let mut tool = BrowserTool::with_policy(
&BrowserConfig { headless: true, ..Default::default() },
true, // session_bypasses_approval
false, // evaluate_full_power
false, // evaluate_persistent_login
None,
None,
None,
);
tool.takeover_controller_mut().enabled = true;
tool.takeover_controller_mut().force_resolution = Some(TakeoverResolution::Confirmed);
// Navigate.
let nav = tool
.execute(json!({"action": "navigate", "url": fixture_url("e2e-form.html")}))
.await;
eprintln!("takeover smoke: navigate -> {}", nav.content);
assert!(!nav.is_error, "navigate: {}", nav.content);
// Observe.
let obs = tool.execute(json!({"action": "observe"})).await;
assert!(!obs.is_error, "observe: {}", obs.content);
let submit_ref = find_ref(&obs.content, "button", "Submit order");
eprintln!("takeover smoke: submit_ref={submit_ref}");
// Click submit (irreversible in bypass session → takeover → Confirmed → proceeds).
let click = tool
.execute(json!({"action": "click", "ref": submit_ref}))
.await;
eprintln!(
"takeover smoke: click submit -> is_error={} content={}",
click.is_error,
&click.content[..click.content.len().min(200)]
);
// With Confirmed takeover, the action should proceed past the redline gate.
assert!(
!click.content.to_lowercase().contains("blocked"),
"Confirmed takeover must release the submit past the redline gate: {}",
click.content
);
// must_re_observe should be set after the Confirmed takeover.
assert!(
tool.needs_re_observe(),
"must_re_observe should be set after Confirmed takeover"
);
// Re-observe to clear the flag and verify the submit went through.
let obs2 = tool.execute(json!({"action": "observe"})).await;
assert!(!obs2.is_error, "re-observe: {}", obs2.content);
assert!(
!tool.needs_re_observe(),
"must_re_observe should be cleared after observe"
);
let _ = std::fs::remove_dir_all(&data_dir);
}
/// **Task 6 P7B: visual-fallback canvas smoke** (`#[ignore]`, needs `NOMIFUN_CHROME_BINARY`).
///
/// Navigates to a `<canvas>` fixture with NO accessible button in the DOM (the "button"
/// is drawn purely as pixels on the canvas). Asserts:
/// 1. DOM/aria anchoring fails (observe does not expose the canvas "button").
/// 2. With a stub locator returning the known button box coordinates, the visual
/// fallback click lands correctly (verified by checking `#click-result` text).
///
/// This proves the full visual fallback path end-to-end with a real Chrome:
/// navigate → observe (no ref for canvas button) → attempt click with stale/fake ref
/// → NodeStale → visual fallback → locator returns known coords → DPR mapping
/// → click_at_css_point → canvas click handler fires.
///
/// Run:
/// ```sh
/// NOMIFUN_CHROME_BINARY="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" \
/// cargo nextest run -p nomi-browser --run-ignored all -E 'test(visual_fallback_canvas_smoke)'
/// ```
#[tokio::test]
#[ignore = "需 NOMIFUN_CHROME_BINARY(真 Chrome):visual-fallback canvas 冒烟"]
async fn visual_fallback_canvas_smoke() {
use nomi_browser::visual_fallback::{PixelBox, VisualLocateResult, VisualLocator};
use std::sync::Arc;
/// Stub locator that returns the known canvas button center coordinates.
/// The button is drawn at (150, 120) with size (100, 40) — center = (200, 140).
/// In headless Chrome (DPR=1.0), pixel coords == CSS coords.
struct CanvasButtonLocator;
#[async_trait::async_trait]
impl VisualLocator for CanvasButtonLocator {
async fn locate(
&self,
_screenshot: &[u8],
_instruction: &str,
) -> Result<VisualLocateResult, String> {
Ok(VisualLocateResult {
pixel_box: PixelBox {
x: 150.0,
y: 120.0,
width: 100.0,
height: 40.0,
},
confidence: 1.0,
})
}
}
let data_dir = isolated_data_dir("visual-fallback-canvas");
let tool = BrowserTool::with_data_dir(data_dir.clone(), false)
.with_visual_fallback_enabled(true)
.with_visual_locator(Arc::new(CanvasButtonLocator));
// 1. Navigate to the canvas fixture.
let nav = tool
.execute(json!({"action": "navigate", "url": fixture_url("visual-fallback-canvas.html")}))
.await;
eprintln!("navigate -> is_error={} content={:?}", nav.is_error, nav.content);
assert!(!nav.is_error, "navigate must succeed: {}", nav.content);
// 2. Observe — the canvas button should NOT appear in the accessibility tree.
let obs = tool.execute(json!({"action": "observe"})).await;
eprintln!("=== observe output ===\n{}", obs.content);
assert!(!obs.is_error, "observe must succeed: {}", obs.content);
// The canvas is just a generic element — no "Submit" button is exposed.
assert!(
!obs.content.contains("Submit") || obs.content.contains("canvas"),
"observe must NOT expose the canvas-drawn button as an interactive element"
);
// 3. Attempt a click with a deliberately stale ref (from the observe output, there is
// no ref for the canvas button). Use a fake ref that doesn't exist — this will
// trigger NodeStale, which then triggers the visual fallback.
let click_result = tool
.execute(json!({"action": "click", "ref": "f999e999"}))
.await;
eprintln!("click (stale ref) -> is_error={} content={:?}", click_result.is_error, click_result.content);
// The visual fallback should have fired and clicked at (200, 140) CSS pixels
// (center of the button box).
assert!(
click_result.content.contains("via visual fallback"),
"expected visual fallback to fire: {}",
click_result.content
);
// 4. Verify the click actually landed on the canvas button by checking #click-result.
// Wait a moment for the click handler to fire.
tokio::time::sleep(std::time::Duration::from_millis(200)).await;
let obs2 = tool.execute(json!({"action": "observe"})).await;
eprintln!("=== post-click observe ===\n{}", obs2.content);
// The click handler sets #click-result text to "canvas-button-clicked".
assert!(
obs2.content.contains("canvas-button-clicked"),
"the visual fallback click must have landed on the canvas button (expected \
'canvas-button-clicked' in post-click observe): {}",
obs2.content
);
let _ = std::fs::remove_dir_all(&data_dir);
}
/// **P7B SoM (Set-of-Marks) visual-fallback e2e** (`#[ignore]`, needs `NOMIFUN_CHROME_BINARY`).
///
/// Proves the full SoM path on real Chrome: `observe` (with visual fallback on) collects per-ref
/// CSS-pixel boxes → a stale-ref click triggers the fallback → the facade draws a numbered overlay
/// on the screenshot and asks the (stub) locator for a label → the label maps back to the real
/// button's CSS center → the click lands on it. Three stacked real buttons make label numbering
/// deterministic: the topmost ("Alpha") is always label 1, which the stub picks.
///
/// Run: `NOMIFUN_CHROME_BINARY="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" \
/// cargo nextest run -p nomi-browser --run-ignored all -E 'test(visual_fallback_som_smoke)'`
#[tokio::test]
#[ignore = "需 NOMIFUN_CHROME_BINARY(真 Chrome):visual-fallback SoM 冒烟"]
async fn visual_fallback_som_smoke() {
use nomi_browser::visual_fallback::{SomLabelResult, VisualLocateResult, VisualLocator};
use std::sync::Arc;
/// Stub SoM locator: always picks label 1 (the topmost button = "Alpha"). Its `locate`
/// (raw bbox) returns Err so that IF the code fell back to raw instead of SoM, the click
/// would fail — making a green test proof that the SoM path actually ran.
struct PickLabelOne;
#[async_trait::async_trait]
impl VisualLocator for PickLabelOne {
async fn locate(
&self,
_screenshot: &[u8],
_instruction: &str,
) -> Result<VisualLocateResult, String> {
Err("raw bbox path must not be used in the SoM smoke".to_string())
}
async fn locate_labeled(
&self,
_annotated_screenshot: &[u8],
_instruction: &str,
_n_labels: usize,
) -> Result<SomLabelResult, String> {
Ok(SomLabelResult { label: 1, confidence: 1.0 })
}
}
let data_dir = isolated_data_dir("visual-fallback-som");
let tool = BrowserTool::with_data_dir(data_dir.clone(), false)
.with_visual_fallback_enabled(true)
.with_visual_locator(Arc::new(PickLabelOne));
// 1. Navigate to the multi-button fixture.
let nav = tool
.execute(json!({"action": "navigate", "url": fixture_url("som-fallback.html")}))
.await;
assert!(!nav.is_error, "navigate must succeed: {}", nav.content);
// 2. Observe — visual_fallback_enabled ⇒ observe collects per-ref boxes (cached for SoM).
let obs = tool.execute(json!({"action": "observe"})).await;
assert!(!obs.is_error, "observe must succeed: {}", obs.content);
// 3. Click with a deliberately stale ref → NodeStale → visual fallback → SoM mode (boxes
// are cached, count is in range). The stub picks label 1 = the topmost button "Alpha".
let click_result = tool
.execute(json!({"action": "click", "ref": "f999e999"}))
.await;
eprintln!(
"click (stale ref) -> is_error={} content={:?}",
click_result.is_error, click_result.content
);
assert!(
click_result.content.contains("via visual fallback (SoM)"),
"expected the SoM path to fire (not raw bbox): {}",
click_result.content
);
// 4. Verify the click landed on the topmost button (label 1) by its distinct result.
tokio::time::sleep(std::time::Duration::from_millis(200)).await;
let obs2 = tool.execute(json!({"action": "observe"})).await;
eprintln!("=== post-click observe ===\n{}", obs2.content);
assert!(
obs2.content.contains("top-clicked"),
"the SoM click must have landed on the topmost button (label 1) — expected \
'top-clicked' in post-click observe: {}",
obs2.content
);
let _ = std::fs::remove_dir_all(&data_dir);
}