Update: 将子项目从 submodule 转为完整内容
- 移除 GovAI, nomifun-tauri, 算力盒子 的 submodule 引用 - 添加所有子项目的完整源代码 - 保留原始 .git 为 .git.bak 备份
This commit is contained in:
@@ -0,0 +1,269 @@
|
||||
// Acceptance tests for context compression (all three compaction levels).
|
||||
//
|
||||
// TC-A2-01 and TC-A2-03 are purely local (no LLM call).
|
||||
// TC-A2-02 makes a real LLM call and is skipped when OPENAI_API_KEY is absent.
|
||||
|
||||
use nomi_agent::compact::auto::{BOUNDARY_PREFIX, autocompact, should_autocompact};
|
||||
use nomi_agent::compact::emergency::is_at_emergency_limit;
|
||||
use nomi_agent::compact::micro::{CLEARED_TOOL_RESULT, microcompact};
|
||||
use nomi_agent::compact::state::CompactState;
|
||||
use nomi_config::compact::CompactConfig;
|
||||
use nomi_types::message::{ContentBlock, Message, Role};
|
||||
use serde_json::json;
|
||||
|
||||
use crate::helpers;
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
fn tool_use_block(id: &str, name: &str) -> ContentBlock {
|
||||
ContentBlock::ToolUse {
|
||||
id: id.to_string(),
|
||||
name: name.to_string(),
|
||||
input: json!({}),
|
||||
extra: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn tool_result_block(id: &str, content: &str) -> ContentBlock {
|
||||
ContentBlock::ToolResult {
|
||||
tool_use_id: id.to_string(),
|
||||
content: content.to_string(),
|
||||
is_error: false,
|
||||
images: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
// ── TC-A2-01: Microcompact clears old tool results (LOCAL) ─────────────────
|
||||
|
||||
/// Construct a message history with more than `micro_keep_recent * 2`
|
||||
/// compactable tool results (each with a matching ToolUse block), run
|
||||
/// microcompact, and verify that old results are cleared while the most
|
||||
/// recent `micro_keep_recent` are preserved.
|
||||
#[test]
|
||||
fn microcompact_clears_old_tool_results() {
|
||||
let keep_recent: usize = 3;
|
||||
// We need MORE than keep_recent * 2 = 6 compactable results, so use 8.
|
||||
let total_results: usize = 8;
|
||||
|
||||
let config = CompactConfig {
|
||||
micro_keep_recent: keep_recent,
|
||||
compactable_tools: vec!["Read".to_string()],
|
||||
..CompactConfig::default()
|
||||
};
|
||||
|
||||
// Build messages: alternating ToolUse (assistant) and ToolResult (user)
|
||||
let mut messages: Vec<Message> = Vec::with_capacity(total_results * 2);
|
||||
for i in 0..total_results {
|
||||
let id = format!("tool_{i}");
|
||||
messages.push(Message::new(
|
||||
Role::Assistant,
|
||||
vec![tool_use_block(&id, "Read")],
|
||||
));
|
||||
messages.push(Message::new(
|
||||
Role::User,
|
||||
vec![tool_result_block(&id, &format!("content of file {i}"))],
|
||||
));
|
||||
}
|
||||
|
||||
let result = microcompact(&mut messages, &config);
|
||||
|
||||
// Verify cleared count is positive
|
||||
assert!(
|
||||
result.cleared_count > 0,
|
||||
"microcompact should clear at least one tool result, got cleared_count=0"
|
||||
);
|
||||
|
||||
// Exactly total_results - keep_recent should be cleared
|
||||
let expected_cleared = total_results - keep_recent;
|
||||
assert_eq!(
|
||||
result.cleared_count, expected_cleared,
|
||||
"expected {expected_cleared} cleared, got {}",
|
||||
result.cleared_count
|
||||
);
|
||||
|
||||
// Verify old results (first `expected_cleared`) are replaced with placeholder
|
||||
for i in 0..expected_cleared {
|
||||
let user_msg_idx = i * 2 + 1; // user messages are at odd indices
|
||||
match &messages[user_msg_idx].content[0] {
|
||||
ContentBlock::ToolResult { content, .. } => {
|
||||
assert_eq!(
|
||||
content, CLEARED_TOOL_RESULT,
|
||||
"tool result at index {i} should be cleared"
|
||||
);
|
||||
}
|
||||
other => panic!("expected ToolResult at index {user_msg_idx}, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
// Verify most recent `keep_recent` results are preserved
|
||||
for i in expected_cleared..total_results {
|
||||
let user_msg_idx = i * 2 + 1;
|
||||
match &messages[user_msg_idx].content[0] {
|
||||
ContentBlock::ToolResult { content, .. } => {
|
||||
let expected = format!("content of file {i}");
|
||||
assert_eq!(
|
||||
content, &expected,
|
||||
"tool result at index {i} should be preserved with original content"
|
||||
);
|
||||
}
|
||||
other => panic!("expected ToolResult at index {user_msg_idx}, got {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── TC-A2-02: Autocompact triggers LLM summary (REAL API CALL) ────────────
|
||||
|
||||
/// Set a very low autocompact threshold, verify should_autocompact triggers,
|
||||
/// then call autocompact with a real LLM provider and verify the result
|
||||
/// contains the boundary prefix marker.
|
||||
#[tokio::test]
|
||||
async fn autocompact_triggers_llm_summary() {
|
||||
let api_key = match helpers::openai_api_key() {
|
||||
Some(k) => k,
|
||||
None => {
|
||||
eprintln!("[acceptance] OPENAI_API_KEY not set — skipping");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Use gpt-4.1-mini which supports up to 32768 output tokens.
|
||||
// The autocompact function requests COMPACT_MAX_OUTPUT_TOKENS (20000),
|
||||
// which exceeds gpt-4o-mini's 16384 limit.
|
||||
let config = {
|
||||
let base = helpers::openai_config(&api_key);
|
||||
nomi_config::config::Config {
|
||||
model: "gpt-4.1-mini".to_string(),
|
||||
..base
|
||||
}
|
||||
};
|
||||
|
||||
let compact_config = CompactConfig {
|
||||
context_window: 1000,
|
||||
output_reserve: 100,
|
||||
autocompact_buffer: 100,
|
||||
// threshold = 1000 - 100 - 100 = 800
|
||||
..CompactConfig::default()
|
||||
};
|
||||
|
||||
// Verify should_autocompact detects the threshold is exceeded
|
||||
assert!(
|
||||
should_autocompact(900, &compact_config),
|
||||
"900 tokens should exceed the threshold of 800"
|
||||
);
|
||||
assert!(
|
||||
!should_autocompact(700, &compact_config),
|
||||
"700 tokens should be below the threshold of 800"
|
||||
);
|
||||
|
||||
// Build a simple conversation
|
||||
let messages = vec![
|
||||
Message::new(
|
||||
Role::User,
|
||||
vec![ContentBlock::Text {
|
||||
text: "Hello".to_string(),
|
||||
}],
|
||||
),
|
||||
Message::new(
|
||||
Role::Assistant,
|
||||
vec![ContentBlock::Text {
|
||||
text: "Hi there!".to_string(),
|
||||
}],
|
||||
),
|
||||
Message::new(
|
||||
Role::User,
|
||||
vec![ContentBlock::Text {
|
||||
text: "What is 2+2?".to_string(),
|
||||
}],
|
||||
),
|
||||
Message::new(
|
||||
Role::Assistant,
|
||||
vec![ContentBlock::Text {
|
||||
text: "4".to_string(),
|
||||
}],
|
||||
),
|
||||
];
|
||||
|
||||
// Create a real provider and run autocompact
|
||||
let provider = nomi_providers::create_provider(&config);
|
||||
|
||||
let state = CompactState {
|
||||
last_input_tokens: 900, // above the threshold of 800
|
||||
..CompactState::default()
|
||||
};
|
||||
// autocompact takes &mut state for recording success/failure
|
||||
let mut state = state;
|
||||
|
||||
let result = autocompact(
|
||||
provider.as_ref(),
|
||||
&messages,
|
||||
&config.model,
|
||||
&compact_config,
|
||||
&mut state,
|
||||
)
|
||||
.await;
|
||||
|
||||
let compact_result = result.expect("autocompact should succeed with a real LLM");
|
||||
|
||||
// Verify the result messages contain the boundary prefix
|
||||
let has_boundary = compact_result.messages.iter().any(|msg| {
|
||||
msg.content.iter().any(|block| {
|
||||
if let ContentBlock::Text { text } = block {
|
||||
text.starts_with(BOUNDARY_PREFIX)
|
||||
} else {
|
||||
false
|
||||
}
|
||||
})
|
||||
});
|
||||
assert!(
|
||||
has_boundary,
|
||||
"autocompact result should contain a message with the boundary prefix"
|
||||
);
|
||||
|
||||
// Verify metadata
|
||||
assert_eq!(compact_result.messages_summarized, messages.len());
|
||||
assert_eq!(compact_result.pre_compact_tokens, 900);
|
||||
}
|
||||
|
||||
// ── TC-A2-03: Emergency truncation detection (LOCAL) ───────────────────────
|
||||
|
||||
/// Verify that is_at_emergency_limit correctly detects when the token count
|
||||
/// is within the emergency buffer of the context window, and that it works
|
||||
/// even when compact is disabled.
|
||||
#[test]
|
||||
fn emergency_truncation_detection() {
|
||||
let config = CompactConfig {
|
||||
context_window: 1000,
|
||||
emergency_buffer: 100,
|
||||
// limit = 1000 - 100 = 900
|
||||
..CompactConfig::default()
|
||||
};
|
||||
|
||||
// 950 >= 900 → true (at emergency limit)
|
||||
assert!(
|
||||
is_at_emergency_limit(950, &config),
|
||||
"950 tokens should be at the emergency limit (threshold = 900)"
|
||||
);
|
||||
|
||||
// 800 < 900 → false (below emergency limit)
|
||||
assert!(
|
||||
!is_at_emergency_limit(800, &config),
|
||||
"800 tokens should be below the emergency limit (threshold = 900)"
|
||||
);
|
||||
|
||||
// Verify emergency check works even when config.enabled = false
|
||||
let disabled_config = CompactConfig {
|
||||
context_window: 1000,
|
||||
emergency_buffer: 100,
|
||||
enabled: false,
|
||||
..CompactConfig::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
is_at_emergency_limit(950, &disabled_config),
|
||||
"emergency limit should apply even when compact is disabled"
|
||||
);
|
||||
assert!(
|
||||
!is_at_emergency_limit(800, &disabled_config),
|
||||
"below-limit should still return false when compact is disabled"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
// TC-AX-01: Multi-feature collaboration test (LOCAL, no LLM).
|
||||
//
|
||||
// Exercises memory + compression + file cache + tool description all at once.
|
||||
|
||||
use nomi_agent::compact::micro::{CLEARED_TOOL_RESULT, microcompact};
|
||||
use nomi_agent::context::{SystemPromptCache, build_system_prompt};
|
||||
use nomi_config::compact::CompactConfig;
|
||||
use nomi_config::file_cache::FileCacheConfig;
|
||||
use nomi_tools::Tool;
|
||||
use nomi_tools::file_cache::FileStateCache;
|
||||
use nomi_tools::read::ReadTool;
|
||||
use nomi_types::message::{ContentBlock, Message, Role};
|
||||
use serde_json::json;
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_ax_01_multi_feature_collaboration() {
|
||||
// ── Step 1: Setup memory directory with MEMORY.md and an entry ──
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let mem_dir = tmp.path().join("memory");
|
||||
std::fs::create_dir_all(&mem_dir).unwrap();
|
||||
std::fs::write(
|
||||
mem_dir.join("MEMORY.md"),
|
||||
"- [Preferences](prefs.md) \u{2014} user prefers dark theme\n",
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(
|
||||
mem_dir.join("prefs.md"),
|
||||
"The user prefers dark theme and compact layout.\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// ── Step 2: Build system prompt with memory ──
|
||||
let system_prompt = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
Some(&mem_dir),
|
||||
false, // plan_mode_active = false
|
||||
false,
|
||||
false, // browser_enabled
|
||||
);
|
||||
|
||||
// Assert: system prompt contains memory content
|
||||
assert!(
|
||||
system_prompt.contains("auto memory"),
|
||||
"system prompt should contain memory system display name"
|
||||
);
|
||||
assert!(
|
||||
system_prompt.contains("prefs.md"),
|
||||
"system prompt should reference the memory entry file"
|
||||
);
|
||||
|
||||
// Assert: system prompt contains tool guidance
|
||||
assert!(
|
||||
system_prompt.contains("# Using your tools"),
|
||||
"system prompt should contain tool usage guidance heading"
|
||||
);
|
||||
|
||||
// ── Step 3: ReadTool dedup via FileStateCache ──
|
||||
let cache_config = FileCacheConfig {
|
||||
max_entries: 100,
|
||||
max_size_bytes: 25 * 1024 * 1024,
|
||||
enabled: true,
|
||||
};
|
||||
let cache = Arc::new(RwLock::new(FileStateCache::new(&cache_config)));
|
||||
let read_tool = ReadTool::new(Some(Arc::clone(&cache)), None);
|
||||
|
||||
let test_file = tmp.path().join("test_read.txt");
|
||||
std::fs::write(&test_file, "line one\nline two\nline three\n").unwrap();
|
||||
|
||||
let input = json!({ "file_path": test_file.to_str().unwrap() });
|
||||
|
||||
// First read: full content
|
||||
let r1 = read_tool.execute(input.clone()).await;
|
||||
assert!(!r1.is_error, "first read should succeed");
|
||||
assert!(
|
||||
r1.content.contains("line one"),
|
||||
"first read should return file content"
|
||||
);
|
||||
|
||||
// Second read: dedup stub (file unchanged)
|
||||
let r2 = read_tool.execute(input).await;
|
||||
assert!(!r2.is_error, "second read should succeed");
|
||||
assert!(
|
||||
r2.content.contains("unchanged since last read"),
|
||||
"second read should return dedup stub, got: {}",
|
||||
r2.content
|
||||
);
|
||||
|
||||
// ── Step 4: Microcompact clears old tool results ──
|
||||
let mut messages = Vec::new();
|
||||
for i in 0..8 {
|
||||
let id = format!("t{i}");
|
||||
messages.push(Message::new(
|
||||
Role::Assistant,
|
||||
vec![ContentBlock::ToolUse {
|
||||
id: id.clone(),
|
||||
name: "Read".to_string(),
|
||||
input: json!({}),
|
||||
extra: None,
|
||||
}],
|
||||
));
|
||||
messages.push(Message::new(
|
||||
Role::User,
|
||||
vec![ContentBlock::ToolResult {
|
||||
tool_use_id: id,
|
||||
content: format!("file-content-{i}"),
|
||||
is_error: false,
|
||||
images: Vec::new(),
|
||||
}],
|
||||
));
|
||||
}
|
||||
|
||||
let compact_config = CompactConfig {
|
||||
micro_keep_recent: 3,
|
||||
..CompactConfig::default()
|
||||
};
|
||||
let result = microcompact(&mut messages, &compact_config);
|
||||
|
||||
// Assert: microcompact cleared some old tool results
|
||||
assert!(
|
||||
result.cleared_count > 0,
|
||||
"microcompact should clear old tool results, got cleared_count={}",
|
||||
result.cleared_count
|
||||
);
|
||||
|
||||
// Verify cleared results contain the placeholder
|
||||
let cleared_results: Vec<_> = messages
|
||||
.iter()
|
||||
.flat_map(|m| &m.content)
|
||||
.filter(|b| matches!(b, ContentBlock::ToolResult { content, .. } if content == CLEARED_TOOL_RESULT))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
cleared_results.len(),
|
||||
result.cleared_count,
|
||||
"number of cleared placeholders should match cleared_count"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
// Acceptance tests for file cache dedup and cross-tool integration.
|
||||
//
|
||||
// These are LOCAL tests — no LLM call required.
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
use serde_json::json;
|
||||
|
||||
use nomi_config::file_cache::FileCacheConfig;
|
||||
use nomi_tools::Tool;
|
||||
use nomi_tools::edit::EditTool;
|
||||
use nomi_tools::file_cache::FileStateCache;
|
||||
use nomi_tools::read::ReadTool;
|
||||
use nomi_tools::write::WriteTool;
|
||||
|
||||
fn make_cache() -> Arc<RwLock<FileStateCache>> {
|
||||
let config = FileCacheConfig::default();
|
||||
Arc::new(RwLock::new(FileStateCache::new(&config)))
|
||||
}
|
||||
|
||||
/// TC-A5-01: Read dedup (LOCAL, no LLM).
|
||||
///
|
||||
/// Verifies that a second read of an unchanged file returns a short dedup stub
|
||||
/// instead of re-sending the full content.
|
||||
#[tokio::test]
|
||||
async fn read_dedup_returns_stub_on_second_read() {
|
||||
let cache = make_cache();
|
||||
let read_tool = ReadTool::new(Some(cache.clone()), None);
|
||||
|
||||
// Create a temporary file with known content.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let file_path = dir.path().join("dedup_test.txt");
|
||||
std::fs::write(&file_path, "line one\nline two\nline three\n").unwrap();
|
||||
let path_str = file_path.to_str().unwrap();
|
||||
|
||||
let input = json!({ "file_path": path_str });
|
||||
|
||||
// First read: should return full line-numbered content.
|
||||
let r1 = read_tool.execute(input.clone()).await;
|
||||
assert!(!r1.is_error, "first read should succeed: {}", r1.content);
|
||||
assert!(
|
||||
r1.content.contains("1\tline one"),
|
||||
"first read should contain line-numbered content, got: {}",
|
||||
r1.content
|
||||
);
|
||||
assert!(
|
||||
r1.content.contains("2\tline two"),
|
||||
"first read should contain line 2"
|
||||
);
|
||||
assert!(
|
||||
r1.content.contains("3\tline three"),
|
||||
"first read should contain line 3"
|
||||
);
|
||||
|
||||
// Second read WITHOUT modifying the file: should return dedup stub.
|
||||
let r2 = read_tool.execute(input).await;
|
||||
assert!(!r2.is_error, "second read should succeed: {}", r2.content);
|
||||
assert!(
|
||||
r2.content.contains("unchanged since last read"),
|
||||
"second read should return dedup stub, got: {}",
|
||||
r2.content
|
||||
);
|
||||
}
|
||||
|
||||
/// TC-A5-02: Write -> Edit chain (LOCAL, no LLM).
|
||||
///
|
||||
/// Verifies that WriteTool populates the cache so EditTool can immediately
|
||||
/// edit the file without a separate Read call (no "must Read first" error).
|
||||
#[tokio::test]
|
||||
async fn write_then_edit_chain_succeeds() {
|
||||
let cache = make_cache();
|
||||
let write_tool = WriteTool::new(Some(cache.clone()));
|
||||
let edit_tool = EditTool::new(Some(cache.clone()));
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let file_path = dir.path().join("write_edit_chain.txt");
|
||||
let path_str = file_path.to_str().unwrap();
|
||||
|
||||
// Write a file via WriteTool.
|
||||
let write_result = write_tool
|
||||
.execute(json!({
|
||||
"file_path": path_str,
|
||||
"content": "hello world\n"
|
||||
}))
|
||||
.await;
|
||||
assert!(
|
||||
!write_result.is_error,
|
||||
"write should succeed: {}",
|
||||
write_result.content
|
||||
);
|
||||
|
||||
// Immediately edit via EditTool — should NOT get "must Read first" error.
|
||||
let edit_result = edit_tool
|
||||
.execute(json!({
|
||||
"file_path": path_str,
|
||||
"old_string": "hello",
|
||||
"new_string": "goodbye"
|
||||
}))
|
||||
.await;
|
||||
assert!(
|
||||
!edit_result.is_error,
|
||||
"edit after write should succeed without 'must Read first' error: {}",
|
||||
edit_result.content
|
||||
);
|
||||
|
||||
// Verify file content on disk.
|
||||
let content = std::fs::read_to_string(&file_path).unwrap();
|
||||
assert_eq!(
|
||||
content, "goodbye world\n",
|
||||
"file content should reflect the edit"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
// Shared helpers for acceptance tests: provider detection and config builders.
|
||||
|
||||
use nomi_config::compat::ProviderCompat;
|
||||
use nomi_config::config::{BedrockConfig, Config, ProviderType, SessionConfig, ToolsConfig};
|
||||
use nomi_config::hooks::HooksConfig;
|
||||
use nomi_mcp::config::McpConfig;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Provider detection
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Returns the OpenAI API key if set and non-empty.
|
||||
pub fn openai_api_key() -> Option<String> {
|
||||
std::env::var("OPENAI_API_KEY")
|
||||
.ok()
|
||||
.filter(|k| !k.is_empty())
|
||||
}
|
||||
|
||||
/// Returns true when AWS Bedrock is configured for use.
|
||||
pub fn bedrock_configured() -> bool {
|
||||
let has_profile = std::env::var("AWS_PROFILE")
|
||||
.ok()
|
||||
.filter(|v| !v.is_empty())
|
||||
.is_some();
|
||||
let bedrock_flag = std::env::var("CLAUDE_CODE_USE_BEDROCK")
|
||||
.ok()
|
||||
.filter(|v| v == "1")
|
||||
.is_some();
|
||||
has_profile && bedrock_flag
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Skip macros
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Skips the current test if OPENAI_API_KEY is not set.
|
||||
/// Usage: `skip_if_no_openai!();` at the start of a test function.
|
||||
macro_rules! skip_if_no_openai {
|
||||
() => {
|
||||
#[allow(unused_variables)]
|
||||
let openai_api_key = match $crate::helpers::openai_api_key() {
|
||||
Some(k) => k,
|
||||
None => {
|
||||
eprintln!("[acceptance] OPENAI_API_KEY not set — skipping");
|
||||
return;
|
||||
}
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
/// Skips the current test if Bedrock is not configured.
|
||||
/// Usage: `skip_if_no_bedrock!();` at the start of a test function.
|
||||
macro_rules! skip_if_no_bedrock {
|
||||
() => {
|
||||
if !$crate::helpers::bedrock_configured() {
|
||||
eprintln!("[acceptance] Bedrock not configured — skipping");
|
||||
return;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Config builders
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Build a Config for the OpenAI provider (gpt-4o-mini, cheap for tests).
|
||||
pub fn openai_config(api_key: &str) -> Config {
|
||||
Config {
|
||||
provider: ProviderType::OpenAI,
|
||||
provider_label: "openai".to_string(),
|
||||
api_key: api_key.to_string(),
|
||||
base_url: "https://api.openai.com".to_string(),
|
||||
model: "gpt-4o-mini".to_string(),
|
||||
max_tokens: 256,
|
||||
max_turns: Some(3),
|
||||
system_prompt: Some("You are a helpful assistant. Be concise.".to_string()),
|
||||
thinking: None,
|
||||
prompt_caching: false,
|
||||
compat: ProviderCompat::openai_defaults(),
|
||||
tools: ToolsConfig {
|
||||
auto_approve: true,
|
||||
allow_list: vec![],
|
||||
..ToolsConfig::default()
|
||||
},
|
||||
session: SessionConfig {
|
||||
enabled: false,
|
||||
directory: "/tmp/nomi-acceptance".to_string(),
|
||||
max_sessions: 1,
|
||||
},
|
||||
compact: nomi_config::compact::CompactConfig::default(),
|
||||
plan: nomi_config::plan::PlanConfig::default(),
|
||||
file_cache: nomi_config::file_cache::FileCacheConfig::default(),
|
||||
hooks: HooksConfig::default(),
|
||||
bedrock: None,
|
||||
vertex: None,
|
||||
mcp: McpConfig::default(),
|
||||
logging: nomi_config::logging::LoggingConfig::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a Config for the AWS Bedrock provider (Claude Haiku).
|
||||
pub fn bedrock_config() -> Config {
|
||||
Config {
|
||||
provider: ProviderType::Bedrock,
|
||||
provider_label: "bedrock".to_string(),
|
||||
api_key: String::new(), // Bedrock uses AWS credentials, not API key
|
||||
base_url: String::new(),
|
||||
model: "us.anthropic.claude-haiku-4-20250514-v1:0".to_string(),
|
||||
max_tokens: 256,
|
||||
max_turns: Some(3),
|
||||
system_prompt: Some("You are a helpful assistant. Be concise.".to_string()),
|
||||
thinking: None,
|
||||
prompt_caching: false,
|
||||
compat: ProviderCompat::anthropic_defaults(),
|
||||
tools: ToolsConfig {
|
||||
auto_approve: true,
|
||||
allow_list: vec![],
|
||||
..ToolsConfig::default()
|
||||
},
|
||||
session: SessionConfig {
|
||||
enabled: false,
|
||||
directory: "/tmp/nomi-acceptance".to_string(),
|
||||
max_sessions: 1,
|
||||
},
|
||||
compact: nomi_config::compact::CompactConfig::default(),
|
||||
plan: nomi_config::plan::PlanConfig::default(),
|
||||
file_cache: nomi_config::file_cache::FileCacheConfig::default(),
|
||||
hooks: HooksConfig::default(),
|
||||
bedrock: Some(BedrockConfig::default()),
|
||||
vertex: None,
|
||||
mcp: McpConfig::default(),
|
||||
logging: nomi_config::logging::LoggingConfig::default(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
// Acceptance tests for the memory system end-to-end.
|
||||
//
|
||||
// These tests verify that the memory system's file I/O, index management,
|
||||
// and prompt building work together correctly. No LLM API calls are needed.
|
||||
|
||||
use nomi_agent::context::{SystemPromptCache, build_system_prompt};
|
||||
use nomi_memory::index::{append_index_entry, remove_index_entry};
|
||||
use nomi_memory::paths::ENTRYPOINT_NAME;
|
||||
use nomi_memory::store::{delete_memory, write_memory};
|
||||
use nomi_memory::types::{MemoryEntry, MemoryType};
|
||||
|
||||
/// TC-A1-01: Memory injection into system prompt.
|
||||
///
|
||||
/// Verifies that when a memory directory exists with an index file and a
|
||||
/// memory entry, `build_system_prompt()` produces output containing both
|
||||
/// the compact behavioral instructions and the MEMORY.md index content.
|
||||
#[test]
|
||||
fn memory_injection_into_system_prompt() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let mem_dir = tmp.path().join("memory");
|
||||
std::fs::create_dir_all(&mem_dir).unwrap();
|
||||
|
||||
// Write MEMORY.md index with one entry
|
||||
let index_path = mem_dir.join(ENTRYPOINT_NAME);
|
||||
std::fs::write(
|
||||
&index_path,
|
||||
"# Memory Index\n\n- [User role](user_role.md) \u{2014} senior Rust engineer\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Write a corresponding memory entry file
|
||||
std::fs::write(
|
||||
mem_dir.join("user_role.md"),
|
||||
"---\nname: user_role\ndescription: senior Rust engineer\ntype: user\n---\n\nThe user is a senior Rust engineer.\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let prompt = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
Some(&mem_dir),
|
||||
false,
|
||||
false,
|
||||
false, // browser_enabled
|
||||
);
|
||||
|
||||
// Behavioral instructions must be present
|
||||
assert!(
|
||||
prompt.contains("auto memory"),
|
||||
"system prompt should contain the memory display name"
|
||||
);
|
||||
assert!(
|
||||
prompt.contains("Memory types:"),
|
||||
"system prompt should contain the compact memory type summary"
|
||||
);
|
||||
|
||||
// MEMORY.md content must be injected
|
||||
assert!(
|
||||
prompt.contains("user_role.md"),
|
||||
"system prompt should contain the MEMORY.md index filename reference"
|
||||
);
|
||||
assert!(
|
||||
prompt.contains("senior Rust engineer"),
|
||||
"system prompt should contain the MEMORY.md index summary"
|
||||
);
|
||||
}
|
||||
|
||||
/// TC-A1-02: Memory full lifecycle (create, index, verify, delete, verify gone).
|
||||
///
|
||||
/// Exercises the complete lifecycle of a memory entry through the public API:
|
||||
/// 1. write_memory() -> create the file
|
||||
/// 2. append_index_entry() -> add to MEMORY.md
|
||||
/// 3. build_system_prompt() -> verify the content appears
|
||||
/// 4. delete_memory() -> remove the file
|
||||
/// 5. remove_index_entry() -> clean the index
|
||||
/// 6. build_system_prompt() -> verify the content is gone
|
||||
#[test]
|
||||
fn memory_full_lifecycle() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let mem_dir = tmp.path().join("memory");
|
||||
std::fs::create_dir_all(&mem_dir).unwrap();
|
||||
|
||||
let index_path = mem_dir.join(ENTRYPOINT_NAME);
|
||||
|
||||
// -- Phase 1: Create memory entry via the store API -----------------------
|
||||
|
||||
let entry = MemoryEntry::build(
|
||||
"project status",
|
||||
"current sprint goals",
|
||||
MemoryType::Project,
|
||||
"We are migrating the auth service to the new provider.",
|
||||
);
|
||||
|
||||
let entry_path = write_memory(&mem_dir, &entry).unwrap();
|
||||
assert!(entry_path.exists(), "memory file should be created on disk");
|
||||
|
||||
let entry_filename = entry_path.file_name().unwrap().to_str().unwrap().to_owned();
|
||||
|
||||
// -- Phase 2: Add the entry to the MEMORY.md index ------------------------
|
||||
|
||||
append_index_entry(
|
||||
&index_path,
|
||||
"Project status",
|
||||
&entry_filename,
|
||||
"current sprint goals",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let index_content = std::fs::read_to_string(&index_path).unwrap();
|
||||
assert!(
|
||||
index_content.contains(&entry_filename),
|
||||
"MEMORY.md should reference the new entry file"
|
||||
);
|
||||
|
||||
// -- Phase 3: Verify system prompt includes the memory content ------------
|
||||
|
||||
let prompt_with_memory = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
Some(&mem_dir),
|
||||
false,
|
||||
false,
|
||||
false, // browser_enabled
|
||||
);
|
||||
|
||||
assert!(
|
||||
prompt_with_memory.contains("auto memory"),
|
||||
"prompt should contain behavioral instructions"
|
||||
);
|
||||
assert!(
|
||||
prompt_with_memory.contains("Memory types:"),
|
||||
"prompt should contain compact memory type summary"
|
||||
);
|
||||
assert!(
|
||||
prompt_with_memory.contains(&entry_filename),
|
||||
"prompt should contain the memory entry filename from the index"
|
||||
);
|
||||
assert!(
|
||||
prompt_with_memory.contains("current sprint goals"),
|
||||
"prompt should contain the index summary"
|
||||
);
|
||||
|
||||
// -- Phase 4: Delete the memory file --------------------------------------
|
||||
|
||||
delete_memory(&entry_path).unwrap();
|
||||
assert!(
|
||||
!entry_path.exists(),
|
||||
"memory file should be removed from disk"
|
||||
);
|
||||
|
||||
// -- Phase 5: Remove the entry from the MEMORY.md index -------------------
|
||||
|
||||
remove_index_entry(&index_path, &entry_filename).unwrap();
|
||||
|
||||
let index_after = std::fs::read_to_string(&index_path).unwrap();
|
||||
assert!(
|
||||
!index_after.contains(&entry_filename),
|
||||
"MEMORY.md should no longer reference the deleted entry"
|
||||
);
|
||||
|
||||
// -- Phase 6: Verify the content is gone from the system prompt -----------
|
||||
|
||||
let prompt_after_delete = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
Some(&mem_dir),
|
||||
false,
|
||||
false,
|
||||
false, // browser_enabled
|
||||
);
|
||||
|
||||
assert!(
|
||||
!prompt_after_delete.contains(&entry_filename),
|
||||
"prompt should no longer contain the deleted entry filename"
|
||||
);
|
||||
assert!(
|
||||
!prompt_after_delete.contains("current sprint goals"),
|
||||
"prompt should no longer contain the deleted entry summary"
|
||||
);
|
||||
// With everything removed, the index is empty — the prompt should show the empty state
|
||||
assert!(
|
||||
prompt_after_delete.contains("currently empty"),
|
||||
"prompt should show empty memory state after all entries are removed"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
#![allow(dead_code, unused_macros, unused_imports)]
|
||||
/// Acceptance tests for evolution features (Phase 6).
|
||||
///
|
||||
/// These tests validate end-to-end behavior of each evolution feature
|
||||
/// against real LLM providers. They are skipped when provider credentials
|
||||
/// are absent, making them safe to run in any environment.
|
||||
///
|
||||
/// Required env vars (at least one):
|
||||
/// OPENAI_API_KEY — runs OpenAI provider tests
|
||||
/// AWS_PROFILE + CLAUDE_CODE_USE_BEDROCK=1 — runs Bedrock provider tests
|
||||
///
|
||||
/// Run manually:
|
||||
/// OPENAI_API_KEY=sk-... cargo nextest run -p nomi-agent --profile e2e --test acceptance
|
||||
#[macro_use]
|
||||
mod helpers;
|
||||
mod compact_test;
|
||||
mod cross_feature_test;
|
||||
mod file_cache_test;
|
||||
mod memory_test;
|
||||
mod plan_mode_test;
|
||||
mod tool_desc_test;
|
||||
@@ -0,0 +1,226 @@
|
||||
// Acceptance tests for Plan Mode tool filtering and prompt injection (Task 6.4).
|
||||
//
|
||||
// These tests are LOCAL (no LLM required) and verify that:
|
||||
// - Tool registry filtering produces the correct tool sets for normal vs plan mode
|
||||
// - System prompt correctly includes/excludes plan mode instructions
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use nomi_agent::context::{SystemPromptCache, build_system_prompt};
|
||||
use nomi_agent::plan::tools::{EnterPlanModeTool, ExitPlanModeTool};
|
||||
use nomi_protocol::events::ToolCategory;
|
||||
use nomi_tools::Tool;
|
||||
use nomi_tools::registry::ToolRegistry;
|
||||
use nomi_types::tool::ToolResult;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers: mock tool with configurable category
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
struct CategoryMockTool {
|
||||
tool_name: String,
|
||||
cat: ToolCategory,
|
||||
}
|
||||
|
||||
impl CategoryMockTool {
|
||||
fn new(name: &str, cat: ToolCategory) -> Self {
|
||||
Self {
|
||||
tool_name: name.to_string(),
|
||||
cat,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for CategoryMockTool {
|
||||
fn name(&self) -> &str {
|
||||
&self.tool_name
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"mock"
|
||||
}
|
||||
|
||||
fn input_schema(&self) -> Value {
|
||||
json!({"type": "object"})
|
||||
}
|
||||
|
||||
fn category(&self) -> ToolCategory {
|
||||
self.cat
|
||||
}
|
||||
|
||||
fn is_concurrency_safe(&self, _input: &Value) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
async fn execute(&self, _input: Value) -> ToolResult {
|
||||
ToolResult {
|
||||
content: String::new(),
|
||||
is_error: false,
|
||||
images: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-A3-01: Plan Mode tool filtering (LOCAL, no LLM)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_a3_01_plan_mode_tool_filtering() {
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let mut registry = ToolRegistry::new();
|
||||
|
||||
// Info category tools
|
||||
registry.register(Box::new(CategoryMockTool::new("Read", ToolCategory::Info)));
|
||||
registry.register(Box::new(CategoryMockTool::new("Grep", ToolCategory::Info)));
|
||||
registry.register(Box::new(EnterPlanModeTool::new(Arc::clone(&flag))));
|
||||
registry.register(Box::new(ExitPlanModeTool::new(Arc::clone(&flag))));
|
||||
|
||||
// Edit category tool
|
||||
registry.register(Box::new(CategoryMockTool::new("Write", ToolCategory::Edit)));
|
||||
|
||||
// Exec category tool
|
||||
registry.register(Box::new(CategoryMockTool::new("Bash", ToolCategory::Exec)));
|
||||
|
||||
// --- Normal mode: all tools except ExitPlanMode ---
|
||||
let normal_defs = registry.to_tool_defs_filtered(|t| t.name() != "ExitPlanMode");
|
||||
let normal_names: Vec<&str> = normal_defs.iter().map(|d| d.name.as_str()).collect();
|
||||
|
||||
assert!(
|
||||
!normal_names.contains(&"ExitPlanMode"),
|
||||
"ExitPlanMode should be excluded in normal mode"
|
||||
);
|
||||
assert!(
|
||||
normal_names.contains(&"Read"),
|
||||
"Read should be present in normal mode"
|
||||
);
|
||||
assert!(
|
||||
normal_names.contains(&"Grep"),
|
||||
"Grep should be present in normal mode"
|
||||
);
|
||||
assert!(
|
||||
normal_names.contains(&"Write"),
|
||||
"Write should be present in normal mode"
|
||||
);
|
||||
assert!(
|
||||
normal_names.contains(&"Bash"),
|
||||
"Bash should be present in normal mode"
|
||||
);
|
||||
assert!(
|
||||
normal_names.contains(&"EnterPlanMode"),
|
||||
"EnterPlanMode should be present in normal mode"
|
||||
);
|
||||
|
||||
// --- Plan mode: only Info tools, excluding EnterPlanMode ---
|
||||
let plan_defs = registry.to_tool_defs_filtered(|t| {
|
||||
t.category() == ToolCategory::Info && t.name() != "EnterPlanMode"
|
||||
});
|
||||
let plan_names: Vec<&str> = plan_defs.iter().map(|d| d.name.as_str()).collect();
|
||||
|
||||
// Info tools should be present
|
||||
assert!(
|
||||
plan_names.contains(&"Read"),
|
||||
"Read (Info) should be available in plan mode"
|
||||
);
|
||||
assert!(
|
||||
plan_names.contains(&"Grep"),
|
||||
"Grep (Info) should be available in plan mode"
|
||||
);
|
||||
assert!(
|
||||
plan_names.contains(&"ExitPlanMode"),
|
||||
"ExitPlanMode (Info) should be available in plan mode"
|
||||
);
|
||||
|
||||
// EnterPlanMode should be excluded
|
||||
assert!(
|
||||
!plan_names.contains(&"EnterPlanMode"),
|
||||
"EnterPlanMode should be excluded in plan mode"
|
||||
);
|
||||
|
||||
// Edit and Exec tools should be excluded
|
||||
assert!(
|
||||
!plan_names.contains(&"Write"),
|
||||
"Write (Edit) should be excluded in plan mode"
|
||||
);
|
||||
assert!(
|
||||
!plan_names.contains(&"Bash"),
|
||||
"Bash (Exec) should be excluded in plan mode"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-A3-02: Plan Mode system prompt injection (LOCAL, no LLM)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_a3_02_plan_mode_system_prompt_injection() {
|
||||
// --- Plan mode active: prompt should contain plan mode instructions ---
|
||||
let active_prompt = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
None,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
|
||||
assert!(
|
||||
active_prompt.contains("# Plan Mode"),
|
||||
"active prompt should contain plan mode heading"
|
||||
);
|
||||
assert!(
|
||||
active_prompt.contains("Understand"),
|
||||
"active prompt should mention Phase 1: Understand"
|
||||
);
|
||||
assert!(
|
||||
active_prompt.contains("Design"),
|
||||
"active prompt should mention Phase 2: Design"
|
||||
);
|
||||
assert!(
|
||||
active_prompt.contains("Write the plan"),
|
||||
"active prompt should mention Phase 3: Write the plan"
|
||||
);
|
||||
assert!(
|
||||
active_prompt.contains("Submit for review"),
|
||||
"active prompt should mention Phase 4: Submit for review"
|
||||
);
|
||||
assert!(
|
||||
active_prompt.contains("Forbidden"),
|
||||
"active prompt should mention forbidden actions"
|
||||
);
|
||||
assert!(
|
||||
active_prompt.contains("ExitPlanMode"),
|
||||
"active prompt should reference ExitPlanMode tool"
|
||||
);
|
||||
|
||||
// --- Plan mode inactive: prompt should NOT contain plan mode instructions ---
|
||||
let inactive_prompt = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
|
||||
assert!(
|
||||
!inactive_prompt.contains("# Plan Mode"),
|
||||
"inactive prompt should NOT contain plan mode heading"
|
||||
);
|
||||
assert!(
|
||||
!inactive_prompt.contains("Forbidden actions"),
|
||||
"inactive prompt should NOT contain forbidden actions section"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
// Acceptance test for tool usage guidance in the system prompt (TC-A4-01).
|
||||
//
|
||||
// This is a LOCAL test — no LLM call required.
|
||||
|
||||
use nomi_agent::context::{SystemPromptCache, build_system_prompt};
|
||||
|
||||
/// TC-A4-01: System prompt contains tool guidance.
|
||||
///
|
||||
/// Calls `build_system_prompt` with minimal arguments and verifies that the
|
||||
/// returned prompt includes the tool-usage guidance section with all expected
|
||||
/// content: heading, Bash prohibition mappings, parallel call guidance,
|
||||
/// Edit-over-Write preference, and Read-before-Edit rule.
|
||||
#[test]
|
||||
fn system_prompt_contains_tool_guidance() {
|
||||
let prompt = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
|
||||
// 1. Heading
|
||||
assert!(
|
||||
prompt.contains("# Using your tools"),
|
||||
"system prompt must contain the '# Using your tools' heading"
|
||||
);
|
||||
|
||||
// 2. Bash prohibition mappings — dedicated tool replacements
|
||||
assert!(
|
||||
prompt.contains("Glob"),
|
||||
"should mention Glob as replacement for find/ls"
|
||||
);
|
||||
assert!(
|
||||
prompt.contains("Grep"),
|
||||
"should mention Grep as replacement for grep/rg"
|
||||
);
|
||||
assert!(
|
||||
prompt.contains("Read"),
|
||||
"should mention Read as replacement for cat/head/tail"
|
||||
);
|
||||
assert!(
|
||||
prompt.contains("Edit"),
|
||||
"should mention Edit as replacement for sed/awk"
|
||||
);
|
||||
assert!(
|
||||
prompt.contains("Write"),
|
||||
"should mention Write as replacement for echo redirection"
|
||||
);
|
||||
|
||||
// 3. Parallel call guidance
|
||||
assert!(
|
||||
prompt.contains("parallel"),
|
||||
"should contain parallel call guidance"
|
||||
);
|
||||
|
||||
// 4. Edit-over-Write preference
|
||||
assert!(
|
||||
prompt.contains("Prefer Edit over Write"),
|
||||
"should contain Edit-over-Write preference"
|
||||
);
|
||||
|
||||
// 5. Read-before-Edit rule
|
||||
assert!(
|
||||
prompt.contains("Read a file before editing"),
|
||||
"should contain Read-before-Edit rule"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,512 @@
|
||||
//! Black-box integration tests for the autocompact subsystem.
|
||||
//!
|
||||
//! These tests correspond to TC-2.4-* in the test plan.
|
||||
//! They exercise the public autocompact API with a mock LLM provider,
|
||||
//! validating trigger logic, summary formatting, boundary markers,
|
||||
//! circuit breaker, and PTL retry behaviour.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use nomi_agent::compact::auto::{
|
||||
CompactError, autocompact, extract_compact_metadata, is_compact_boundary, should_autocompact,
|
||||
};
|
||||
use nomi_agent::compact::prompt::{
|
||||
build_compact_prompt, build_summary_content, format_compact_summary,
|
||||
};
|
||||
use nomi_agent::compact::state::CompactState;
|
||||
use nomi_config::compact::CompactConfig;
|
||||
use nomi_providers::{LlmProvider, ProviderError};
|
||||
use nomi_types::compact::CompactTrigger;
|
||||
use nomi_types::llm::{LlmEvent, LlmRequest};
|
||||
use nomi_types::message::{ContentBlock, Message, Role, StopReason, TokenUsage};
|
||||
|
||||
// ── Mock provider ───────────────────────────────────────────────────────────
|
||||
|
||||
/// A mock LLM provider that returns pre-configured responses in order.
|
||||
struct MockProvider {
|
||||
responses: Mutex<VecDeque<Result<Vec<LlmEvent>, ProviderError>>>,
|
||||
}
|
||||
|
||||
impl MockProvider {
|
||||
fn new(responses: Vec<Result<Vec<LlmEvent>, ProviderError>>) -> Self {
|
||||
Self {
|
||||
responses: Mutex::new(VecDeque::from(responses)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a provider that returns a successful summary response.
|
||||
fn with_summary(summary: &str) -> Self {
|
||||
Self::new(vec![Ok(vec![
|
||||
LlmEvent::TextDelta(summary.to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 50_000,
|
||||
output_tokens: 2_000,
|
||||
..Default::default()
|
||||
},
|
||||
},
|
||||
])])
|
||||
}
|
||||
|
||||
/// Create a provider that returns an error.
|
||||
fn with_error(error: ProviderError) -> Self {
|
||||
Self::new(vec![Err(error)])
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl LlmProvider for MockProvider {
|
||||
async fn stream(
|
||||
&self,
|
||||
_request: &LlmRequest,
|
||||
) -> Result<mpsc::Receiver<LlmEvent>, ProviderError> {
|
||||
let response = self
|
||||
.responses
|
||||
.lock()
|
||||
.unwrap()
|
||||
.pop_front()
|
||||
.expect("MockProvider: no more responses queued");
|
||||
|
||||
match response {
|
||||
Ok(events) => {
|
||||
let (tx, rx) = mpsc::channel(events.len() + 1);
|
||||
for event in events {
|
||||
tx.send(event).await.ok();
|
||||
}
|
||||
Ok(rx)
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
fn text_msg(role: Role, content: &str) -> Message {
|
||||
Message::new(
|
||||
role,
|
||||
vec![ContentBlock::Text {
|
||||
text: content.to_string(),
|
||||
}],
|
||||
)
|
||||
}
|
||||
|
||||
fn sample_conversation(n: usize) -> Vec<Message> {
|
||||
(0..n)
|
||||
.map(|i| {
|
||||
let role = if i % 2 == 0 {
|
||||
Role::User
|
||||
} else {
|
||||
Role::Assistant
|
||||
};
|
||||
text_msg(role, &format!("message-{i}"))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn default_config() -> CompactConfig {
|
||||
CompactConfig::default()
|
||||
}
|
||||
|
||||
// ── TC-2.4-01: Watermark above threshold triggers ───────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_4_01_above_threshold_triggers() {
|
||||
// effective_window = 200k - 20k = 180k, threshold = 180k - 13k = 167k
|
||||
assert!(should_autocompact(170_000, &default_config()));
|
||||
}
|
||||
|
||||
// ── TC-2.4-02: Below threshold does not trigger ─────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_4_02_below_threshold_does_not_trigger() {
|
||||
assert!(!should_autocompact(160_000, &default_config()));
|
||||
}
|
||||
|
||||
// ── TC-2.4-03: Exact threshold triggers ─────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_4_03_at_exact_threshold_triggers() {
|
||||
assert!(should_autocompact(167_000, &default_config()));
|
||||
}
|
||||
|
||||
// ── TC-2.4-04: Circuit breaker initial state ────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_4_04_initial_state_not_broken() {
|
||||
let state = CompactState::new();
|
||||
assert_eq!(state.consecutive_failures, 0);
|
||||
assert!(!state.is_circuit_broken(&default_config()));
|
||||
}
|
||||
|
||||
// ── TC-2.4-05: Circuit breaker trips ────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_4_05_circuit_breaker_trips() {
|
||||
let config = default_config();
|
||||
let mut state = CompactState::new();
|
||||
state.record_failure();
|
||||
state.record_failure();
|
||||
state.record_failure();
|
||||
assert!(state.is_circuit_broken(&config));
|
||||
}
|
||||
|
||||
// ── TC-2.4-06: Circuit breaker resets ───────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_4_06_circuit_breaker_resets_on_success() {
|
||||
let config = default_config();
|
||||
let mut state = CompactState::new();
|
||||
state.record_failure();
|
||||
state.record_failure();
|
||||
state.record_success();
|
||||
assert_eq!(state.consecutive_failures, 0);
|
||||
assert!(!state.is_circuit_broken(&config));
|
||||
}
|
||||
|
||||
// ── TC-2.4-07: Circuit breaker blocks autocompact ───────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_2_4_07_circuit_breaker_blocks_autocompact() {
|
||||
let provider = MockProvider::with_summary("<summary>should not be called</summary>");
|
||||
let messages = sample_conversation(10);
|
||||
let config = default_config();
|
||||
let mut state = CompactState::new();
|
||||
state.record_failure();
|
||||
state.record_failure();
|
||||
state.record_failure();
|
||||
|
||||
let result = autocompact(&provider, &messages, "test-model", &config, &mut state).await;
|
||||
assert!(matches!(result, Err(CompactError::CircuitBroken { .. })));
|
||||
}
|
||||
|
||||
// ── TC-2.4-08: Prompt contains all 9 sections ──────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_4_08_prompt_contains_all_sections() {
|
||||
let prompt = build_compact_prompt();
|
||||
for i in 1..=9 {
|
||||
assert!(prompt.contains(&format!("{i}.")), "Missing section {i}");
|
||||
}
|
||||
assert!(prompt.contains("CRITICAL: Respond with TEXT ONLY"));
|
||||
}
|
||||
|
||||
// ── TC-2.4-09: Summary formatting (normal) ──────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_4_09_format_strips_analysis_extracts_summary() {
|
||||
let raw = "<analysis>thinking</analysis>\n<summary>result</summary>";
|
||||
assert_eq!(format_compact_summary(raw), "Summary:\nresult");
|
||||
}
|
||||
|
||||
// ── TC-2.4-10: Summary formatting (no analysis) ────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_4_10_format_without_analysis() {
|
||||
let raw = "<summary>result</summary>";
|
||||
assert_eq!(format_compact_summary(raw), "Summary:\nresult");
|
||||
}
|
||||
|
||||
// ── TC-2.4-11: Summary formatting (no tags) ────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_4_11_format_graceful_degradation() {
|
||||
let raw = "plain text without tags";
|
||||
assert_eq!(format_compact_summary(raw), "plain text without tags");
|
||||
}
|
||||
|
||||
// ── TC-2.4-12: Post-compact message structure ───────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_2_4_12_post_compact_message_structure() {
|
||||
let summary = "<analysis>thinking</analysis>\n<summary>Detailed summary here</summary>";
|
||||
let provider = MockProvider::with_summary(summary);
|
||||
let messages = sample_conversation(20);
|
||||
let config = default_config();
|
||||
let mut state = CompactState::new();
|
||||
state.last_input_tokens = 170_000;
|
||||
|
||||
let result = autocompact(&provider, &messages, "test-model", &config, &mut state)
|
||||
.await
|
||||
.expect("autocompact should succeed");
|
||||
|
||||
// Should have 2 messages: boundary + summary
|
||||
assert_eq!(result.messages.len(), 2);
|
||||
assert_eq!(result.messages_summarized, 20);
|
||||
|
||||
// First message is the boundary marker
|
||||
assert!(is_compact_boundary(&result.messages[0]));
|
||||
assert_eq!(result.messages[0].role, Role::User);
|
||||
|
||||
// Second message is the summary
|
||||
assert_eq!(result.messages[1].role, Role::User);
|
||||
match &result.messages[1].content[0] {
|
||||
ContentBlock::Text { text } => {
|
||||
assert!(text.contains("Detailed summary here"));
|
||||
assert!(text.contains("This session is being continued"));
|
||||
}
|
||||
_ => panic!("expected Text block"),
|
||||
}
|
||||
}
|
||||
|
||||
// ── TC-2.4-13: Boundary marker metadata ─────────────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_2_4_13_boundary_metadata() {
|
||||
let provider = MockProvider::with_summary("<summary>summary</summary>");
|
||||
let messages = sample_conversation(15);
|
||||
let config = default_config();
|
||||
let mut state = CompactState::new();
|
||||
state.last_input_tokens = 170_000;
|
||||
|
||||
let result = autocompact(&provider, &messages, "test-model", &config, &mut state)
|
||||
.await
|
||||
.expect("autocompact should succeed");
|
||||
|
||||
let metadata = extract_compact_metadata(&result.messages[0]).expect("should have metadata");
|
||||
assert_eq!(metadata.trigger, CompactTrigger::Auto);
|
||||
assert_eq!(metadata.pre_compact_tokens, 170_000);
|
||||
assert_eq!(metadata.messages_summarized, 15);
|
||||
}
|
||||
|
||||
// ── TC-2.4-14: Disabled config skips (tested via should_autocompact) ────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_4_14_disabled_config_skips() {
|
||||
let config = CompactConfig {
|
||||
enabled: false,
|
||||
..default_config()
|
||||
};
|
||||
assert!(!should_autocompact(999_999, &config));
|
||||
}
|
||||
|
||||
// ── TC-2.4-15: Prompt forbids tool calls ────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_4_15_prompt_forbids_tool_calls() {
|
||||
let prompt = build_compact_prompt();
|
||||
assert!(prompt.contains("Do NOT call any tools"));
|
||||
}
|
||||
|
||||
// ── TC-2.4-16: Success resets failure counter ───────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_2_4_16_success_resets_failure_counter() {
|
||||
let provider = MockProvider::with_summary("<summary>summary</summary>");
|
||||
let messages = sample_conversation(10);
|
||||
let config = default_config();
|
||||
let mut state = CompactState::new();
|
||||
state.consecutive_failures = 2;
|
||||
state.last_input_tokens = 170_000;
|
||||
|
||||
autocompact(&provider, &messages, "test-model", &config, &mut state)
|
||||
.await
|
||||
.expect("autocompact should succeed");
|
||||
|
||||
assert_eq!(state.consecutive_failures, 0);
|
||||
}
|
||||
|
||||
// ── TC-2.4-17: Failure increments failure counter ───────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_2_4_17_failure_increments_counter() {
|
||||
let provider = MockProvider::with_error(ProviderError::Api {
|
||||
status: 500,
|
||||
message: "Internal error".to_string(),
|
||||
});
|
||||
let messages = sample_conversation(10);
|
||||
let config = default_config();
|
||||
let mut state = CompactState::new();
|
||||
|
||||
let result = autocompact(&provider, &messages, "test-model", &config, &mut state).await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(state.consecutive_failures, 1);
|
||||
}
|
||||
|
||||
// ── TC-2.4-18: PTL retry succeeds on second attempt ────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_2_4_18_ptl_retry_succeeds() {
|
||||
let provider = MockProvider::new(vec![
|
||||
// First attempt: prompt too long
|
||||
Err(ProviderError::PromptTooLong(
|
||||
"prompt exceeds limit".to_string(),
|
||||
)),
|
||||
// Second attempt (after truncation): success
|
||||
Ok(vec![
|
||||
LlmEvent::TextDelta("<summary>retried summary</summary>".to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage::default(),
|
||||
},
|
||||
]),
|
||||
]);
|
||||
|
||||
let messages = sample_conversation(20);
|
||||
let config = default_config();
|
||||
let mut state = CompactState::new();
|
||||
state.last_input_tokens = 170_000;
|
||||
|
||||
let result = autocompact(&provider, &messages, "test-model", &config, &mut state)
|
||||
.await
|
||||
.expect("autocompact should succeed after retry");
|
||||
|
||||
assert_eq!(result.messages.len(), 2);
|
||||
assert_eq!(state.consecutive_failures, 0);
|
||||
|
||||
// Verify summary content
|
||||
match &result.messages[1].content[0] {
|
||||
ContentBlock::Text { text } => {
|
||||
assert!(text.contains("retried summary"));
|
||||
}
|
||||
_ => panic!("expected Text block"),
|
||||
}
|
||||
}
|
||||
|
||||
// ── TC-2.4-19: PTL retry exhausted ─────────────────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_2_4_19_ptl_retry_exhausted() {
|
||||
let provider = MockProvider::new(vec![
|
||||
Err(ProviderError::PromptTooLong("too long 1".to_string())),
|
||||
Err(ProviderError::PromptTooLong("too long 2".to_string())),
|
||||
Err(ProviderError::PromptTooLong("too long 3".to_string())),
|
||||
]);
|
||||
|
||||
let messages = sample_conversation(20);
|
||||
let config = default_config();
|
||||
let mut state = CompactState::new();
|
||||
|
||||
let result = autocompact(&provider, &messages, "test-model", &config, &mut state).await;
|
||||
assert!(matches!(result, Err(CompactError::PromptTooLong { .. })));
|
||||
assert_eq!(state.consecutive_failures, 1);
|
||||
}
|
||||
|
||||
// ── TC-2.4-20: PTL retry truncates messages ─────────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_2_4_20_ptl_retry_truncates_messages() {
|
||||
// Track the request message count on each attempt
|
||||
let request_counts: std::sync::Arc<Mutex<Vec<usize>>> =
|
||||
std::sync::Arc::new(Mutex::new(Vec::new()));
|
||||
let counts_clone = request_counts.clone();
|
||||
|
||||
// Custom mock that records message counts
|
||||
struct CountingProvider {
|
||||
counts: std::sync::Arc<Mutex<Vec<usize>>>,
|
||||
attempt: Mutex<u32>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl LlmProvider for CountingProvider {
|
||||
async fn stream(
|
||||
&self,
|
||||
request: &LlmRequest,
|
||||
) -> Result<mpsc::Receiver<LlmEvent>, ProviderError> {
|
||||
// Scope the lock so the MutexGuard is dropped before the await
|
||||
let current_attempt = {
|
||||
let mut attempt = self.attempt.lock().unwrap();
|
||||
self.counts.lock().unwrap().push(request.messages.len());
|
||||
let val = *attempt;
|
||||
*attempt += 1;
|
||||
val
|
||||
};
|
||||
|
||||
if current_attempt == 0 {
|
||||
return Err(ProviderError::PromptTooLong("too long".to_string()));
|
||||
}
|
||||
|
||||
// Second attempt: succeed
|
||||
let (tx, rx) = mpsc::channel(2);
|
||||
tx.send(LlmEvent::TextDelta(
|
||||
"<summary>truncated summary</summary>".to_string(),
|
||||
))
|
||||
.await
|
||||
.ok();
|
||||
tx.send(LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage::default(),
|
||||
})
|
||||
.await
|
||||
.ok();
|
||||
Ok(rx)
|
||||
}
|
||||
}
|
||||
|
||||
let provider = CountingProvider {
|
||||
counts: counts_clone,
|
||||
attempt: Mutex::new(0),
|
||||
};
|
||||
|
||||
let messages = sample_conversation(20);
|
||||
let config = default_config();
|
||||
let mut state = CompactState::new();
|
||||
state.last_input_tokens = 170_000;
|
||||
|
||||
autocompact(&provider, &messages, "test-model", &config, &mut state)
|
||||
.await
|
||||
.expect("should succeed after retry");
|
||||
|
||||
let counts = request_counts.lock().unwrap();
|
||||
assert_eq!(counts.len(), 2, "should have 2 attempts");
|
||||
|
||||
// First attempt: 20 conversation + 1 prompt = 21
|
||||
assert_eq!(counts[0], 21);
|
||||
|
||||
// Second attempt: truncated (~20% dropped from 20 = 4 dropped) + 1 prompt
|
||||
// 20 - 4 = 16, + 1 prompt = 17
|
||||
assert_eq!(counts[1], 17);
|
||||
}
|
||||
|
||||
// ── Additional edge cases ───────────────────────────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_response_fails() {
|
||||
// Provider returns Done without any TextDelta
|
||||
let provider = MockProvider::new(vec![Ok(vec![LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage::default(),
|
||||
}])]);
|
||||
|
||||
let messages = sample_conversation(10);
|
||||
let config = default_config();
|
||||
let mut state = CompactState::new();
|
||||
|
||||
let result = autocompact(&provider, &messages, "test-model", &config, &mut state).await;
|
||||
assert!(matches!(result, Err(CompactError::EmptyResponse)));
|
||||
assert_eq!(state.consecutive_failures, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stream_error_fails() {
|
||||
let provider = MockProvider::new(vec![Ok(vec![
|
||||
LlmEvent::TextDelta("partial".to_string()),
|
||||
LlmEvent::Error("connection reset".to_string()),
|
||||
])]);
|
||||
|
||||
let messages = sample_conversation(10);
|
||||
let config = default_config();
|
||||
let mut state = CompactState::new();
|
||||
|
||||
let result = autocompact(&provider, &messages, "test-model", &config, &mut state).await;
|
||||
assert!(matches!(result, Err(CompactError::StreamError(_))));
|
||||
assert_eq!(state.consecutive_failures, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summary_content_auto_has_continuation() {
|
||||
let content = build_summary_content("Summary:\ntest", true);
|
||||
assert!(content.contains("Continue the conversation"));
|
||||
assert!(content.contains("as if the break never happened"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summary_content_manual_no_continuation() {
|
||||
let content = build_summary_content("Summary:\ntest", false);
|
||||
assert!(!content.contains("Continue the conversation"));
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use nomi_agent::bootstrap::AgentBootstrap;
|
||||
use nomi_agent::output::null_sink::NullSink;
|
||||
use nomi_config::compat::ProviderCompat;
|
||||
use nomi_config::config::{Config, ProviderType};
|
||||
|
||||
fn minimal_config() -> Config {
|
||||
Config {
|
||||
provider_label: "openai".into(),
|
||||
provider: ProviderType::OpenAI,
|
||||
api_key: "sk-test".into(),
|
||||
base_url: "http://localhost:0".into(),
|
||||
model: "gpt-test-model".into(),
|
||||
max_tokens: 1024,
|
||||
max_turns: Some(5),
|
||||
system_prompt: None,
|
||||
thinking: None,
|
||||
prompt_caching: false,
|
||||
compat: ProviderCompat::openai_defaults(),
|
||||
tools: Default::default(),
|
||||
session: Default::default(),
|
||||
compact: Default::default(),
|
||||
plan: Default::default(),
|
||||
file_cache: Default::default(),
|
||||
hooks: Default::default(),
|
||||
bedrock: None,
|
||||
vertex: None,
|
||||
mcp: Default::default(),
|
||||
logging: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn null_output() -> Arc<dyn nomi_agent::output::OutputSink> {
|
||||
Arc::new(NullSink)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_builds_engine_with_model_in_prompt() {
|
||||
let config = minimal_config();
|
||||
let result = AgentBootstrap::new(config, "/tmp/test-workspace", null_output())
|
||||
.build()
|
||||
.await
|
||||
.expect("bootstrap should succeed");
|
||||
|
||||
assert!(!result.engine.tool_names().is_empty());
|
||||
assert!(!result.has_mcp);
|
||||
assert!(result.mcp_managers.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_registers_all_expected_tools() {
|
||||
let config = minimal_config();
|
||||
let result = AgentBootstrap::new(config, "/tmp/test-workspace", null_output())
|
||||
.build()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let names = result.engine.tool_names();
|
||||
|
||||
for expected in &["Read", "Write", "Edit", "Bash", "Grep", "Glob"] {
|
||||
assert!(
|
||||
names.iter().any(|n| n == expected),
|
||||
"missing built-in tool: {expected}"
|
||||
);
|
||||
}
|
||||
|
||||
assert!(
|
||||
names.iter().any(|n| n == "Skill"),
|
||||
"SkillTool should be registered"
|
||||
);
|
||||
assert!(
|
||||
names.iter().any(|n| n == "Spawn"),
|
||||
"SpawnTool should be registered"
|
||||
);
|
||||
assert!(
|
||||
names.iter().any(|n| n == "ToolSearch"),
|
||||
"ToolSearchTool should be registered"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_plan_tools_when_enabled() {
|
||||
let mut config = minimal_config();
|
||||
config.plan.enabled = true;
|
||||
|
||||
let result = AgentBootstrap::new(config, "/tmp/test-workspace", null_output())
|
||||
.build()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let names = result.engine.tool_names();
|
||||
assert!(
|
||||
names.iter().any(|n| n == "EnterPlanMode"),
|
||||
"EnterPlanMode should be registered when plan.enabled"
|
||||
);
|
||||
assert!(
|
||||
names.iter().any(|n| n == "ExitPlanMode"),
|
||||
"ExitPlanMode should be registered when plan.enabled"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_no_plan_tools_when_disabled() {
|
||||
let mut config = minimal_config();
|
||||
config.plan.enabled = false;
|
||||
|
||||
let result = AgentBootstrap::new(config, "/tmp/test-workspace", null_output())
|
||||
.build()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let names = result.engine.tool_names();
|
||||
assert!(
|
||||
!names.iter().any(|n| n == "EnterPlanMode"),
|
||||
"EnterPlanMode should NOT be registered when plan.disabled"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_no_mcp_when_no_servers() {
|
||||
let config = minimal_config();
|
||||
let result = AgentBootstrap::new(config, "/tmp/test-workspace", null_output())
|
||||
.build()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(!result.has_mcp);
|
||||
assert!(result.mcp_managers.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_with_custom_system_prompt() {
|
||||
let mut config = minimal_config();
|
||||
config.system_prompt = Some("You are a pirate assistant.".into());
|
||||
|
||||
let _result = AgentBootstrap::new(config, "/tmp/test-workspace", null_output())
|
||||
.build()
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_with_agents_md_in_workspace() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let workspace = tmp.path();
|
||||
std::fs::write(workspace.join("AGENTS.md"), "PROJECT_RULES_MARKER").unwrap();
|
||||
|
||||
let config = minimal_config();
|
||||
let _result = AgentBootstrap::new(config, workspace.to_string_lossy().as_ref(), null_output())
|
||||
.build()
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_config_accessor_returns_config() {
|
||||
let config = minimal_config();
|
||||
let bootstrap = AgentBootstrap::new(config, "/tmp/ws", null_output());
|
||||
assert_eq!(bootstrap.config().model, "gpt-test-model");
|
||||
assert_eq!(bootstrap.config().max_tokens, 1024);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_with_external_provider() {
|
||||
let config = minimal_config();
|
||||
let provider = nomi_providers::create_provider(&config);
|
||||
|
||||
let result = AgentBootstrap::new(config, "/tmp/test-workspace", null_output())
|
||||
.provider(provider)
|
||||
.build()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(!result.engine.tool_names().is_empty());
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
// Shared test utilities for integration tests.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde_json::{Value, json};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use nomi_agent::confirm::ToolConfirmer;
|
||||
use nomi_config::config::{Config, ProviderType, SessionConfig, ToolsConfig};
|
||||
use nomi_config::hooks::HooksConfig;
|
||||
use nomi_mcp::config::McpConfig;
|
||||
use nomi_protocol::events::ToolCategory;
|
||||
use nomi_providers::{LlmProvider, ProviderError};
|
||||
use nomi_tools::Tool;
|
||||
use nomi_types::llm::{LlmEvent, LlmRequest};
|
||||
use nomi_types::message::{StopReason, TokenUsage};
|
||||
use nomi_types::tool::ToolResult;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MockLlmProvider — deterministic LLM for engine / spawn tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// A mock LLM provider that emits a pre-configured sequence of events.
|
||||
/// Each call to `stream` pops the first sequence from `responses`.
|
||||
/// When `responses` is empty it falls back to a single EndTurn with empty text.
|
||||
pub struct MockLlmProvider {
|
||||
responses: Mutex<Vec<Vec<LlmEvent>>>,
|
||||
}
|
||||
|
||||
impl MockLlmProvider {
|
||||
/// Create a provider that returns a single text response then ends.
|
||||
pub fn with_text_response(text: &str) -> Self {
|
||||
let events = vec![
|
||||
LlmEvent::TextDelta(text.to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 100,
|
||||
output_tokens: 50,
|
||||
cache_creation_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
},
|
||||
},
|
||||
];
|
||||
Self {
|
||||
responses: Mutex::new(vec![events]),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a provider that returns a single tool_use then ends with ToolUse stop reason.
|
||||
pub fn with_tool_use(id: &str, name: &str, input: Value) -> Self {
|
||||
let events = vec![
|
||||
LlmEvent::ToolUse {
|
||||
id: id.to_string(),
|
||||
name: name.to_string(),
|
||||
input,
|
||||
extra: None,
|
||||
},
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::ToolUse,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 80,
|
||||
output_tokens: 30,
|
||||
cache_creation_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
},
|
||||
},
|
||||
];
|
||||
Self {
|
||||
responses: Mutex::new(vec![events]),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a provider with multiple turns of pre-configured event sequences.
|
||||
/// Each call to `stream` consumes the next sequence.
|
||||
pub fn with_turns(turns: Vec<Vec<LlmEvent>>) -> Self {
|
||||
Self {
|
||||
responses: Mutex::new(turns),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a provider that returns custom events.
|
||||
pub fn with_events(events: Vec<LlmEvent>) -> Self {
|
||||
Self {
|
||||
responses: Mutex::new(vec![events]),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl LlmProvider for MockLlmProvider {
|
||||
async fn stream(
|
||||
&self,
|
||||
_request: &LlmRequest,
|
||||
) -> Result<mpsc::Receiver<LlmEvent>, ProviderError> {
|
||||
let events = {
|
||||
let mut responses = self.responses.lock().unwrap();
|
||||
if responses.is_empty() {
|
||||
// Fallback: end turn with empty text
|
||||
vec![LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage::default(),
|
||||
}]
|
||||
} else {
|
||||
responses.remove(0)
|
||||
}
|
||||
};
|
||||
|
||||
let (tx, rx) = mpsc::channel(64);
|
||||
tokio::spawn(async move {
|
||||
for event in events {
|
||||
let _ = tx.send(event).await;
|
||||
}
|
||||
});
|
||||
Ok(rx)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MockTool — deterministic tool for orchestration tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// A simple mock tool that returns a pre-configured result.
|
||||
pub struct MockTool {
|
||||
pub tool_name: String,
|
||||
pub tool_description: String,
|
||||
pub concurrent_safe: bool,
|
||||
pub result: Mutex<ToolResult>,
|
||||
}
|
||||
|
||||
impl MockTool {
|
||||
pub fn new(name: &str, result: &str, is_error: bool) -> Self {
|
||||
Self {
|
||||
tool_name: name.to_string(),
|
||||
tool_description: format!("Mock tool: {}", name),
|
||||
concurrent_safe: true,
|
||||
result: Mutex::new(ToolResult {
|
||||
content: result.to_string(),
|
||||
is_error,
|
||||
images: Vec::new(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn sequential(name: &str, result: &str) -> Self {
|
||||
Self {
|
||||
tool_name: name.to_string(),
|
||||
tool_description: format!("Mock sequential tool: {}", name),
|
||||
concurrent_safe: false,
|
||||
result: Mutex::new(ToolResult {
|
||||
content: result.to_string(),
|
||||
is_error: false,
|
||||
images: Vec::new(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for MockTool {
|
||||
fn name(&self) -> &str {
|
||||
&self.tool_name
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
&self.tool_description
|
||||
}
|
||||
|
||||
fn input_schema(&self) -> Value {
|
||||
json!({"type": "object"})
|
||||
}
|
||||
|
||||
fn category(&self) -> ToolCategory {
|
||||
ToolCategory::Info
|
||||
}
|
||||
|
||||
fn is_concurrency_safe(&self, _input: &Value) -> bool {
|
||||
self.concurrent_safe
|
||||
}
|
||||
|
||||
async fn execute(&self, _input: Value) -> ToolResult {
|
||||
self.result.lock().unwrap().clone()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ExecMockTool — mock tool with Exec category (requires approval)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// A mock tool that returns a pre-configured result, with Exec category.
|
||||
pub struct ExecMockTool {
|
||||
pub tool_name: String,
|
||||
pub result: Mutex<ToolResult>,
|
||||
}
|
||||
|
||||
impl ExecMockTool {
|
||||
pub fn new(name: &str, result: &str) -> Self {
|
||||
Self {
|
||||
tool_name: name.to_string(),
|
||||
result: Mutex::new(ToolResult {
|
||||
content: result.to_string(),
|
||||
is_error: false,
|
||||
images: Vec::new(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for ExecMockTool {
|
||||
fn name(&self) -> &str {
|
||||
&self.tool_name
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"Mock exec tool"
|
||||
}
|
||||
|
||||
fn input_schema(&self) -> Value {
|
||||
json!({"type": "object"})
|
||||
}
|
||||
|
||||
fn category(&self) -> ToolCategory {
|
||||
ToolCategory::Exec
|
||||
}
|
||||
|
||||
fn is_concurrency_safe(&self, _input: &Value) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
async fn execute(&self, _input: Value) -> ToolResult {
|
||||
self.result.lock().unwrap().clone()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helper: build a minimal Config for testing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub fn test_config() -> Config {
|
||||
Config {
|
||||
provider_label: "anthropic".to_string(),
|
||||
provider: ProviderType::Anthropic,
|
||||
api_key: "test-key".to_string(),
|
||||
base_url: "http://localhost:0".to_string(),
|
||||
model: "test-model".to_string(),
|
||||
max_tokens: 4096,
|
||||
max_turns: Some(10),
|
||||
system_prompt: Some("You are a test assistant.".to_string()),
|
||||
thinking: None,
|
||||
prompt_caching: false,
|
||||
compat: nomi_config::compat::ProviderCompat::anthropic_defaults(),
|
||||
tools: ToolsConfig {
|
||||
auto_approve: true,
|
||||
allow_list: vec![],
|
||||
..ToolsConfig::default()
|
||||
},
|
||||
session: SessionConfig {
|
||||
enabled: false,
|
||||
directory: "/tmp/nomi-test-sessions".to_string(),
|
||||
max_sessions: 5,
|
||||
},
|
||||
compact: nomi_config::compact::CompactConfig::default(),
|
||||
plan: nomi_config::plan::PlanConfig::default(),
|
||||
file_cache: nomi_config::file_cache::FileCacheConfig::default(),
|
||||
hooks: HooksConfig::default(),
|
||||
bedrock: None,
|
||||
vertex: None,
|
||||
mcp: McpConfig::default(),
|
||||
logging: nomi_config::logging::LoggingConfig::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a ToolConfirmer that auto-approves everything.
|
||||
pub fn auto_approve_confirmer() -> Arc<Mutex<ToolConfirmer>> {
|
||||
Arc::new(Mutex::new(ToolConfirmer::new(true, vec![])))
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
//! Black-box integration tests for compact types (TC-2.2-04 through TC-2.2-06).
|
||||
//!
|
||||
//! These test Message.timestamp serialization and CompactMetadata roundtrip
|
||||
//! from a consumer's perspective.
|
||||
|
||||
use nomi_types::compact::{CompactMetadata, CompactTrigger};
|
||||
use nomi_types::message::{ContentBlock, Message, Role};
|
||||
|
||||
/// TC-2.2-04: Message timestamp serialization — ISO 8601 format.
|
||||
#[test]
|
||||
fn tc_2_2_04_message_timestamp_serialization() {
|
||||
let msg = Message::now(
|
||||
Role::User,
|
||||
vec![ContentBlock::Text {
|
||||
text: "hello".into(),
|
||||
}],
|
||||
);
|
||||
|
||||
let json = serde_json::to_string(&msg).unwrap();
|
||||
assert!(
|
||||
json.contains("\"timestamp\""),
|
||||
"JSON should contain timestamp"
|
||||
);
|
||||
|
||||
// Verify ISO 8601 format (contains 'T' separator and '+' or 'Z' timezone)
|
||||
let value: serde_json::Value = serde_json::from_str(&json).unwrap();
|
||||
let ts_str = value["timestamp"].as_str().unwrap();
|
||||
assert!(
|
||||
ts_str.contains('T'),
|
||||
"timestamp should be ISO 8601 with T separator"
|
||||
);
|
||||
}
|
||||
|
||||
/// TC-2.2-05: Message timestamp backward compatibility — old JSON without
|
||||
/// timestamp deserializes with timestamp = None.
|
||||
#[test]
|
||||
fn tc_2_2_05_message_timestamp_backward_compat() {
|
||||
let old_json = r#"{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "hello"}]
|
||||
}"#;
|
||||
|
||||
let msg: Message = serde_json::from_str(old_json).unwrap();
|
||||
assert!(
|
||||
msg.timestamp.is_none(),
|
||||
"old JSON without timestamp should deserialize to None"
|
||||
);
|
||||
assert_eq!(msg.role, Role::User);
|
||||
assert_eq!(msg.content.len(), 1);
|
||||
}
|
||||
|
||||
/// TC-2.2-06: CompactMetadata serialization/deserialization roundtrip.
|
||||
#[test]
|
||||
fn tc_2_2_06_compact_metadata_roundtrip() {
|
||||
let meta = CompactMetadata {
|
||||
trigger: CompactTrigger::Auto,
|
||||
pre_compact_tokens: 150_000,
|
||||
messages_summarized: 42,
|
||||
};
|
||||
|
||||
let json = serde_json::to_value(&meta).unwrap();
|
||||
let back: CompactMetadata = serde_json::from_value(json).unwrap();
|
||||
|
||||
assert_eq!(back.trigger, CompactTrigger::Auto);
|
||||
assert_eq!(back.pre_compact_tokens, 150_000);
|
||||
assert_eq!(back.messages_summarized, 42);
|
||||
}
|
||||
|
||||
/// Additional: CompactState circuit breaker integration with CompactConfig.
|
||||
#[test]
|
||||
fn compact_state_circuit_breaker_integration() {
|
||||
use nomi_agent::compact::state::CompactState;
|
||||
use nomi_config::compact::CompactConfig;
|
||||
|
||||
let config = CompactConfig {
|
||||
max_failures: 3,
|
||||
..Default::default()
|
||||
};
|
||||
let mut state = CompactState::new();
|
||||
|
||||
// Not broken initially
|
||||
assert!(!state.is_circuit_broken(&config));
|
||||
|
||||
// Record failures up to the limit
|
||||
for _ in 0..3 {
|
||||
state.record_failure();
|
||||
}
|
||||
assert!(state.is_circuit_broken(&config));
|
||||
|
||||
// One success resets
|
||||
state.record_success();
|
||||
assert!(!state.is_circuit_broken(&config));
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
//! Integration tests verifying that tools use the injected workspace cwd
|
||||
//! rather than the process working directory.
|
||||
|
||||
use std::fs;
|
||||
|
||||
use nomi_tools::Tool;
|
||||
use nomi_tools::bash::BashTool;
|
||||
use nomi_tools::glob::GlobTool;
|
||||
use nomi_tools::grep::GrepTool;
|
||||
use serde_json::json;
|
||||
use tempfile::tempdir;
|
||||
|
||||
// Windows `cd` outputs 8.3 short names (RUNNER~1) that don't match canonicalized paths;
|
||||
// bash_tool_with_file_operations_uses_correct_cwd covers the same behavior reliably.
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
async fn bash_tool_executes_in_injected_cwd_not_process_cwd() {
|
||||
let workspace = tempdir().unwrap();
|
||||
let tool = BashTool::new(workspace.path().to_path_buf());
|
||||
|
||||
let result = tool.execute(json!({"command": "pwd"})).await;
|
||||
|
||||
assert!(!result.is_error, "unexpected error: {}", result.content);
|
||||
let expected = workspace
|
||||
.path()
|
||||
.canonicalize()
|
||||
.unwrap_or_else(|_| workspace.path().to_path_buf());
|
||||
assert!(
|
||||
result.content.contains(expected.to_string_lossy().as_ref()),
|
||||
"BashTool should run in injected cwd '{}', got: {}",
|
||||
expected.display(),
|
||||
result.content
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn glob_tool_finds_files_relative_to_injected_cwd() {
|
||||
let workspace = tempdir().unwrap();
|
||||
fs::write(workspace.path().join("cwd_marker.txt"), "hello").unwrap();
|
||||
|
||||
let tool = GlobTool::new(workspace.path().to_path_buf());
|
||||
let result = tool.execute(json!({"pattern": "cwd_marker.txt"})).await;
|
||||
|
||||
assert!(!result.is_error, "unexpected error: {}", result.content);
|
||||
assert!(
|
||||
result.content.contains("cwd_marker.txt"),
|
||||
"GlobTool should find file relative to injected cwd, got: {}",
|
||||
result.content
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn grep_tool_searches_relative_to_injected_cwd() {
|
||||
let workspace = tempdir().unwrap();
|
||||
fs::write(
|
||||
workspace.path().join("searchable.txt"),
|
||||
"unique_cwd_injection_marker_99",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let tool = GrepTool::new(workspace.path().to_path_buf());
|
||||
let result = tool
|
||||
.execute(json!({"pattern": "unique_cwd_injection_marker_99", "path": "."}))
|
||||
.await;
|
||||
|
||||
assert!(!result.is_error, "unexpected error: {}", result.content);
|
||||
assert!(
|
||||
result.content.contains("unique_cwd_injection_marker_99"),
|
||||
"GrepTool should search in injected cwd, got: {}",
|
||||
result.content
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bash_tool_with_file_operations_uses_correct_cwd() {
|
||||
let workspace = tempdir().unwrap();
|
||||
fs::write(workspace.path().join("canary.txt"), "found_it").unwrap();
|
||||
|
||||
let tool = BashTool::new(workspace.path().to_path_buf());
|
||||
let result = tool.execute(json!({"command": "cat canary.txt"})).await;
|
||||
|
||||
assert!(!result.is_error, "unexpected error: {}", result.content);
|
||||
assert!(
|
||||
result.content.contains("found_it"),
|
||||
"BashTool should be able to read files in injected cwd, got: {}",
|
||||
result.content
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use nomi_agent::engine::AgentEngine;
|
||||
use nomi_agent::output::OutputSink;
|
||||
use nomi_agent::output::terminal::TerminalSink;
|
||||
use nomi_config::compat::ProviderCompat;
|
||||
use nomi_config::config::{Config, ProviderType, SessionConfig, ToolsConfig};
|
||||
use nomi_config::hooks::HooksConfig;
|
||||
use nomi_mcp::config::McpConfig;
|
||||
use nomi_providers::create_provider;
|
||||
use nomi_tools::read::ReadTool;
|
||||
use nomi_tools::registry::ToolRegistry;
|
||||
|
||||
/// Skip the test if ANTHROPIC_API_KEY is not set.
|
||||
fn anthropic_api_key() -> Option<String> {
|
||||
std::env::var("ANTHROPIC_API_KEY")
|
||||
.ok()
|
||||
.filter(|k| !k.is_empty())
|
||||
}
|
||||
|
||||
fn anthropic_config(api_key: &str) -> Config {
|
||||
Config {
|
||||
provider: ProviderType::Anthropic,
|
||||
provider_label: "anthropic".to_string(),
|
||||
api_key: api_key.to_string(),
|
||||
base_url: "https://api.anthropic.com".to_string(),
|
||||
model: "claude-haiku-4-20250514".to_string(), // cheapest for e2e
|
||||
max_tokens: 256,
|
||||
max_turns: Some(3),
|
||||
system_prompt: Some("You are a helpful assistant. Be concise.".to_string()),
|
||||
thinking: None,
|
||||
prompt_caching: false,
|
||||
compat: ProviderCompat::anthropic_defaults(),
|
||||
tools: ToolsConfig {
|
||||
auto_approve: true,
|
||||
allow_list: vec![],
|
||||
..ToolsConfig::default()
|
||||
},
|
||||
session: SessionConfig {
|
||||
enabled: false,
|
||||
directory: "/tmp".to_string(),
|
||||
max_sessions: 1,
|
||||
},
|
||||
compact: nomi_config::compact::CompactConfig::default(),
|
||||
plan: nomi_config::plan::PlanConfig::default(),
|
||||
file_cache: nomi_config::file_cache::FileCacheConfig::default(),
|
||||
hooks: HooksConfig::default(),
|
||||
bedrock: None,
|
||||
vertex: None,
|
||||
mcp: McpConfig::default(),
|
||||
logging: nomi_config::logging::LoggingConfig::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Smoke test: single-turn text completion returns non-empty text.
|
||||
#[tokio::test]
|
||||
async fn test_anthropic_single_turn_completion() {
|
||||
let Some(api_key) = anthropic_api_key() else {
|
||||
eprintln!("[e2e] ANTHROPIC_API_KEY not set — skipping");
|
||||
return;
|
||||
};
|
||||
|
||||
let config = anthropic_config(&api_key);
|
||||
let provider = create_provider(&config);
|
||||
let output: Arc<dyn OutputSink> = Arc::new(TerminalSink::new(true));
|
||||
let registry = ToolRegistry::new();
|
||||
|
||||
let mut engine =
|
||||
AgentEngine::new_with_provider(provider, config, registry, output, std::env::temp_dir());
|
||||
let result = engine
|
||||
.run("Say 'hello world' and nothing else.", "")
|
||||
.await
|
||||
.expect("engine.run should not fail for a valid request");
|
||||
|
||||
assert!(!result.text.is_empty(), "response text should not be empty");
|
||||
assert!(result.turns >= 1, "should complete in at least 1 turn");
|
||||
assert!(result.usage.output_tokens > 0, "should have output tokens");
|
||||
|
||||
eprintln!(
|
||||
"[e2e] anthropic single-turn: {} tokens in / {} out",
|
||||
result.usage.input_tokens, result.usage.output_tokens
|
||||
);
|
||||
}
|
||||
|
||||
/// Tool-use smoke test: agent calls Read tool when asked to read a file.
|
||||
#[tokio::test]
|
||||
async fn test_anthropic_tool_use() {
|
||||
let Some(api_key) = anthropic_api_key() else {
|
||||
eprintln!("[e2e] ANTHROPIC_API_KEY not set — skipping");
|
||||
return;
|
||||
};
|
||||
|
||||
// Write a temp file to read
|
||||
let tmp = tempfile::NamedTempFile::new().expect("tempfile");
|
||||
std::fs::write(tmp.path(), "e2e-test-content-42").expect("write tempfile");
|
||||
let path = tmp.path().to_string_lossy().to_string();
|
||||
|
||||
let config = anthropic_config(&api_key);
|
||||
let provider = create_provider(&config);
|
||||
let output: Arc<dyn OutputSink> = Arc::new(TerminalSink::new(true));
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(ReadTool::new(None, None)));
|
||||
|
||||
let mut engine =
|
||||
AgentEngine::new_with_provider(provider, config, registry, output, std::env::temp_dir());
|
||||
let prompt = format!(
|
||||
"Read the file at path '{}' and tell me what it contains. Be brief.",
|
||||
path
|
||||
);
|
||||
let result = engine
|
||||
.run(&prompt, "")
|
||||
.await
|
||||
.expect("engine.run should not fail");
|
||||
|
||||
assert!(!result.text.is_empty(), "response text should not be empty");
|
||||
// The model should have called Read and seen our content
|
||||
assert!(
|
||||
result.text.contains("e2e-test-content-42") || result.turns > 1,
|
||||
"model should either echo the content or have used multiple turns (tool call): {}",
|
||||
result.text
|
||||
);
|
||||
|
||||
eprintln!(
|
||||
"[e2e] anthropic tool-use: {} turns, {} tokens out",
|
||||
result.turns, result.usage.output_tokens
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,399 @@
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use nomi_agent::confirm::ToolConfirmer;
|
||||
use nomi_agent::engine::AgentEngine;
|
||||
use nomi_agent::orchestration::execute_tool_calls;
|
||||
use nomi_agent::output::OutputSink;
|
||||
use nomi_agent::output::null_sink::NullSink;
|
||||
use nomi_compact::CompactionLevel;
|
||||
use nomi_config::compat::ProviderCompat;
|
||||
use nomi_config::config::{Config, ProviderType, SessionConfig, ToolsConfig};
|
||||
use nomi_config::hooks::HooksConfig;
|
||||
use nomi_mcp::config::McpConfig;
|
||||
use nomi_providers::create_provider;
|
||||
use nomi_tools::registry::ToolRegistry;
|
||||
use nomi_types::message::ContentBlock;
|
||||
use serde_json::json;
|
||||
|
||||
const TEST_OUTPUT: &str = "\x1b[32mSTATUS: OK\x1b[0m\n\n\n\n50%\r100%\nCompiling dep-0 v1.0.0\nCompiling dep-1 v1.0.0\nCompiling dep-2 v1.0.0\nCompiling dep-3 v1.0.0\nCompiling dep-4 v1.0.0\n{\n \"id\": 1,\n \"name\": \"Alice Wonderland\",\n \"email\": \"alice@example.com\",\n \"age\": 30,\n \"address\": \"123 Main Street, Anytown, USA 12345\",\n \"phone\": \"+1-555-0123\"\n}";
|
||||
|
||||
const TOON_INPUT: &str =
|
||||
r#"[{"id":1,"name":"Alice","role":"admin"},{"id":2,"name":"Bob","role":"user"}]"#;
|
||||
|
||||
fn openai_api_key() -> Option<String> {
|
||||
std::env::var("OPENAI_API_KEY")
|
||||
.ok()
|
||||
.filter(|k| !k.is_empty())
|
||||
}
|
||||
|
||||
fn openai_config(api_key: &str) -> Config {
|
||||
Config {
|
||||
provider: ProviderType::OpenAI,
|
||||
provider_label: "openai".to_string(),
|
||||
api_key: api_key.to_string(),
|
||||
base_url: "https://api.openai.com".to_string(),
|
||||
model: "gpt-4o-mini".to_string(),
|
||||
max_tokens: 256,
|
||||
max_turns: Some(3),
|
||||
system_prompt: Some(
|
||||
"You are a helpful assistant. Be concise. Answer exactly what is asked.".to_string(),
|
||||
),
|
||||
thinking: None,
|
||||
prompt_caching: false,
|
||||
compat: ProviderCompat::openai_defaults(),
|
||||
tools: ToolsConfig {
|
||||
auto_approve: true,
|
||||
allow_list: vec![],
|
||||
..ToolsConfig::default()
|
||||
},
|
||||
session: SessionConfig {
|
||||
enabled: false,
|
||||
directory: "/tmp".to_string(),
|
||||
max_sessions: 1,
|
||||
},
|
||||
compact: nomi_config::compact::CompactConfig::default(),
|
||||
plan: nomi_config::plan::PlanConfig::default(),
|
||||
file_cache: nomi_config::file_cache::FileCacheConfig::default(),
|
||||
hooks: HooksConfig::default(),
|
||||
bedrock: None,
|
||||
vertex: None,
|
||||
mcp: McpConfig::default(),
|
||||
logging: nomi_config::logging::LoggingConfig::default(),
|
||||
}
|
||||
}
|
||||
|
||||
struct FixedOutputTool {
|
||||
name: String,
|
||||
output: String,
|
||||
}
|
||||
|
||||
impl FixedOutputTool {
|
||||
fn new(name: &str, output: &str) -> Self {
|
||||
Self {
|
||||
name: name.to_string(),
|
||||
output: output.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl nomi_tools::Tool for FixedOutputTool {
|
||||
fn name(&self) -> &str {
|
||||
&self.name
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"Returns fixed output for testing"
|
||||
}
|
||||
|
||||
fn input_schema(&self) -> serde_json::Value {
|
||||
json!({"type": "object", "properties": {}, "required": []})
|
||||
}
|
||||
|
||||
fn category(&self) -> nomi_protocol::events::ToolCategory {
|
||||
nomi_protocol::events::ToolCategory::Info
|
||||
}
|
||||
|
||||
fn is_concurrency_safe(&self, _input: &serde_json::Value) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
async fn execute(&self, _input: serde_json::Value) -> nomi_types::tool::ToolResult {
|
||||
nomi_types::tool::ToolResult {
|
||||
content: self.output.clone(),
|
||||
is_error: false,
|
||||
images: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_tool_result_content(blocks: &[ContentBlock]) -> Option<String> {
|
||||
for block in blocks {
|
||||
if let ContentBlock::ToolResult { content, .. } = block {
|
||||
return Some(content.clone());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// C Layer: Case 9 (Off vs Safe content comparison)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn case_9_off_vs_safe_content() {
|
||||
let Some(api_key) = openai_api_key() else {
|
||||
eprintln!("[e2e:compaction] OPENAI_API_KEY not set — skipping");
|
||||
return;
|
||||
};
|
||||
|
||||
eprintln!("[e2e:compaction] === Case 9: Off vs Safe content comparison ===");
|
||||
|
||||
let confirmer = Arc::new(Mutex::new(ToolConfirmer::new(true, vec![])));
|
||||
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(FixedOutputTool::new("check_tool", TEST_OUTPUT)));
|
||||
let tool_calls = vec![ContentBlock::ToolUse {
|
||||
id: "t1".to_string(),
|
||||
name: "check_tool".to_string(),
|
||||
input: json!({}),
|
||||
extra: None,
|
||||
}];
|
||||
|
||||
// Off
|
||||
let outcome_off = execute_tool_calls(
|
||||
®istry,
|
||||
&tool_calls,
|
||||
&confirmer,
|
||||
None,
|
||||
CompactionLevel::Off,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("should succeed");
|
||||
let content_off = extract_tool_result_content(&outcome_off).unwrap();
|
||||
|
||||
// Safe
|
||||
let outcome_safe = execute_tool_calls(
|
||||
®istry,
|
||||
&tool_calls,
|
||||
&confirmer,
|
||||
None,
|
||||
CompactionLevel::Safe,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("should succeed");
|
||||
let content_safe = extract_tool_result_content(&outcome_safe).unwrap();
|
||||
|
||||
eprintln!("[e2e:compaction] Off content ({} chars)", content_off.len());
|
||||
eprintln!(
|
||||
"[e2e:compaction] Safe content ({} chars)",
|
||||
content_safe.len()
|
||||
);
|
||||
|
||||
assert!(
|
||||
content_off.contains("\x1b"),
|
||||
"Off should preserve ANSI escapes"
|
||||
);
|
||||
assert!(
|
||||
!content_safe.contains("\x1b"),
|
||||
"Safe should strip ANSI escapes"
|
||||
);
|
||||
|
||||
// LLM question (secondary evidence)
|
||||
let mut config = openai_config(&api_key);
|
||||
config.compact.compaction = CompactionLevel::Safe;
|
||||
|
||||
let provider = create_provider(&config);
|
||||
let mut registry2 = ToolRegistry::new();
|
||||
registry2.register(Box::new(FixedOutputTool::new("check_tool", TEST_OUTPUT)));
|
||||
let output: Arc<dyn OutputSink> = Arc::new(NullSink);
|
||||
let mut engine =
|
||||
AgentEngine::new_with_provider(provider, config, registry2, output, std::env::temp_dir());
|
||||
|
||||
let prompt = "Call check_tool, then answer: does the tool output contain ANSI color escape codes (sequences starting with \\x1b)? Answer only 'yes' or 'no'.";
|
||||
let result = engine
|
||||
.run(prompt, "")
|
||||
.await
|
||||
.expect("engine.run should succeed");
|
||||
|
||||
eprintln!("[e2e:compaction] LLM question: does Safe output contain ANSI?");
|
||||
eprintln!("[e2e:compaction] LLM answer: {}", result.text);
|
||||
eprintln!(
|
||||
"[e2e:compaction] Token usage: {} input / {} output",
|
||||
result.usage.input_tokens, result.usage.output_tokens
|
||||
);
|
||||
|
||||
let answer = result.text.to_lowercase();
|
||||
if answer.contains("no") {
|
||||
eprintln!("[e2e:compaction] ✓ LLM confirms no ANSI in Safe output");
|
||||
} else {
|
||||
eprintln!(
|
||||
"[e2e:compaction] ⚠ LLM answer unexpected (non-deterministic, logged for review)"
|
||||
);
|
||||
}
|
||||
|
||||
eprintln!("[e2e:compaction] ✓ PASS (primary: content assertions passed)");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// C Layer: Case 10 (Off vs Full token savings)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn case_10_off_vs_full_token_savings() {
|
||||
let Some(api_key) = openai_api_key() else {
|
||||
eprintln!("[e2e:compaction] OPENAI_API_KEY not set — skipping");
|
||||
return;
|
||||
};
|
||||
|
||||
eprintln!("[e2e:compaction] === Case 10: Off vs Full token savings ===");
|
||||
|
||||
let mut large_output = String::new();
|
||||
for i in 0..20 {
|
||||
large_output.push_str(&format!(
|
||||
"Compiling dependency-{i} v0.1.0 (registry+https://github.com/rust-lang/crates.io-index)\n"
|
||||
));
|
||||
}
|
||||
large_output.push_str("{\n \"users\": [\n");
|
||||
for i in 0..10 {
|
||||
large_output.push_str(&format!(
|
||||
" {{\n \"id\": {i},\n \"name\": \"User {i}\",\n \"email\": \"user{i}@example.com\"\n }}{}\n",
|
||||
if i < 9 { "," } else { "" }
|
||||
));
|
||||
}
|
||||
large_output.push_str(" ]\n}");
|
||||
|
||||
// Off
|
||||
let mut config_off = openai_config(&api_key);
|
||||
config_off.compact.compaction = CompactionLevel::Off;
|
||||
let provider_off = create_provider(&config_off);
|
||||
let mut registry_off = ToolRegistry::new();
|
||||
registry_off.register(Box::new(FixedOutputTool::new("big_tool", &large_output)));
|
||||
let output_off: Arc<dyn OutputSink> = Arc::new(NullSink);
|
||||
let mut engine_off = AgentEngine::new_with_provider(
|
||||
provider_off,
|
||||
config_off,
|
||||
registry_off,
|
||||
output_off,
|
||||
std::env::temp_dir(),
|
||||
);
|
||||
|
||||
let prompt = "Call big_tool, then say 'done'.";
|
||||
let result_off = engine_off
|
||||
.run(prompt, "")
|
||||
.await
|
||||
.expect("engine.run should succeed");
|
||||
|
||||
// Full
|
||||
let mut config_full = openai_config(&api_key);
|
||||
config_full.compact.compaction = CompactionLevel::Full;
|
||||
let provider_full = create_provider(&config_full);
|
||||
let mut registry_full = ToolRegistry::new();
|
||||
registry_full.register(Box::new(FixedOutputTool::new("big_tool", &large_output)));
|
||||
let output_full: Arc<dyn OutputSink> = Arc::new(NullSink);
|
||||
let mut engine_full = AgentEngine::new_with_provider(
|
||||
provider_full,
|
||||
config_full,
|
||||
registry_full,
|
||||
output_full,
|
||||
std::env::temp_dir(),
|
||||
);
|
||||
|
||||
let result_full = engine_full
|
||||
.run(prompt, "")
|
||||
.await
|
||||
.expect("engine.run should succeed");
|
||||
|
||||
eprintln!(
|
||||
"[e2e:compaction] Off input_tokens: {}",
|
||||
result_off.usage.input_tokens
|
||||
);
|
||||
eprintln!(
|
||||
"[e2e:compaction] Full input_tokens: {}",
|
||||
result_full.usage.input_tokens
|
||||
);
|
||||
eprintln!(
|
||||
"[e2e:compaction] Savings: {} tokens ({:.1}%)",
|
||||
result_off
|
||||
.usage
|
||||
.input_tokens
|
||||
.saturating_sub(result_full.usage.input_tokens),
|
||||
if result_off.usage.input_tokens > 0 {
|
||||
(1.0 - result_full.usage.input_tokens as f64 / result_off.usage.input_tokens as f64)
|
||||
* 100.0
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
);
|
||||
|
||||
assert!(
|
||||
result_full.usage.input_tokens < result_off.usage.input_tokens,
|
||||
"Full compaction should use fewer input tokens: full={} vs off={}",
|
||||
result_full.usage.input_tokens,
|
||||
result_off.usage.input_tokens
|
||||
);
|
||||
|
||||
eprintln!("[e2e:compaction] ✓ PASS");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// C Layer: Case 11 (TOON comprehension + system prompt)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn case_11_toon_comprehension_and_system_prompt() {
|
||||
let Some(api_key) = openai_api_key() else {
|
||||
eprintln!("[e2e:compaction] OPENAI_API_KEY not set — skipping");
|
||||
return;
|
||||
};
|
||||
|
||||
eprintln!("[e2e:compaction] === Case 11: TOON comprehension + system prompt ===");
|
||||
|
||||
// Direct content check (deterministic)
|
||||
let confirmer = Arc::new(Mutex::new(ToolConfirmer::new(true, vec![])));
|
||||
let mut registry_check = ToolRegistry::new();
|
||||
registry_check.register(Box::new(FixedOutputTool::new("data_tool", TOON_INPUT)));
|
||||
let tool_calls = vec![ContentBlock::ToolUse {
|
||||
id: "t1".to_string(),
|
||||
name: "data_tool".to_string(),
|
||||
input: json!({}),
|
||||
extra: None,
|
||||
}];
|
||||
|
||||
let outcome = execute_tool_calls(
|
||||
®istry_check,
|
||||
&tool_calls,
|
||||
&confirmer,
|
||||
None,
|
||||
CompactionLevel::Full,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.expect("should succeed");
|
||||
let content = extract_tool_result_content(&outcome).unwrap();
|
||||
|
||||
eprintln!("[e2e:compaction] TOON-encoded content: {content}");
|
||||
assert!(
|
||||
content.contains("[2]{id,name,role}:"),
|
||||
"should contain TOON header: {content}"
|
||||
);
|
||||
|
||||
// LLM comprehension test
|
||||
let mut config = openai_config(&api_key);
|
||||
config.compact.compaction = CompactionLevel::Full;
|
||||
config.compact.toon = true;
|
||||
|
||||
let provider = create_provider(&config);
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(FixedOutputTool::new("data_tool", TOON_INPUT)));
|
||||
let output: Arc<dyn OutputSink> = Arc::new(NullSink);
|
||||
let mut engine =
|
||||
AgentEngine::new_with_provider(provider, config, registry, output, std::env::temp_dir());
|
||||
|
||||
let prompt = "Call data_tool, then answer: what is the name of the second record? Answer with just the name, nothing else.";
|
||||
let result = engine
|
||||
.run(prompt, "")
|
||||
.await
|
||||
.expect("engine.run should succeed");
|
||||
|
||||
eprintln!("[e2e:compaction] LLM question: name of second record?");
|
||||
eprintln!("[e2e:compaction] LLM answer: {}", result.text);
|
||||
eprintln!(
|
||||
"[e2e:compaction] Token usage: {} input / {} output",
|
||||
result.usage.input_tokens, result.usage.output_tokens
|
||||
);
|
||||
|
||||
let answer = result.text.to_lowercase();
|
||||
if answer.contains("bob") {
|
||||
eprintln!("[e2e:compaction] ✓ LLM correctly understood TOON format");
|
||||
} else {
|
||||
eprintln!(
|
||||
"[e2e:compaction] ⚠ LLM answer: '{}' (expected 'Bob', logged for review)",
|
||||
result.text
|
||||
);
|
||||
}
|
||||
|
||||
eprintln!("[e2e:compaction] ✓ PASS (primary: TOON content assertion passed)");
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
/// End-to-end tests that hit real LLM provider APIs.
|
||||
///
|
||||
/// These tests are skipped when the required environment variable is absent,
|
||||
/// making them safe to compile and run in any environment while still providing
|
||||
/// full coverage in CI when secrets are available.
|
||||
///
|
||||
/// Required env vars (at least one):
|
||||
/// ANTHROPIC_API_KEY — runs Anthropic provider tests
|
||||
/// OPENAI_API_KEY — runs OpenAI provider tests
|
||||
///
|
||||
/// Run manually:
|
||||
/// ANTHROPIC_API_KEY=sk-ant-... cargo test -p nomi-agent --test e2e -- --nocapture
|
||||
mod anthropic;
|
||||
mod compaction;
|
||||
mod openai;
|
||||
@@ -0,0 +1,123 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use nomi_agent::engine::AgentEngine;
|
||||
use nomi_agent::output::OutputSink;
|
||||
use nomi_agent::output::terminal::TerminalSink;
|
||||
use nomi_config::compat::ProviderCompat;
|
||||
use nomi_config::config::{Config, ProviderType, SessionConfig, ToolsConfig};
|
||||
use nomi_config::hooks::HooksConfig;
|
||||
use nomi_mcp::config::McpConfig;
|
||||
use nomi_providers::create_provider;
|
||||
use nomi_tools::read::ReadTool;
|
||||
use nomi_tools::registry::ToolRegistry;
|
||||
|
||||
fn openai_api_key() -> Option<String> {
|
||||
std::env::var("OPENAI_API_KEY")
|
||||
.ok()
|
||||
.filter(|k| !k.is_empty())
|
||||
}
|
||||
|
||||
fn openai_config(api_key: &str) -> Config {
|
||||
Config {
|
||||
provider: ProviderType::OpenAI,
|
||||
provider_label: "openai".to_string(),
|
||||
api_key: api_key.to_string(),
|
||||
base_url: "https://api.openai.com".to_string(),
|
||||
model: "gpt-4o-mini".to_string(), // cheapest for e2e
|
||||
max_tokens: 256,
|
||||
max_turns: Some(3),
|
||||
system_prompt: Some("You are a helpful assistant. Be concise.".to_string()),
|
||||
thinking: None,
|
||||
prompt_caching: false,
|
||||
compat: ProviderCompat::openai_defaults(),
|
||||
tools: ToolsConfig {
|
||||
auto_approve: true,
|
||||
allow_list: vec![],
|
||||
..ToolsConfig::default()
|
||||
},
|
||||
session: SessionConfig {
|
||||
enabled: false,
|
||||
directory: "/tmp".to_string(),
|
||||
max_sessions: 1,
|
||||
},
|
||||
compact: nomi_config::compact::CompactConfig::default(),
|
||||
plan: nomi_config::plan::PlanConfig::default(),
|
||||
file_cache: nomi_config::file_cache::FileCacheConfig::default(),
|
||||
hooks: HooksConfig::default(),
|
||||
bedrock: None,
|
||||
vertex: None,
|
||||
mcp: McpConfig::default(),
|
||||
logging: nomi_config::logging::LoggingConfig::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Smoke test: single-turn text completion.
|
||||
#[tokio::test]
|
||||
async fn test_openai_single_turn_completion() {
|
||||
let Some(api_key) = openai_api_key() else {
|
||||
eprintln!("[e2e] OPENAI_API_KEY not set — skipping");
|
||||
return;
|
||||
};
|
||||
|
||||
let config = openai_config(&api_key);
|
||||
let provider = create_provider(&config);
|
||||
let output: Arc<dyn OutputSink> = Arc::new(TerminalSink::new(true));
|
||||
let registry = ToolRegistry::new();
|
||||
|
||||
let mut engine =
|
||||
AgentEngine::new_with_provider(provider, config, registry, output, std::env::temp_dir());
|
||||
let result = engine
|
||||
.run("Say 'hello world' and nothing else.", "")
|
||||
.await
|
||||
.expect("engine.run should not fail");
|
||||
|
||||
assert!(!result.text.is_empty(), "response text should not be empty");
|
||||
assert!(result.usage.output_tokens > 0);
|
||||
|
||||
eprintln!(
|
||||
"[e2e] openai single-turn: {} tokens in / {} out",
|
||||
result.usage.input_tokens, result.usage.output_tokens
|
||||
);
|
||||
}
|
||||
|
||||
/// Tool-use smoke test: agent calls Read tool when asked to read a file.
|
||||
#[tokio::test]
|
||||
async fn test_openai_tool_use() {
|
||||
let Some(api_key) = openai_api_key() else {
|
||||
eprintln!("[e2e] OPENAI_API_KEY not set — skipping");
|
||||
return;
|
||||
};
|
||||
|
||||
let tmp = tempfile::NamedTempFile::new().expect("tempfile");
|
||||
std::fs::write(tmp.path(), "e2e-openai-content-99").expect("write tempfile");
|
||||
let path = tmp.path().to_string_lossy().to_string();
|
||||
|
||||
let config = openai_config(&api_key);
|
||||
let provider = create_provider(&config);
|
||||
let output: Arc<dyn OutputSink> = Arc::new(TerminalSink::new(true));
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(ReadTool::new(None, None)));
|
||||
|
||||
let mut engine =
|
||||
AgentEngine::new_with_provider(provider, config, registry, output, std::env::temp_dir());
|
||||
let prompt = format!(
|
||||
"Read the file at '{}' and tell me what it contains. Be brief.",
|
||||
path
|
||||
);
|
||||
let result = engine
|
||||
.run(&prompt, "")
|
||||
.await
|
||||
.expect("engine.run should not fail");
|
||||
|
||||
assert!(!result.text.is_empty());
|
||||
assert!(
|
||||
result.text.contains("e2e-openai-content-99") || result.turns > 1,
|
||||
"model should echo the content or use multiple turns: {}",
|
||||
result.text
|
||||
);
|
||||
|
||||
eprintln!(
|
||||
"[e2e] openai tool-use: {} turns, {} tokens out",
|
||||
result.turns, result.usage.output_tokens
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
//! Black-box integration tests for emergency truncation (TC-2.5-01 .. TC-2.5-04).
|
||||
//!
|
||||
//! These tests treat `is_at_emergency_limit` as a public API and verify
|
||||
//! functional requirements from test-plan.md without relying on internal details.
|
||||
|
||||
use nomi_agent::compact::emergency::{EMERGENCY_USER_MESSAGE, is_at_emergency_limit};
|
||||
use nomi_config::compact::CompactConfig;
|
||||
|
||||
// ── TC-2.5-01: Below emergency threshold ───────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_5_01_below_emergency_threshold() {
|
||||
// context_window=200_000, emergency_buffer=3_000
|
||||
// emergency_limit = 200k - 3k = 197k
|
||||
// 190k < 197k → false
|
||||
let config = CompactConfig::default();
|
||||
assert!(
|
||||
!is_at_emergency_limit(190_000, &config),
|
||||
"190k tokens should be below the 197k emergency limit"
|
||||
);
|
||||
}
|
||||
|
||||
// ── TC-2.5-02: Above emergency threshold ───────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_5_02_above_emergency_threshold() {
|
||||
// 198k >= 197k → true
|
||||
let config = CompactConfig::default();
|
||||
assert!(
|
||||
is_at_emergency_limit(198_000, &config),
|
||||
"198k tokens should exceed the 197k emergency limit"
|
||||
);
|
||||
}
|
||||
|
||||
// ── TC-2.5-03: Exactly at emergency threshold ──────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_5_03_at_exact_emergency_threshold() {
|
||||
// 197k >= 197k → true
|
||||
let config = CompactConfig::default();
|
||||
assert!(
|
||||
is_at_emergency_limit(197_000, &config),
|
||||
"197k tokens should trigger at exactly the emergency limit"
|
||||
);
|
||||
}
|
||||
|
||||
// ── TC-2.5-04: Small context window ────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_5_04_small_context_window() {
|
||||
// context_window=8_000, emergency_buffer=3_000
|
||||
// emergency_limit = 8k - 3k = 5k
|
||||
// 6k >= 5k → true
|
||||
let config = CompactConfig {
|
||||
context_window: 8_000,
|
||||
emergency_buffer: 3_000,
|
||||
..CompactConfig::default()
|
||||
};
|
||||
assert!(
|
||||
is_at_emergency_limit(6_000, &config),
|
||||
"6k tokens should exceed 5k emergency limit on an 8k context window"
|
||||
);
|
||||
}
|
||||
|
||||
// ── Additional integration-level checks ────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn emergency_check_ignores_enabled_flag() {
|
||||
// Emergency is the safety net — it fires even when compact is disabled
|
||||
let config = CompactConfig {
|
||||
enabled: false,
|
||||
..CompactConfig::default()
|
||||
};
|
||||
assert!(
|
||||
is_at_emergency_limit(198_000, &config),
|
||||
"emergency check must fire regardless of the enabled flag"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn user_message_is_actionable() {
|
||||
// The message should tell the user what to do
|
||||
assert!(
|
||||
EMERGENCY_USER_MESSAGE.contains("/compact"),
|
||||
"emergency message should mention /compact"
|
||||
);
|
||||
assert!(
|
||||
EMERGENCY_USER_MESSAGE.contains("new conversation"),
|
||||
"emergency message should mention starting a new conversation"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn autocompact_fires_before_emergency() {
|
||||
// Verify that the autocompact threshold is lower than the emergency limit
|
||||
// so autocompact gets a chance to run before the safety net kicks in.
|
||||
use nomi_agent::compact::auto::should_autocompact;
|
||||
|
||||
let config = CompactConfig::default();
|
||||
|
||||
// Pick a token count that triggers autocompact but not emergency
|
||||
let token_count: u64 = 170_000;
|
||||
let autocompact_triggers = should_autocompact(token_count, &config);
|
||||
let emergency_triggers = is_at_emergency_limit(token_count, &config);
|
||||
|
||||
assert!(
|
||||
autocompact_triggers && !emergency_triggers,
|
||||
"at 170k tokens, autocompact should trigger (threshold 167k) \
|
||||
but emergency should not (limit 197k)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn both_trigger_near_limit() {
|
||||
// When very close to the limit, both autocompact and emergency should fire
|
||||
use nomi_agent::compact::auto::should_autocompact;
|
||||
|
||||
let config = CompactConfig::default();
|
||||
let token_count: u64 = 198_000;
|
||||
|
||||
assert!(should_autocompact(token_count, &config));
|
||||
assert!(is_at_emergency_limit(token_count, &config));
|
||||
}
|
||||
@@ -0,0 +1,878 @@
|
||||
//! Black-box integration tests for engine compaction integration (TC-2.6-*).
|
||||
//!
|
||||
//! These tests exercise the full `AgentEngine::run()` loop and verify
|
||||
//! that the compaction pipeline (microcompact → autocompact → emergency)
|
||||
//! is correctly wired into the agentic loop.
|
||||
|
||||
mod common;
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use nomi_agent::engine::{AgentEngine, AgentError};
|
||||
use nomi_agent::output::OutputSink;
|
||||
use nomi_agent::output::terminal::TerminalSink;
|
||||
use nomi_agent::session::SessionManager;
|
||||
use nomi_config::compact::CompactConfig;
|
||||
use nomi_providers::{LlmProvider, ProviderError};
|
||||
use nomi_tools::registry::ToolRegistry;
|
||||
use nomi_types::llm::{LlmEvent, LlmRequest};
|
||||
use nomi_types::message::{StopReason, TokenUsage};
|
||||
use tempfile::tempdir;
|
||||
|
||||
use common::test_config;
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
fn silent_output() -> Arc<dyn OutputSink> {
|
||||
Arc::new(TerminalSink::new(true))
|
||||
}
|
||||
|
||||
/// A mock provider that returns configurable per-turn events.
|
||||
/// Tracks the number of stream() calls for order verification.
|
||||
struct CompactMockProvider {
|
||||
turns: Mutex<VecDeque<Vec<LlmEvent>>>,
|
||||
call_count: Mutex<usize>,
|
||||
}
|
||||
|
||||
impl CompactMockProvider {
|
||||
fn new(turns: Vec<Vec<LlmEvent>>) -> Self {
|
||||
Self {
|
||||
turns: Mutex::new(VecDeque::from(turns)),
|
||||
call_count: Mutex::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
fn call_count(&self) -> usize {
|
||||
*self.call_count.lock().unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl LlmProvider for CompactMockProvider {
|
||||
async fn stream(
|
||||
&self,
|
||||
_request: &LlmRequest,
|
||||
) -> Result<mpsc::Receiver<LlmEvent>, ProviderError> {
|
||||
*self.call_count.lock().unwrap() += 1;
|
||||
let events = self.turns.lock().unwrap().pop_front().unwrap_or_else(|| {
|
||||
vec![LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage::default(),
|
||||
}]
|
||||
});
|
||||
|
||||
let (tx, rx) = mpsc::channel(64);
|
||||
tokio::spawn(async move {
|
||||
for event in events {
|
||||
let _ = tx.send(event).await;
|
||||
}
|
||||
});
|
||||
Ok(rx)
|
||||
}
|
||||
}
|
||||
|
||||
/// Build events for a simple text response with configurable input_tokens.
|
||||
fn text_turn(text: &str, input_tokens: u64) -> Vec<LlmEvent> {
|
||||
vec![
|
||||
LlmEvent::TextDelta(text.to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage {
|
||||
input_tokens,
|
||||
output_tokens: 100,
|
||||
..Default::default()
|
||||
},
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
/// Build events for a summary LLM call (used by autocompact internally).
|
||||
fn summary_turn(summary_text: &str) -> Vec<LlmEvent> {
|
||||
vec![
|
||||
LlmEvent::TextDelta(summary_text.to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 5_000,
|
||||
output_tokens: 2_000,
|
||||
..Default::default()
|
||||
},
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
// ── TC-2.6-01: First turn does not trigger compaction ──────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_2_6_01_first_turn_no_compaction() {
|
||||
// On the first turn last_input_tokens is 0, so neither autocompact
|
||||
// nor emergency should fire.
|
||||
let provider = Arc::new(CompactMockProvider::new(vec![text_turn("Hello", 50_000)]));
|
||||
|
||||
let config = test_config();
|
||||
let registry = ToolRegistry::new();
|
||||
let output = silent_output();
|
||||
|
||||
let mut engine = AgentEngine::new_with_provider(
|
||||
provider.clone(),
|
||||
config,
|
||||
registry,
|
||||
output,
|
||||
std::env::temp_dir(),
|
||||
);
|
||||
let result = engine.run("Hi", "msg-1").await.expect("should succeed");
|
||||
|
||||
assert_eq!(result.text, "Hello");
|
||||
assert_eq!(result.turns, 1);
|
||||
// Only one call to stream() — no compaction call
|
||||
assert_eq!(provider.call_count(), 1);
|
||||
}
|
||||
|
||||
// ── TC-2.6-03: Emergency truncation returns error ──────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_2_6_03_emergency_returns_error() {
|
||||
// Emergency is the last safety net — it fires when autocompact is
|
||||
// disabled or circuit-broken. We disable compact so only emergency
|
||||
// is active, then push input_tokens above the emergency limit.
|
||||
//
|
||||
// Turn 1: tool use, returns input_tokens above emergency threshold
|
||||
// Turn 2: emergency fires before the API call → ContextTooLong
|
||||
let turn1 = vec![
|
||||
LlmEvent::ToolUse {
|
||||
id: "t1".to_string(),
|
||||
name: "mock_tool".to_string(),
|
||||
input: serde_json::json!({}),
|
||||
extra: None,
|
||||
},
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::ToolUse,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 198_000, // above emergency limit (197k)
|
||||
output_tokens: 100,
|
||||
..Default::default()
|
||||
},
|
||||
},
|
||||
];
|
||||
// Turn 2 events are queued but should never be consumed
|
||||
let turn2 = text_turn("Should not reach", 50_000);
|
||||
|
||||
let provider = Arc::new(CompactMockProvider::new(vec![turn1, turn2]));
|
||||
let mut config = test_config();
|
||||
config.compact.enabled = false; // disable auto/micro so emergency is the only gate
|
||||
config.compact.context_window = 200_000;
|
||||
config.compact.emergency_buffer = 3_000;
|
||||
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(common::MockTool::new(
|
||||
"mock_tool",
|
||||
"result",
|
||||
false,
|
||||
)));
|
||||
let output = silent_output();
|
||||
|
||||
let mut engine = AgentEngine::new_with_provider(
|
||||
provider.clone(),
|
||||
config,
|
||||
registry,
|
||||
output,
|
||||
std::env::temp_dir(),
|
||||
);
|
||||
let err = engine.run("Do something", "msg-1").await.unwrap_err();
|
||||
|
||||
match err {
|
||||
AgentError::ContextTooLong {
|
||||
input_tokens,
|
||||
limit,
|
||||
} => {
|
||||
assert_eq!(input_tokens, 198_000);
|
||||
assert_eq!(limit, 197_000);
|
||||
}
|
||||
other => panic!("expected ContextTooLong, got: {:?}", other),
|
||||
}
|
||||
|
||||
// Only one call to stream() — second call blocked by emergency
|
||||
assert_eq!(provider.call_count(), 1);
|
||||
}
|
||||
|
||||
// ── TC-2.6-04: Autocompact then continue ───────────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_2_6_04_autocompact_then_continue() {
|
||||
// Turn 1: tool use, returns input_tokens=170k (above autocompact threshold 167k)
|
||||
// Before turn 2: autocompact fires → LLM summary call → messages replaced
|
||||
// Turn 2 (after compact): text response with low input_tokens
|
||||
let turn1 = vec![
|
||||
LlmEvent::ToolUse {
|
||||
id: "t1".to_string(),
|
||||
name: "mock_tool".to_string(),
|
||||
input: serde_json::json!({}),
|
||||
extra: None,
|
||||
},
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::ToolUse,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 170_000,
|
||||
output_tokens: 100,
|
||||
..Default::default()
|
||||
},
|
||||
},
|
||||
];
|
||||
let compact_summary = summary_turn("<summary>Conversation summary</summary>");
|
||||
let turn2_after_compact = text_turn("Continuing after compact", 10_000);
|
||||
|
||||
let provider = Arc::new(CompactMockProvider::new(vec![
|
||||
turn1,
|
||||
compact_summary,
|
||||
turn2_after_compact,
|
||||
]));
|
||||
|
||||
let mut config = test_config();
|
||||
config.compact = CompactConfig::default();
|
||||
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(common::MockTool::new(
|
||||
"mock_tool",
|
||||
"result",
|
||||
false,
|
||||
)));
|
||||
let output = silent_output();
|
||||
|
||||
let mut engine = AgentEngine::new_with_provider(
|
||||
provider.clone(),
|
||||
config,
|
||||
registry,
|
||||
output,
|
||||
std::env::temp_dir(),
|
||||
);
|
||||
let result = engine
|
||||
.run("Start work", "msg-1")
|
||||
.await
|
||||
.expect("should succeed after compact");
|
||||
|
||||
assert_eq!(result.text, "Continuing after compact");
|
||||
assert_eq!(result.turns, 2);
|
||||
// 3 calls: turn1 + compact summary + turn2
|
||||
assert_eq!(provider.call_count(), 3);
|
||||
}
|
||||
|
||||
// ── TC-2.6-05: Session save includes compacted messages ────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_2_6_05_session_save_after_compact() {
|
||||
let dir = tempdir().expect("tempdir");
|
||||
|
||||
let turn1 = vec![
|
||||
LlmEvent::ToolUse {
|
||||
id: "t1".to_string(),
|
||||
name: "mock_tool".to_string(),
|
||||
input: serde_json::json!({}),
|
||||
extra: None,
|
||||
},
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::ToolUse,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 170_000,
|
||||
output_tokens: 100,
|
||||
..Default::default()
|
||||
},
|
||||
},
|
||||
];
|
||||
let compact_summary = summary_turn("<summary>Session summary</summary>");
|
||||
let turn2 = text_turn("After compact", 10_000);
|
||||
|
||||
let provider = Arc::new(CompactMockProvider::new(vec![
|
||||
turn1,
|
||||
compact_summary,
|
||||
turn2,
|
||||
]));
|
||||
|
||||
let mut config = test_config();
|
||||
config.compact = CompactConfig::default();
|
||||
config.session.enabled = true;
|
||||
config.session.directory = dir.path().to_string_lossy().into_owned();
|
||||
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(common::MockTool::new(
|
||||
"mock_tool",
|
||||
"result",
|
||||
false,
|
||||
)));
|
||||
let output = silent_output();
|
||||
|
||||
let mut engine =
|
||||
AgentEngine::new_with_provider(provider, config, registry, output, std::env::temp_dir());
|
||||
engine
|
||||
.init_session("test", "/tmp", None)
|
||||
.expect("init session");
|
||||
|
||||
engine.run("Start", "msg-1").await.expect("should succeed");
|
||||
|
||||
// Load the saved session
|
||||
let mgr = SessionManager::new(dir.path().to_path_buf(), 10);
|
||||
let session = mgr.load("latest").expect("load session");
|
||||
|
||||
// After compaction + turn2, messages should include the compact boundary,
|
||||
// summary, and the post-compact assistant/user messages.
|
||||
// The exact count depends on implementation, but should be small (not
|
||||
// the full pre-compact count).
|
||||
assert!(
|
||||
session.messages.len() < 10,
|
||||
"session should have compacted messages, got {}",
|
||||
session.messages.len()
|
||||
);
|
||||
|
||||
// Verify at least one message contains compact boundary marker
|
||||
let has_boundary = session.messages.iter().any(|m| {
|
||||
m.content.iter().any(|b| {
|
||||
matches!(b, nomi_types::message::ContentBlock::Text { text } if text.contains("[Conversation compacted]"))
|
||||
})
|
||||
});
|
||||
assert!(
|
||||
has_boundary,
|
||||
"session should contain compact boundary marker"
|
||||
);
|
||||
}
|
||||
|
||||
// ── TC-2.6-06: Disabled skips all except emergency ─────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_2_6_06_disabled_skips_micro_auto() {
|
||||
// With compact disabled, a text response that reports high usage
|
||||
// should not trigger autocompact (only emergency if at limit).
|
||||
let provider = Arc::new(CompactMockProvider::new(vec![
|
||||
// Returns high but not emergency-level tokens
|
||||
text_turn("Normal response", 170_000),
|
||||
]));
|
||||
|
||||
let mut config = test_config();
|
||||
config.compact.enabled = false;
|
||||
|
||||
let registry = ToolRegistry::new();
|
||||
let output = silent_output();
|
||||
|
||||
let mut engine = AgentEngine::new_with_provider(
|
||||
provider.clone(),
|
||||
config,
|
||||
registry,
|
||||
output,
|
||||
std::env::temp_dir(),
|
||||
);
|
||||
let result = engine.run("Hi", "msg-1").await.expect("should succeed");
|
||||
|
||||
assert_eq!(result.text, "Normal response");
|
||||
// Only 1 call — no compact summary call
|
||||
assert_eq!(provider.call_count(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_2_6_06b_disabled_still_fires_emergency() {
|
||||
let turn1 = vec![
|
||||
LlmEvent::ToolUse {
|
||||
id: "t1".to_string(),
|
||||
name: "mock_tool".to_string(),
|
||||
input: serde_json::json!({}),
|
||||
extra: None,
|
||||
},
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::ToolUse,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 198_000,
|
||||
output_tokens: 100,
|
||||
..Default::default()
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
let provider = Arc::new(CompactMockProvider::new(vec![
|
||||
turn1,
|
||||
text_turn("unreachable", 0),
|
||||
]));
|
||||
|
||||
let mut config = test_config();
|
||||
config.compact.enabled = false;
|
||||
config.compact.context_window = 200_000;
|
||||
config.compact.emergency_buffer = 3_000;
|
||||
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(common::MockTool::new(
|
||||
"mock_tool",
|
||||
"result",
|
||||
false,
|
||||
)));
|
||||
let output = silent_output();
|
||||
|
||||
let mut engine =
|
||||
AgentEngine::new_with_provider(provider, config, registry, output, std::env::temp_dir());
|
||||
let err = engine.run("Go", "msg-1").await.unwrap_err();
|
||||
|
||||
assert!(
|
||||
matches!(err, AgentError::ContextTooLong { .. }),
|
||||
"emergency should fire even when disabled"
|
||||
);
|
||||
}
|
||||
|
||||
// ── TC-2.6-07: input_tokens correctly tracked ──────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_2_6_07_input_tokens_tracked() {
|
||||
// Two turns: first returns 50k tokens, second returns 60k tokens.
|
||||
// We verify that the engine updates compact state after each turn.
|
||||
let turn1 = vec![
|
||||
LlmEvent::ToolUse {
|
||||
id: "t1".to_string(),
|
||||
name: "mock_tool".to_string(),
|
||||
input: serde_json::json!({}),
|
||||
extra: None,
|
||||
},
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::ToolUse,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 50_000,
|
||||
output_tokens: 100,
|
||||
..Default::default()
|
||||
},
|
||||
},
|
||||
];
|
||||
let turn2 = text_turn("Done", 60_000);
|
||||
|
||||
let provider = Arc::new(CompactMockProvider::new(vec![turn1, turn2]));
|
||||
|
||||
let config = test_config();
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(common::MockTool::new(
|
||||
"mock_tool",
|
||||
"result",
|
||||
false,
|
||||
)));
|
||||
let output = silent_output();
|
||||
|
||||
let mut engine =
|
||||
AgentEngine::new_with_provider(provider, config, registry, output, std::env::temp_dir());
|
||||
let result = engine.run("Work", "msg-1").await.expect("should succeed");
|
||||
|
||||
assert_eq!(result.turns, 2);
|
||||
// Total usage should accumulate: 50k + 60k = 110k input tokens
|
||||
assert_eq!(result.usage.input_tokens, 110_000);
|
||||
}
|
||||
|
||||
// ── TC-2.6-02: Execution order — micro before auto ────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_2_6_02_micro_before_auto_execution_order() {
|
||||
// Build a scenario where both microcompact and autocompact trigger
|
||||
// in the same compaction cycle. A custom provider captures the
|
||||
// messages sent to the autocompact LLM call so we can verify that
|
||||
// microcompact already cleared old tool results before autocompact
|
||||
// was invoked.
|
||||
|
||||
let captured: Arc<Mutex<Option<Vec<nomi_types::message::Message>>>> =
|
||||
Arc::new(Mutex::new(None));
|
||||
let capture_ref = captured.clone();
|
||||
|
||||
struct OrderProvider {
|
||||
regular_count: Mutex<usize>,
|
||||
captured: Arc<Mutex<Option<Vec<nomi_types::message::Message>>>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl LlmProvider for OrderProvider {
|
||||
async fn stream(
|
||||
&self,
|
||||
request: &LlmRequest,
|
||||
) -> Result<mpsc::Receiver<LlmEvent>, ProviderError> {
|
||||
let is_compact = request.tools.is_empty();
|
||||
|
||||
if is_compact {
|
||||
// Capture messages that autocompact sends to the LLM
|
||||
*self.captured.lock().unwrap() = Some(request.messages.clone());
|
||||
|
||||
let events = vec![
|
||||
LlmEvent::TextDelta("<summary>Order test summary</summary>".to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 5_000,
|
||||
output_tokens: 2_000,
|
||||
..Default::default()
|
||||
},
|
||||
},
|
||||
];
|
||||
let (tx, rx) = mpsc::channel(64);
|
||||
tokio::spawn(async move {
|
||||
for e in events {
|
||||
let _ = tx.send(e).await;
|
||||
}
|
||||
});
|
||||
return Ok(rx);
|
||||
}
|
||||
|
||||
let count = {
|
||||
let mut c = self.regular_count.lock().unwrap();
|
||||
let v = *c;
|
||||
*c += 1;
|
||||
v
|
||||
};
|
||||
|
||||
// Turns 0-6: tool use. Turn 6 reports high input_tokens
|
||||
// so that micro and auto both trigger in the SAME cycle
|
||||
// (turn 7's run_compaction).
|
||||
// Turn 7 (after compact): text to end the run.
|
||||
//
|
||||
// micro_keep_recent = 3 → count threshold = 6.
|
||||
// After 7 tool-use turns: 7 > 6 → micro fires.
|
||||
// After turn 6: last_input_tokens = 170k > 167k → auto fires.
|
||||
let events = if count < 7 {
|
||||
let input_tokens = if count == 6 { 170_000 } else { 10_000 };
|
||||
vec![
|
||||
LlmEvent::ToolUse {
|
||||
id: format!("t{count}"),
|
||||
name: "mock_tool".to_string(),
|
||||
input: serde_json::json!({}),
|
||||
extra: None,
|
||||
},
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::ToolUse,
|
||||
usage: TokenUsage {
|
||||
input_tokens,
|
||||
output_tokens: 100,
|
||||
..Default::default()
|
||||
},
|
||||
},
|
||||
]
|
||||
} else {
|
||||
vec![
|
||||
LlmEvent::TextDelta("Done after compact".to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 5_000,
|
||||
output_tokens: 100,
|
||||
..Default::default()
|
||||
},
|
||||
},
|
||||
]
|
||||
};
|
||||
|
||||
let (tx, rx) = mpsc::channel(64);
|
||||
tokio::spawn(async move {
|
||||
for e in events {
|
||||
let _ = tx.send(e).await;
|
||||
}
|
||||
});
|
||||
Ok(rx)
|
||||
}
|
||||
}
|
||||
|
||||
let provider = Arc::new(OrderProvider {
|
||||
regular_count: Mutex::new(0),
|
||||
captured: capture_ref,
|
||||
});
|
||||
|
||||
let mut config = test_config();
|
||||
config.compact = CompactConfig {
|
||||
micro_keep_recent: 3,
|
||||
compactable_tools: vec!["mock_tool".into()],
|
||||
context_window: 200_000,
|
||||
emergency_buffer: 3_000,
|
||||
..Default::default()
|
||||
};
|
||||
config.max_turns = Some(20);
|
||||
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(common::MockTool::new(
|
||||
"mock_tool",
|
||||
"tool output data",
|
||||
false,
|
||||
)));
|
||||
let output = silent_output();
|
||||
|
||||
let mut engine =
|
||||
AgentEngine::new_with_provider(provider, config, registry, output, std::env::temp_dir());
|
||||
let result = engine.run("Start", "msg-1").await.expect("should succeed");
|
||||
|
||||
assert_eq!(result.text, "Done after compact");
|
||||
|
||||
// Verify: the messages that autocompact received should contain
|
||||
// tool results cleared by microcompact (proving micro ran first
|
||||
// within the SAME compaction cycle).
|
||||
let msgs = captured.lock().unwrap();
|
||||
let msgs = msgs.as_ref().expect("autocompact should have been called");
|
||||
|
||||
let cleared_count = msgs
|
||||
.iter()
|
||||
.flat_map(|m| m.content.iter())
|
||||
.filter(|b| {
|
||||
matches!(
|
||||
b,
|
||||
nomi_types::message::ContentBlock::ToolResult { content, .. }
|
||||
if content == nomi_agent::compact::micro::CLEARED_TOOL_RESULT
|
||||
)
|
||||
})
|
||||
.count();
|
||||
|
||||
// 7 tool results total, keep_recent=3 → 4 cleared by micro
|
||||
// before auto received the messages.
|
||||
assert_eq!(
|
||||
cleared_count, 4,
|
||||
"microcompact should have cleared 4 tool results before autocompact ran"
|
||||
);
|
||||
}
|
||||
|
||||
// ── TC-2.6-E2E-02: Microcompact + autocompact cooperative scenario ────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_2_6_e2e_02_micro_and_auto_cooperative() {
|
||||
// Verify that microcompact and autocompact cooperate in the same
|
||||
// compaction cycle. Microcompact frees some tokens from old tool
|
||||
// results, and autocompact still fires because the input token
|
||||
// watermark (which is not reduced by micro) remains above threshold.
|
||||
|
||||
let compact_call_count: Arc<Mutex<usize>> = Arc::new(Mutex::new(0));
|
||||
let counter_ref = compact_call_count.clone();
|
||||
|
||||
struct CoopProvider {
|
||||
regular_count: Mutex<usize>,
|
||||
compact_calls: Arc<Mutex<usize>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl LlmProvider for CoopProvider {
|
||||
async fn stream(
|
||||
&self,
|
||||
request: &LlmRequest,
|
||||
) -> Result<mpsc::Receiver<LlmEvent>, ProviderError> {
|
||||
let is_compact = request.tools.is_empty();
|
||||
|
||||
if is_compact {
|
||||
*self.compact_calls.lock().unwrap() += 1;
|
||||
|
||||
let events = vec![
|
||||
LlmEvent::TextDelta("<summary>Cooperative summary</summary>".to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 5_000,
|
||||
output_tokens: 2_000,
|
||||
..Default::default()
|
||||
},
|
||||
},
|
||||
];
|
||||
let (tx, rx) = mpsc::channel(64);
|
||||
tokio::spawn(async move {
|
||||
for e in events {
|
||||
let _ = tx.send(e).await;
|
||||
}
|
||||
});
|
||||
return Ok(rx);
|
||||
}
|
||||
|
||||
let count = {
|
||||
let mut c = self.regular_count.lock().unwrap();
|
||||
let v = *c;
|
||||
*c += 1;
|
||||
v
|
||||
};
|
||||
|
||||
// 7 tool-use turns (count 0-6). Turn 6 returns high tokens.
|
||||
// micro_keep_recent = 3 → count threshold = 6.
|
||||
// After 7 tool results: 7 > 6 → micro fires.
|
||||
// After turn 6: last_input_tokens = 170k > 167k → auto fires.
|
||||
let events = if count < 7 {
|
||||
let input_tokens = if count == 6 { 170_000 } else { 10_000 };
|
||||
vec![
|
||||
LlmEvent::ToolUse {
|
||||
id: format!("t{count}"),
|
||||
name: "mock_tool".to_string(),
|
||||
input: serde_json::json!({}),
|
||||
extra: None,
|
||||
},
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::ToolUse,
|
||||
usage: TokenUsage {
|
||||
input_tokens,
|
||||
output_tokens: 100,
|
||||
..Default::default()
|
||||
},
|
||||
},
|
||||
]
|
||||
} else {
|
||||
vec![
|
||||
LlmEvent::TextDelta("After cooperative compact".to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 5_000,
|
||||
output_tokens: 100,
|
||||
..Default::default()
|
||||
},
|
||||
},
|
||||
]
|
||||
};
|
||||
|
||||
let (tx, rx) = mpsc::channel(64);
|
||||
tokio::spawn(async move {
|
||||
for e in events {
|
||||
let _ = tx.send(e).await;
|
||||
}
|
||||
});
|
||||
Ok(rx)
|
||||
}
|
||||
}
|
||||
|
||||
let provider = Arc::new(CoopProvider {
|
||||
regular_count: Mutex::new(0),
|
||||
compact_calls: counter_ref,
|
||||
});
|
||||
|
||||
let mut config = test_config();
|
||||
config.compact = CompactConfig {
|
||||
micro_keep_recent: 3,
|
||||
compactable_tools: vec!["mock_tool".into()],
|
||||
context_window: 200_000,
|
||||
emergency_buffer: 3_000,
|
||||
..Default::default()
|
||||
};
|
||||
config.max_turns = Some(20);
|
||||
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(common::MockTool::new(
|
||||
"mock_tool",
|
||||
"tool output data",
|
||||
false,
|
||||
)));
|
||||
let output = silent_output();
|
||||
|
||||
let mut engine =
|
||||
AgentEngine::new_with_provider(provider, config, registry, output, std::env::temp_dir());
|
||||
let result = engine.run("Work", "msg-1").await.expect("should succeed");
|
||||
|
||||
assert_eq!(result.text, "After cooperative compact");
|
||||
|
||||
// Autocompact was called exactly once (micro freed tokens but
|
||||
// did not reduce last_input_tokens, so auto still fired).
|
||||
let calls = *compact_call_count.lock().unwrap();
|
||||
assert_eq!(
|
||||
calls, 1,
|
||||
"autocompact should fire exactly once despite microcompact running first"
|
||||
);
|
||||
|
||||
// Total turns: 7 tool-use + 1 post-compact text = 8 engine turns,
|
||||
// plus 1 internal compact LLM call = 9 provider calls.
|
||||
assert_eq!(result.turns, 8);
|
||||
}
|
||||
|
||||
// ── TC-2.6-E2E-03: Circuit breaker after repeated failures ─────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_2_6_e2e_03_circuit_breaker_stops_retries() {
|
||||
// Simulate: 3 turns where autocompact would trigger but fails each time.
|
||||
// After 3 failures the circuit breaker trips and autocompact stops.
|
||||
//
|
||||
// We use a provider that always fails the compact summary call with
|
||||
// a generic API error, but succeeds for regular conversation turns.
|
||||
|
||||
struct CircuitBreakerProvider {
|
||||
call_index: Mutex<usize>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl LlmProvider for CircuitBreakerProvider {
|
||||
async fn stream(
|
||||
&self,
|
||||
request: &LlmRequest,
|
||||
) -> Result<mpsc::Receiver<LlmEvent>, ProviderError> {
|
||||
let idx = {
|
||||
let mut i = self.call_index.lock().unwrap();
|
||||
let v = *i;
|
||||
*i += 1;
|
||||
v
|
||||
};
|
||||
|
||||
// Compact summary calls have no tools defined and include the
|
||||
// compact prompt in messages. We detect them by checking tools.is_empty().
|
||||
let is_compact_call = request.tools.is_empty();
|
||||
|
||||
if is_compact_call {
|
||||
return Err(ProviderError::Api {
|
||||
status: 500,
|
||||
message: "Internal error".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// Regular conversation turns: tool use on odd calls, text on even
|
||||
let events = if idx % 2 == 0 {
|
||||
// Tool use turn → keeps the loop going
|
||||
vec![
|
||||
LlmEvent::ToolUse {
|
||||
id: format!("t{idx}"),
|
||||
name: "mock_tool".to_string(),
|
||||
input: serde_json::json!({}),
|
||||
extra: None,
|
||||
},
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::ToolUse,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 170_000, // above autocompact threshold
|
||||
output_tokens: 100,
|
||||
..Default::default()
|
||||
},
|
||||
},
|
||||
]
|
||||
} else {
|
||||
// Text turn → ends the loop
|
||||
vec![
|
||||
LlmEvent::TextDelta("Final".to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 170_000,
|
||||
output_tokens: 100,
|
||||
..Default::default()
|
||||
},
|
||||
},
|
||||
]
|
||||
};
|
||||
|
||||
let (tx, rx) = mpsc::channel(64);
|
||||
tokio::spawn(async move {
|
||||
for event in events {
|
||||
let _ = tx.send(event).await;
|
||||
}
|
||||
});
|
||||
Ok(rx)
|
||||
}
|
||||
}
|
||||
|
||||
let provider = Arc::new(CircuitBreakerProvider {
|
||||
call_index: Mutex::new(0),
|
||||
});
|
||||
|
||||
let mut config = test_config();
|
||||
config.compact = CompactConfig {
|
||||
max_failures: 3,
|
||||
// Set emergency very high so it doesn't interfere
|
||||
context_window: 500_000,
|
||||
emergency_buffer: 3_000,
|
||||
..Default::default()
|
||||
};
|
||||
config.max_turns = Some(10);
|
||||
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(common::MockTool::new(
|
||||
"mock_tool",
|
||||
"result",
|
||||
false,
|
||||
)));
|
||||
let output = silent_output();
|
||||
|
||||
let mut engine =
|
||||
AgentEngine::new_with_provider(provider, config, registry, output, std::env::temp_dir());
|
||||
let result = engine.run("Work", "msg-1").await.expect("should succeed");
|
||||
|
||||
assert_eq!(result.text, "Final");
|
||||
}
|
||||
@@ -0,0 +1,589 @@
|
||||
mod common;
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use nomi_agent::engine::{AgentEngine, AgentError};
|
||||
use nomi_agent::output::OutputSink;
|
||||
use nomi_agent::output::terminal::TerminalSink;
|
||||
use nomi_agent::session::SessionManager;
|
||||
use nomi_providers::{LlmProvider, ProviderError};
|
||||
use nomi_tools::registry::ToolRegistry;
|
||||
use nomi_types::llm::{LlmEvent, LlmRequest};
|
||||
use nomi_types::message::{ContentBlock, Message, Role, StopReason, TokenUsage};
|
||||
use serde_json::json;
|
||||
use tempfile::tempdir;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use common::{MockLlmProvider, MockTool, test_config};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helper: build a no-color OutputFormatter for silent test output
|
||||
// ---------------------------------------------------------------------------
|
||||
fn silent_output() -> Arc<dyn OutputSink> {
|
||||
Arc::new(TerminalSink::new(true))
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct RecordingOutputSink {
|
||||
tool_calls: Mutex<Vec<(String, String)>>,
|
||||
tool_results: Mutex<Vec<(String, String, bool)>>,
|
||||
}
|
||||
|
||||
impl OutputSink for RecordingOutputSink {
|
||||
fn emit_text_delta(&self, _text: &str, _msg_id: &str) {}
|
||||
fn emit_thinking(&self, _text: &str, _msg_id: &str) {}
|
||||
|
||||
fn emit_tool_call(&self, tool_use_id: &str, name: &str, _input: &str) {
|
||||
self.tool_calls
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push((tool_use_id.to_owned(), name.to_owned()));
|
||||
}
|
||||
|
||||
fn emit_tool_result(&self, tool_use_id: &str, name: &str, is_error: bool, _content: &str) {
|
||||
self.tool_results
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push((tool_use_id.to_owned(), name.to_owned(), is_error));
|
||||
}
|
||||
|
||||
fn emit_stream_start(&self, _msg_id: &str) {}
|
||||
fn emit_stream_end(
|
||||
&self,
|
||||
_msg_id: &str,
|
||||
_turns: usize,
|
||||
_input_tokens: u64,
|
||||
_output_tokens: u64,
|
||||
_cache_creation_tokens: u64,
|
||||
_cache_read_tokens: u64,
|
||||
) {
|
||||
}
|
||||
fn emit_error(&self, _msg: &str) {}
|
||||
fn emit_info(&self, _msg: &str) {}
|
||||
}
|
||||
|
||||
struct RecordingRequestProvider {
|
||||
requests: Arc<Mutex<Vec<Vec<Message>>>>,
|
||||
responses: Mutex<Vec<Vec<LlmEvent>>>,
|
||||
}
|
||||
|
||||
impl RecordingRequestProvider {
|
||||
fn new(responses: Vec<Vec<LlmEvent>>) -> Self {
|
||||
Self {
|
||||
requests: Arc::new(Mutex::new(Vec::new())),
|
||||
responses: Mutex::new(responses),
|
||||
}
|
||||
}
|
||||
|
||||
fn requests(&self) -> Arc<Mutex<Vec<Vec<Message>>>> {
|
||||
Arc::clone(&self.requests)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl LlmProvider for RecordingRequestProvider {
|
||||
async fn stream(
|
||||
&self,
|
||||
request: &LlmRequest,
|
||||
) -> Result<mpsc::Receiver<LlmEvent>, ProviderError> {
|
||||
self.requests.lock().unwrap().push(request.messages.clone());
|
||||
let events = self.responses.lock().unwrap().remove(0);
|
||||
let (tx, rx) = mpsc::channel(64);
|
||||
tokio::spawn(async move {
|
||||
for event in events {
|
||||
let _ = tx.send(event).await;
|
||||
}
|
||||
});
|
||||
Ok(rx)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// test_engine_text_response_ends_turn
|
||||
//
|
||||
// Verifies that when the LLM returns a pure text response the engine:
|
||||
// - captures the full text
|
||||
// - reports StopReason::EndTurn
|
||||
// - completes in a single turn
|
||||
// ---------------------------------------------------------------------------
|
||||
#[tokio::test]
|
||||
async fn test_engine_text_response_ends_turn() {
|
||||
let provider = Arc::new(MockLlmProvider::with_text_response("Hello, world!"));
|
||||
let config = test_config();
|
||||
let registry = ToolRegistry::new();
|
||||
let output = silent_output();
|
||||
|
||||
let mut engine =
|
||||
AgentEngine::new_with_provider(provider, config, registry, output, std::env::temp_dir());
|
||||
let result = engine.run("Hi", "").await.expect("engine should succeed");
|
||||
|
||||
assert_eq!(result.text, "Hello, world!");
|
||||
assert_eq!(result.stop_reason, StopReason::EndTurn);
|
||||
assert_eq!(result.turns, 1);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// test_engine_tool_use_executes_and_continues
|
||||
//
|
||||
// Verifies the agentic loop when the LLM first requests a tool then, after
|
||||
// receiving the tool result, produces a final text answer.
|
||||
// - Turn 1: LLM emits ToolUse for "mock_tool"
|
||||
// - Turn 2: LLM emits TextDelta("Done") + EndTurn
|
||||
// - result.turns == 2 and result.text == "Done"
|
||||
// ---------------------------------------------------------------------------
|
||||
#[tokio::test]
|
||||
async fn test_engine_tool_use_executes_and_continues() {
|
||||
let turn1 = vec![
|
||||
LlmEvent::ToolUse {
|
||||
id: "tool-1".to_string(),
|
||||
name: "mock_tool".to_string(),
|
||||
input: json!({}),
|
||||
extra: None,
|
||||
},
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::ToolUse,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 80,
|
||||
output_tokens: 30,
|
||||
cache_creation_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
},
|
||||
},
|
||||
];
|
||||
let turn2 = vec![
|
||||
LlmEvent::TextDelta("Done".to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 100,
|
||||
output_tokens: 50,
|
||||
cache_creation_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
let provider = Arc::new(MockLlmProvider::with_turns(vec![turn1, turn2]));
|
||||
let config = test_config();
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(MockTool::new("mock_tool", "tool output", false)));
|
||||
let output = silent_output();
|
||||
|
||||
let mut engine =
|
||||
AgentEngine::new_with_provider(provider, config, registry, output, std::env::temp_dir());
|
||||
let result = engine
|
||||
.run("Use the tool", "")
|
||||
.await
|
||||
.expect("engine should succeed");
|
||||
|
||||
assert_eq!(result.turns, 2);
|
||||
assert_eq!(result.text, "Done");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_engine_round_trips_thinking_signature_into_tool_followup_request() {
|
||||
let provider = Arc::new(RecordingRequestProvider::new(vec![
|
||||
vec![
|
||||
LlmEvent::ThinkingDelta("need a tool".to_string()),
|
||||
LlmEvent::ThinkingSignature("sig-123".to_string()),
|
||||
LlmEvent::ToolUse {
|
||||
id: "call_1".to_string(),
|
||||
name: "mock_tool".to_string(),
|
||||
input: json!({}),
|
||||
extra: None,
|
||||
},
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::ToolUse,
|
||||
usage: TokenUsage::default(),
|
||||
},
|
||||
],
|
||||
vec![
|
||||
LlmEvent::TextDelta("done".to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage::default(),
|
||||
},
|
||||
],
|
||||
]));
|
||||
let requests = provider.requests();
|
||||
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(MockTool::new("mock_tool", "tool result", false)));
|
||||
|
||||
let mut engine = AgentEngine::new_with_provider(
|
||||
provider,
|
||||
test_config(),
|
||||
registry,
|
||||
silent_output(),
|
||||
std::env::temp_dir(),
|
||||
);
|
||||
|
||||
let result = engine
|
||||
.run("use tool", "")
|
||||
.await
|
||||
.expect("engine should succeed");
|
||||
|
||||
assert_eq!(result.text, "done");
|
||||
let requests = requests.lock().unwrap();
|
||||
assert_eq!(requests.len(), 2);
|
||||
|
||||
let followup_messages = &requests[1];
|
||||
let assistant_message = followup_messages
|
||||
.iter()
|
||||
.find(|message| message.role == Role::Assistant)
|
||||
.expect("assistant message should be present");
|
||||
|
||||
match &assistant_message.content[0] {
|
||||
ContentBlock::Thinking {
|
||||
thinking,
|
||||
signature,
|
||||
} => {
|
||||
assert_eq!(thinking, "need a tool");
|
||||
assert_eq!(signature.as_deref(), Some("sig-123"));
|
||||
}
|
||||
other => panic!("expected thinking block, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn duplicate_tool_names_emit_distinct_tool_use_ids() {
|
||||
let turn1 = vec![
|
||||
LlmEvent::ToolUse {
|
||||
id: "call_a".to_string(),
|
||||
name: "Glob".to_string(),
|
||||
input: json!({"pattern": "*.rs"}),
|
||||
extra: None,
|
||||
},
|
||||
LlmEvent::ToolUse {
|
||||
id: "call_b".to_string(),
|
||||
name: "Glob".to_string(),
|
||||
input: json!({"pattern": "*.toml"}),
|
||||
extra: None,
|
||||
},
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::ToolUse,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 80,
|
||||
output_tokens: 30,
|
||||
cache_creation_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
},
|
||||
},
|
||||
];
|
||||
let turn2 = vec![
|
||||
LlmEvent::TextDelta("Done".to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 100,
|
||||
output_tokens: 50,
|
||||
cache_creation_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
let provider = Arc::new(MockLlmProvider::with_turns(vec![turn1, turn2]));
|
||||
let config = test_config();
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(MockTool::new("Glob", "tool output", false)));
|
||||
let output = Arc::new(RecordingOutputSink::default());
|
||||
|
||||
let mut engine = AgentEngine::new_with_provider(
|
||||
provider,
|
||||
config,
|
||||
registry,
|
||||
output.clone(),
|
||||
std::env::temp_dir(),
|
||||
);
|
||||
let result = engine
|
||||
.run("Use Glob twice", "")
|
||||
.await
|
||||
.expect("engine should succeed");
|
||||
|
||||
assert_eq!(result.text, "Done");
|
||||
assert_eq!(
|
||||
*output.tool_calls.lock().unwrap(),
|
||||
vec![
|
||||
("call_a".to_string(), "Glob".to_string()),
|
||||
("call_b".to_string(), "Glob".to_string()),
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
*output.tool_results.lock().unwrap(),
|
||||
vec![
|
||||
("call_a".to_string(), "Glob".to_string(), false),
|
||||
("call_b".to_string(), "Glob".to_string(), false),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// test_engine_max_tokens_handling
|
||||
//
|
||||
// Verifies that a MaxTokens stop reason is surfaced correctly when the LLM
|
||||
// hits its token limit mid-response.
|
||||
// ---------------------------------------------------------------------------
|
||||
#[tokio::test]
|
||||
async fn test_engine_max_tokens_handling() {
|
||||
let events = vec![
|
||||
LlmEvent::TextDelta("partial".to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::MaxTokens,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 200,
|
||||
output_tokens: 100,
|
||||
cache_creation_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
let provider = Arc::new(MockLlmProvider::with_events(events));
|
||||
let config = test_config();
|
||||
let registry = ToolRegistry::new();
|
||||
let output = silent_output();
|
||||
|
||||
let mut engine =
|
||||
AgentEngine::new_with_provider(provider, config, registry, output, std::env::temp_dir());
|
||||
let result = engine
|
||||
.run("Give me a long answer", "")
|
||||
.await
|
||||
.expect("engine should succeed");
|
||||
|
||||
assert_eq!(result.stop_reason, StopReason::MaxTokens);
|
||||
assert_eq!(result.text, "partial");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// test_engine_message_accumulation
|
||||
//
|
||||
// Verifies that consecutive calls to `run` accumulate messages across turns.
|
||||
// Session persistence is used to observe the messages externally since
|
||||
// engine.messages is private.
|
||||
//
|
||||
// After two independent `run` calls the persisted session must contain
|
||||
// exactly 4 messages: [user, assistant, user, assistant].
|
||||
// ---------------------------------------------------------------------------
|
||||
#[tokio::test]
|
||||
async fn test_engine_message_accumulation() {
|
||||
let dir = tempdir().expect("tempdir should be created");
|
||||
|
||||
// Provider needs two responses (one per run() call)
|
||||
let provider = Arc::new(MockLlmProvider::with_turns(vec![
|
||||
vec![
|
||||
LlmEvent::TextDelta("Response 1".to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 10,
|
||||
output_tokens: 5,
|
||||
cache_creation_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
},
|
||||
},
|
||||
],
|
||||
vec![
|
||||
LlmEvent::TextDelta("Response 2".to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 10,
|
||||
output_tokens: 5,
|
||||
cache_creation_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
},
|
||||
},
|
||||
],
|
||||
]));
|
||||
|
||||
let mut config = test_config();
|
||||
config.session.enabled = true;
|
||||
config.session.directory = dir.path().to_string_lossy().into_owned();
|
||||
|
||||
let registry = ToolRegistry::new();
|
||||
let output = silent_output();
|
||||
|
||||
let mut engine = AgentEngine::new_with_provider(
|
||||
provider,
|
||||
config.clone(),
|
||||
registry,
|
||||
output,
|
||||
std::env::temp_dir(),
|
||||
);
|
||||
|
||||
// Initialize session so save_session() has a session to persist
|
||||
engine
|
||||
.init_session("test-provider", "/tmp", None)
|
||||
.expect("init_session should succeed");
|
||||
|
||||
engine
|
||||
.run("First message", "")
|
||||
.await
|
||||
.expect("first run should succeed");
|
||||
engine
|
||||
.run("Second message", "")
|
||||
.await
|
||||
.expect("second run should succeed");
|
||||
|
||||
// Load the persisted session and count accumulated messages
|
||||
let session_manager = SessionManager::new(dir.path().to_path_buf(), 10);
|
||||
let session = session_manager
|
||||
.load("latest")
|
||||
.expect("session should be loadable");
|
||||
|
||||
// Expected layout: user, assistant, user, assistant
|
||||
assert_eq!(
|
||||
session.messages.len(),
|
||||
4,
|
||||
"expected 4 messages (user+assistant for each run), got {}",
|
||||
session.messages.len()
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// test_engine_token_usage_tracking
|
||||
//
|
||||
// Verifies that token usage is accumulated correctly across multiple turns.
|
||||
// - Turn 1: ToolUse with usage(80 in, 30 out)
|
||||
// - Turn 2: EndTurn with usage(100 in, 50 out)
|
||||
// - Expected total: input=180, output=80
|
||||
// ---------------------------------------------------------------------------
|
||||
#[tokio::test]
|
||||
async fn test_engine_token_usage_tracking() {
|
||||
let turn1 = vec![
|
||||
LlmEvent::ToolUse {
|
||||
id: "tool-1".to_string(),
|
||||
name: "mock_tool".to_string(),
|
||||
input: json!({}),
|
||||
extra: None,
|
||||
},
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::ToolUse,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 80,
|
||||
output_tokens: 30,
|
||||
cache_creation_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
},
|
||||
},
|
||||
];
|
||||
let turn2 = vec![
|
||||
LlmEvent::TextDelta("Final answer".to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 100,
|
||||
output_tokens: 50,
|
||||
cache_creation_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
let provider = Arc::new(MockLlmProvider::with_turns(vec![turn1, turn2]));
|
||||
let config = test_config();
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(MockTool::new("mock_tool", "result", false)));
|
||||
let output = silent_output();
|
||||
|
||||
let mut engine =
|
||||
AgentEngine::new_with_provider(provider, config, registry, output, std::env::temp_dir());
|
||||
let result = engine
|
||||
.run("Do work", "")
|
||||
.await
|
||||
.expect("engine should succeed");
|
||||
|
||||
assert_eq!(
|
||||
result.usage.input_tokens, 180,
|
||||
"input tokens should accumulate across turns"
|
||||
);
|
||||
assert_eq!(
|
||||
result.usage.output_tokens, 80,
|
||||
"output tokens should accumulate across turns"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// test_engine_max_turns_returns_ok
|
||||
//
|
||||
// Verifies that the engine returns Ok with StopReason::MaxTurns when the
|
||||
// LLM keeps requesting tools beyond the configured max_turns limit.
|
||||
//
|
||||
// With max_turns=1 the engine executes one turn. If that turn has tool
|
||||
// calls it processes them, then loops back and hits the limit.
|
||||
// ---------------------------------------------------------------------------
|
||||
#[tokio::test]
|
||||
async fn test_engine_max_turns_returns_ok() {
|
||||
let tool_use_turn = || {
|
||||
vec![
|
||||
LlmEvent::ToolUse {
|
||||
id: "tool-1".to_string(),
|
||||
name: "mock_tool".to_string(),
|
||||
input: json!({}),
|
||||
extra: None,
|
||||
},
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::ToolUse,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 50,
|
||||
output_tokens: 20,
|
||||
cache_creation_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
},
|
||||
},
|
||||
]
|
||||
};
|
||||
|
||||
let provider = Arc::new(MockLlmProvider::with_turns(vec![
|
||||
tool_use_turn(),
|
||||
tool_use_turn(),
|
||||
]));
|
||||
|
||||
let mut config = test_config();
|
||||
config.max_turns = Some(1);
|
||||
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(MockTool::new("mock_tool", "result", false)));
|
||||
let output = silent_output();
|
||||
|
||||
let mut engine =
|
||||
AgentEngine::new_with_provider(provider, config, registry, output, std::env::temp_dir());
|
||||
let result = engine
|
||||
.run("Keep calling tools", "")
|
||||
.await
|
||||
.expect("should return Ok, not Err");
|
||||
|
||||
assert_eq!(result.stop_reason, StopReason::MaxTurns);
|
||||
assert_eq!(result.turns, 1);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// test_engine_api_error_handling
|
||||
//
|
||||
// Verifies that an LlmEvent::Error propagates as AgentError::ApiError with
|
||||
// the original error message intact.
|
||||
// ---------------------------------------------------------------------------
|
||||
#[tokio::test]
|
||||
async fn test_engine_api_error_handling() {
|
||||
let events = vec![LlmEvent::Error("test error".to_string())];
|
||||
|
||||
let provider = Arc::new(MockLlmProvider::with_events(events));
|
||||
let config = test_config();
|
||||
let registry = ToolRegistry::new();
|
||||
let output = silent_output();
|
||||
|
||||
let mut engine =
|
||||
AgentEngine::new_with_provider(provider, config, registry, output, std::env::temp_dir());
|
||||
let err = engine
|
||||
.run("Hello", "")
|
||||
.await
|
||||
.map(|_| panic!("expected error, got Ok"))
|
||||
.unwrap_err();
|
||||
|
||||
match err {
|
||||
AgentError::ApiError(msg) => assert_eq!(msg, "test error"),
|
||||
other => panic!("expected ApiError(\"test error\"), got: {:?}", other),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
mod common;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_json::json;
|
||||
|
||||
use nomi_agent::engine::AgentEngine;
|
||||
use nomi_agent::output::OutputSink;
|
||||
use nomi_agent::output::terminal::TerminalSink;
|
||||
use nomi_protocol::writer::ProtocolWriter;
|
||||
use nomi_protocol::{ToolApprovalManager, ToolApprovalResult};
|
||||
use nomi_tools::registry::ToolRegistry;
|
||||
use nomi_types::llm::LlmEvent;
|
||||
use nomi_types::message::{StopReason, TokenUsage};
|
||||
|
||||
use common::{ExecMockTool, MockLlmProvider, test_config};
|
||||
|
||||
fn silent_output() -> Arc<dyn OutputSink> {
|
||||
Arc::new(TerminalSink::new(true))
|
||||
}
|
||||
|
||||
fn token_usage(input: u64, output: u64) -> TokenUsage {
|
||||
TokenUsage {
|
||||
input_tokens: input,
|
||||
output_tokens: output,
|
||||
cache_creation_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// test: tool approval approve flow
|
||||
//
|
||||
// LLM requests exec_tool → engine pauses at approval_manager.request_approval
|
||||
// → background task resolves with Approved → tool executes → LLM continues
|
||||
// ---------------------------------------------------------------------------
|
||||
#[tokio::test]
|
||||
async fn test_tool_approval_approve_flow() {
|
||||
let turn1 = vec![
|
||||
LlmEvent::ToolUse {
|
||||
id: "call-1".to_string(),
|
||||
name: "exec_tool".to_string(),
|
||||
input: json!({}),
|
||||
extra: None,
|
||||
},
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::ToolUse,
|
||||
usage: token_usage(80, 30),
|
||||
},
|
||||
];
|
||||
let turn2 = vec![
|
||||
LlmEvent::TextDelta("Done".to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: token_usage(100, 50),
|
||||
},
|
||||
];
|
||||
|
||||
let provider = Arc::new(MockLlmProvider::with_turns(vec![turn1, turn2]));
|
||||
let mut config = test_config();
|
||||
config.tools.auto_approve = false;
|
||||
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(ExecMockTool::new("exec_tool", "tool output")));
|
||||
|
||||
let output = silent_output();
|
||||
let approval_manager = Arc::new(ToolApprovalManager::new());
|
||||
let writer = Arc::new(ProtocolWriter::new());
|
||||
|
||||
let mut engine =
|
||||
AgentEngine::new_with_provider(provider, config, registry, output, std::env::temp_dir());
|
||||
engine.set_approval_manager(approval_manager.clone());
|
||||
engine.set_protocol_writer(writer);
|
||||
|
||||
// Spawn a task that approves the tool call after a short delay
|
||||
let am = approval_manager.clone();
|
||||
tokio::spawn(async move {
|
||||
// Wait until the approval request appears
|
||||
loop {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
let has_pending = {
|
||||
// Check if there's a pending request by trying to resolve a known id
|
||||
// We know the call_id is "call-1" from the mock
|
||||
true
|
||||
};
|
||||
if has_pending {
|
||||
am.resolve("call-1", ToolApprovalResult::Approved);
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let result = engine
|
||||
.run("Use the tool", "msg-1")
|
||||
.await
|
||||
.expect("should succeed");
|
||||
assert_eq!(result.text, "Done");
|
||||
assert_eq!(result.turns, 2);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// test: tool approval deny flow
|
||||
//
|
||||
// LLM requests exec_tool → engine pauses → background resolves with Denied
|
||||
// → tool_cancelled → denial fed back to LLM → LLM responds with text
|
||||
// ---------------------------------------------------------------------------
|
||||
#[tokio::test]
|
||||
async fn test_tool_approval_deny_flow() {
|
||||
let turn1 = vec![
|
||||
LlmEvent::ToolUse {
|
||||
id: "call-2".to_string(),
|
||||
name: "exec_tool".to_string(),
|
||||
input: json!({}),
|
||||
extra: None,
|
||||
},
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::ToolUse,
|
||||
usage: token_usage(80, 30),
|
||||
},
|
||||
];
|
||||
let turn2 = vec![
|
||||
LlmEvent::TextDelta("Cannot run tool".to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: token_usage(100, 50),
|
||||
},
|
||||
];
|
||||
|
||||
let provider = Arc::new(MockLlmProvider::with_turns(vec![turn1, turn2]));
|
||||
let mut config = test_config();
|
||||
config.tools.auto_approve = false;
|
||||
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(ExecMockTool::new("exec_tool", "tool output")));
|
||||
|
||||
let output = silent_output();
|
||||
let approval_manager = Arc::new(ToolApprovalManager::new());
|
||||
let writer = Arc::new(ProtocolWriter::new());
|
||||
|
||||
let mut engine =
|
||||
AgentEngine::new_with_provider(provider, config, registry, output, std::env::temp_dir());
|
||||
engine.set_approval_manager(approval_manager.clone());
|
||||
engine.set_protocol_writer(writer);
|
||||
|
||||
let am = approval_manager.clone();
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
am.resolve(
|
||||
"call-2",
|
||||
ToolApprovalResult::Denied {
|
||||
reason: "policy violation".into(),
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
let result = engine
|
||||
.run("Use the tool", "msg-2")
|
||||
.await
|
||||
.expect("should succeed");
|
||||
assert_eq!(result.text, "Cannot run tool");
|
||||
assert_eq!(result.turns, 2);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// test: auto_approve bypasses approval wait
|
||||
//
|
||||
// With auto_approve=true, exec category tools should execute immediately
|
||||
// without waiting for approval.
|
||||
// ---------------------------------------------------------------------------
|
||||
#[tokio::test]
|
||||
async fn test_auto_approve_bypasses_approval() {
|
||||
let turn1 = vec![
|
||||
LlmEvent::ToolUse {
|
||||
id: "call-3".to_string(),
|
||||
name: "exec_tool".to_string(),
|
||||
input: json!({}),
|
||||
extra: None,
|
||||
},
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::ToolUse,
|
||||
usage: token_usage(80, 30),
|
||||
},
|
||||
];
|
||||
let turn2 = vec![
|
||||
LlmEvent::TextDelta("Auto done".to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: token_usage(100, 50),
|
||||
},
|
||||
];
|
||||
|
||||
let provider = Arc::new(MockLlmProvider::with_turns(vec![turn1, turn2]));
|
||||
let mut config = test_config();
|
||||
config.tools.auto_approve = true;
|
||||
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(ExecMockTool::new("exec_tool", "tool output")));
|
||||
|
||||
let output = silent_output();
|
||||
let approval_manager = Arc::new(ToolApprovalManager::new());
|
||||
let writer = Arc::new(ProtocolWriter::new());
|
||||
|
||||
let mut engine =
|
||||
AgentEngine::new_with_provider(provider, config, registry, output, std::env::temp_dir());
|
||||
engine.set_approval_manager(approval_manager.clone());
|
||||
engine.set_protocol_writer(writer);
|
||||
|
||||
// No background task to approve — should not hang
|
||||
let result = engine
|
||||
.run("Use the tool", "msg-3")
|
||||
.await
|
||||
.expect("should succeed");
|
||||
assert_eq!(result.text, "Auto done");
|
||||
assert_eq!(result.turns, 2);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// test: session auto-approve (scope=always) bypasses future approvals
|
||||
//
|
||||
// After add_auto_approve("exec"), exec tools skip the approval wait.
|
||||
// ---------------------------------------------------------------------------
|
||||
#[tokio::test]
|
||||
async fn test_session_auto_approve_category() {
|
||||
let turn1 = vec![
|
||||
LlmEvent::ToolUse {
|
||||
id: "call-4".to_string(),
|
||||
name: "exec_tool".to_string(),
|
||||
input: json!({}),
|
||||
extra: None,
|
||||
},
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::ToolUse,
|
||||
usage: token_usage(80, 30),
|
||||
},
|
||||
];
|
||||
let turn2 = vec![
|
||||
LlmEvent::TextDelta("Session auto".to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: token_usage(100, 50),
|
||||
},
|
||||
];
|
||||
|
||||
let provider = Arc::new(MockLlmProvider::with_turns(vec![turn1, turn2]));
|
||||
let mut config = test_config();
|
||||
config.tools.auto_approve = false;
|
||||
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(ExecMockTool::new("exec_tool", "tool output")));
|
||||
|
||||
let output = silent_output();
|
||||
let approval_manager = Arc::new(ToolApprovalManager::new());
|
||||
// Pre-approve the "exec" category
|
||||
approval_manager.add_auto_approve("exec");
|
||||
let writer = Arc::new(ProtocolWriter::new());
|
||||
|
||||
let mut engine =
|
||||
AgentEngine::new_with_provider(provider, config, registry, output, std::env::temp_dir());
|
||||
engine.set_approval_manager(approval_manager.clone());
|
||||
engine.set_protocol_writer(writer);
|
||||
|
||||
// No background task to approve — should not hang
|
||||
let result = engine
|
||||
.run("Use the tool", "msg-4")
|
||||
.await
|
||||
.expect("should succeed");
|
||||
assert_eq!(result.text, "Session auto");
|
||||
assert_eq!(result.turns, 2);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// test: client disconnect (channel drop) causes UserAborted
|
||||
//
|
||||
// If the approval channel sender is dropped before resolve, the engine
|
||||
// should return an abort error.
|
||||
// ---------------------------------------------------------------------------
|
||||
#[tokio::test]
|
||||
async fn test_client_disconnect_aborts() {
|
||||
let turn1 = vec![
|
||||
LlmEvent::ToolUse {
|
||||
id: "call-5".to_string(),
|
||||
name: "exec_tool".to_string(),
|
||||
input: json!({}),
|
||||
extra: None,
|
||||
},
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::ToolUse,
|
||||
usage: token_usage(80, 30),
|
||||
},
|
||||
];
|
||||
|
||||
let provider = Arc::new(MockLlmProvider::with_turns(vec![turn1]));
|
||||
let mut config = test_config();
|
||||
config.tools.auto_approve = false;
|
||||
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(ExecMockTool::new("exec_tool", "tool output")));
|
||||
|
||||
let output = silent_output();
|
||||
let approval_manager = Arc::new(ToolApprovalManager::new());
|
||||
let writer = Arc::new(ProtocolWriter::new());
|
||||
|
||||
let mut engine =
|
||||
AgentEngine::new_with_provider(provider, config, registry, output, std::env::temp_dir());
|
||||
engine.set_approval_manager(approval_manager.clone());
|
||||
engine.set_protocol_writer(writer);
|
||||
|
||||
// Simulate client disconnect: drop the pending sender without resolving
|
||||
let am = approval_manager.clone();
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
am.drop_pending("call-5");
|
||||
});
|
||||
|
||||
let err = engine.run("Use the tool", "msg-5").await.unwrap_err();
|
||||
assert!(
|
||||
format!("{:?}", err).contains("UserAborted"),
|
||||
"expected UserAborted, got: {:?}",
|
||||
err
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
// Integration tests for memory system context assembly (TC-7).
|
||||
//
|
||||
// These are black-box tests that verify the memory system is correctly
|
||||
// integrated into the system prompt assembly pipeline.
|
||||
|
||||
use std::fs;
|
||||
|
||||
use nomi_agent::context::{SystemPromptCache, build_system_prompt};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-7.1: With memory_dir, system prompt includes memory content
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_7_1_memory_dir_with_content_injects_prompt() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let mem_dir = tmp.path().join("memory");
|
||||
fs::create_dir_all(&mem_dir).unwrap();
|
||||
fs::write(
|
||||
mem_dir.join("MEMORY.md"),
|
||||
"- [Role](user_role.md) \u{2014} senior engineer\n\
|
||||
- [Policy](feedback_tests.md) \u{2014} always use real DB\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let result = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
Some(&mem_dir),
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
|
||||
// Should contain minimal memory system sections
|
||||
assert!(
|
||||
result.contains("auto memory"),
|
||||
"should contain memory system display name"
|
||||
);
|
||||
assert!(
|
||||
result.contains("Memory types:"),
|
||||
"should contain compact type summary"
|
||||
);
|
||||
assert!(
|
||||
result.contains("MEMORY.md is the index"),
|
||||
"should contain compact save guidance"
|
||||
);
|
||||
|
||||
// Should contain MEMORY.md content
|
||||
assert!(
|
||||
result.contains("user_role.md"),
|
||||
"should contain MEMORY.md entries"
|
||||
);
|
||||
assert!(
|
||||
result.contains("senior engineer"),
|
||||
"should contain entry descriptions"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-7.2: Without memory_dir, no memory injection
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_7_2_no_memory_dir_no_injection() {
|
||||
let result = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
|
||||
assert!(
|
||||
!result.contains("auto memory"),
|
||||
"no memory content when memory_dir is None"
|
||||
);
|
||||
assert!(
|
||||
!result.contains("Types of memory"),
|
||||
"no type definitions when memory_dir is None"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-7.3: Memory appears after AGENTS.md, before skills
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_7_3_section_ordering() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let cwd = tmp.path();
|
||||
|
||||
// Create AGENTS.md
|
||||
fs::write(cwd.join("AGENTS.md"), "PROJECT_RULES_CONTENT").unwrap();
|
||||
|
||||
// Create memory dir
|
||||
let mem_dir = tmp.path().join("memory");
|
||||
fs::create_dir_all(&mem_dir).unwrap();
|
||||
fs::write(mem_dir.join("MEMORY.md"), "- [A](a.md) \u{2014} test\n").unwrap();
|
||||
|
||||
// Create a minimal skill metadata
|
||||
use nomi_skills::types::{ExecutionContext, LoadedFrom, SkillMetadata, SkillSource};
|
||||
let skill = SkillMetadata {
|
||||
name: "test-skill".to_string(),
|
||||
display_name: None,
|
||||
description: "A test skill".to_string(),
|
||||
has_user_specified_description: false,
|
||||
allowed_tools: vec![],
|
||||
argument_hint: None,
|
||||
argument_names: vec![],
|
||||
when_to_use: None,
|
||||
version: None,
|
||||
model: None,
|
||||
disable_model_invocation: false,
|
||||
user_invocable: true,
|
||||
execution_context: ExecutionContext::Inline,
|
||||
agent: None,
|
||||
effort: None,
|
||||
shell: None,
|
||||
paths: vec![],
|
||||
hooks_raw: None,
|
||||
source: SkillSource::User,
|
||||
loaded_from: LoadedFrom::Skills,
|
||||
content: String::new(),
|
||||
content_length: 0,
|
||||
skill_root: None,
|
||||
};
|
||||
|
||||
let result = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
&cwd.to_string_lossy(),
|
||||
"test-model",
|
||||
&[skill],
|
||||
None,
|
||||
Some(&mem_dir),
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
|
||||
let agents_pos = result
|
||||
.find("PROJECT_RULES_CONTENT")
|
||||
.expect("AGENTS.md content should be present");
|
||||
let memory_pos = result
|
||||
.find("auto memory")
|
||||
.expect("memory section should be present");
|
||||
let skills_pos = result
|
||||
.find("test-skill")
|
||||
.expect("skills section should be present");
|
||||
|
||||
assert!(
|
||||
agents_pos < memory_pos,
|
||||
"AGENTS.md content should appear before memory section"
|
||||
);
|
||||
assert!(
|
||||
memory_pos < skills_pos,
|
||||
"memory section should appear before skills listing"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-7.4: Non-existent memory_dir degrades gracefully
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_7_4_nonexistent_dir_graceful_degradation() {
|
||||
let result = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
Some(std::path::Path::new("/nonexistent/memory/dir")),
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
|
||||
// Should not panic
|
||||
assert!(
|
||||
result.contains("currently empty"),
|
||||
"nonexistent memory dir should show empty state"
|
||||
);
|
||||
assert!(
|
||||
result.contains("auto memory"),
|
||||
"memory section should still be present (with empty state)"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-7.5: MEMORY.md content correctly injected
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_7_5_memory_md_content_injected() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let mem_dir = tmp.path().join("memory");
|
||||
fs::create_dir_all(&mem_dir).unwrap();
|
||||
fs::write(
|
||||
mem_dir.join("MEMORY.md"),
|
||||
"- [User Role](user_role.md) \u{2014} senior engineer\n\
|
||||
- [Test Policy](feedback_tests.md) \u{2014} always use real DB\n\
|
||||
- [Sprint](project_sprint.md) \u{2014} sprint 42 ends Friday\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let result = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
Some(&mem_dir),
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
|
||||
assert!(
|
||||
result.contains("user_role.md"),
|
||||
"should contain first entry"
|
||||
);
|
||||
assert!(
|
||||
result.contains("feedback_tests.md"),
|
||||
"should contain second entry"
|
||||
);
|
||||
assert!(
|
||||
result.contains("project_sprint.md"),
|
||||
"should contain third entry"
|
||||
);
|
||||
assert!(
|
||||
result.contains("sprint 42 ends Friday"),
|
||||
"should contain entry descriptions"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-7.6: No MEMORY.md shows empty state
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_7_6_no_memory_md_shows_empty() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let mem_dir = tmp.path().join("memory");
|
||||
fs::create_dir_all(&mem_dir).unwrap();
|
||||
// No MEMORY.md created
|
||||
|
||||
let result = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
Some(&mem_dir),
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
|
||||
assert!(
|
||||
result.contains("currently empty"),
|
||||
"should show empty state when MEMORY.md doesn't exist"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-7.7: No bb brand identifiers in integrated prompt
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_7_7_no_bb_brand_in_integrated_prompt() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let mem_dir = tmp.path().join("memory");
|
||||
fs::create_dir_all(&mem_dir).unwrap();
|
||||
fs::write(
|
||||
mem_dir.join("MEMORY.md"),
|
||||
"- [Test](test.md) \u{2014} entry\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let result = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
Some(&mem_dir),
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
|
||||
assert!(
|
||||
!result.contains("~/.claude"),
|
||||
"should not contain bb brand path ~/.claude"
|
||||
);
|
||||
assert!(
|
||||
!result.contains("CLAUDE.md"),
|
||||
"should not reference CLAUDE.md"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,338 @@
|
||||
//! Black-box integration tests for the microcompact subsystem.
|
||||
//!
|
||||
//! These tests correspond to TC-2.3-01 through TC-2.3-11 in the test plan.
|
||||
//! They treat `should_microcompact` and `microcompact` as opaque functions
|
||||
//! and validate observable behaviour only (inputs → outputs).
|
||||
|
||||
use chrono::{Duration, Utc};
|
||||
use nomi_agent::compact::micro::{
|
||||
CLEARED_TOOL_RESULT, MicrocompactResult, microcompact, should_microcompact,
|
||||
};
|
||||
use nomi_config::compact::CompactConfig;
|
||||
use nomi_types::message::{ContentBlock, Message, Role};
|
||||
use serde_json::json;
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
fn tool_use(id: &str, name: &str) -> ContentBlock {
|
||||
ContentBlock::ToolUse {
|
||||
id: id.into(),
|
||||
name: name.into(),
|
||||
input: json!({}),
|
||||
extra: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn tool_result(id: &str, content: &str) -> ContentBlock {
|
||||
ContentBlock::ToolResult {
|
||||
tool_use_id: id.into(),
|
||||
content: content.into(),
|
||||
is_error: false,
|
||||
images: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn text(s: &str) -> ContentBlock {
|
||||
ContentBlock::Text { text: s.into() }
|
||||
}
|
||||
|
||||
fn assistant(blocks: Vec<ContentBlock>) -> Message {
|
||||
Message::new(Role::Assistant, blocks)
|
||||
}
|
||||
|
||||
fn user(blocks: Vec<ContentBlock>) -> Message {
|
||||
Message::new(Role::User, blocks)
|
||||
}
|
||||
|
||||
fn assistant_at(blocks: Vec<ContentBlock>, ts: chrono::DateTime<Utc>) -> Message {
|
||||
Message {
|
||||
role: Role::Assistant,
|
||||
content: blocks,
|
||||
timestamp: Some(ts),
|
||||
}
|
||||
}
|
||||
|
||||
fn get_tool_result_content(msg: &Message, block_idx: usize) -> &str {
|
||||
match &msg.content[block_idx] {
|
||||
ContentBlock::ToolResult { content, .. } => content.as_str(),
|
||||
other => panic!("expected ToolResult, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
// ── TC-2.3-01: Basic clearing ───────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_3_01_basic_clearing() {
|
||||
// 10 messages containing 8 tool results (Read x3, Bash x3, Grep x2).
|
||||
// keep_recent = 3 → oldest 5 cleared.
|
||||
let tool_specs = [
|
||||
("r1", "Read"),
|
||||
("b1", "Bash"),
|
||||
("g1", "Grep"),
|
||||
("r2", "Read"),
|
||||
("b2", "Bash"),
|
||||
("g2", "Grep"),
|
||||
("r3", "Read"),
|
||||
("b3", "Bash"),
|
||||
];
|
||||
let mut msgs: Vec<Message> = Vec::new();
|
||||
for (id, name) in &tool_specs {
|
||||
msgs.push(assistant(vec![tool_use(id, name)]));
|
||||
msgs.push(user(vec![tool_result(id, &format!("output-{id}"))]));
|
||||
}
|
||||
|
||||
let config = CompactConfig {
|
||||
micro_keep_recent: 3,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let result = microcompact(&mut msgs, &config);
|
||||
assert_eq!(result.cleared_count, 5);
|
||||
|
||||
// First 5 user messages (indices 1, 3, 5, 7, 9) are cleared.
|
||||
for i in 0..5 {
|
||||
let user_msg_idx = i * 2 + 1;
|
||||
assert_eq!(
|
||||
get_tool_result_content(&msgs[user_msg_idx], 0),
|
||||
CLEARED_TOOL_RESULT,
|
||||
"tool result at msg index {user_msg_idx} should be cleared"
|
||||
);
|
||||
}
|
||||
// Last 3 user messages (indices 11, 13, 15) retain original content.
|
||||
for (idx, &(id, _name)) in tool_specs.iter().enumerate().skip(5) {
|
||||
let user_msg_idx = idx * 2 + 1;
|
||||
assert_eq!(
|
||||
get_tool_result_content(&msgs[user_msg_idx], 0),
|
||||
format!("output-{id}"),
|
||||
"tool result at msg index {user_msg_idx} should be preserved"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── TC-2.3-02: Tool results insufficient — no clearing ──────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_3_02_insufficient_results_no_clearing() {
|
||||
let mut msgs = vec![
|
||||
assistant(vec![tool_use("t1", "Read")]),
|
||||
user(vec![tool_result("t1", "data-1")]),
|
||||
assistant(vec![tool_use("t2", "Bash")]),
|
||||
user(vec![tool_result("t2", "data-2")]),
|
||||
];
|
||||
let config = CompactConfig {
|
||||
micro_keep_recent: 5,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let result = microcompact(&mut msgs, &config);
|
||||
assert_eq!(result.cleared_count, 0);
|
||||
assert_eq!(
|
||||
result,
|
||||
MicrocompactResult {
|
||||
cleared_count: 0,
|
||||
estimated_tokens_freed: 0,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
// ── TC-2.3-03: Only compactable tools are cleared ───────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_3_03_only_compactable_tools_cleared() {
|
||||
let mut msgs = vec![
|
||||
assistant(vec![tool_use("t1", "Read")]),
|
||||
user(vec![tool_result("t1", "read-output")]),
|
||||
assistant(vec![tool_use("t2", "Bash")]),
|
||||
user(vec![tool_result("t2", "bash-output")]),
|
||||
assistant(vec![tool_use("t3", "Skill")]),
|
||||
user(vec![tool_result("t3", "skill-output")]),
|
||||
assistant(vec![tool_use("t4", "Read")]),
|
||||
user(vec![tool_result("t4", "read-output-2")]),
|
||||
];
|
||||
|
||||
// compactable_tools does NOT include "Skill".
|
||||
let config = CompactConfig {
|
||||
micro_keep_recent: 1,
|
||||
compactable_tools: vec!["Read".into(), "Bash".into()],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let result = microcompact(&mut msgs, &config);
|
||||
// 3 compactable (t1-Read, t2-Bash, t4-Read), keep 1 → clear 2.
|
||||
assert_eq!(result.cleared_count, 2);
|
||||
|
||||
// Skill result (t3) must be untouched.
|
||||
assert_eq!(get_tool_result_content(&msgs[5], 0), "skill-output");
|
||||
// Most recent compactable (t4) must be preserved.
|
||||
assert_eq!(get_tool_result_content(&msgs[7], 0), "read-output-2");
|
||||
}
|
||||
|
||||
// ── TC-2.3-04: Time trigger — exceeds threshold ────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_3_04_time_trigger_exceeds_threshold() {
|
||||
let old_ts = Utc::now() - Duration::seconds(3660); // 61 minutes ago
|
||||
let msgs = vec![assistant_at(vec![text("thinking")], old_ts)];
|
||||
let config = CompactConfig {
|
||||
micro_gap_seconds: 3600,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(should_microcompact(&msgs, &config));
|
||||
}
|
||||
|
||||
// ── TC-2.3-05: Time trigger — within threshold ─────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_3_05_time_trigger_within_threshold() {
|
||||
let recent_ts = Utc::now() - Duration::seconds(1800); // 30 minutes ago
|
||||
let msgs = vec![assistant_at(vec![text("thinking")], recent_ts)];
|
||||
let config = CompactConfig {
|
||||
micro_gap_seconds: 3600,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!should_microcompact(&msgs, &config));
|
||||
}
|
||||
|
||||
// ── TC-2.3-06: Count trigger ────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_3_06_count_trigger() {
|
||||
// 12 compactable results, keep_recent=5 → threshold = 10.
|
||||
// 12 > 10 → should trigger.
|
||||
let mut msgs = Vec::new();
|
||||
for i in 0..12 {
|
||||
let id = format!("t{i}");
|
||||
msgs.push(assistant(vec![tool_use(&id, "Read")]));
|
||||
msgs.push(user(vec![tool_result(&id, "data")]));
|
||||
}
|
||||
let config = CompactConfig {
|
||||
micro_keep_recent: 5,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(should_microcompact(&msgs, &config));
|
||||
}
|
||||
|
||||
// ── TC-2.3-07: No timestamp — time check skipped ───────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_3_07_no_timestamp_skips_time_check() {
|
||||
// All messages have no timestamp.
|
||||
// Only 2 compactable results with keep_recent=5 → count trigger also false.
|
||||
let msgs = vec![
|
||||
assistant(vec![tool_use("t1", "Read")]),
|
||||
user(vec![tool_result("t1", "data-1")]),
|
||||
assistant(vec![tool_use("t2", "Read")]),
|
||||
user(vec![tool_result("t2", "data-2")]),
|
||||
];
|
||||
let config = CompactConfig {
|
||||
micro_keep_recent: 5,
|
||||
micro_gap_seconds: 3600,
|
||||
..Default::default()
|
||||
};
|
||||
// No timestamp → time trigger skipped.
|
||||
// 2 results ≤ 5*2=10 → count trigger false.
|
||||
assert!(!should_microcompact(&msgs, &config));
|
||||
}
|
||||
|
||||
// ── TC-2.3-08: Token estimation after clearing ──────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_3_08_token_estimation() {
|
||||
// 3 tool results with known content lengths, clear all but 1.
|
||||
let content_a = "x".repeat(200); // 50 tokens
|
||||
let content_b = "y".repeat(400); // 100 tokens
|
||||
let content_c = "z".repeat(80); // 20 tokens — kept
|
||||
let mut msgs = vec![
|
||||
assistant(vec![tool_use("a", "Read")]),
|
||||
user(vec![tool_result("a", &content_a)]),
|
||||
assistant(vec![tool_use("b", "Bash")]),
|
||||
user(vec![tool_result("b", &content_b)]),
|
||||
assistant(vec![tool_use("c", "Grep")]),
|
||||
user(vec![tool_result("c", &content_c)]),
|
||||
];
|
||||
let config = CompactConfig {
|
||||
micro_keep_recent: 1,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let result = microcompact(&mut msgs, &config);
|
||||
assert_eq!(result.cleared_count, 2);
|
||||
assert!(result.estimated_tokens_freed > 0);
|
||||
// 200/4 + 400/4 = 50 + 100 = 150
|
||||
assert_eq!(result.estimated_tokens_freed, 150);
|
||||
}
|
||||
|
||||
// ── TC-2.3-09: Already cleared content not re-cleared ───────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_3_09_already_cleared_not_recounted() {
|
||||
let mut msgs = vec![
|
||||
assistant(vec![tool_use("t1", "Read")]),
|
||||
user(vec![tool_result("t1", CLEARED_TOOL_RESULT)]),
|
||||
assistant(vec![tool_use("t2", "Read")]),
|
||||
user(vec![tool_result("t2", "live-data")]),
|
||||
];
|
||||
let config = CompactConfig {
|
||||
micro_keep_recent: 1,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let result = microcompact(&mut msgs, &config);
|
||||
// t1 already cleared, only t2 is compactable and is the most recent → keep.
|
||||
assert_eq!(result.cleared_count, 0);
|
||||
assert_eq!(result.estimated_tokens_freed, 0);
|
||||
}
|
||||
|
||||
// ── TC-2.3-10: Empty message list ───────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_3_10_empty_messages() {
|
||||
let mut msgs: Vec<Message> = vec![];
|
||||
let result = microcompact(&mut msgs, &CompactConfig::default());
|
||||
assert_eq!(
|
||||
result,
|
||||
MicrocompactResult {
|
||||
cleared_count: 0,
|
||||
estimated_tokens_freed: 0,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
// ── TC-2.3-11: Message order preserved ──────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tc_2_3_11_message_order_preserved() {
|
||||
let mut msgs = vec![
|
||||
assistant(vec![tool_use("t1", "Read")]),
|
||||
user(vec![tool_result("t1", "data-1")]),
|
||||
assistant(vec![text("thinking about it")]),
|
||||
user(vec![text("please continue")]),
|
||||
assistant(vec![tool_use("t2", "Bash")]),
|
||||
user(vec![tool_result("t2", "bash-out")]),
|
||||
assistant(vec![tool_use("t3", "Grep")]),
|
||||
user(vec![tool_result("t3", "grep-out")]),
|
||||
];
|
||||
let original_len = msgs.len();
|
||||
let original_roles: Vec<Role> = msgs.iter().map(|m| m.role).collect();
|
||||
|
||||
let config = CompactConfig {
|
||||
micro_keep_recent: 1,
|
||||
..Default::default()
|
||||
};
|
||||
microcompact(&mut msgs, &config);
|
||||
|
||||
// Message count unchanged.
|
||||
assert_eq!(msgs.len(), original_len);
|
||||
// Role sequence unchanged.
|
||||
let after_roles: Vec<Role> = msgs.iter().map(|m| m.role).collect();
|
||||
assert_eq!(after_roles, original_roles);
|
||||
// Non-tool-result content blocks unchanged.
|
||||
match &msgs[2].content[0] {
|
||||
ContentBlock::Text { text } => assert_eq!(text, "thinking about it"),
|
||||
_ => panic!("expected Text"),
|
||||
}
|
||||
match &msgs[3].content[0] {
|
||||
ContentBlock::Text { text } => assert_eq!(text, "please continue"),
|
||||
_ => panic!("expected Text"),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,363 @@
|
||||
mod common;
|
||||
|
||||
use common::{MockTool, auto_approve_confirmer};
|
||||
use nomi_agent::orchestration::execute_tool_calls;
|
||||
use nomi_compact::CompactionLevel;
|
||||
use nomi_config::hooks::{HookDef, HookEngine, HooksConfig};
|
||||
use nomi_tools::registry::ToolRegistry;
|
||||
use nomi_types::message::ContentBlock;
|
||||
use serde_json::json;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn make_tool_use(id: &str, name: &str) -> ContentBlock {
|
||||
ContentBlock::ToolUse {
|
||||
id: id.to_string(),
|
||||
name: name.to_string(),
|
||||
input: json!({}),
|
||||
extra: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn make_pre_hook(name: &str, tool_match: &str, command: &str) -> HookDef {
|
||||
HookDef {
|
||||
name: name.to_string(),
|
||||
tool_match: vec![tool_match.to_string()],
|
||||
file_match: vec![],
|
||||
command: command.to_string(),
|
||||
timeout_ms: 5_000,
|
||||
}
|
||||
}
|
||||
|
||||
fn make_post_hook(name: &str, tool_match: &str, command: &str) -> HookDef {
|
||||
HookDef {
|
||||
name: name.to_string(),
|
||||
tool_match: vec![tool_match.to_string()],
|
||||
file_match: vec![],
|
||||
command: command.to_string(),
|
||||
timeout_ms: 5_000,
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Single tool call executes and returns the expected result
|
||||
#[tokio::test]
|
||||
async fn test_execute_single_tool_call() {
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(MockTool::new("echo", "hello", false)));
|
||||
|
||||
let tool_calls = vec![make_tool_use("call-1", "echo")];
|
||||
let confirmer = auto_approve_confirmer();
|
||||
|
||||
let results = execute_tool_calls(
|
||||
®istry,
|
||||
&tool_calls,
|
||||
&confirmer,
|
||||
None,
|
||||
CompactionLevel::Off,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("execution should succeed");
|
||||
|
||||
assert_eq!(results.len(), 1);
|
||||
match &results[0] {
|
||||
ContentBlock::ToolResult {
|
||||
tool_use_id,
|
||||
content,
|
||||
is_error,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(tool_use_id, "call-1");
|
||||
assert_eq!(content, "hello");
|
||||
assert!(!is_error);
|
||||
}
|
||||
other => panic!("expected ToolResult, got {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
/// Two concurrent-safe tools execute in parallel and both return results
|
||||
#[tokio::test]
|
||||
async fn test_execute_concurrent_safe_tools() {
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(MockTool::new("tool_a", "result_a", false)));
|
||||
registry.register(Box::new(MockTool::new("tool_b", "result_b", false)));
|
||||
|
||||
let tool_calls = vec![
|
||||
make_tool_use("id-a", "tool_a"),
|
||||
make_tool_use("id-b", "tool_b"),
|
||||
];
|
||||
let confirmer = auto_approve_confirmer();
|
||||
|
||||
let results = execute_tool_calls(
|
||||
®istry,
|
||||
&tool_calls,
|
||||
&confirmer,
|
||||
None,
|
||||
CompactionLevel::Off,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("execution should succeed");
|
||||
|
||||
assert_eq!(results.len(), 2);
|
||||
|
||||
// Collect content strings keyed by tool_use_id for order-independent assertion
|
||||
let content_map: std::collections::HashMap<_, _> = results
|
||||
.iter()
|
||||
.filter_map(|r| match r {
|
||||
ContentBlock::ToolResult {
|
||||
tool_use_id,
|
||||
content,
|
||||
..
|
||||
} => Some((tool_use_id.as_str(), content.as_str())),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
|
||||
assert_eq!(content_map.get("id-a"), Some(&"result_a"));
|
||||
assert_eq!(content_map.get("id-b"), Some(&"result_b"));
|
||||
}
|
||||
|
||||
/// Two sequential (non-concurrent) tools execute one after the other and both succeed
|
||||
#[tokio::test]
|
||||
async fn test_execute_non_concurrent_tools_sequential() {
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(MockTool::sequential("seq_a", "seq_result_a")));
|
||||
registry.register(Box::new(MockTool::sequential("seq_b", "seq_result_b")));
|
||||
|
||||
let tool_calls = vec![
|
||||
make_tool_use("id-a", "seq_a"),
|
||||
make_tool_use("id-b", "seq_b"),
|
||||
];
|
||||
let confirmer = auto_approve_confirmer();
|
||||
|
||||
let results = execute_tool_calls(
|
||||
®istry,
|
||||
&tool_calls,
|
||||
&confirmer,
|
||||
None,
|
||||
CompactionLevel::Off,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("execution should succeed");
|
||||
|
||||
assert_eq!(results.len(), 2);
|
||||
|
||||
let content_map: std::collections::HashMap<_, _> = results
|
||||
.iter()
|
||||
.filter_map(|r| match r {
|
||||
ContentBlock::ToolResult {
|
||||
tool_use_id,
|
||||
content,
|
||||
..
|
||||
} => Some((tool_use_id.as_str(), content.as_str())),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
|
||||
assert_eq!(content_map.get("id-a"), Some(&"seq_result_a"));
|
||||
assert_eq!(content_map.get("id-b"), Some(&"seq_result_b"));
|
||||
}
|
||||
|
||||
/// Calling a tool that is not registered returns an error ToolResult with "Unknown tool"
|
||||
#[tokio::test]
|
||||
async fn test_unknown_tool_returns_error() {
|
||||
let registry = ToolRegistry::new(); // empty registry
|
||||
|
||||
let tool_calls = vec![make_tool_use("id-x", "nonexistent_tool")];
|
||||
let confirmer = auto_approve_confirmer();
|
||||
|
||||
let results = execute_tool_calls(
|
||||
®istry,
|
||||
&tool_calls,
|
||||
&confirmer,
|
||||
None,
|
||||
CompactionLevel::Off,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("execute_tool_calls itself should not fail");
|
||||
|
||||
assert_eq!(results.len(), 1);
|
||||
match &results[0] {
|
||||
ContentBlock::ToolResult {
|
||||
content, is_error, ..
|
||||
} => {
|
||||
assert!(*is_error, "unknown tool should produce is_error = true");
|
||||
assert!(
|
||||
content.contains("Unknown tool"),
|
||||
"error message should mention 'Unknown tool', got: {}",
|
||||
content
|
||||
);
|
||||
}
|
||||
other => panic!("expected ToolResult, got {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
/// A tool that signals an error surfaces is_error = true in the result
|
||||
#[tokio::test]
|
||||
async fn test_tool_error_returns_error_result() {
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(MockTool::new("fail_tool", "error message", true)));
|
||||
|
||||
let tool_calls = vec![make_tool_use("id-fail", "fail_tool")];
|
||||
let confirmer = auto_approve_confirmer();
|
||||
|
||||
let results = execute_tool_calls(
|
||||
®istry,
|
||||
&tool_calls,
|
||||
&confirmer,
|
||||
None,
|
||||
CompactionLevel::Off,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("execution should succeed");
|
||||
|
||||
assert_eq!(results.len(), 1);
|
||||
match &results[0] {
|
||||
ContentBlock::ToolResult {
|
||||
content, is_error, ..
|
||||
} => {
|
||||
assert!(*is_error, "tool error should propagate as is_error = true");
|
||||
assert_eq!(content, "error message");
|
||||
}
|
||||
other => panic!("expected ToolResult, got {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
/// A pre-tool-use hook that exits with a non-zero status blocks tool execution
|
||||
#[tokio::test]
|
||||
async fn test_pre_hook_blocks_tool() {
|
||||
let hook_config = HooksConfig {
|
||||
pre_tool_use: vec![make_pre_hook("blocker", "echo", "exit 1")],
|
||||
post_tool_use: vec![],
|
||||
stop: vec![],
|
||||
};
|
||||
let mut hook_engine = HookEngine::new(hook_config, std::env::temp_dir());
|
||||
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(MockTool::new("echo", "should not appear", false)));
|
||||
|
||||
let tool_calls = vec![make_tool_use("id-blocked", "echo")];
|
||||
let confirmer = auto_approve_confirmer();
|
||||
|
||||
let results = execute_tool_calls(
|
||||
®istry,
|
||||
&tool_calls,
|
||||
&confirmer,
|
||||
Some(&mut hook_engine),
|
||||
CompactionLevel::Off,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("execute_tool_calls itself should not fail");
|
||||
|
||||
assert_eq!(results.len(), 1);
|
||||
match &results[0] {
|
||||
ContentBlock::ToolResult {
|
||||
content, is_error, ..
|
||||
} => {
|
||||
assert!(
|
||||
*is_error,
|
||||
"blocked execution should produce is_error = true"
|
||||
);
|
||||
assert!(
|
||||
content.contains("Blocked by hook"),
|
||||
"result should mention 'Blocked by hook', got: {}",
|
||||
content
|
||||
);
|
||||
}
|
||||
other => panic!("expected ToolResult, got {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
/// A post-tool-use hook runs after the tool but does not alter the tool's result
|
||||
#[tokio::test]
|
||||
async fn test_post_hook_runs_after_tool() {
|
||||
let hook_config = HooksConfig {
|
||||
pre_tool_use: vec![],
|
||||
post_tool_use: vec![make_post_hook("post-logger", "echo", "echo done")],
|
||||
stop: vec![],
|
||||
};
|
||||
let mut hook_engine = HookEngine::new(hook_config, std::env::temp_dir());
|
||||
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(MockTool::new("echo", "result", false)));
|
||||
|
||||
let tool_calls = vec![make_tool_use("id-post", "echo")];
|
||||
let confirmer = auto_approve_confirmer();
|
||||
|
||||
let results = execute_tool_calls(
|
||||
®istry,
|
||||
&tool_calls,
|
||||
&confirmer,
|
||||
Some(&mut hook_engine),
|
||||
CompactionLevel::Off,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("execution should succeed");
|
||||
|
||||
assert_eq!(results.len(), 1);
|
||||
match &results[0] {
|
||||
ContentBlock::ToolResult {
|
||||
content, is_error, ..
|
||||
} => {
|
||||
// Post-hooks must not mutate the tool result
|
||||
assert!(!is_error);
|
||||
assert_eq!(content, "result");
|
||||
}
|
||||
other => panic!("expected ToolResult, got {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
/// Results that exceed max_result_size are truncated with a "[truncated N chars]" marker
|
||||
#[tokio::test]
|
||||
async fn test_tool_result_truncation() {
|
||||
// Default max_result_size is 50_000; build a result that exceeds it
|
||||
let long_result: String = "x".repeat(60_000);
|
||||
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(MockTool::new("big_tool", &long_result, false)));
|
||||
|
||||
let tool_calls = vec![make_tool_use("id-big", "big_tool")];
|
||||
let confirmer = auto_approve_confirmer();
|
||||
|
||||
let results = execute_tool_calls(
|
||||
®istry,
|
||||
&tool_calls,
|
||||
&confirmer,
|
||||
None,
|
||||
CompactionLevel::Off,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("execution should succeed");
|
||||
|
||||
assert_eq!(results.len(), 1);
|
||||
match &results[0] {
|
||||
ContentBlock::ToolResult {
|
||||
content, is_error, ..
|
||||
} => {
|
||||
assert!(!is_error);
|
||||
assert!(
|
||||
content.len() < long_result.len(),
|
||||
"truncated result should be shorter than the original"
|
||||
);
|
||||
assert!(
|
||||
content.contains("truncated"),
|
||||
"truncated result should contain the word 'truncated', got length {}",
|
||||
content.len()
|
||||
);
|
||||
}
|
||||
other => panic!("expected ToolResult, got {:?}", other),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,494 @@
|
||||
mod common;
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use common::{MockLlmProvider, MockTool, auto_approve_confirmer, test_config};
|
||||
use nomi_agent::context::{SystemPromptCache, build_system_prompt};
|
||||
use nomi_agent::engine::AgentEngine;
|
||||
use nomi_agent::orchestration::execute_tool_calls;
|
||||
use nomi_agent::output::OutputSink;
|
||||
use nomi_agent::output::null_sink::NullSink;
|
||||
use nomi_compact::CompactionLevel;
|
||||
use nomi_providers::{LlmProvider, ProviderError};
|
||||
use nomi_tools::registry::ToolRegistry;
|
||||
use nomi_types::llm::{LlmEvent, LlmRequest};
|
||||
use nomi_types::message::{ContentBlock, StopReason, TokenUsage};
|
||||
use serde_json::json;
|
||||
|
||||
const TEST_OUTPUT: &str = "\x1b[32mSTATUS: OK\x1b[0m\n\n\n\n50%\r100%\nCompiling dep-0 v1.0.0\nCompiling dep-1 v1.0.0\nCompiling dep-2 v1.0.0\nCompiling dep-3 v1.0.0\nCompiling dep-4 v1.0.0\n{\n \"id\": 1,\n \"name\": \"Alice Wonderland\",\n \"email\": \"alice@example.com\",\n \"age\": 30,\n \"address\": \"123 Main Street, Anytown, USA 12345\",\n \"phone\": \"+1-555-0123\"\n}";
|
||||
|
||||
const TOON_INPUT: &str =
|
||||
r#"[{"id":1,"name":"Alice","role":"admin"},{"id":2,"name":"Bob","role":"user"}]"#;
|
||||
|
||||
fn make_tool_use(id: &str, name: &str) -> ContentBlock {
|
||||
ContentBlock::ToolUse {
|
||||
id: id.to_string(),
|
||||
name: name.to_string(),
|
||||
input: json!({}),
|
||||
extra: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_tool_result_content(blocks: &[ContentBlock]) -> &str {
|
||||
for block in blocks {
|
||||
if let ContentBlock::ToolResult { content, .. } = block {
|
||||
return content;
|
||||
}
|
||||
}
|
||||
panic!("no ToolResult found in blocks");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// A Layer: Case 1-3 (Off / Safe / Full)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn case_1_off_passthrough() {
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(MockTool::new("test_tool", TEST_OUTPUT, false)));
|
||||
|
||||
let tool_calls = vec![make_tool_use("c1", "test_tool")];
|
||||
let confirmer = auto_approve_confirmer();
|
||||
|
||||
let outcome = execute_tool_calls(
|
||||
®istry,
|
||||
&tool_calls,
|
||||
&confirmer,
|
||||
None,
|
||||
CompactionLevel::Off,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("should succeed");
|
||||
|
||||
let content = extract_tool_result_content(&outcome);
|
||||
eprintln!("[compaction:A] === Case 1: Off passthrough ===");
|
||||
eprintln!(
|
||||
"[compaction:A] raw ({} chars): {:?}",
|
||||
TEST_OUTPUT.len(),
|
||||
&TEST_OUTPUT[..60]
|
||||
);
|
||||
eprintln!(
|
||||
"[compaction:A] result ({} chars): {:?}",
|
||||
content.len(),
|
||||
&content[..60]
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
content, TEST_OUTPUT,
|
||||
"Off level should pass content through unchanged"
|
||||
);
|
||||
eprintln!("[compaction:A] ✓ content unchanged");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn case_2_safe_sanitizes() {
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(MockTool::new("test_tool", TEST_OUTPUT, false)));
|
||||
|
||||
let tool_calls = vec![make_tool_use("c2", "test_tool")];
|
||||
let confirmer = auto_approve_confirmer();
|
||||
|
||||
let outcome = execute_tool_calls(
|
||||
®istry,
|
||||
&tool_calls,
|
||||
&confirmer,
|
||||
None,
|
||||
CompactionLevel::Safe,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("should succeed");
|
||||
|
||||
let content = extract_tool_result_content(&outcome);
|
||||
eprintln!("[compaction:A] === Case 2: Safe sanitizes ===");
|
||||
eprintln!("[compaction:A] raw ({} chars)", TEST_OUTPUT.len());
|
||||
eprintln!(
|
||||
"[compaction:A] result ({} chars): {:?}",
|
||||
content.len(),
|
||||
content
|
||||
);
|
||||
|
||||
assert!(!content.contains("\x1b"), "Safe should strip ANSI escapes");
|
||||
assert!(!content.contains("\n\n\n"), "Safe should merge blank lines");
|
||||
assert!(!content.contains("\r"), "Safe should collapse CR lines");
|
||||
assert!(
|
||||
content.contains("Compiling dep-0"),
|
||||
"Safe should keep all repeated lines"
|
||||
);
|
||||
assert!(
|
||||
content.contains("Compiling dep-4"),
|
||||
"Safe should keep all repeated lines"
|
||||
);
|
||||
assert!(
|
||||
content.contains(" \"id\""),
|
||||
"Safe should preserve original JSON indentation"
|
||||
);
|
||||
|
||||
eprintln!(
|
||||
"[compaction:A] ✓ ANSI stripped, blanks merged, CR collapsed, repeats & JSON untouched"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn case_3_full_folds_and_compacts() {
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(MockTool::new("test_tool", TEST_OUTPUT, false)));
|
||||
|
||||
let tool_calls = vec![make_tool_use("c3", "test_tool")];
|
||||
let confirmer = auto_approve_confirmer();
|
||||
|
||||
let outcome = execute_tool_calls(
|
||||
®istry,
|
||||
&tool_calls,
|
||||
&confirmer,
|
||||
None,
|
||||
CompactionLevel::Full,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("should succeed");
|
||||
|
||||
let content = extract_tool_result_content(&outcome);
|
||||
eprintln!("[compaction:A] === Case 3: Full folds and compacts ===");
|
||||
eprintln!("[compaction:A] raw ({} chars)", TEST_OUTPUT.len());
|
||||
eprintln!(
|
||||
"[compaction:A] result ({} chars): {:?}",
|
||||
content.len(),
|
||||
content
|
||||
);
|
||||
|
||||
assert!(!content.contains("\x1b"), "Full should strip ANSI");
|
||||
assert!(
|
||||
content.contains("similar lines") || content.contains("identical lines"),
|
||||
"Full should fold repeated lines: {content}"
|
||||
);
|
||||
assert!(
|
||||
content.len() < TEST_OUTPUT.len(),
|
||||
"Full should produce shorter output: {} vs {}",
|
||||
content.len(),
|
||||
TEST_OUTPUT.len()
|
||||
);
|
||||
|
||||
eprintln!("[compaction:A] ✓ ANSI stripped, lines folded, output shorter");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// A Layer: Case 4-5 (TOON on / off)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn case_4_toon_encodes_array() {
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(MockTool::new("test_tool", TOON_INPUT, false)));
|
||||
|
||||
let tool_calls = vec![make_tool_use("c4", "test_tool")];
|
||||
let confirmer = auto_approve_confirmer();
|
||||
|
||||
let outcome = execute_tool_calls(
|
||||
®istry,
|
||||
&tool_calls,
|
||||
&confirmer,
|
||||
None,
|
||||
CompactionLevel::Full,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.expect("should succeed");
|
||||
|
||||
let content = extract_tool_result_content(&outcome);
|
||||
eprintln!("[compaction:A] === Case 4: TOON encodes array ===");
|
||||
eprintln!("[compaction:A] raw: {TOON_INPUT}");
|
||||
eprintln!("[compaction:A] result: {content}");
|
||||
|
||||
assert!(
|
||||
content.contains("[2]{id,name,role}:"),
|
||||
"TOON should produce header: {content}"
|
||||
);
|
||||
assert!(content.contains("Alice"), "TOON should contain data");
|
||||
assert!(content.contains("Bob"), "TOON should contain data");
|
||||
|
||||
eprintln!("[compaction:A] ✓ TOON header present with data rows");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn case_5_toon_disabled_no_encoding() {
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(MockTool::new("test_tool", TOON_INPUT, false)));
|
||||
|
||||
let tool_calls = vec![make_tool_use("c5", "test_tool")];
|
||||
let confirmer = auto_approve_confirmer();
|
||||
|
||||
let outcome = execute_tool_calls(
|
||||
®istry,
|
||||
&tool_calls,
|
||||
&confirmer,
|
||||
None,
|
||||
CompactionLevel::Full,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("should succeed");
|
||||
|
||||
let content = extract_tool_result_content(&outcome);
|
||||
eprintln!("[compaction:A] === Case 5: TOON disabled ===");
|
||||
eprintln!("[compaction:A] raw: {TOON_INPUT}");
|
||||
eprintln!("[compaction:A] result: {content}");
|
||||
|
||||
assert!(
|
||||
!content.contains("[2]{id,name,role}:"),
|
||||
"TOON off should not produce TOON header: {content}"
|
||||
);
|
||||
|
||||
eprintln!("[compaction:A] ✓ no TOON encoding when disabled");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CapturingProvider — wraps MockLlmProvider, records each LlmRequest
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
struct CapturingProvider {
|
||||
inner: MockLlmProvider,
|
||||
captured: Arc<Mutex<Vec<LlmRequest>>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl LlmProvider for CapturingProvider {
|
||||
async fn stream(
|
||||
&self,
|
||||
request: &LlmRequest,
|
||||
) -> Result<mpsc::Receiver<LlmEvent>, ProviderError> {
|
||||
self.captured.lock().unwrap().push(request.clone());
|
||||
self.inner.stream(request).await
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// B Layer: Case 6 (compressed content reaches LLM)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn case_6_compressed_content_reaches_llm() {
|
||||
let captured: Arc<Mutex<Vec<LlmRequest>>> = Arc::new(Mutex::new(Vec::new()));
|
||||
|
||||
let provider = CapturingProvider {
|
||||
inner: MockLlmProvider::with_turns(vec![
|
||||
vec![
|
||||
LlmEvent::ToolUse {
|
||||
id: "t1".to_string(),
|
||||
name: "test_tool".to_string(),
|
||||
input: json!({}),
|
||||
extra: None,
|
||||
},
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::ToolUse,
|
||||
usage: TokenUsage::default(),
|
||||
},
|
||||
],
|
||||
vec![
|
||||
LlmEvent::TextDelta("done".to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage::default(),
|
||||
},
|
||||
],
|
||||
]),
|
||||
captured: captured.clone(),
|
||||
};
|
||||
|
||||
let mut config = test_config();
|
||||
config.compact.compaction = CompactionLevel::Full;
|
||||
config.compact.toon = false;
|
||||
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(MockTool::new("test_tool", TEST_OUTPUT, false)));
|
||||
|
||||
let output: Arc<dyn OutputSink> = Arc::new(NullSink);
|
||||
let mut engine = AgentEngine::new_with_provider(
|
||||
Arc::new(provider),
|
||||
config,
|
||||
registry,
|
||||
output,
|
||||
std::env::temp_dir(),
|
||||
);
|
||||
|
||||
engine
|
||||
.run("call test_tool", "")
|
||||
.await
|
||||
.expect("engine.run should succeed");
|
||||
|
||||
let requests = captured.lock().unwrap();
|
||||
eprintln!("[compaction:B] === Case 6: Compressed content reaches LLM ===");
|
||||
eprintln!("[compaction:B] captured {} LlmRequests", requests.len());
|
||||
assert!(
|
||||
requests.len() >= 2,
|
||||
"should have at least 2 requests (initial + after tool)"
|
||||
);
|
||||
|
||||
let second_req = &requests[1];
|
||||
let mut found_tool_result = false;
|
||||
for msg in &second_req.messages {
|
||||
for block in &msg.content {
|
||||
if let ContentBlock::ToolResult { content, .. } = block {
|
||||
eprintln!(
|
||||
"[compaction:B] tool_result content ({} chars): {:?}",
|
||||
content.len(),
|
||||
content
|
||||
);
|
||||
assert!(!content.contains("\x1b"), "LLM should not see ANSI escapes");
|
||||
assert!(
|
||||
content.contains("similar lines") || content.contains("identical lines"),
|
||||
"LLM should see folded lines: {content}"
|
||||
);
|
||||
found_tool_result = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
found_tool_result,
|
||||
"second request should contain a ToolResult"
|
||||
);
|
||||
|
||||
eprintln!("[compaction:B] ✓ LLM received compressed content");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// B Layer: Case 7 (runtime compaction switch)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn case_7_runtime_compaction_switch() {
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(MockTool::new("test_tool", TEST_OUTPUT, false)));
|
||||
|
||||
let tool_calls = vec![make_tool_use("c7", "test_tool")];
|
||||
let confirmer = auto_approve_confirmer();
|
||||
|
||||
let outcome_off = execute_tool_calls(
|
||||
®istry,
|
||||
&tool_calls,
|
||||
&confirmer,
|
||||
None,
|
||||
CompactionLevel::Off,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("should succeed");
|
||||
let content_off = extract_tool_result_content(&outcome_off).to_string();
|
||||
|
||||
let outcome_full = execute_tool_calls(
|
||||
®istry,
|
||||
&tool_calls,
|
||||
&confirmer,
|
||||
None,
|
||||
CompactionLevel::Full,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("should succeed");
|
||||
let content_full = extract_tool_result_content(&outcome_full).to_string();
|
||||
|
||||
eprintln!("[compaction:B] === Case 7: Runtime compaction switch ===");
|
||||
eprintln!("[compaction:B] Off content ({} chars)", content_off.len());
|
||||
eprintln!("[compaction:B] Full content ({} chars)", content_full.len());
|
||||
|
||||
assert_ne!(
|
||||
content_off, content_full,
|
||||
"Off and Full should produce different content"
|
||||
);
|
||||
assert!(content_off.contains("\x1b"), "Off should preserve ANSI");
|
||||
assert!(!content_full.contains("\x1b"), "Full should strip ANSI");
|
||||
assert!(
|
||||
content_full.contains("similar lines") || content_full.contains("identical lines"),
|
||||
"Full should fold lines"
|
||||
);
|
||||
|
||||
// Verify apply_config_update works on the engine
|
||||
let mut config = test_config();
|
||||
config.compact.compaction = CompactionLevel::Off;
|
||||
let registry_engine = ToolRegistry::new();
|
||||
let output: Arc<dyn OutputSink> = Arc::new(NullSink);
|
||||
let mut engine = AgentEngine::new_with_provider(
|
||||
Arc::new(MockLlmProvider::with_text_response("ok")),
|
||||
config,
|
||||
registry_engine,
|
||||
output,
|
||||
std::env::temp_dir(),
|
||||
);
|
||||
assert_eq!(engine.compaction_level(), CompactionLevel::Off);
|
||||
|
||||
let changes = engine.apply_config_update(None, None, None, None, Some("full".to_string()));
|
||||
assert!(!changes.is_empty(), "should report changes");
|
||||
assert_eq!(engine.compaction_level(), CompactionLevel::Full);
|
||||
eprintln!("[compaction:B] apply_config_update changes: {:?}", changes);
|
||||
|
||||
eprintln!("[compaction:B] ✓ runtime switch from Off to Full verified");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// B Layer: Case 8 (TOON system prompt injection)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn case_8_toon_system_prompt_injection() {
|
||||
eprintln!("[compaction:B] === Case 8: TOON system prompt injection ===");
|
||||
|
||||
// TOON enabled
|
||||
let mut cache_on = SystemPromptCache::new();
|
||||
let prompt_on = build_system_prompt(
|
||||
&mut cache_on,
|
||||
Some("You are a test assistant."),
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
true, // toon_enabled
|
||||
false, // browser_enabled
|
||||
);
|
||||
|
||||
eprintln!(
|
||||
"[compaction:B] TOON=true system prompt length: {} chars",
|
||||
prompt_on.len()
|
||||
);
|
||||
|
||||
assert!(
|
||||
prompt_on.contains("TOON"),
|
||||
"TOON enabled: system prompt should mention TOON"
|
||||
);
|
||||
assert!(
|
||||
prompt_on.contains("Token-Oriented Object Notation"),
|
||||
"should contain full TOON description"
|
||||
);
|
||||
|
||||
// TOON disabled
|
||||
let mut cache_off = SystemPromptCache::new();
|
||||
let prompt_off = build_system_prompt(
|
||||
&mut cache_off,
|
||||
Some("You are a test assistant."),
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
false, // toon_enabled
|
||||
false, // browser_enabled
|
||||
);
|
||||
|
||||
eprintln!(
|
||||
"[compaction:B] TOON=false system prompt length: {} chars",
|
||||
prompt_off.len()
|
||||
);
|
||||
|
||||
assert!(
|
||||
!prompt_off.contains("TOON"),
|
||||
"TOON disabled: system prompt should NOT mention TOON"
|
||||
);
|
||||
|
||||
eprintln!("[compaction:B] ✓ TOON system prompt injection verified");
|
||||
}
|
||||
@@ -0,0 +1,387 @@
|
||||
//! End-to-end integration tests for Plan Mode (task 3.6).
|
||||
//!
|
||||
//! Tests are numbered to match the test-plan.md identifiers (TC-3.6-E2E-*).
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use nomi_agent::plan::prompt::plan_mode_instructions;
|
||||
use nomi_agent::plan::tools::{EnterPlanModeTool, ExitPlanModeTool};
|
||||
use nomi_protocol::events::ToolCategory;
|
||||
use nomi_tools::Tool;
|
||||
use nomi_tools::registry::ToolRegistry;
|
||||
use nomi_types::skill_types::PlanModeTransition;
|
||||
use serde_json::json;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
struct MockTool {
|
||||
tool_name: String,
|
||||
tool_category: ToolCategory,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for MockTool {
|
||||
fn name(&self) -> &str {
|
||||
&self.tool_name
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"mock tool"
|
||||
}
|
||||
|
||||
fn input_schema(&self) -> serde_json::Value {
|
||||
json!({"type": "object", "properties": {}, "required": []})
|
||||
}
|
||||
|
||||
fn is_concurrency_safe(&self, _input: &serde_json::Value) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
async fn execute(&self, _input: serde_json::Value) -> nomi_types::tool::ToolResult {
|
||||
nomi_types::tool::ToolResult {
|
||||
content: format!("{} executed", self.tool_name),
|
||||
is_error: false,
|
||||
images: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn category(&self) -> ToolCategory {
|
||||
self.tool_category
|
||||
}
|
||||
}
|
||||
|
||||
fn mock_tool(name: &str, category: ToolCategory) -> Box<MockTool> {
|
||||
Box::new(MockTool {
|
||||
tool_name: name.to_string(),
|
||||
tool_category: category,
|
||||
})
|
||||
}
|
||||
|
||||
/// Simulate the plan mode filter the engine applies in its run() loop.
|
||||
fn plan_mode_filter(registry: &ToolRegistry) -> Vec<String> {
|
||||
registry
|
||||
.to_tool_defs_filtered(|t| {
|
||||
t.category() == ToolCategory::Info && t.name() != "EnterPlanMode"
|
||||
})
|
||||
.iter()
|
||||
.map(|d| d.name.clone())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Simulate the normal mode filter the engine applies in its run() loop.
|
||||
fn normal_mode_filter(registry: &ToolRegistry) -> Vec<String> {
|
||||
registry
|
||||
.to_tool_defs_filtered(|t| t.name() != "ExitPlanMode")
|
||||
.iter()
|
||||
.map(|d| d.name.clone())
|
||||
.collect()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.6-E2E-01: Full plan mode lifecycle
|
||||
//
|
||||
// Verifies the complete flow: normal → enter plan mode → use read-only tools
|
||||
// → exit plan mode → write tools restored.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_3_6_e2e_01_full_plan_mode_lifecycle() {
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let mut registry = ToolRegistry::new();
|
||||
|
||||
// Register standard tools
|
||||
registry.register(mock_tool("Read", ToolCategory::Info));
|
||||
registry.register(mock_tool("Grep", ToolCategory::Info));
|
||||
registry.register(mock_tool("Glob", ToolCategory::Info));
|
||||
registry.register(mock_tool("Skill", ToolCategory::Info));
|
||||
registry.register(mock_tool("Write", ToolCategory::Edit));
|
||||
registry.register(mock_tool("Edit", ToolCategory::Edit));
|
||||
registry.register(mock_tool("Bash", ToolCategory::Exec));
|
||||
registry.register(Box::new(EnterPlanModeTool::new(Arc::clone(&flag))));
|
||||
registry.register(Box::new(ExitPlanModeTool::new(Arc::clone(&flag))));
|
||||
|
||||
// Step 1: Verify normal mode — all tools except ExitPlanMode
|
||||
let normal_tools = normal_mode_filter(®istry);
|
||||
assert!(normal_tools.contains(&"Read".to_string()));
|
||||
assert!(normal_tools.contains(&"Write".to_string()));
|
||||
assert!(normal_tools.contains(&"Bash".to_string()));
|
||||
assert!(normal_tools.contains(&"EnterPlanMode".to_string()));
|
||||
assert!(!normal_tools.contains(&"ExitPlanMode".to_string()));
|
||||
|
||||
// Step 2: LLM calls EnterPlanMode
|
||||
let enter_tool = EnterPlanModeTool::new(Arc::clone(&flag));
|
||||
let result = enter_tool.execute(json!({})).await;
|
||||
assert!(!result.is_error, "EnterPlanMode should succeed");
|
||||
|
||||
// Verify context modifier signals Enter transition
|
||||
let cm = enter_tool.context_modifier_for(&json!({})).unwrap();
|
||||
assert_eq!(cm.plan_mode_transition, Some(PlanModeTransition::Enter));
|
||||
|
||||
// Simulate engine processing the transition
|
||||
flag.store(true, Ordering::Release);
|
||||
|
||||
// Step 3: Verify plan mode — only read-only tools + ExitPlanMode
|
||||
let plan_tools = plan_mode_filter(®istry);
|
||||
assert!(plan_tools.contains(&"Read".to_string()));
|
||||
assert!(plan_tools.contains(&"Grep".to_string()));
|
||||
assert!(plan_tools.contains(&"Glob".to_string()));
|
||||
assert!(plan_tools.contains(&"Skill".to_string()));
|
||||
assert!(plan_tools.contains(&"ExitPlanMode".to_string()));
|
||||
assert!(!plan_tools.contains(&"Write".to_string()));
|
||||
assert!(!plan_tools.contains(&"Edit".to_string()));
|
||||
assert!(!plan_tools.contains(&"Bash".to_string()));
|
||||
assert!(!plan_tools.contains(&"EnterPlanMode".to_string()));
|
||||
|
||||
// Step 4: Verify read-only tools execute successfully in plan mode
|
||||
let read_tool = mock_tool("Read", ToolCategory::Info);
|
||||
let read_result = read_tool.execute(json!({})).await;
|
||||
assert!(!read_result.is_error, "Read should work in plan mode");
|
||||
|
||||
// Step 5: Verify double-enter is rejected
|
||||
let double_enter = enter_tool.execute(json!({})).await;
|
||||
assert!(double_enter.is_error, "double-enter should be rejected");
|
||||
|
||||
// Step 6: LLM calls ExitPlanMode
|
||||
let exit_tool = ExitPlanModeTool::new(Arc::clone(&flag));
|
||||
let exit_result = exit_tool.execute(json!({})).await;
|
||||
assert!(!exit_result.is_error, "ExitPlanMode should succeed");
|
||||
|
||||
// Verify context modifier signals Exit transition
|
||||
let exit_cm = exit_tool.context_modifier_for(&json!({})).unwrap();
|
||||
assert!(matches!(
|
||||
exit_cm.plan_mode_transition,
|
||||
Some(PlanModeTransition::Exit { .. })
|
||||
));
|
||||
|
||||
// Simulate engine processing the transition
|
||||
flag.store(false, Ordering::Release);
|
||||
|
||||
// Step 7: Verify normal mode restored — write tools available again
|
||||
let restored_tools = normal_mode_filter(®istry);
|
||||
assert!(restored_tools.contains(&"Write".to_string()));
|
||||
assert!(restored_tools.contains(&"Edit".to_string()));
|
||||
assert!(restored_tools.contains(&"Bash".to_string()));
|
||||
assert!(restored_tools.contains(&"EnterPlanMode".to_string()));
|
||||
assert!(!restored_tools.contains(&"ExitPlanMode".to_string()));
|
||||
|
||||
// Step 8: Verify double-exit is rejected
|
||||
let double_exit = exit_tool.execute(json!({})).await;
|
||||
assert!(double_exit.is_error, "double-exit should be rejected");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.6-E2E-02: Plan mode + compaction don't conflict
|
||||
//
|
||||
// Verifies that microcompact does not interfere with plan mode state.
|
||||
// Uses tool result clearing to simulate compaction activity.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_3_6_e2e_02_plan_mode_and_compaction_independent() {
|
||||
use nomi_agent::compact::micro::microcompact;
|
||||
use nomi_agent::plan::state::PlanState;
|
||||
use nomi_config::compact::CompactConfig;
|
||||
use nomi_types::message::{ContentBlock, Message, Role};
|
||||
|
||||
// Build messages with compactable tool results
|
||||
let mut messages = Vec::new();
|
||||
for i in 0..8 {
|
||||
let id = format!("t{i}");
|
||||
messages.push(Message::new(
|
||||
Role::Assistant,
|
||||
vec![ContentBlock::ToolUse {
|
||||
id: id.clone(),
|
||||
name: "Read".to_string(),
|
||||
input: json!({}),
|
||||
extra: None,
|
||||
}],
|
||||
));
|
||||
messages.push(Message::new(
|
||||
Role::User,
|
||||
vec![ContentBlock::ToolResult {
|
||||
tool_use_id: id,
|
||||
content: format!("data-{i}"),
|
||||
is_error: false,
|
||||
images: Vec::new(),
|
||||
}],
|
||||
));
|
||||
}
|
||||
|
||||
// Create plan state simulating active plan mode
|
||||
let plan_state = PlanState {
|
||||
is_active: true,
|
||||
pre_plan_allow_list: vec!["Read".to_string(), "Grep".to_string()],
|
||||
};
|
||||
|
||||
// Actually run microcompact
|
||||
let config = CompactConfig {
|
||||
micro_keep_recent: 3,
|
||||
..CompactConfig::default()
|
||||
};
|
||||
let result = microcompact(&mut messages, &config);
|
||||
|
||||
// Microcompact should have cleared some results
|
||||
assert!(
|
||||
result.cleared_count > 0,
|
||||
"microcompact should clear old results"
|
||||
);
|
||||
|
||||
// Plan state should be completely unaffected (microcompact only touches messages)
|
||||
assert!(plan_state.is_active, "plan mode should remain active");
|
||||
assert_eq!(
|
||||
plan_state.pre_plan_allow_list,
|
||||
vec!["Read".to_string(), "Grep".to_string()],
|
||||
"allow list should be unchanged"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.6-E2E-03: SkillTool available in plan mode
|
||||
//
|
||||
// Verifies that Skill (category: Info) is available in plan mode.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_3_6_e2e_03_skill_tool_available_in_plan_mode() {
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let mut registry = ToolRegistry::new();
|
||||
|
||||
registry.register(mock_tool("Read", ToolCategory::Info));
|
||||
registry.register(mock_tool("Skill", ToolCategory::Info));
|
||||
registry.register(mock_tool("Write", ToolCategory::Edit));
|
||||
registry.register(mock_tool("Bash", ToolCategory::Exec));
|
||||
registry.register(Box::new(EnterPlanModeTool::new(Arc::clone(&flag))));
|
||||
registry.register(Box::new(ExitPlanModeTool::new(Arc::clone(&flag))));
|
||||
|
||||
// Plan mode filter
|
||||
let plan_tools = plan_mode_filter(®istry);
|
||||
|
||||
assert!(
|
||||
plan_tools.contains(&"Skill".to_string()),
|
||||
"Skill tool (Info category) should be available in plan mode"
|
||||
);
|
||||
assert!(
|
||||
plan_tools.contains(&"Read".to_string()),
|
||||
"Read should be available"
|
||||
);
|
||||
assert!(
|
||||
!plan_tools.contains(&"Write".to_string()),
|
||||
"Write should not be available"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.6-E2E-04: Plan mode state is runtime-only, not persisted
|
||||
//
|
||||
// Verifies that PlanState defaults to inactive — session resume starts fresh.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_3_6_e2e_04_plan_state_not_persisted_across_sessions() {
|
||||
use nomi_agent::plan::state::PlanState;
|
||||
|
||||
// Simulate a "previous session" where plan mode was active
|
||||
let active_state = PlanState {
|
||||
is_active: true,
|
||||
pre_plan_allow_list: vec!["Read".into(), "Bash".into()],
|
||||
};
|
||||
assert!(active_state.is_active);
|
||||
|
||||
// On session resume, engine creates PlanState::default() (see engine.rs resume_with_provider)
|
||||
let resumed_state = PlanState::default();
|
||||
|
||||
assert!(
|
||||
!resumed_state.is_active,
|
||||
"plan state should be inactive after session resume"
|
||||
);
|
||||
assert!(
|
||||
resumed_state.pre_plan_allow_list.is_empty(),
|
||||
"allow list should be empty after resume"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Additional: System prompt reflects plan mode transitions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn system_prompt_tracks_plan_mode_transitions() {
|
||||
let base_prompt = "You are an AI assistant.";
|
||||
let instructions = plan_mode_instructions();
|
||||
|
||||
// Normal mode: no plan instructions
|
||||
assert!(
|
||||
!base_prompt.contains("Plan Mode"),
|
||||
"base prompt should not mention plan mode"
|
||||
);
|
||||
|
||||
// Enter plan mode: instructions appended
|
||||
let active_prompt = format!("{}\n\n{}", base_prompt, instructions);
|
||||
assert!(active_prompt.contains("# Plan Mode"));
|
||||
assert!(active_prompt.contains("MUST NOT"));
|
||||
assert!(active_prompt.contains("ExitPlanMode"));
|
||||
assert!(active_prompt.contains(base_prompt));
|
||||
|
||||
// Exit plan mode: back to base prompt only
|
||||
let exited_prompt = base_prompt.to_string();
|
||||
assert!(!exited_prompt.contains("# Plan Mode"));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Additional: Multiple enter-exit cycles maintain consistency
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn multiple_plan_mode_cycles_consistent() {
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let enter = EnterPlanModeTool::new(Arc::clone(&flag));
|
||||
let exit = ExitPlanModeTool::new(Arc::clone(&flag));
|
||||
|
||||
for cycle in 0..3 {
|
||||
// Enter should succeed
|
||||
let r = enter.execute(json!({})).await;
|
||||
assert!(!r.is_error, "enter should succeed on cycle {cycle}");
|
||||
|
||||
flag.store(true, Ordering::Release);
|
||||
|
||||
// Exit should succeed
|
||||
let r = exit.execute(json!({})).await;
|
||||
assert!(!r.is_error, "exit should succeed on cycle {cycle}");
|
||||
|
||||
flag.store(false, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Additional: Plan mode context_modifier fields are orthogonal
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn plan_mode_modifiers_do_not_interfere_with_other_fields() {
|
||||
let enter = EnterPlanModeTool::new(Arc::new(AtomicBool::new(false)));
|
||||
let exit = ExitPlanModeTool::new(Arc::new(AtomicBool::new(true)));
|
||||
|
||||
// Enter modifier should only set plan_mode_transition
|
||||
let enter_cm = enter.context_modifier_for(&json!({})).unwrap();
|
||||
assert!(enter_cm.model.is_none());
|
||||
assert!(enter_cm.effort.is_none());
|
||||
assert!(enter_cm.allowed_tools.is_empty());
|
||||
assert_eq!(
|
||||
enter_cm.plan_mode_transition,
|
||||
Some(PlanModeTransition::Enter)
|
||||
);
|
||||
|
||||
// Exit modifier should only set plan_mode_transition
|
||||
let exit_cm = exit.context_modifier_for(&json!({})).unwrap();
|
||||
assert!(exit_cm.model.is_none());
|
||||
assert!(exit_cm.effort.is_none());
|
||||
assert!(exit_cm.allowed_tools.is_empty());
|
||||
assert!(matches!(
|
||||
exit_cm.plan_mode_transition,
|
||||
Some(PlanModeTransition::Exit { .. })
|
||||
));
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
//! Integration tests for Plan Mode engine integration (task 3.5).
|
||||
//!
|
||||
//! Tests are numbered to match the test-plan.md identifiers (TC-3.5-*).
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use nomi_agent::plan::tools::{EnterPlanModeTool, ExitPlanModeTool};
|
||||
use nomi_protocol::events::ToolCategory;
|
||||
use nomi_tools::Tool;
|
||||
use nomi_tools::registry::ToolRegistry;
|
||||
use serde_json::json;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers: mock tools with configurable categories
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
struct MockTool {
|
||||
tool_name: String,
|
||||
tool_category: ToolCategory,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for MockTool {
|
||||
fn name(&self) -> &str {
|
||||
&self.tool_name
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"mock tool"
|
||||
}
|
||||
|
||||
fn input_schema(&self) -> serde_json::Value {
|
||||
json!({"type": "object", "properties": {}, "required": []})
|
||||
}
|
||||
|
||||
fn is_concurrency_safe(&self, _input: &serde_json::Value) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
async fn execute(&self, _input: serde_json::Value) -> nomi_types::tool::ToolResult {
|
||||
nomi_types::tool::ToolResult {
|
||||
content: "ok".to_string(),
|
||||
is_error: false,
|
||||
images: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn category(&self) -> ToolCategory {
|
||||
self.tool_category
|
||||
}
|
||||
}
|
||||
|
||||
fn mock_tool(name: &str, category: ToolCategory) -> Box<MockTool> {
|
||||
Box::new(MockTool {
|
||||
tool_name: name.to_string(),
|
||||
tool_category: category,
|
||||
})
|
||||
}
|
||||
|
||||
/// Build a registry with typical tools + plan mode tools
|
||||
fn build_test_registry() -> ToolRegistry {
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let mut registry = ToolRegistry::new();
|
||||
|
||||
// Info tools
|
||||
registry.register(mock_tool("Read", ToolCategory::Info));
|
||||
registry.register(mock_tool("Grep", ToolCategory::Info));
|
||||
registry.register(mock_tool("Glob", ToolCategory::Info));
|
||||
registry.register(mock_tool("Skill", ToolCategory::Info));
|
||||
|
||||
// Edit tools
|
||||
registry.register(mock_tool("Write", ToolCategory::Edit));
|
||||
registry.register(mock_tool("Edit", ToolCategory::Edit));
|
||||
|
||||
// Exec tools
|
||||
registry.register(mock_tool("Bash", ToolCategory::Exec));
|
||||
|
||||
// Plan mode tools (both are Info category)
|
||||
registry.register(Box::new(EnterPlanModeTool::new(Arc::clone(&flag))));
|
||||
registry.register(Box::new(ExitPlanModeTool::new(Arc::clone(&flag))));
|
||||
|
||||
registry
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.5-01: Plan mode filters to only Info tools + ExitPlanMode
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_3_5_01_plan_mode_only_info_tools_plus_exit() {
|
||||
let registry = build_test_registry();
|
||||
|
||||
// Plan mode filter: Info category except EnterPlanMode
|
||||
let defs = registry.to_tool_defs_filtered(|t| {
|
||||
t.category() == ToolCategory::Info && t.name() != "EnterPlanMode"
|
||||
});
|
||||
|
||||
let names: Vec<&str> = defs.iter().map(|d| d.name.as_str()).collect();
|
||||
|
||||
// Should include Info tools
|
||||
assert!(
|
||||
names.contains(&"Read"),
|
||||
"Read should be available in plan mode"
|
||||
);
|
||||
assert!(
|
||||
names.contains(&"Grep"),
|
||||
"Grep should be available in plan mode"
|
||||
);
|
||||
assert!(
|
||||
names.contains(&"Glob"),
|
||||
"Glob should be available in plan mode"
|
||||
);
|
||||
assert!(
|
||||
names.contains(&"Skill"),
|
||||
"Skill should be available in plan mode"
|
||||
);
|
||||
assert!(
|
||||
names.contains(&"ExitPlanMode"),
|
||||
"ExitPlanMode should be available in plan mode"
|
||||
);
|
||||
|
||||
// Should NOT include write/exec/EnterPlanMode
|
||||
assert!(
|
||||
!names.contains(&"Write"),
|
||||
"Write should NOT be in plan mode"
|
||||
);
|
||||
assert!(!names.contains(&"Edit"), "Edit should NOT be in plan mode");
|
||||
assert!(!names.contains(&"Bash"), "Bash should NOT be in plan mode");
|
||||
assert!(
|
||||
!names.contains(&"EnterPlanMode"),
|
||||
"EnterPlanMode should NOT be in plan mode"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.5-02: Normal mode includes all tools except ExitPlanMode
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_3_5_02_normal_mode_all_tools_except_exit_plan_mode() {
|
||||
let registry = build_test_registry();
|
||||
|
||||
// Normal mode filter: everything except ExitPlanMode
|
||||
let defs = registry.to_tool_defs_filtered(|t| t.name() != "ExitPlanMode");
|
||||
|
||||
let names: Vec<&str> = defs.iter().map(|d| d.name.as_str()).collect();
|
||||
|
||||
// Should include all standard tools and EnterPlanMode
|
||||
assert!(names.contains(&"Read"));
|
||||
assert!(names.contains(&"Grep"));
|
||||
assert!(names.contains(&"Glob"));
|
||||
assert!(names.contains(&"Skill"));
|
||||
assert!(names.contains(&"Write"));
|
||||
assert!(names.contains(&"Edit"));
|
||||
assert!(names.contains(&"Bash"));
|
||||
assert!(
|
||||
names.contains(&"EnterPlanMode"),
|
||||
"EnterPlanMode should be in normal mode"
|
||||
);
|
||||
|
||||
// Should NOT include ExitPlanMode
|
||||
assert!(
|
||||
!names.contains(&"ExitPlanMode"),
|
||||
"ExitPlanMode should NOT be in normal mode"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.5-07: to_tool_defs_filtered correctly filters mixed categories
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_3_5_07_to_tool_defs_filtered_mixed_categories() {
|
||||
let mut registry = ToolRegistry::new();
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
|
||||
registry.register(mock_tool("Read", ToolCategory::Info));
|
||||
registry.register(mock_tool("Write", ToolCategory::Edit));
|
||||
registry.register(mock_tool("Bash", ToolCategory::Exec));
|
||||
registry.register(Box::new(ExitPlanModeTool::new(flag)));
|
||||
|
||||
let defs = registry.to_tool_defs_filtered(|t| {
|
||||
t.category() == ToolCategory::Info || t.name() == "ExitPlanMode"
|
||||
});
|
||||
|
||||
let names: Vec<&str> = defs.iter().map(|d| d.name.as_str()).collect();
|
||||
assert!(names.contains(&"Read"), "Info tool should be included");
|
||||
assert!(
|
||||
names.contains(&"ExitPlanMode"),
|
||||
"ExitPlanMode should be included"
|
||||
);
|
||||
assert!(!names.contains(&"Write"), "Edit tool should be excluded");
|
||||
assert!(!names.contains(&"Bash"), "Exec tool should be excluded");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.5-08: System prompt dynamically includes plan mode instructions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_3_5_08_system_prompt_includes_plan_mode_when_active() {
|
||||
use nomi_agent::plan::prompt::plan_mode_instructions;
|
||||
|
||||
let base_prompt = "You are an AI assistant.";
|
||||
let instructions = plan_mode_instructions();
|
||||
|
||||
// Simulate plan mode active: system prompt should contain plan instructions
|
||||
let active_prompt = format!("{}\n\n{}", base_prompt, instructions);
|
||||
assert!(
|
||||
active_prompt.contains("Plan Mode"),
|
||||
"active prompt should contain plan mode instructions"
|
||||
);
|
||||
assert!(
|
||||
active_prompt.contains("MUST NOT"),
|
||||
"should contain plan mode restrictions"
|
||||
);
|
||||
assert!(
|
||||
active_prompt.contains(base_prompt),
|
||||
"should still contain base prompt"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tc_3_5_08_system_prompt_excludes_plan_mode_when_inactive() {
|
||||
let base_prompt = "You are an AI assistant.";
|
||||
|
||||
// Normal mode: system prompt is just the base
|
||||
assert!(
|
||||
!base_prompt.contains("Plan Mode"),
|
||||
"inactive prompt should NOT contain plan mode instructions"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.5-06: PlanConfig disabled means no plan tools registered
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_3_5_06_disabled_config_no_plan_tools() {
|
||||
// Simulate PlanConfig.enabled = false — do not register plan tools
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(mock_tool("Read", ToolCategory::Info));
|
||||
registry.register(mock_tool("Write", ToolCategory::Edit));
|
||||
// Intentionally NOT registering EnterPlanMode/ExitPlanMode
|
||||
|
||||
let defs = registry.to_tool_defs();
|
||||
let names: Vec<&str> = defs.iter().map(|d| d.name.as_str()).collect();
|
||||
|
||||
assert!(
|
||||
!names.contains(&"EnterPlanMode"),
|
||||
"EnterPlanMode should not be registered when disabled"
|
||||
);
|
||||
assert!(
|
||||
!names.contains(&"ExitPlanMode"),
|
||||
"ExitPlanMode should not be registered when disabled"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Additional: plan mode filter with MCP tools
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn plan_mode_filter_excludes_mcp_tools() {
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(mock_tool("Read", ToolCategory::Info));
|
||||
registry.register(mock_tool("mcp_server_tool", ToolCategory::Mcp));
|
||||
|
||||
let defs = registry.to_tool_defs_filtered(|t| {
|
||||
t.category() == ToolCategory::Info && t.name() != "EnterPlanMode"
|
||||
});
|
||||
|
||||
let names: Vec<&str> = defs.iter().map(|d| d.name.as_str()).collect();
|
||||
assert!(names.contains(&"Read"));
|
||||
assert!(
|
||||
!names.contains(&"mcp_server_tool"),
|
||||
"MCP tools should be excluded in plan mode"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Additional: full enter-exit cycle verifies tool set transitions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tool_set_transitions_through_plan_mode_cycle() {
|
||||
let registry = build_test_registry();
|
||||
|
||||
// Normal mode: all except ExitPlanMode
|
||||
let normal_defs = registry.to_tool_defs_filtered(|t| t.name() != "ExitPlanMode");
|
||||
let normal_names: Vec<&str> = normal_defs.iter().map(|d| d.name.as_str()).collect();
|
||||
assert!(normal_names.contains(&"Write"));
|
||||
assert!(normal_names.contains(&"EnterPlanMode"));
|
||||
assert!(!normal_names.contains(&"ExitPlanMode"));
|
||||
|
||||
// Enter plan mode: only Info (minus EnterPlanMode)
|
||||
let plan_defs = registry.to_tool_defs_filtered(|t| {
|
||||
t.category() == ToolCategory::Info && t.name() != "EnterPlanMode"
|
||||
});
|
||||
let plan_names: Vec<&str> = plan_defs.iter().map(|d| d.name.as_str()).collect();
|
||||
assert!(!plan_names.contains(&"Write"));
|
||||
assert!(!plan_names.contains(&"EnterPlanMode"));
|
||||
assert!(plan_names.contains(&"ExitPlanMode"));
|
||||
assert!(plan_names.contains(&"Read"));
|
||||
|
||||
// Exit plan mode: back to normal
|
||||
let back_to_normal = registry.to_tool_defs_filtered(|t| t.name() != "ExitPlanMode");
|
||||
let back_names: Vec<&str> = back_to_normal.iter().map(|d| d.name.as_str()).collect();
|
||||
assert_eq!(
|
||||
normal_names, back_names,
|
||||
"tool set should be identical after exit"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
//! Integration tests for Plan Mode prompts and file management (task 3.4).
|
||||
//!
|
||||
//! Tests are numbered to match the test-plan.md identifiers (TC-3.4-*).
|
||||
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use nomi_agent::context::{SystemPromptCache, build_system_prompt};
|
||||
use nomi_agent::plan::file::{plan_file_path, read_plan, write_plan};
|
||||
use nomi_agent::plan::prompt::plan_mode_instructions;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.4-01 plan_mode_instructions content
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_3_4_01_instructions_not_empty() {
|
||||
let text = plan_mode_instructions();
|
||||
assert!(!text.is_empty(), "instructions should not be empty");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tc_3_4_01_instructions_guide_code_reading() {
|
||||
let text = plan_mode_instructions();
|
||||
assert!(
|
||||
text.contains("Read") && text.contains("Grep") && text.contains("Glob"),
|
||||
"instructions should reference read-only tools"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tc_3_4_01_instructions_guide_plan_creation() {
|
||||
let text = plan_mode_instructions();
|
||||
// Should mention planning/design phases
|
||||
assert!(
|
||||
text.contains("plan") || text.contains("Plan"),
|
||||
"instructions should guide plan creation"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tc_3_4_01_instructions_mention_exit_tool() {
|
||||
let text = plan_mode_instructions();
|
||||
assert!(
|
||||
text.contains("ExitPlanMode"),
|
||||
"instructions should mention ExitPlanMode tool"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tc_3_4_01_instructions_forbid_writes() {
|
||||
let text = plan_mode_instructions();
|
||||
assert!(
|
||||
text.contains("MUST NOT") || text.contains("Forbidden"),
|
||||
"instructions should forbid write operations"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.4-03 System prompt with plan mode active
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_3_4_03_system_prompt_includes_plan_instructions_when_active() {
|
||||
let result = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
None,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
|
||||
// Should contain plan mode instructions
|
||||
assert!(
|
||||
result.contains("Plan Mode"),
|
||||
"active plan mode should inject plan mode instructions"
|
||||
);
|
||||
assert!(
|
||||
result.contains("ExitPlanMode"),
|
||||
"plan mode instructions should mention ExitPlanMode"
|
||||
);
|
||||
assert!(
|
||||
result.contains("MUST NOT"),
|
||||
"plan mode instructions should contain restrictions"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.4-04 System prompt without plan mode
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_3_4_04_system_prompt_excludes_plan_instructions_when_inactive() {
|
||||
let result = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
|
||||
// Should NOT contain plan mode instructions
|
||||
assert!(
|
||||
!result.contains("# Plan Mode"),
|
||||
"inactive plan mode should not inject plan mode heading"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.4-05 Plan file write
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_3_4_05_write_plan_creates_file_and_parents() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let path = tmp.path().join("deep").join("nested").join("plan.md");
|
||||
|
||||
write_plan(&path, "# My Plan\n\n## Steps\n1. Do thing").unwrap();
|
||||
|
||||
assert!(path.exists(), "plan file should be created");
|
||||
let content = fs::read_to_string(&path).unwrap();
|
||||
assert_eq!(content, "# My Plan\n\n## Steps\n1. Do thing");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.4-06 Plan file read
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_3_4_06_read_plan_returns_content() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let path = tmp.path().join("plan.md");
|
||||
fs::write(&path, "# My Plan\nStep 1").unwrap();
|
||||
|
||||
let result = read_plan(&path).unwrap();
|
||||
assert_eq!(result, Some("# My Plan\nStep 1".to_string()));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.4-07 Plan file read when not exists
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_3_4_07_read_plan_nonexistent_returns_none() {
|
||||
let result = read_plan(Path::new("/nonexistent/path/plan.md")).unwrap();
|
||||
assert_eq!(result, None, "reading nonexistent plan should return None");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.4-08 Plan file path generation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_3_4_08_plan_file_path_format() {
|
||||
let path = plan_file_path(Path::new("/tmp/plans"), "session-abc");
|
||||
assert_eq!(
|
||||
path,
|
||||
std::path::PathBuf::from("/tmp/plans/session-abc.md"),
|
||||
"plan file path should be {{dir}}/{{session_id}}.md"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.4-09 No bb brand identifiers in plan mode instructions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_3_4_09_no_bb_brand_in_instructions() {
|
||||
let text = plan_mode_instructions();
|
||||
assert!(
|
||||
!text.contains("Claude"),
|
||||
"instructions should not contain Claude brand"
|
||||
);
|
||||
assert!(
|
||||
!text.contains("claude"),
|
||||
"instructions should not contain lowercase claude"
|
||||
);
|
||||
assert!(
|
||||
!text.contains("~/.claude"),
|
||||
"instructions should not contain bb config path"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Additional: write-then-read roundtrip
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn write_then_read_roundtrip() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let dir = tmp.path().join("plans");
|
||||
let path = plan_file_path(&dir, "test-session");
|
||||
|
||||
let content = "# Implementation Plan\n\n## Context\nRefactor auth module\n\n## Files\n- src/auth.rs\n- src/middleware.rs";
|
||||
write_plan(&path, content).unwrap();
|
||||
|
||||
let result = read_plan(&path).unwrap();
|
||||
assert_eq!(result, Some(content.to_string()));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Additional: plan mode instructions appear in correct position in system prompt
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn plan_instructions_appear_after_memory_before_skills() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let mem_dir = tmp.path().join("memory");
|
||||
std::fs::create_dir_all(&mem_dir).unwrap();
|
||||
std::fs::write(mem_dir.join("MEMORY.md"), "- [A](a.md) \u{2014} test\n").unwrap();
|
||||
|
||||
let result = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
Some(&mem_dir),
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
|
||||
let memory_pos = result
|
||||
.find("auto memory")
|
||||
.expect("memory section should be present");
|
||||
let plan_pos = result
|
||||
.find("# Plan Mode")
|
||||
.expect("plan mode instructions should be present");
|
||||
|
||||
assert!(
|
||||
memory_pos < plan_pos,
|
||||
"memory should appear before plan mode instructions"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Additional: write_plan overwrites existing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn write_plan_overwrites_existing_content() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let path = tmp.path().join("plan.md");
|
||||
|
||||
write_plan(&path, "version 1").unwrap();
|
||||
write_plan(&path, "version 2").unwrap();
|
||||
|
||||
let result = read_plan(&path).unwrap();
|
||||
assert_eq!(result, Some("version 2".to_string()));
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
//! Integration tests for Plan Mode tools (task 3.3).
|
||||
//!
|
||||
//! Tests are numbered to match the test-plan.md identifiers (TC-3.3-*).
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use serde_json::json;
|
||||
|
||||
use nomi_agent::plan::state::PlanState;
|
||||
use nomi_agent::plan::tools::{EnterPlanModeTool, ExitPlanModeTool};
|
||||
use nomi_protocol::events::ToolCategory;
|
||||
use nomi_tools::Tool;
|
||||
use nomi_types::skill_types::PlanModeTransition;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.3-01 PlanState initial state
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_3_3_01_plan_state_default_is_inactive() {
|
||||
let state = PlanState::default();
|
||||
assert!(!state.is_active);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tc_3_3_01_plan_state_default_allow_list_empty() {
|
||||
let state = PlanState::default();
|
||||
assert!(state.pre_plan_allow_list.is_empty());
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.3-02 EnterPlanMode normal execution
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_3_3_02_enter_plan_mode_succeeds_when_not_active() {
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let tool = EnterPlanModeTool::new(flag);
|
||||
|
||||
let result = tool.execute(json!({})).await;
|
||||
|
||||
assert!(!result.is_error, "should succeed when not in plan mode");
|
||||
assert!(
|
||||
result.content.contains("plan mode"),
|
||||
"confirmation message should mention plan mode"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.3-03 EnterPlanMode duplicate entry rejected
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_3_3_03_enter_plan_mode_rejects_when_already_active() {
|
||||
let flag = Arc::new(AtomicBool::new(true));
|
||||
let tool = EnterPlanModeTool::new(flag);
|
||||
|
||||
let result = tool.execute(json!({})).await;
|
||||
|
||||
assert!(result.is_error, "should fail when already in plan mode");
|
||||
assert!(
|
||||
result.content.contains("Already in plan mode"),
|
||||
"error message should indicate already in plan mode"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.3-04 ExitPlanMode normal execution
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_3_3_04_exit_plan_mode_succeeds_when_active() {
|
||||
let flag = Arc::new(AtomicBool::new(true));
|
||||
let tool = ExitPlanModeTool::new(flag);
|
||||
|
||||
let result = tool.execute(json!({})).await;
|
||||
|
||||
assert!(!result.is_error, "should succeed when in plan mode");
|
||||
assert!(
|
||||
result.content.contains("Exited plan mode"),
|
||||
"confirmation message should mention exiting"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.3-05 ExitPlanMode when not in plan mode
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn tc_3_3_05_exit_plan_mode_rejects_when_not_active() {
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let tool = ExitPlanModeTool::new(flag);
|
||||
|
||||
let result = tool.execute(json!({})).await;
|
||||
|
||||
assert!(result.is_error, "should fail when not in plan mode");
|
||||
assert!(
|
||||
result.content.contains("Not in plan mode"),
|
||||
"error message should indicate not in plan mode"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.3-06 EnterPlanMode context_modifier returns Enter
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_3_3_06_enter_context_modifier_returns_enter_transition() {
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let tool = EnterPlanModeTool::new(flag);
|
||||
|
||||
let modifier = tool.context_modifier_for(&json!({}));
|
||||
|
||||
assert!(modifier.is_some(), "should return a context modifier");
|
||||
let cm = modifier.unwrap();
|
||||
assert_eq!(
|
||||
cm.plan_mode_transition,
|
||||
Some(PlanModeTransition::Enter),
|
||||
"transition should be Enter"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.3-07 ExitPlanMode context_modifier returns Exit
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_3_3_07_exit_context_modifier_returns_exit_transition() {
|
||||
let flag = Arc::new(AtomicBool::new(true));
|
||||
let tool = ExitPlanModeTool::new(flag);
|
||||
|
||||
let modifier = tool.context_modifier_for(&json!({}));
|
||||
|
||||
assert!(modifier.is_some(), "should return a context modifier");
|
||||
let cm = modifier.unwrap();
|
||||
assert!(
|
||||
matches!(
|
||||
cm.plan_mode_transition,
|
||||
Some(PlanModeTransition::Exit { .. })
|
||||
),
|
||||
"transition should be Exit variant"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.3-08 EnterPlanMode tool metadata
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_3_3_08_enter_tool_name() {
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let tool = EnterPlanModeTool::new(flag);
|
||||
assert_eq!(tool.name(), "EnterPlanMode");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tc_3_3_08_enter_tool_category_is_info() {
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let tool = EnterPlanModeTool::new(flag);
|
||||
assert!(matches!(tool.category(), ToolCategory::Info));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tc_3_3_08_enter_tool_is_concurrency_safe() {
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let tool = EnterPlanModeTool::new(flag);
|
||||
assert!(tool.is_concurrency_safe(&json!({})));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tc_3_3_08_enter_tool_schema_no_required_params() {
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let tool = EnterPlanModeTool::new(flag);
|
||||
let schema = tool.input_schema();
|
||||
let required = schema["required"]
|
||||
.as_array()
|
||||
.expect("required should be an array");
|
||||
assert!(required.is_empty(), "no required parameters expected");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-3.3-09 ExitPlanMode tool metadata
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_3_3_09_exit_tool_name() {
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let tool = ExitPlanModeTool::new(flag);
|
||||
assert_eq!(tool.name(), "ExitPlanMode");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tc_3_3_09_exit_tool_category_is_info() {
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let tool = ExitPlanModeTool::new(flag);
|
||||
assert!(matches!(tool.category(), ToolCategory::Info));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tc_3_3_09_exit_tool_is_concurrency_safe() {
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let tool = ExitPlanModeTool::new(flag);
|
||||
assert!(tool.is_concurrency_safe(&json!({})));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tc_3_3_09_exit_tool_schema_no_required_params() {
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let tool = ExitPlanModeTool::new(flag);
|
||||
let schema = tool.input_schema();
|
||||
let required = schema["required"]
|
||||
.as_array()
|
||||
.expect("required should be an array");
|
||||
assert!(required.is_empty(), "no required parameters expected");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Additional: full enter-exit cycle with shared flag
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn enter_exit_cycle_with_shared_flag() {
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let enter = EnterPlanModeTool::new(Arc::clone(&flag));
|
||||
let exit = ExitPlanModeTool::new(Arc::clone(&flag));
|
||||
|
||||
// Phase 1: enter should succeed
|
||||
let r = enter.execute(json!({})).await;
|
||||
assert!(!r.is_error);
|
||||
|
||||
// Simulate engine applying the transition
|
||||
flag.store(true, Ordering::Release);
|
||||
|
||||
// Phase 2: double-enter should fail
|
||||
let r = enter.execute(json!({})).await;
|
||||
assert!(r.is_error);
|
||||
|
||||
// Phase 3: exit should succeed
|
||||
let r = exit.execute(json!({})).await;
|
||||
assert!(!r.is_error);
|
||||
|
||||
// Simulate engine applying the transition
|
||||
flag.store(false, Ordering::Release);
|
||||
|
||||
// Phase 4: double-exit should fail
|
||||
let r = exit.execute(json!({})).await;
|
||||
assert!(r.is_error);
|
||||
|
||||
// Phase 5: re-enter should succeed
|
||||
let r = enter.execute(json!({})).await;
|
||||
assert!(!r.is_error);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Additional: context_modifier fields are default except plan_mode_transition
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn enter_context_modifier_other_fields_are_default() {
|
||||
let tool = EnterPlanModeTool::new(Arc::new(AtomicBool::new(false)));
|
||||
let cm = tool.context_modifier_for(&json!({})).unwrap();
|
||||
assert!(cm.model.is_none());
|
||||
assert!(cm.effort.is_none());
|
||||
assert!(cm.allowed_tools.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exit_context_modifier_other_fields_are_default() {
|
||||
let tool = ExitPlanModeTool::new(Arc::new(AtomicBool::new(false)));
|
||||
let cm = tool.context_modifier_for(&json!({})).unwrap();
|
||||
assert!(cm.model.is_none());
|
||||
assert!(cm.effort.is_none());
|
||||
assert!(cm.allowed_tools.is_empty());
|
||||
}
|
||||
@@ -0,0 +1,345 @@
|
||||
//! End-to-end skill tests using real files on disk.
|
||||
//!
|
||||
//! Each test creates skill files in a temporary directory that mirrors the
|
||||
//! `.nomi/skills/` and `.nomi/commands/` layout, then exercises the full
|
||||
//! pipeline: discovery -> loading -> system prompt injection -> SkillTool execution.
|
||||
//!
|
||||
//! Tests use `load_all_skills` with `add_dirs` or a temp cwd to avoid depending
|
||||
//! on any pre-existing files in the repo or user home directory.
|
||||
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use nomi_agent::context::{SystemPromptCache, build_system_prompt};
|
||||
use nomi_agent::skill_tool::SkillTool;
|
||||
use nomi_agent::skills::loader::load_all_skills;
|
||||
use nomi_agent::skills::permissions::SkillPermissionChecker;
|
||||
use nomi_agent::skills::types::SkillMetadata;
|
||||
use nomi_tools::Tool;
|
||||
use serde_json::json;
|
||||
use tempfile::TempDir;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn find_skill<'a>(skills: &'a [SkillMetadata], name: &str) -> Option<&'a SkillMetadata> {
|
||||
skills.iter().find(|s| s.name == name)
|
||||
}
|
||||
|
||||
/// Create a project-like temp directory with `.git` marker and `.nomi/skills/` + `.nomi/commands/`.
|
||||
/// Returns (TempDir guard, root path).
|
||||
fn make_project() -> (TempDir, PathBuf) {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let root = tmp.path().to_path_buf();
|
||||
|
||||
// Git root marker so walk_up stops here
|
||||
fs::create_dir(root.join(".git")).unwrap();
|
||||
|
||||
// Skills directory
|
||||
let skills_dir = root.join(".nomi").join("skills");
|
||||
fs::create_dir_all(&skills_dir).unwrap();
|
||||
|
||||
// Commands directory (legacy)
|
||||
let commands_dir = root.join(".nomi").join("commands");
|
||||
fs::create_dir_all(&commands_dir).unwrap();
|
||||
|
||||
// --- greet skill ---
|
||||
let greet_dir = skills_dir.join("greet");
|
||||
fs::create_dir_all(&greet_dir).unwrap();
|
||||
fs::write(
|
||||
greet_dir.join("SKILL.md"),
|
||||
"---\nname: greet\ndescription: Greet a user by name\n---\n\nHello, $ARGUMENTS! Welcome to the project.\n",
|
||||
).unwrap();
|
||||
|
||||
// --- db:migrate (nested namespace) ---
|
||||
let migrate_dir = skills_dir.join("db").join("migrate");
|
||||
fs::create_dir_all(&migrate_dir).unwrap();
|
||||
fs::write(
|
||||
migrate_dir.join("SKILL.md"),
|
||||
"---\nname: db:migrate\ndescription: Run database migrations\n---\n\nRunning migrations for: $ARGUMENTS\nSkill directory: ${NOMI_SKILL_DIR}\n",
|
||||
).unwrap();
|
||||
|
||||
// --- rust-review (conditional paths) ---
|
||||
let review_dir = skills_dir.join("rust-review");
|
||||
fs::create_dir_all(&review_dir).unwrap();
|
||||
fs::write(
|
||||
review_dir.join("SKILL.md"),
|
||||
"---\nname: rust-review\ndescription: Rust-specific code review checklist\npaths:\n - \"**/*.rs\"\n - \"Cargo.toml\"\n---\n\nWhen reviewing Rust code, check:\n- No unwrap() in library code\n",
|
||||
).unwrap();
|
||||
|
||||
// --- shell-demo (shell expansion) ---
|
||||
let shell_dir = skills_dir.join("shell-demo");
|
||||
fs::create_dir_all(&shell_dir).unwrap();
|
||||
fs::write(
|
||||
shell_dir.join("SKILL.md"),
|
||||
"---\nname: shell-demo\ndescription: Demonstrate shell command expansion\n---\n\nCurrent date: !`date +%Y-%m-%d`\n",
|
||||
).unwrap();
|
||||
|
||||
// --- legacy command (flat .md in commands/) ---
|
||||
fs::write(
|
||||
commands_dir.join("legacy-cmd.md"),
|
||||
"---\nname: legacy-cmd\ndescription: A legacy command for backward compatibility testing\n---\n\nThis is a legacy command loaded from .nomi/commands/\nArguments: $ARGUMENTS\n",
|
||||
).unwrap();
|
||||
|
||||
(tmp, root)
|
||||
}
|
||||
|
||||
fn make_tool(skills: Vec<SkillMetadata>, cwd: &str) -> SkillTool {
|
||||
SkillTool::new(
|
||||
Arc::new(skills),
|
||||
cwd.to_string(),
|
||||
SkillPermissionChecker::new(vec![], vec![], false),
|
||||
)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// E1: Project-level skill discovery
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn e1_project_skill_discovered() {
|
||||
let (_guard, root) = make_project();
|
||||
let skills = load_all_skills(&root, &[], false, None).await;
|
||||
|
||||
let greet = find_skill(&skills, "greet");
|
||||
assert!(greet.is_some(), "E1 FAIL: 'greet' skill not discovered");
|
||||
assert_eq!(greet.unwrap().description, "Greet a user by name");
|
||||
println!("E1 PASS: project-level skill 'greet' discovered with correct description");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// E2: Legacy commands discovery
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn e2_legacy_commands_discovered() {
|
||||
let (_guard, root) = make_project();
|
||||
let skills = load_all_skills(&root, &[], false, None).await;
|
||||
|
||||
let legacy = find_skill(&skills, "legacy-cmd");
|
||||
assert!(legacy.is_some(), "E2 FAIL: 'legacy-cmd' not discovered");
|
||||
println!("E2 PASS: legacy command 'legacy-cmd' discovered from .nomi/commands/");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// E3: Nested namespace (db:migrate)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn e3_nested_namespace() {
|
||||
let (_guard, root) = make_project();
|
||||
let skills = load_all_skills(&root, &[], false, None).await;
|
||||
|
||||
let migrate = find_skill(&skills, "db:migrate");
|
||||
assert!(migrate.is_some(), "E3 FAIL: 'db:migrate' not discovered");
|
||||
println!("E3 PASS: nested skill 'db:migrate' discovered with colon namespace");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// E4: Variable substitution ($ARGUMENTS)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn e4_variable_substitution() {
|
||||
let (_guard, root) = make_project();
|
||||
let cwd = root.to_string_lossy().to_string();
|
||||
let skills = load_all_skills(&root, &[], false, None).await;
|
||||
let tool = make_tool(skills, &cwd);
|
||||
|
||||
let result = tool
|
||||
.execute(json!({"skill": "greet", "args": "Alice"}))
|
||||
.await;
|
||||
assert!(!result.is_error, "E4 FAIL: error: {}", result.content);
|
||||
assert!(
|
||||
result.content.contains("Hello, Alice!"),
|
||||
"E4 FAIL: $ARGUMENTS not substituted. Got: {}",
|
||||
result.content
|
||||
);
|
||||
println!("E4 PASS: $ARGUMENTS substituted correctly");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// E5: Shell command expansion
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
#[cfg(not(windows))] // Shell expansion uses Unix commands; skip on Windows
|
||||
async fn e5_shell_expansion() {
|
||||
let (_guard, root) = make_project();
|
||||
let cwd = root.to_string_lossy().to_string();
|
||||
let skills = load_all_skills(&root, &[], false, None).await;
|
||||
let tool = make_tool(skills, &cwd);
|
||||
|
||||
let result = tool.execute(json!({"skill": "shell-demo"})).await;
|
||||
assert!(!result.is_error, "E5 FAIL: error: {}", result.content);
|
||||
|
||||
let today = chrono::Local::now().format("%Y-%m-%d").to_string();
|
||||
assert!(
|
||||
result.content.contains(&today),
|
||||
"E5 FAIL: shell expansion did not produce today's date. Got: {}",
|
||||
result.content
|
||||
);
|
||||
println!("E5 PASS: shell expansion produced today's date ({})", today);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// E6: Conditional activation (paths filter)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn e6_conditional_activation() {
|
||||
let (_guard, root) = make_project();
|
||||
let skills = load_all_skills(&root, &[], false, None).await;
|
||||
|
||||
let rust_review = find_skill(&skills, "rust-review").expect("E6 FAIL: 'rust-review' not found");
|
||||
assert!(
|
||||
!rust_review.paths.is_empty(),
|
||||
"E6 FAIL: paths should not be empty"
|
||||
);
|
||||
assert!(
|
||||
rust_review.paths.iter().any(|p| p.contains("*.rs")),
|
||||
"E6 FAIL: paths should contain '*.rs'. Got: {:?}",
|
||||
rust_review.paths
|
||||
);
|
||||
println!(
|
||||
"E6 PASS: 'rust-review' has conditional paths: {:?}",
|
||||
rust_review.paths
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// E7: System prompt injection
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn e7_system_prompt_injection() {
|
||||
let (_guard, root) = make_project();
|
||||
let cwd = root.to_string_lossy().to_string();
|
||||
let skills = load_all_skills(&root, &[], false, None).await;
|
||||
|
||||
let prompt = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
&cwd,
|
||||
"test-model",
|
||||
&skills,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
assert!(
|
||||
prompt.contains("greet"),
|
||||
"E7 FAIL: 'greet' not in system prompt"
|
||||
);
|
||||
assert!(
|
||||
prompt.contains("db:migrate"),
|
||||
"E7 FAIL: 'db:migrate' not in system prompt"
|
||||
);
|
||||
assert!(
|
||||
prompt.contains("system-reminder"),
|
||||
"E7 FAIL: missing <system-reminder> wrapper"
|
||||
);
|
||||
println!("E7 PASS: skills injected into system prompt");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// E8: Full SkillTool execution (db:migrate with $ARGUMENTS + ${NOMI_SKILL_DIR})
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn e8_full_execution() {
|
||||
let (_guard, root) = make_project();
|
||||
let cwd = root.to_string_lossy().to_string();
|
||||
let skills = load_all_skills(&root, &[], false, None).await;
|
||||
let tool = make_tool(skills, &cwd);
|
||||
|
||||
let result = tool
|
||||
.execute(json!({"skill": "db:migrate", "args": "production"}))
|
||||
.await;
|
||||
assert!(!result.is_error, "E8 FAIL: error: {}", result.content);
|
||||
assert!(
|
||||
result
|
||||
.content
|
||||
.contains("Running migrations for: production"),
|
||||
"E8 FAIL: $ARGUMENTS not substituted. Got: {}",
|
||||
result.content
|
||||
);
|
||||
assert!(
|
||||
!result.content.contains("${NOMI_SKILL_DIR}"),
|
||||
"E8 FAIL: ${{NOMI_SKILL_DIR}} not expanded. Got: {}",
|
||||
result.content
|
||||
);
|
||||
println!("E8 PASS: full execution with $ARGUMENTS and ${{NOMI_SKILL_DIR}} substitution");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// E9: Deduplication
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn e9_deduplication() {
|
||||
let (_guard, root) = make_project();
|
||||
let skills = load_all_skills(&root, &[], false, None).await;
|
||||
|
||||
let mut name_counts = std::collections::HashMap::new();
|
||||
for skill in &skills {
|
||||
*name_counts.entry(skill.name.as_str()).or_insert(0usize) += 1;
|
||||
}
|
||||
for (name, count) in &name_counts {
|
||||
assert_eq!(*count, 1, "E9 FAIL: '{}' appears {} times", name, count);
|
||||
}
|
||||
println!("E9 PASS: all {} skills have unique names", skills.len());
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// E10: Skill not found error
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn e10_skill_not_found() {
|
||||
let (_guard, root) = make_project();
|
||||
let cwd = root.to_string_lossy().to_string();
|
||||
let skills = load_all_skills(&root, &[], false, None).await;
|
||||
let tool = make_tool(skills, &cwd);
|
||||
|
||||
let result = tool.execute(json!({"skill": "nonexistent-skill"})).await;
|
||||
assert!(result.is_error, "E10 FAIL: should return error");
|
||||
assert!(
|
||||
result.content.contains("not found"),
|
||||
"E10 FAIL: got: {}",
|
||||
result.content
|
||||
);
|
||||
println!("E10 PASS: nonexistent skill returns clear error message");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// E11: Legacy command execution
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn e11_legacy_command_execution() {
|
||||
let (_guard, root) = make_project();
|
||||
let cwd = root.to_string_lossy().to_string();
|
||||
let skills = load_all_skills(&root, &[], false, None).await;
|
||||
let tool = make_tool(skills, &cwd);
|
||||
|
||||
let result = tool
|
||||
.execute(json!({"skill": "legacy-cmd", "args": "test-arg"}))
|
||||
.await;
|
||||
assert!(!result.is_error, "E11 FAIL: error: {}", result.content);
|
||||
assert!(
|
||||
result.content.contains("legacy command"),
|
||||
"E11 FAIL: got: {}",
|
||||
result.content
|
||||
);
|
||||
assert!(
|
||||
result.content.contains("test-arg"),
|
||||
"E11 FAIL: $ARGUMENTS not substituted. Got: {}",
|
||||
result.content
|
||||
);
|
||||
println!("E11 PASS: legacy command executed with variable substitution");
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
//! Integration tests for Spawn tool description (TC-4.2-07).
|
||||
//!
|
||||
//! Verifies the enhanced Spawn tool description contains capacity limits
|
||||
//! and usage guidance as specified in the test plan.
|
||||
|
||||
mod common;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use common::{MockLlmProvider, test_config};
|
||||
use nomi_agent::spawn_tool::SpawnTool;
|
||||
use nomi_agent::spawner::AgentSpawner;
|
||||
use nomi_tools::Tool;
|
||||
|
||||
fn make_spawn_tool() -> SpawnTool {
|
||||
let provider = Arc::new(MockLlmProvider::with_text_response("ok"));
|
||||
let spawner = Arc::new(AgentSpawner::new(
|
||||
provider,
|
||||
test_config(),
|
||||
std::env::temp_dir(),
|
||||
));
|
||||
SpawnTool::new(spawner)
|
||||
}
|
||||
|
||||
// --- TC-4.2-07: Spawn tool description contains capacity limits ---
|
||||
|
||||
#[test]
|
||||
fn spawn_description_mentions_max_agents() {
|
||||
let tool = make_spawn_tool();
|
||||
let desc = tool.description();
|
||||
assert!(
|
||||
desc.contains('5'),
|
||||
"Spawn description should mention the 5 sub-agent limit"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spawn_description_mentions_parallel() {
|
||||
let tool = make_spawn_tool();
|
||||
let desc = tool.description();
|
||||
assert!(
|
||||
desc.contains("parallel"),
|
||||
"Spawn description should mention parallel execution"
|
||||
);
|
||||
}
|
||||
|
||||
// --- R-4.1-01: Spawn description should document max_turns and max_tokens ---
|
||||
|
||||
#[test]
|
||||
fn spawn_description_mentions_max_turns() {
|
||||
let tool = make_spawn_tool();
|
||||
let desc = tool.description();
|
||||
assert!(
|
||||
desc.contains("200"),
|
||||
"Spawn description should mention the 200 turn limit per sub-agent"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spawn_description_mentions_max_tokens() {
|
||||
let tool = make_spawn_tool();
|
||||
let desc = tool.description();
|
||||
assert!(
|
||||
desc.contains("4096"),
|
||||
"Spawn description should mention the 4096 token limit per sub-agent"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
mod common;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use common::{MockLlmProvider, test_config};
|
||||
use nomi_agent::spawner::{AgentSpawner, SubAgentConfig};
|
||||
use nomi_types::llm::LlmEvent;
|
||||
use nomi_types::message::{StopReason, TokenUsage};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helper: build a minimal SubAgentConfig for testing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn make_sub_config(name: &str) -> SubAgentConfig {
|
||||
SubAgentConfig {
|
||||
name: name.to_string(),
|
||||
prompt: format!("Task for {}", name),
|
||||
max_turns: 5,
|
||||
max_tokens: 1024,
|
||||
system_prompt: None,
|
||||
allowed_tools: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Single sub-agent executes and returns the expected text result.
|
||||
#[tokio::test]
|
||||
async fn test_spawn_single_agent() {
|
||||
let provider = Arc::new(MockLlmProvider::with_text_response("Sub-agent done"));
|
||||
let spawner = AgentSpawner::new(provider, test_config(), std::env::temp_dir());
|
||||
|
||||
let result = spawner.spawn_one(make_sub_config("agent-1")).await;
|
||||
|
||||
assert_eq!(result.text, "Sub-agent done");
|
||||
assert!(!result.is_error, "expected no error, got: {}", result.text);
|
||||
assert_eq!(result.turns, 1);
|
||||
assert_eq!(result.name, "agent-1");
|
||||
}
|
||||
|
||||
/// Parallel sub-agents all complete successfully and return distinct results.
|
||||
#[tokio::test]
|
||||
async fn test_spawn_parallel_agents() {
|
||||
// Provide one turn sequence per sub-agent; each stream() call pops one entry.
|
||||
let make_turn = |text: &str| {
|
||||
vec![
|
||||
LlmEvent::TextDelta(text.to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 10,
|
||||
output_tokens: 5,
|
||||
cache_creation_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
},
|
||||
},
|
||||
]
|
||||
};
|
||||
|
||||
let provider = Arc::new(MockLlmProvider::with_turns(vec![
|
||||
make_turn("result-A"),
|
||||
make_turn("result-B"),
|
||||
make_turn("result-C"),
|
||||
]));
|
||||
|
||||
let spawner = AgentSpawner::new(provider, test_config(), std::env::temp_dir());
|
||||
|
||||
let sub_configs = vec![
|
||||
make_sub_config("agent-A"),
|
||||
make_sub_config("agent-B"),
|
||||
make_sub_config("agent-C"),
|
||||
];
|
||||
|
||||
let results = spawner.spawn_parallel(sub_configs).await;
|
||||
|
||||
assert_eq!(results.len(), 3, "expected 3 results from 3 sub-agents");
|
||||
|
||||
for result in &results {
|
||||
assert!(
|
||||
!result.is_error,
|
||||
"sub-agent '{}' returned an error: {}",
|
||||
result.name, result.text
|
||||
);
|
||||
}
|
||||
|
||||
// Each result should contain one of the expected texts (order may vary due
|
||||
// to concurrent scheduling, so we just verify the full set is covered).
|
||||
let texts: std::collections::HashSet<&str> = results.iter().map(|r| r.text.as_str()).collect();
|
||||
assert!(texts.contains("result-A"), "missing result-A");
|
||||
assert!(texts.contains("result-B"), "missing result-B");
|
||||
assert!(texts.contains("result-C"), "missing result-C");
|
||||
}
|
||||
|
||||
/// The same provider Arc is reused across sequentially spawned sub-agents.
|
||||
#[tokio::test]
|
||||
async fn test_spawn_shares_provider() {
|
||||
// Two turns: one for each sequential sub-agent call.
|
||||
let provider = Arc::new(MockLlmProvider::with_turns(vec![
|
||||
vec![
|
||||
LlmEvent::TextDelta("first".to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 10,
|
||||
output_tokens: 5,
|
||||
cache_creation_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
},
|
||||
},
|
||||
],
|
||||
vec![
|
||||
LlmEvent::TextDelta("second".to_string()),
|
||||
LlmEvent::Done {
|
||||
stop_reason: StopReason::EndTurn,
|
||||
usage: TokenUsage {
|
||||
input_tokens: 10,
|
||||
output_tokens: 5,
|
||||
cache_creation_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
},
|
||||
},
|
||||
],
|
||||
]));
|
||||
|
||||
// Both sub-agents share the same underlying provider via Arc.
|
||||
let provider_dyn: Arc<dyn nomi_providers::LlmProvider> = provider;
|
||||
let spawner = AgentSpawner::new(
|
||||
Arc::clone(&provider_dyn),
|
||||
test_config(),
|
||||
std::env::temp_dir(),
|
||||
);
|
||||
|
||||
let result1 = spawner.spawn_one(make_sub_config("seq-1")).await;
|
||||
let result2 = spawner.spawn_one(make_sub_config("seq-2")).await;
|
||||
|
||||
assert!(!result1.is_error, "seq-1 errored: {}", result1.text);
|
||||
assert!(!result2.is_error, "seq-2 errored: {}", result2.text);
|
||||
assert_eq!(result1.text, "first");
|
||||
assert_eq!(result2.text, "second");
|
||||
}
|
||||
|
||||
/// An LLM error event causes the sub-agent result to be marked as an error.
|
||||
#[tokio::test]
|
||||
async fn test_spawn_agent_error_captured() {
|
||||
// Emit an Error event — the engine converts this to AgentError::ApiError,
|
||||
// which spawner catches and stores in SubAgentResult::is_error.
|
||||
let provider = Arc::new(MockLlmProvider::with_events(vec![LlmEvent::Error(
|
||||
"provider failed".to_string(),
|
||||
)]));
|
||||
|
||||
let spawner = AgentSpawner::new(provider, test_config(), std::env::temp_dir());
|
||||
|
||||
let result = spawner.spawn_one(make_sub_config("error-agent")).await;
|
||||
|
||||
assert!(result.is_error, "expected is_error=true");
|
||||
assert!(
|
||||
result.text.to_lowercase().contains("error"),
|
||||
"expected error message to contain 'error', got: {}",
|
||||
result.text
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,375 @@
|
||||
//! Integration tests for system prompt tool usage guidance (TC-4.3-01 through TC-4.3-08).
|
||||
//!
|
||||
//! Black-box tests verifying the "# Using your tools" section is correctly
|
||||
//! assembled into the system prompt with proper content and ordering.
|
||||
|
||||
use std::fs;
|
||||
|
||||
use nomi_agent::context::{SystemPromptCache, build_system_prompt};
|
||||
use nomi_skills::types::{ExecutionContext, LoadedFrom, SkillMetadata, SkillSource};
|
||||
|
||||
fn make_skill(name: &str, description: &str) -> SkillMetadata {
|
||||
SkillMetadata {
|
||||
name: name.to_string(),
|
||||
display_name: None,
|
||||
description: description.to_string(),
|
||||
has_user_specified_description: false,
|
||||
allowed_tools: vec![],
|
||||
argument_hint: None,
|
||||
argument_names: vec![],
|
||||
when_to_use: None,
|
||||
version: None,
|
||||
model: None,
|
||||
disable_model_invocation: false,
|
||||
user_invocable: true,
|
||||
execution_context: ExecutionContext::Inline,
|
||||
agent: None,
|
||||
effort: None,
|
||||
shell: None,
|
||||
paths: vec![],
|
||||
hooks_raw: None,
|
||||
source: SkillSource::User,
|
||||
loaded_from: LoadedFrom::Skills,
|
||||
content: String::new(),
|
||||
content_length: 0,
|
||||
skill_root: None,
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-4.3-01: Tool guidance section exists
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_4_3_01_tool_guidance_section_exists() {
|
||||
let result = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
false, // browser_enabled
|
||||
);
|
||||
assert!(
|
||||
result.contains("# Using your tools"),
|
||||
"system prompt should contain the tool guidance section heading"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-4.3-02: Bash prohibition list with dedicated tool alternatives
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_4_3_02_bash_prohibition_list() {
|
||||
let result = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
false, // browser_enabled
|
||||
);
|
||||
|
||||
// Glob replaces find/ls
|
||||
assert!(
|
||||
result.contains("Glob") && result.contains("find"),
|
||||
"should map Glob as replacement for find"
|
||||
);
|
||||
// Grep replaces grep/rg
|
||||
assert!(
|
||||
result.contains("Grep") && result.contains("grep"),
|
||||
"should map Grep as replacement for grep"
|
||||
);
|
||||
// Read replaces cat/head/tail
|
||||
assert!(
|
||||
result.contains("Read") && result.contains("cat"),
|
||||
"should map Read as replacement for cat"
|
||||
);
|
||||
// Edit replaces sed/awk
|
||||
assert!(
|
||||
result.contains("Edit") && result.contains("sed"),
|
||||
"should map Edit as replacement for sed"
|
||||
);
|
||||
// Write replaces echo/heredoc
|
||||
assert!(
|
||||
result.contains("Write") && result.contains("echo"),
|
||||
"should map Write as replacement for echo"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-4.3-03: Parallel call guidance
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_4_3_03_parallel_call_guidance() {
|
||||
let result = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
false, // browser_enabled
|
||||
);
|
||||
assert!(
|
||||
result.contains("parallel"),
|
||||
"should contain parallel call guidance"
|
||||
);
|
||||
assert!(
|
||||
result.contains("sequentially"),
|
||||
"should explain when to run sequentially (dependencies)"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-4.3-04: Edit-over-Write and Read-before-Edit rules
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_4_3_04_edit_write_read_rules() {
|
||||
let result = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
false, // browser_enabled
|
||||
);
|
||||
assert!(
|
||||
result.contains("Prefer Edit over Write"),
|
||||
"should contain Edit-over-Write preference"
|
||||
);
|
||||
assert!(
|
||||
result.contains("Read a file before editing"),
|
||||
"should contain Read-before-Edit rule"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-4.3-05: Section order — guidance after intro, before custom prompt
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_4_3_05_order_after_intro_before_custom() {
|
||||
let result = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
Some("CUSTOM_PROMPT_MARKER"),
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
false, // browser_enabled
|
||||
);
|
||||
|
||||
let intro_pos = result
|
||||
.find("Working directory")
|
||||
.expect("intro should contain 'Working directory'");
|
||||
let guidance_pos = result
|
||||
.find("# Using your tools")
|
||||
.expect("tool guidance section should exist");
|
||||
let custom_pos = result
|
||||
.find("CUSTOM_PROMPT_MARKER")
|
||||
.expect("custom prompt should exist");
|
||||
|
||||
assert!(
|
||||
guidance_pos > intro_pos,
|
||||
"tool guidance should appear after the base intro"
|
||||
);
|
||||
assert!(
|
||||
guidance_pos < custom_pos,
|
||||
"tool guidance should appear before custom prompt"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-4.3-06: Section order — guidance before skills and memory
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_4_3_06_order_before_skills() {
|
||||
let skills = vec![make_skill("order-test-skill", "Order test")];
|
||||
let result = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&skills,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
false, // browser_enabled
|
||||
);
|
||||
|
||||
let guidance_pos = result
|
||||
.find("# Using your tools")
|
||||
.expect("tool guidance should exist");
|
||||
let skills_pos = result
|
||||
.find("order-test-skill")
|
||||
.expect("skill should be listed");
|
||||
|
||||
assert!(
|
||||
guidance_pos < skills_pos,
|
||||
"tool guidance should appear before skills listing"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tc_4_3_06_order_before_memory() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let mem_dir = tmp.path().join("memory");
|
||||
fs::create_dir_all(&mem_dir).unwrap();
|
||||
fs::write(
|
||||
mem_dir.join("MEMORY.md"),
|
||||
"- [Note](note.md) \u{2014} some note\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let result = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
Some(&mem_dir),
|
||||
false,
|
||||
false,
|
||||
false, // browser_enabled
|
||||
);
|
||||
|
||||
let guidance_pos = result
|
||||
.find("# Using your tools")
|
||||
.expect("tool guidance should exist");
|
||||
let memory_pos = result
|
||||
.find("auto memory")
|
||||
.expect("memory section should exist");
|
||||
|
||||
assert!(
|
||||
guidance_pos < memory_pos,
|
||||
"tool guidance should appear before memory section"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-4.3-07: All sections coexist with correct ordering
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_4_3_07_all_sections_coexist() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let cwd = tmp.path();
|
||||
|
||||
// Create AGENTS.md
|
||||
fs::write(cwd.join("AGENTS.md"), "PROJECT_RULES_COEXIST").unwrap();
|
||||
|
||||
// Create memory
|
||||
let mem_dir = tmp.path().join("memory");
|
||||
fs::create_dir_all(&mem_dir).unwrap();
|
||||
fs::write(
|
||||
mem_dir.join("MEMORY.md"),
|
||||
"- [Item](item.md) \u{2014} coexist test\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let skills = vec![make_skill("coexist-skill", "Coexist test")];
|
||||
|
||||
let result = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
Some("CUSTOM_COEXIST"),
|
||||
&cwd.to_string_lossy(),
|
||||
"test-model",
|
||||
&skills,
|
||||
None,
|
||||
Some(&mem_dir),
|
||||
true, // plan mode active
|
||||
false,
|
||||
false, // browser_enabled
|
||||
);
|
||||
|
||||
// All sections should exist
|
||||
assert!(result.contains("Working directory"), "intro should exist");
|
||||
assert!(
|
||||
result.contains("# Using your tools"),
|
||||
"tool guidance should exist"
|
||||
);
|
||||
assert!(
|
||||
result.contains("CUSTOM_COEXIST"),
|
||||
"custom prompt should exist"
|
||||
);
|
||||
assert!(
|
||||
result.contains("PROJECT_RULES_COEXIST"),
|
||||
"AGENTS.md should exist"
|
||||
);
|
||||
assert!(
|
||||
result.contains("auto memory"),
|
||||
"memory section should exist"
|
||||
);
|
||||
assert!(
|
||||
result.contains("coexist-skill"),
|
||||
"skills listing should exist"
|
||||
);
|
||||
|
||||
// Verify ordering: intro < guidance < custom < agents.md < memory < skills
|
||||
let intro_pos = result.find("Working directory").unwrap();
|
||||
let guidance_pos = result.find("# Using your tools").unwrap();
|
||||
let custom_pos = result.find("CUSTOM_COEXIST").unwrap();
|
||||
let agents_pos = result.find("PROJECT_RULES_COEXIST").unwrap();
|
||||
let memory_pos = result.find("auto memory").unwrap();
|
||||
let skills_pos = result.find("coexist-skill").unwrap();
|
||||
|
||||
assert!(guidance_pos > intro_pos, "guidance after intro");
|
||||
assert!(custom_pos > guidance_pos, "custom after guidance");
|
||||
assert!(agents_pos > custom_pos, "agents.md after custom");
|
||||
assert!(memory_pos > agents_pos, "memory after agents.md");
|
||||
assert!(skills_pos > memory_pos, "skills after memory");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC-4.3-08: Tool guidance present in plan mode
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tc_4_3_08_guidance_in_plan_mode() {
|
||||
let result = build_system_prompt(
|
||||
&mut SystemPromptCache::new(),
|
||||
None,
|
||||
"/tmp",
|
||||
"test-model",
|
||||
&[],
|
||||
None,
|
||||
None,
|
||||
true,
|
||||
false,
|
||||
false, // browser_enabled
|
||||
);
|
||||
assert!(
|
||||
result.contains("# Using your tools"),
|
||||
"tool guidance should be present even in plan mode"
|
||||
);
|
||||
// Plan mode instructions should also be present
|
||||
assert!(
|
||||
result.contains("plan") || result.contains("Plan"),
|
||||
"plan mode instructions should coexist with tool guidance"
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user