Update: 将子项目从 submodule 转为完整内容
- 移除 GovAI, nomifun-tauri, 算力盒子 的 submodule 引用 - 添加所有子项目的完整源代码 - 保留原始 .git 为 .git.bak 备份
This commit is contained in:
freedak
@@ -0,0 +1,12 @@
|
||||
[package]
|
||||
name = "nomi-compact"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
regex = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
@@ -0,0 +1,122 @@
|
||||
const MIN_FOLD_COUNT: usize = 3;
|
||||
const MIN_PREFIX_RATIO: f64 = 0.5;
|
||||
|
||||
fn common_prefix_len(a: &str, b: &str) -> usize {
|
||||
a.chars()
|
||||
.zip(b.chars())
|
||||
.take_while(|(ca, cb)| ca == cb)
|
||||
.count()
|
||||
}
|
||||
|
||||
fn lines_are_similar(a: &str, b: &str) -> bool {
|
||||
if a.is_empty() || b.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let prefix = common_prefix_len(a, b);
|
||||
let min_len = a.len().min(b.len());
|
||||
prefix as f64 / min_len as f64 >= MIN_PREFIX_RATIO
|
||||
}
|
||||
|
||||
pub fn fold_repeated_lines(text: &str) -> String {
|
||||
if text.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
let lines: Vec<&str> = text.split('\n').collect();
|
||||
let mut result: Vec<String> = Vec::new();
|
||||
let mut i = 0;
|
||||
|
||||
while i < lines.len() {
|
||||
let mut j = i + 1;
|
||||
while j < lines.len() && lines_are_similar(lines[i], lines[j]) {
|
||||
j += 1;
|
||||
}
|
||||
|
||||
let group_len = j - i;
|
||||
if group_len >= MIN_FOLD_COUNT {
|
||||
let folded = group_len - 2;
|
||||
result.push(lines[i].to_string());
|
||||
let identical = (i + 1..j).all(|k| lines[k] == lines[i]);
|
||||
if identical {
|
||||
result.push(format!("[... {folded} identical lines]"));
|
||||
} else {
|
||||
result.push(format!("[... {folded} similar lines]"));
|
||||
}
|
||||
result.push(lines[j - 1].to_string());
|
||||
} else {
|
||||
for line in &lines[i..j] {
|
||||
result.push(line.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
i = j;
|
||||
}
|
||||
|
||||
result.join("\n")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn fold_identical_consecutive_lines() {
|
||||
let input = "ok\nok\nok\nok\nok\ndone";
|
||||
let result = fold_repeated_lines(input);
|
||||
assert!(result.contains("[... 3 identical lines]"));
|
||||
assert!(result.contains("ok"));
|
||||
assert!(result.contains("done"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fold_no_repeats_unchanged() {
|
||||
let input = "apple\nbanana\ncherry";
|
||||
assert_eq!(fold_repeated_lines(input), input);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fold_similar_prefix_lines() {
|
||||
let lines: Vec<String> = (0..10)
|
||||
.map(|i| format!("Compiling crate-{i} v0.1.0"))
|
||||
.collect();
|
||||
let input = lines.join("\n");
|
||||
let result = fold_repeated_lines(&input);
|
||||
assert!(result.contains("[... 8 similar lines]"));
|
||||
assert!(result.contains("Compiling crate-0"));
|
||||
assert!(result.contains("Compiling crate-9"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fold_below_threshold_unchanged() {
|
||||
let input = "Compiling a v0.1.0\nCompiling b v0.1.0\ndone";
|
||||
assert_eq!(fold_repeated_lines(input), input);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fold_mixed_groups() {
|
||||
let mut lines = Vec::new();
|
||||
for i in 0..6 {
|
||||
lines.push(format!("Downloading dep-{i}..."));
|
||||
}
|
||||
lines.push("Install complete".to_string());
|
||||
for i in 0..5 {
|
||||
lines.push(format!("Compiling mod-{i}"));
|
||||
}
|
||||
let input = lines.join("\n");
|
||||
let result = fold_repeated_lines(&input);
|
||||
assert!(
|
||||
result.contains("[... 4 similar lines]"),
|
||||
"first group folded: {result}"
|
||||
);
|
||||
assert!(result.contains("Install complete"));
|
||||
assert!(
|
||||
result.contains("[... 3 similar lines]"),
|
||||
"second group folded: {result}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fold_empty_input() {
|
||||
assert_eq!(fold_repeated_lines(""), "");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
const INLINE_THRESHOLD: usize = 80;
|
||||
|
||||
fn compact_value(value: &serde_json::Value) -> String {
|
||||
format_value(value, 0)
|
||||
}
|
||||
|
||||
fn format_value(value: &serde_json::Value, depth: usize) -> String {
|
||||
match value {
|
||||
serde_json::Value::Object(map) => {
|
||||
let oneliner = serde_json::to_string(value).unwrap_or_default();
|
||||
if oneliner.len() <= INLINE_THRESHOLD && !oneliner.contains('\n') {
|
||||
return oneliner;
|
||||
}
|
||||
let indent = " ".repeat(depth + 1);
|
||||
let close_indent = " ".repeat(depth);
|
||||
let entries: Vec<String> = map
|
||||
.iter()
|
||||
.map(|(k, v)| format!("{indent}\"{k}\": {}", format_value(v, depth + 1)))
|
||||
.collect();
|
||||
format!("{{\n{}\n{close_indent}}}", entries.join(",\n"))
|
||||
}
|
||||
serde_json::Value::Array(arr) => {
|
||||
let oneliner = serde_json::to_string(value).unwrap_or_default();
|
||||
if oneliner.len() <= INLINE_THRESHOLD {
|
||||
return oneliner;
|
||||
}
|
||||
let indent = " ".repeat(depth + 1);
|
||||
let close_indent = " ".repeat(depth);
|
||||
let items: Vec<String> = arr
|
||||
.iter()
|
||||
.map(|v| format!("{indent}{}", format_value(v, depth + 1)))
|
||||
.collect();
|
||||
format!("[\n{}\n{close_indent}]", items.join(",\n"))
|
||||
}
|
||||
other => serde_json::to_string(other).unwrap_or_default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn compact_json(text: &str) -> String {
|
||||
if text.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
let trimmed = text.trim();
|
||||
|
||||
if (trimmed.starts_with('{') || trimmed.starts_with('['))
|
||||
&& let Ok(value) = serde_json::from_str::<serde_json::Value>(trimmed)
|
||||
{
|
||||
let compacted = compact_value(&value);
|
||||
if compacted.len() < trimmed.len() {
|
||||
return compacted;
|
||||
}
|
||||
return text.to_string();
|
||||
}
|
||||
|
||||
if let Some(start) = trimmed.find(['{', '[']) {
|
||||
let candidate = &trimmed[start..];
|
||||
if let Ok(value) = serde_json::from_str::<serde_json::Value>(candidate) {
|
||||
let compacted = compact_value(&value);
|
||||
if compacted.len() < candidate.len() {
|
||||
return format!("{}{}", &trimmed[..start], compacted);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
text.to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn compact_4space_to_2space() {
|
||||
let input = r#"{
|
||||
"name": "Alice Wonderland",
|
||||
"email": "alice@example.com",
|
||||
"age": 30,
|
||||
"address": "123 Main Street, Anytown, USA 12345",
|
||||
"phone": "+1-555-0123"
|
||||
}"#;
|
||||
let result = compact_json(input);
|
||||
assert!(
|
||||
result.contains(" \"name\""),
|
||||
"should use 2-space indent: {result}"
|
||||
);
|
||||
assert!(
|
||||
!result.contains(" \"name\""),
|
||||
"should not have 4-space indent"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compact_short_object_inline() {
|
||||
let input = r#"{
|
||||
"user": {
|
||||
"id": 1,
|
||||
"name": "Alice"
|
||||
}
|
||||
}"#;
|
||||
let result = compact_json(input);
|
||||
assert!(
|
||||
result.contains(r#"{"id":1,"name":"Alice"}"#)
|
||||
|| result.contains(r#"{"id": 1, "name": "Alice"}"#),
|
||||
"short nested object should be inlined: {result}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compact_non_json_unchanged() {
|
||||
let input = "This is not JSON\njust plain text";
|
||||
assert_eq!(compact_json(input), input);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compact_already_minified() {
|
||||
let input = r#"{"id":1,"name":"Alice"}"#;
|
||||
let result = compact_json(input);
|
||||
assert_eq!(
|
||||
result.len(),
|
||||
input.len(),
|
||||
"already compact JSON should not grow"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compact_preserves_array_structure() {
|
||||
let input = r#"[
|
||||
{
|
||||
"id": 1,
|
||||
"name": "Alice"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "Bob"
|
||||
}
|
||||
]"#;
|
||||
let result = compact_json(input);
|
||||
assert!(result.len() < input.len(), "should be shorter than input");
|
||||
let parsed: serde_json::Value = serde_json::from_str(&result).unwrap();
|
||||
assert_eq!(parsed[0]["name"], "Alice");
|
||||
assert_eq!(parsed[1]["name"], "Bob");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compact_mixed_text_with_json_block() {
|
||||
let input = "Exit code: 0\nSTDOUT:\n{\n \"status\": \"ok\"\n}\nSTDERR:\n";
|
||||
let result = compact_json(input);
|
||||
assert!(result.contains("\"status\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compact_empty_input() {
|
||||
assert_eq!(compact_json(""), "");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
use std::fmt;
|
||||
use std::str::FromStr;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum CompactionLevel {
|
||||
Off,
|
||||
#[default]
|
||||
Safe,
|
||||
Full,
|
||||
}
|
||||
|
||||
impl fmt::Display for CompactionLevel {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Off => write!(f, "off"),
|
||||
Self::Safe => write!(f, "safe"),
|
||||
Self::Full => write!(f, "full"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for CompactionLevel {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s.to_lowercase().as_str() {
|
||||
"off" => Ok(Self::Off),
|
||||
"safe" => Ok(Self::Safe),
|
||||
"full" => Ok(Self::Full),
|
||||
other => Err(format!(
|
||||
"unknown compaction level: '{other}' (expected: off, safe, full)"
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn default_is_safe() {
|
||||
assert_eq!(CompactionLevel::default(), CompactionLevel::Safe);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn display_fromstr_roundtrip() {
|
||||
for level in [
|
||||
CompactionLevel::Off,
|
||||
CompactionLevel::Safe,
|
||||
CompactionLevel::Full,
|
||||
] {
|
||||
let s = level.to_string();
|
||||
let parsed: CompactionLevel = s.parse().unwrap();
|
||||
assert_eq!(parsed, level);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn case_insensitive_parsing() {
|
||||
assert_eq!(
|
||||
"OFF".parse::<CompactionLevel>().unwrap(),
|
||||
CompactionLevel::Off
|
||||
);
|
||||
assert_eq!(
|
||||
"Safe".parse::<CompactionLevel>().unwrap(),
|
||||
CompactionLevel::Safe
|
||||
);
|
||||
assert_eq!(
|
||||
"FULL".parse::<CompactionLevel>().unwrap(),
|
||||
CompactionLevel::Full
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_input_error() {
|
||||
let err = "unknown".parse::<CompactionLevel>().unwrap_err();
|
||||
assert!(err.contains("unknown compaction level"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serde_roundtrip() {
|
||||
for level in [
|
||||
CompactionLevel::Off,
|
||||
CompactionLevel::Safe,
|
||||
CompactionLevel::Full,
|
||||
] {
|
||||
let json = serde_json::to_string(&level).unwrap();
|
||||
let back: CompactionLevel = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(back, level);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serde_lowercase_format() {
|
||||
assert_eq!(
|
||||
serde_json::to_string(&CompactionLevel::Off).unwrap(),
|
||||
"\"off\""
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_string(&CompactionLevel::Safe).unwrap(),
|
||||
"\"safe\""
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_string(&CompactionLevel::Full).unwrap(),
|
||||
"\"full\""
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
pub mod fold;
|
||||
pub mod json;
|
||||
pub mod level;
|
||||
pub mod sanitize;
|
||||
pub mod toon;
|
||||
|
||||
pub use level::CompactionLevel;
|
||||
pub use toon::toon_format_instructions;
|
||||
|
||||
pub fn compact_output(text: &str, level: CompactionLevel) -> String {
|
||||
match level {
|
||||
CompactionLevel::Off => text.to_string(),
|
||||
CompactionLevel::Safe => sanitize::sanitize(text),
|
||||
CompactionLevel::Full => {
|
||||
let text = sanitize::sanitize(text);
|
||||
let text = fold::fold_repeated_lines(&text);
|
||||
json::compact_json(&text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn compact_output_toon(text: &str) -> String {
|
||||
toon::try_toon_encode(text)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn off_returns_unchanged() {
|
||||
let input = "hello\x1b[31m world\n\n\nfoo";
|
||||
assert_eq!(compact_output(input, CompactionLevel::Off), input);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_strips_ansi() {
|
||||
let input = "\x1b[32mOK\x1b[0m done";
|
||||
let result = compact_output(input, CompactionLevel::Safe);
|
||||
assert_eq!(result, "OK done");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_merges_blank_lines() {
|
||||
let input = "a\n\n\n\nb";
|
||||
let result = compact_output(input, CompactionLevel::Safe);
|
||||
assert_eq!(result, "a\n\nb");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_collapses_cr() {
|
||||
let input = "50%\r100%\nDone";
|
||||
let result = compact_output(input, CompactionLevel::Safe);
|
||||
assert_eq!(result, "100%\nDone");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_folds_repeated_lines() {
|
||||
let lines: Vec<String> = (0..6)
|
||||
.map(|i| format!("Compiling dep-{i} v0.1.0"))
|
||||
.collect();
|
||||
let input = lines.join("\n");
|
||||
let result = compact_output(&input, CompactionLevel::Full);
|
||||
assert!(result.contains("[... 4 similar lines]"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_compacts_json() {
|
||||
let input = "{\n \"id\": 1,\n \"name\": \"Alice\"\n}";
|
||||
let result = compact_output(input, CompactionLevel::Full);
|
||||
assert!(result.len() < input.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_does_not_fold_lines() {
|
||||
let lines: Vec<String> = (0..6)
|
||||
.map(|i| format!("Compiling dep-{i} v0.1.0"))
|
||||
.collect();
|
||||
let input = lines.join("\n");
|
||||
let result = compact_output(&input, CompactionLevel::Safe);
|
||||
assert!(!result.contains("[..."), "Safe level should not fold lines");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use regex::Regex;
|
||||
|
||||
static ANSI_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\x1b\[[0-9;]*[a-zA-Z]").unwrap());
|
||||
|
||||
pub fn strip_ansi(text: &str) -> String {
|
||||
ANSI_RE.replace_all(text, "").into_owned()
|
||||
}
|
||||
|
||||
pub fn collapse_cr_lines(text: &str) -> String {
|
||||
let mut result = String::with_capacity(text.len());
|
||||
for line in text.split('\n') {
|
||||
if !result.is_empty() {
|
||||
result.push('\n');
|
||||
}
|
||||
if let Some(last) = line.rsplit('\r').next() {
|
||||
result.push_str(last);
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub fn merge_blank_lines(text: &str) -> String {
|
||||
let mut result = String::with_capacity(text.len());
|
||||
let mut prev_blank = false;
|
||||
for line in text.split('\n') {
|
||||
let is_blank = line.trim().is_empty();
|
||||
if is_blank {
|
||||
if !prev_blank {
|
||||
if !result.is_empty() {
|
||||
result.push('\n');
|
||||
}
|
||||
result.push('\n');
|
||||
}
|
||||
prev_blank = true;
|
||||
} else {
|
||||
if !result.is_empty() && !prev_blank {
|
||||
result.push('\n');
|
||||
} else if prev_blank && result.ends_with('\n') {
|
||||
// blank section already has trailing newline
|
||||
} else if !result.is_empty() {
|
||||
result.push('\n');
|
||||
}
|
||||
result.push_str(line.trim_end());
|
||||
prev_blank = false;
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub fn trim_trailing_whitespace(text: &str) -> String {
|
||||
text.lines()
|
||||
.map(|line| line.trim_end())
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
pub fn sanitize(text: &str) -> String {
|
||||
let text = strip_ansi(text);
|
||||
let text = collapse_cr_lines(&text);
|
||||
let text = trim_trailing_whitespace(&text);
|
||||
merge_blank_lines(&text)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn strip_ansi_color_codes() {
|
||||
let input = "\x1b[31mError\x1b[0m: something failed";
|
||||
assert_eq!(strip_ansi(input), "Error: something failed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strip_ansi_bold_and_nested() {
|
||||
let input = "\x1b[1m\x1b[32mCompiling\x1b[0m nomi-compact v0.1.0";
|
||||
assert_eq!(strip_ansi(input), "Compiling nomi-compact v0.1.0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strip_ansi_no_codes_unchanged() {
|
||||
let input = "plain text without any codes";
|
||||
assert_eq!(strip_ansi(input), input);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strip_ansi_cursor_movement() {
|
||||
let input = "\x1b[2K\x1b[1G> prompt";
|
||||
assert_eq!(strip_ansi(input), "> prompt");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strip_ansi_empty_input() {
|
||||
assert_eq!(strip_ansi(""), "");
|
||||
}
|
||||
|
||||
// --- collapse_cr_lines ---
|
||||
|
||||
#[test]
|
||||
fn collapse_cr_overwrites() {
|
||||
let input = "Downloading... 10%\rDownloading... 50%\rDownloading... 100%\nDone.";
|
||||
assert_eq!(collapse_cr_lines(input), "Downloading... 100%\nDone.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collapse_cr_no_cr_unchanged() {
|
||||
let input = "line1\nline2\nline3";
|
||||
assert_eq!(collapse_cr_lines(input), input);
|
||||
}
|
||||
|
||||
// --- merge_blank_lines ---
|
||||
|
||||
#[test]
|
||||
fn merge_consecutive_blank_lines() {
|
||||
let input = "a\n\n\n\n\nb";
|
||||
assert_eq!(merge_blank_lines(input), "a\n\nb");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_blank_lines_preserves_single() {
|
||||
let input = "a\n\nb\n\nc";
|
||||
assert_eq!(merge_blank_lines(input), input);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_blank_lines_whitespace_only_lines() {
|
||||
let input = "a\n \n \n\nb";
|
||||
assert_eq!(merge_blank_lines(input), "a\n\nb");
|
||||
}
|
||||
|
||||
// --- trim_trailing_whitespace ---
|
||||
|
||||
#[test]
|
||||
fn trim_trailing_spaces() {
|
||||
let input = "hello \nworld\t\t\nfoo";
|
||||
assert_eq!(trim_trailing_whitespace(input), "hello\nworld\nfoo");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trim_trailing_no_trailing() {
|
||||
let input = "clean\nlines";
|
||||
assert_eq!(trim_trailing_whitespace(input), input);
|
||||
}
|
||||
|
||||
// --- sanitize (combined safe layer) ---
|
||||
|
||||
#[test]
|
||||
fn sanitize_applies_all() {
|
||||
let input = "\x1b[32mCompiling\x1b[0m foo \n\n\n\nbar\rbar done\n";
|
||||
let result = sanitize(input);
|
||||
assert!(!result.contains("\x1b["));
|
||||
assert!(!result.contains("\n\n\n"));
|
||||
assert!(!result.contains(" \n"));
|
||||
assert!(result.contains("bar done"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
pub fn toon_encode_array(value: &serde_json::Value) -> Option<String> {
|
||||
let arr = value.as_array()?;
|
||||
if arr.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let first = arr[0].as_object()?;
|
||||
let fields: Vec<&str> = first.keys().map(|k| k.as_str()).collect();
|
||||
|
||||
if fields.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
for item in arr {
|
||||
let obj = item.as_object()?;
|
||||
if obj.len() != fields.len() {
|
||||
return None;
|
||||
}
|
||||
for field in &fields {
|
||||
let val = obj.get(*field)?;
|
||||
if val.is_object() || val.is_array() {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut result = String::new();
|
||||
let header = format!("[{}]{{{}}}:", arr.len(), fields.join(","));
|
||||
result.push_str(&header);
|
||||
result.push('\n');
|
||||
|
||||
for item in arr {
|
||||
let obj = item.as_object().unwrap();
|
||||
result.push_str(" ");
|
||||
let values: Vec<String> = fields
|
||||
.iter()
|
||||
.map(|f| format_toon_value(obj.get(*f).unwrap()))
|
||||
.collect();
|
||||
result.push_str(&values.join(","));
|
||||
result.push('\n');
|
||||
}
|
||||
|
||||
if result.ends_with('\n') {
|
||||
result.pop();
|
||||
}
|
||||
|
||||
Some(result)
|
||||
}
|
||||
|
||||
fn format_toon_value(value: &serde_json::Value) -> String {
|
||||
match value {
|
||||
serde_json::Value::Null => "null".to_string(),
|
||||
serde_json::Value::Bool(b) => b.to_string(),
|
||||
serde_json::Value::Number(n) => n.to_string(),
|
||||
serde_json::Value::String(s) => {
|
||||
if s.contains(',') || s.contains('\n') || s.contains('"') {
|
||||
format!("\"{}\"", s.replace('"', "\\\""))
|
||||
} else {
|
||||
s.clone()
|
||||
}
|
||||
}
|
||||
_ => serde_json::to_string(value).unwrap_or_default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn try_toon_encode(text: &str) -> String {
|
||||
if text.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
let trimmed = text.trim();
|
||||
|
||||
if trimmed.starts_with('[')
|
||||
&& let Ok(value) = serde_json::from_str::<serde_json::Value>(trimmed)
|
||||
&& value.is_array()
|
||||
{
|
||||
if let Some(encoded) = toon_encode_array(&value) {
|
||||
return encoded;
|
||||
}
|
||||
return text.to_string();
|
||||
}
|
||||
|
||||
if let Some(start) = trimmed.find('[') {
|
||||
let rest = &trimmed[start..];
|
||||
// Try to find the JSON array boundary by looking for matching ']'
|
||||
let mut depth = 0;
|
||||
let mut end = None;
|
||||
for (i, ch) in rest.char_indices() {
|
||||
match ch {
|
||||
'[' => depth += 1,
|
||||
']' => {
|
||||
depth -= 1;
|
||||
if depth == 0 {
|
||||
end = Some(i + 1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if let Some(end_pos) = end {
|
||||
let candidate = &rest[..end_pos];
|
||||
if let Ok(value) = serde_json::from_str::<serde_json::Value>(candidate)
|
||||
&& value.is_array()
|
||||
&& let Some(encoded) = toon_encode_array(&value)
|
||||
{
|
||||
let suffix = &rest[end_pos..];
|
||||
return format!("{}{}{}", &trimmed[..start], encoded, suffix);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
text.to_string()
|
||||
}
|
||||
|
||||
pub fn toon_format_instructions() -> &'static str {
|
||||
"\
|
||||
# TOON Format
|
||||
|
||||
Tool results may contain data in TOON (Token-Oriented Object Notation) tabular format \
|
||||
for token efficiency. Format:
|
||||
|
||||
```
|
||||
[N]{field1,field2,...}:
|
||||
value1,value2,...
|
||||
value1,value2,...
|
||||
```
|
||||
|
||||
- `[N]` is the array length
|
||||
- `{fields}` are column headers
|
||||
- Each indented line is one row, values comma-separated
|
||||
- String values containing commas are quoted
|
||||
|
||||
This is equivalent to a JSON array of objects. Example:
|
||||
```
|
||||
[2]{id,name,role}:
|
||||
1,Alice,admin
|
||||
2,Bob,user
|
||||
```
|
||||
equals `[{\"id\":1,\"name\":\"Alice\",\"role\":\"admin\"},{\"id\":2,\"name\":\"Bob\",\"role\":\"user\"}]`"
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn encode_uniform_array() {
|
||||
let json = r#"[
|
||||
{"id": 1, "name": "Alice", "role": "admin"},
|
||||
{"id": 2, "name": "Bob", "role": "user"}
|
||||
]"#;
|
||||
let value: serde_json::Value = serde_json::from_str(json).unwrap();
|
||||
let result = toon_encode_array(&value);
|
||||
assert!(result.is_some());
|
||||
let encoded = result.unwrap();
|
||||
assert!(
|
||||
encoded.contains("[2]{id,name,role}:"),
|
||||
"should have header: {encoded}"
|
||||
);
|
||||
assert!(encoded.contains("1,Alice,admin"));
|
||||
assert!(encoded.contains("2,Bob,user"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encode_non_uniform_array_returns_none() {
|
||||
let json = r#"[{"id": 1}, {"name": "Bob"}]"#;
|
||||
let value: serde_json::Value = serde_json::from_str(json).unwrap();
|
||||
assert!(toon_encode_array(&value).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encode_nested_values_returns_none() {
|
||||
let json = r#"[{"id": 1, "meta": {"x": 1}}]"#;
|
||||
let value: serde_json::Value = serde_json::from_str(json).unwrap();
|
||||
assert!(toon_encode_array(&value).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encode_empty_array_returns_none() {
|
||||
let value = serde_json::json!([]);
|
||||
assert!(toon_encode_array(&value).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encode_single_element() {
|
||||
let json = r#"[{"id": 1, "name": "Alice"}]"#;
|
||||
let value: serde_json::Value = serde_json::from_str(json).unwrap();
|
||||
let result = toon_encode_array(&value);
|
||||
assert!(result.is_some());
|
||||
let encoded = result.unwrap();
|
||||
assert!(encoded.contains("[1]{id,name}:"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encode_values_with_commas_quoted() {
|
||||
let json = r#"[{"name": "Alice, Jr.", "age": 30}]"#;
|
||||
let value: serde_json::Value = serde_json::from_str(json).unwrap();
|
||||
let result = toon_encode_array(&value);
|
||||
assert!(result.is_some());
|
||||
let encoded = result.unwrap();
|
||||
assert!(
|
||||
encoded.contains("\"Alice, Jr.\""),
|
||||
"comma in value should be quoted: {encoded}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn toon_prompt_instructions_not_empty() {
|
||||
let instructions = toon_format_instructions();
|
||||
assert!(!instructions.is_empty());
|
||||
assert!(instructions.contains("TOON"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn try_toon_encode_text_with_json_array() {
|
||||
let input = "Exit code: 0\nSTDOUT:\n[{\"id\":1,\"name\":\"Alice\",\"role\":\"admin\"},{\"id\":2,\"name\":\"Bob\",\"role\":\"user\"}]\nSTDERR:\n";
|
||||
let result = try_toon_encode(input);
|
||||
assert!(
|
||||
result.contains("[2]{id,name,role}:"),
|
||||
"should contain TOON header: {result}"
|
||||
);
|
||||
assert!(result.contains("Exit code: 0"), "should preserve prefix");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user