From db3b63c49e7a614580d15c39b9a78e06c9a3e9d6 Mon Sep 17 00:00:00 2001 From: aayank13 Date: Sat, 10 Oct 2026 15:39:39 +0530 Subject: [PATCH 1/2] fix: judge only masked checks by their output --- crates/trodden-capture/src/claude_code.rs | 8 ++ crates/trodden-capture/src/command.rs | 44 +++++++++- crates/trodden-capture/src/evidence.rs | 98 ++++++++++++++++++++++- 3 files changed, 146 insertions(+), 4 deletions(-) diff --git a/crates/trodden-capture/src/claude_code.rs b/crates/trodden-capture/src/claude_code.rs index 82ff415..fd2f6db 100644 --- a/crates/trodden-capture/src/claude_code.rs +++ b/crates/trodden-capture/src/claude_code.rs @@ -590,6 +590,14 @@ mod tests { Session::outcome("npm run build | tee build.log", "built in 2.1s", false), ToolOutcome::Succeeded ); + assert_eq!( + Session::outcome( + "sed -n '1,40p' src/report.rs; echo done", + "fn lookup(id: u32) {\nerror: unknown customer id\n}\ndone", + false + ), + ToolOutcome::Succeeded + ); } #[test] diff --git a/crates/trodden-capture/src/command.rs b/crates/trodden-capture/src/command.rs index c226a14..a928c0b 100644 --- a/crates/trodden-capture/src/command.rs +++ b/crates/trodden-capture/src/command.rs @@ -281,7 +281,7 @@ impl Command { pub fn action(&self) -> ToolAction { let program = self.program(); let name = program.split_whitespace().next().unwrap_or_default(); - if Self::READ_PROGRAMS.contains(&name) { + if Self::READ_PROGRAMS.contains(&name) || self.reads_stream(name) { ToolAction::Read } else if Self::SEARCH_PROGRAMS.contains(&name) || program == "git grep" { ToolAction::Search @@ -290,6 +290,23 @@ impl Command { } } + fn reads_stream(&self, name: &str) -> bool { + let argv = self.argv(); + let arguments = argv.get(1..).unwrap_or_default(); + match name { + "sed" => !arguments.iter().any(|argument| { + argument.starts_with("--in-place") + || (argument.starts_with('-') + && !argument.starts_with("--") + && argument.contains('i')) + }), + "awk" | "gawk" => !arguments + .iter() + .any(|argument| argument.contains("inplace")), + _ => false, + } + } + pub fn argv(&self) -> Vec { self.layers().1 } @@ -794,5 +811,30 @@ mod tests { Command::normalize("npm test", "/w").action(), ToolAction::Run ); + for command in [ + "sed -n '1,40p' src/report.rs", + "sed -E 's/-i/x/' src/report.rs", + "awk 'NR<=40' src/report.rs", + "cd src && gawk -F, '{print $2}' totals.csv", + ] { + assert_eq!( + Command::normalize(command, "/w").action(), + ToolAction::Read, + "{command}" + ); + } + for command in [ + "sed -i 's/old/new/' src/report.rs", + "sed -i.bak 's/old/new/' src/report.rs", + "sed -Ei 's/old/new/' src/report.rs", + "sed --in-place=.bak 's/old/new/' src/report.rs", + "gawk -i inplace '{print}' totals.csv", + ] { + assert_eq!( + Command::normalize(command, "/w").action(), + ToolAction::Run, + "{command}" + ); + } } } diff --git a/crates/trodden-capture/src/evidence.rs b/crates/trodden-capture/src/evidence.rs index 998f5a3..9559a8b 100644 --- a/crates/trodden-capture/src/evidence.rs +++ b/crates/trodden-capture/src/evidence.rs @@ -2,7 +2,7 @@ use std::sync::LazyLock; use regex::Regex; -use crate::error::ANSI; +use crate::{command::Command, error::ANSI}; static FAILED: LazyLock = LazyLock::new(|| { Regex::new( @@ -18,12 +18,60 @@ static PASSED: LazyLock = LazyLock::new(|| { .expect("success pattern is valid") }); +static CHECK_WORD: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)(?:^|[/_.-])(?:test|tests|check|lint|verify|typecheck|ci)(?:$|[/_.-])") + .expect("check word pattern is valid") +}); + #[derive(Debug)] pub(crate) struct CheckOutput; impl CheckOutput { + const CHECK_PROGRAMS: &[&str] = &[ + "cargo test", + "cargo check", + "cargo clippy", + "cargo build", + "cargo nextest", + "npm test", + "pnpm test", + "yarn test", + "bun test", + "deno test", + "npm run", + "pnpm run", + "yarn run", + "bun run", + "go test", + "go vet", + "go build", + "python -m pytest", + "python -m unittest", + "python -m mypy", + "pytest", + "tox", + "nox", + "mypy", + "ruff", + "eslint", + "tsc", + "jest", + "vitest", + "mocha", + "rspec", + "mix test", + "dotnet test", + "dotnet build", + "swift test", + "phpunit", + ]; + + const TASK_RUNNERS: &[&str] = &["make", "just", "rake", "mvn", "gradle"]; + + const SHELLS: &[&str] = &["sh", "bash", "zsh"]; + pub(crate) fn verdict(command: &str, output: &str) -> Option { - if !Self::hides_exit_code(command) { + if !Self::hides_exit_code(command) || !Self::runs_check(command) { return None; } let output = ANSI.replace_all(output, ""); @@ -34,6 +82,21 @@ impl CheckOutput { } } + fn runs_check(command: &str) -> bool { + let command = Command::normalize(command, ""); + let program = command.program(); + let runner = program.split_whitespace().next().unwrap_or_default(); + let argv = command.argv(); + let script = if Self::SHELLS.contains(&runner) { + argv.get(1) + } else { + argv.first().filter(|executable| executable.contains('/')) + }; + Self::CHECK_PROGRAMS.contains(&program.as_str()) + || Self::TASK_RUNNERS.contains(&runner) + || script.is_some_and(|script| CHECK_WORD.is_match(script)) + } + fn hides_exit_code(command: &str) -> bool { let chars: Vec = command.chars().collect(); let mut quote = None; @@ -94,9 +157,38 @@ mod tests { } } + #[test] + fn only_checks_are_judged_by_output() { + let printed = "fn lookup(id: u32) {\nerror: unknown customer id\n}\ndone"; + for command in [ + "sed -n '1,40p' src/report.rs; echo done", + "cat src/report.rs | head -40", + "git log --oneline | head", + "echo 'test result: FAILED'; true", + "python scripts/gen.py | tail", + "bash -c 'cat notes.txt' | head", + ] { + assert_eq!(CheckOutput::verdict(command, printed), None, "{command}"); + } + for command in [ + "cd api && cargo test 2>&1 | tail -20", + "npx jest --ci | tail", + "uv run pytest -q || true", + "make check | tail", + "./scripts/test.sh 2>&1 | tail", + "bash ci.sh; echo exit=$?", + ] { + assert_eq!( + CheckOutput::verdict(command, printed), + Some(false), + "{command}" + ); + } + } + #[test] fn reads_common_runner_summaries() { - let piped = "check | tail"; + let piped = "cargo test | tail"; for output in [ "test result: FAILED. 0 passed; 1 failed", "error[E0308]: mismatched types\n --> src/lib.rs:4:5", From 99f33757d39f87605203f2915373e06c42cd0423 Mon Sep 17 00:00:00 2001 From: aayank13 Date: Sat, 10 Oct 2026 16:17:51 +0530 Subject: [PATCH 2/2] fix: need a test result line before passing a masked test run --- crates/trodden-capture/src/evidence.rs | 37 ++++++++++++++++++++------ 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/crates/trodden-capture/src/evidence.rs b/crates/trodden-capture/src/evidence.rs index 9559a8b..e033b7c 100644 --- a/crates/trodden-capture/src/evidence.rs +++ b/crates/trodden-capture/src/evidence.rs @@ -13,11 +13,15 @@ static FAILED: LazyLock = LazyLock::new(|| { static PASSED: LazyLock = LazyLock::new(|| { Regex::new( - r"(?m)^\s*(?:test result: ok\.|# pass [1-9]|ℹ pass [1-9]|ok\s+\S+\s+[\d.]+s|All checks passed!|Finished\b.*\btarget\(s\))|(?:^|[\s(,])[1-9]\d* passed(?:[\s,;.)]|$)", + r"(?m)^\s*(?:test result: ok\.|# pass [1-9]|ℹ pass [1-9]|ok\s+\S+\s+[\d.]+s|All checks passed!)|(?:^|[\s(,])[1-9]\d* passed(?:[\s,;.)]|$)", ) .expect("success pattern is valid") }); +static BUILT: LazyLock = LazyLock::new(|| { + Regex::new(r"(?m)^\s*Finished\b.*\btarget\(s\)").expect("build pattern is valid") +}); + static CHECK_WORD: LazyLock = LazyLock::new(|| { Regex::new(r"(?i)(?:^|[/_.-])(?:test|tests|check|lint|verify|typecheck|ci)(?:$|[/_.-])") .expect("check word pattern is valid") @@ -66,25 +70,31 @@ impl CheckOutput { "phpunit", ]; + const BUILD_PROGRAMS: &[&str] = &["cargo build", "cargo check", "cargo clippy"]; + const TASK_RUNNERS: &[&str] = &["make", "just", "rake", "mvn", "gradle"]; const SHELLS: &[&str] = &["sh", "bash", "zsh"]; pub(crate) fn verdict(command: &str, output: &str) -> Option { - if !Self::hides_exit_code(command) || !Self::runs_check(command) { + if !Self::hides_exit_code(command) { + return None; + } + let command = Command::normalize(command, ""); + let program = command.program(); + if !Self::runs_check(&command, &program) { return None; } let output = ANSI.replace_all(output, ""); if FAILED.is_match(&output) { Some(false) } else { - PASSED.is_match(&output).then_some(true) + let built = Self::BUILD_PROGRAMS.contains(&program.as_str()) && BUILT.is_match(&output); + (built || PASSED.is_match(&output)).then_some(true) } } - fn runs_check(command: &str) -> bool { - let command = Command::normalize(command, ""); - let program = command.program(); + fn runs_check(command: &Command, program: &str) -> bool { let runner = program.split_whitespace().next().unwrap_or_default(); let argv = command.argv(); let script = if Self::SHELLS.contains(&runner) { @@ -92,7 +102,7 @@ impl CheckOutput { } else { argv.first().filter(|executable| executable.contains('/')) }; - Self::CHECK_PROGRAMS.contains(&program.as_str()) + Self::CHECK_PROGRAMS.contains(&program) || Self::TASK_RUNNERS.contains(&runner) || script.is_some_and(|script| CHECK_WORD.is_match(script)) } @@ -210,10 +220,21 @@ mod tests { "# pass 3\n# fail 0", "ok \texample.com/paging\t0.012s", "All checks passed!", - " Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.31s", ] { assert_eq!(CheckOutput::verdict(piped, output), Some(true), "{output}"); } + let built = " Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.31s"; + assert_eq!( + CheckOutput::verdict("cargo build 2>&1 | tail", built), + Some(true) + ); + assert_eq!( + CheckOutput::verdict( + "cargo test 2>&1 | head -6", + &format!("{built}\n Running unittests src/lib.rs\n\nrunning 2 tests") + ), + None + ); assert_eq!( CheckOutput::verdict(piped, "line 18\nline 19\nline 20"), None