From ac33dd49db800609944aa4ebdc0eb77ccedea7ba Mon Sep 17 00:00:00 2001 From: AdityaVG13 Date: Sun, 20 Sep 2026 21:48:36 -0400 Subject: [PATCH 01/26] feat(echo): stateless index-free echolocation + program-wide token diet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sound every file read, grep, file search, and project map with its surroundings — sibling podmates, line-numbered symbols, import roots, resolved file links, and (in the terminal buzz) reverse-link callers — sounded fresh from source with no index, no background work, and no new dependencies. Deterministic (sorted, capped, budgeted) so repeated calls are byte-identical and cache-friendly. Echolocation (new: tools/echolocation.rs, ~2.5k lines incl. tests): - read footer: [Pod] mates + [Sounding] symbols/imports/links/heard-by, 1200B hard budget; miss echo on NotFound (denylist-gated, never fires on refused probes); terminal buzz adds callers after 2 distinct files read in a pod - grep + file_search chart matched dirs (caller's own visibility gates: denylist, excludes, extensions, gitignore); project_map sounds key files; 4 sounding languages (Rust/Python/JS-TS/Go) Token diet (no compression, behavior-preserving): - prefix 14,241 -> 7,118 tokens/turn (-50.0%): skill path cut (load_skill resolves names), MCP prose 1526->548B (scar-tissue phrases kept), agent+workflow deferral, goal tools conditional-eager keyed off the turn's resolved goal (preview/wire-safe) - activation cache: best-match-last push + regex name-rank + 16->24KiB bound (fixes deferral stranding agent behind its own discovery set) - grep empty-context omit, workflow event projection (lossless: duplicates dropped, workspaces hoisted, branch kept) + zero-memo omit - wire-prefix dump moved to PreparedOutboundRequest (exact-final-body) Evidence: targeted echo 46, workflow 131, file 145 green; full tui+workflow suites green except 4 failures reproducing identically on base c75b23397 (pre-existing, unrelated) + known timing flakes proven solo-pass; clippy zero on owned files. Research: codewhale-research/ ECHOLOCATION-THESIS.md (exp-10/11/12 + hardening audit). --- crates/tui/src/client.rs | 6 +- crates/tui/src/client/chat.rs | 91 +- crates/tui/src/client/prepared.rs | 1 + crates/tui/src/core/engine.rs | 19 +- crates/tui/src/core/engine/preview.rs | 1 + crates/tui/src/core/engine/preview/tests.rs | 2 + crates/tui/src/core/engine/tests.rs | 67 +- crates/tui/src/core/engine/tool_catalog.rs | 37 +- .../tui/src/core/engine/tool_catalog/tests.rs | 46 +- crates/tui/src/core/session.rs | 58 +- crates/tui/src/hooks/executor.rs | 19 +- crates/tui/src/skills/mod.rs | 84 +- crates/tui/src/skills/tests.rs | 50 +- crates/tui/src/tools/echolocation.rs | 2588 +++++++++++++++++ crates/tui/src/tools/file.rs | 29 +- crates/tui/src/tools/file/tests.rs | 171 +- crates/tui/src/tools/file_search.rs | 72 +- crates/tui/src/tools/mod.rs | 1 + crates/tui/src/tools/project.rs | 34 + crates/tui/src/tools/search.rs | 33 +- crates/tui/src/tools/search/tests.rs | 34 +- crates/tui/src/tools/spec.rs | 17 + crates/tui/src/tools/subagent/mod.rs | 2 +- crates/tui/src/tools/subagent/tests.rs | 144 +- crates/tui/src/tools/workflow/mod.rs | 148 +- crates/workflow/src/lib.rs | 17 +- 26 files changed, 3492 insertions(+), 279 deletions(-) create mode 100644 crates/tui/src/tools/echolocation.rs diff --git a/crates/tui/src/client.rs b/crates/tui/src/client.rs index c7bfed4982..90a50e7b1f 100644 --- a/crates/tui/src/client.rs +++ b/crates/tui/src/client.rs @@ -7031,14 +7031,14 @@ mod tests { let expected_names = crate::core::engine::default_active_native_tool_names() .iter() .copied() - // Children can inspect the shared goal, but only its owning - // session can create it or change its completion state. + // Children discover goal inspection via search; only the + // owning session can create a goal or change its + // completion state, so those never ride the child surface. .filter(|name| !matches!(*name, "create_goal" | "update_goal")) .chain([crate::core::engine::tool_catalog::TOOL_SEARCH_NAME]) .map(str::to_string) .collect(); assert_eq!(source_names, expected_names); - assert!(source_names.contains("get_goal")); assert!(!source_names.contains("create_goal")); assert!(!source_names.contains("update_goal")); diff --git a/crates/tui/src/client/chat.rs b/crates/tui/src/client/chat.rs index 4d2f759e9b..179a707b35 100644 --- a/crates/tui/src/client/chat.rs +++ b/crates/tui/src/client/chat.rs @@ -1793,7 +1793,6 @@ impl<'a> PromptBuilder<'a> { ), false, ); - dump_system_prompt_if_requested(&messages); if provider == ApiProvider::Arcee { apply_arcee_waf_safe_message_encoding(&mut messages); } @@ -1873,38 +1872,82 @@ const SYSTEM_PROMPT_DUMP_BEGIN: &str = "<<>>"; const SYSTEM_PROMPT_DUMP_END: &str = "<<>>"; const ARCEE_WAF_TEXT_SPLIT_TRIGGERS: &[(&str, &str, &str)] = &[("python -c", "python ", "-c")]; -fn dump_system_prompt_if_requested(messages: &[Value]) { +/// Dump the wire prefix (system region + tools array) when the +/// system-prompt dump flag is set, so prefix-token audits can measure the +/// catalog beside the prompt. Called once from +/// [`PreparedOutboundRequest::new`](super::prepared::PreparedOutboundRequest::new), +/// the single funnel for all three wire dialects, so the bytes are the +/// exact-final-body bytes. Debug-only, zero production impact. +pub(crate) fn dump_wire_prefix_if_requested(body: &Value) { let Ok(flag) = std::env::var(SYSTEM_PROMPT_DUMP_ENV) else { return; }; if !matches!(flag.trim(), "1" | "true" | "TRUE" | "yes" | "YES") { return; } - let Some(prompt) = messages.iter().find_map(system_message_text) else { - return; - }; let mut stderr = std::io::stderr().lock(); - let _ = writeln!(stderr, "{SYSTEM_PROMPT_DUMP_BEGIN}"); - let _ = writeln!(stderr, "{prompt}"); - let _ = writeln!(stderr, "{SYSTEM_PROMPT_DUMP_END}"); -} - -fn system_message_text(message: &Value) -> Option { - if message.get("role").and_then(Value::as_str) != Some("system") { - return None; + if let Some(prompt) = wire_system_text(body) { + let _ = writeln!(stderr, "{SYSTEM_PROMPT_DUMP_BEGIN}"); + let _ = writeln!(stderr, "{prompt}"); + let _ = writeln!(stderr, "{SYSTEM_PROMPT_DUMP_END}"); + } + if let Some(tools) = body.get("tools") { + let _ = writeln!(stderr, "<<>>"); + let _ = writeln!(stderr, "{tools}"); + let _ = writeln!(stderr, "<<>>"); + } +} + +/// Best-effort system-region text across the three wire dialects: +/// Anthropic `system` (string or blocks), Responses `instructions`, and +/// chat `messages[]` with a system/developer role. +fn wire_system_text(body: &Value) -> Option { + if let Some(system) = body.get("system") { + return Some(match system { + Value::String(text) => text.clone(), + Value::Array(parts) => { + let text = parts + .iter() + .filter_map(|part| part.get("text").and_then(Value::as_str)) + .collect::>() + .join("\n"); + if text.is_empty() { + system.to_string() + } else { + text + } + } + _ => system.to_string(), + }); } - match message.get("content")? { - Value::String(text) => Some(text.clone()), - Value::Array(parts) => { - let text = parts - .iter() - .filter_map(|part| part.get("text").and_then(Value::as_str)) - .collect::>() - .join(""); - (!text.is_empty()).then_some(text) - } - _ => None, + if let Some(instructions) = body.get("instructions").and_then(Value::as_str) { + return Some(instructions.to_string()); } + body.get("messages")? + .as_array()? + .iter() + .find_map(|message| { + if !matches!( + message.get("role").and_then(Value::as_str), + Some("system" | "developer") + ) { + return None; + } + match message.get("content")? { + Value::String(text) => Some(text.clone()), + Value::Array(parts) => { + // Same "\n" block join as the Anthropic arm above, so + // the audit text is dialect-consistent. + let text = parts + .iter() + .filter_map(|part| part.get("text").and_then(Value::as_str)) + .collect::>() + .join("\n"); + (!text.is_empty()).then_some(text) + } + _ => None, + } + }) } fn apply_arcee_waf_safe_message_encoding(messages: &mut [Value]) { diff --git a/crates/tui/src/client/prepared.rs b/crates/tui/src/client/prepared.rs index 759f911bdd..8958de7543 100644 --- a/crates/tui/src/client/prepared.rs +++ b/crates/tui/src/client/prepared.rs @@ -305,6 +305,7 @@ impl PreparedOutboundRequest { replay_input_tokens: Option, entrypoint: CallerStreamMode, ) -> Self { + super::chat::dump_wire_prefix_if_requested(&body); let reasoning = ReasoningReceipt::from_body(dialect, &body, requested_effort); Self { dialect, diff --git a/crates/tui/src/core/engine.rs b/crates/tui/src/core/engine.rs index ec0fa23692..42c5e5f13b 100644 --- a/crates/tui/src/core/engine.rs +++ b/crates/tui/src/core/engine.rs @@ -136,7 +136,7 @@ fn agent_list_event(manager: &SubAgentManager, active_session_id: &str) -> Event } const MCP_REGISTRY_FIRST_INSTRUCTION_SOURCE: &str = "runtime:mcp-registry-first"; -const MCP_REGISTRY_FIRST_INSTRUCTION: &str = "## MCP Registry\n\nThe Registry installs and connects a local MCP server when this session lacks a capability. It is a fallback for a capability you do not have, not a step before ordinary work.\n\nPrefer what is already available, in order: tools already in this catalog, the project's own scripts, tests, and dev tooling, and platform capabilities. Creating a file, reading a fixture, running a repo command, and checking your own output are ordinary work — do them directly.\n\nReach for the Registry once you have identified a specific capability that no available tool covers and that you would otherwise install or reimplement, such as a document or media converter, access to an external database or service, or a protocol client. Then call `registry_sync` with a `query` naming that capability; it scores the local Registry snapshot host-side and returns at most eight matches, so the full index never enters the conversation. When a returned server plausibly covers that capability, call `start_registry_mcp_server` with its exact name rather than installing or running its package command through the shell. If nothing matches, refine the query once, then continue with local tools.\n\nBoth Registry tools are deferred: load one with `tool_search` before its first call, and use the returned schema. If a call instead reports that it only loaded the schema, retry once with that schema. Do not go searching for them for work you can already do."; +const MCP_REGISTRY_FIRST_INSTRUCTION: &str = "## MCP Registry\n\nThe Registry installs a local MCP server when this session lacks a capability — a fallback for a capability you do not have, not a step before ordinary work. Prefer what is already available and do ordinary work directly; only reach for `registry_sync` with a `query` naming a specific missing capability. Both Registry tools are deferred: load one with `tool_search` before its first call. Start a match with `start_registry_mcp_server` and its exact name rather than installing or running its package command through the shell."; const ISOLATED_CHAT_ENGINE_PROMPT: &str = "You are Codewhale Chat. Answer the user's request directly and conversationally. This isolated chat-only session has no local workspace, project, memory, skill, account, credential, path, runtime context, or tools."; pub(crate) fn sanitize_isolated_chat_attachments(mut text: String) -> String { @@ -4474,6 +4474,7 @@ impl Engine { mcp_access: McpAccess, route: TurnRouteContext, turn_id: &str, + turn_goal_objective: Option<&str>, ) -> TurnToolBuild { // Account-owned Chat is a text-only inference boundary. Do not build // native/plugin/dynamic registries, connect or snapshot MCP, capture a @@ -4729,7 +4730,20 @@ impl Engine { // The surface budget belongs to the route the request would go to, // which is not necessarily the installed one under auto routing. let capability = route.capability_profile(); - let always_load = self.config.tools_always_load.clone(); + let mut always_load = self.config.tools_always_load.clone(); + // Token diet: get/update_goal cannot succeed without an active goal, + // so they stay deferred until one exists. Keyed off the TURN's + // resolved goal (the same `NextTurnPromptContext.goal_objective` + // the turn fragment renders), never ambient engine config: + // production installs the turn goal before this build while + // preview builds without installing, so ambient state would + // diverge preview from the wire body on detached-goal turns. + // Same-field gating also means no new cache break — the tools + // flip exactly when the fragment flips. + if turn_goal_objective.is_some() { + always_load.insert("get_goal".to_string()); + always_load.insert("update_goal".to_string()); + } self.turn_tool_surface_budget = Some(capability.tool_surface_budget); let catalog = build_model_tool_catalog_with_surface( tool_registry.to_api_tools_with_cache(true), @@ -5332,6 +5346,7 @@ impl Engine { reasoning_effort_auto: self.session.reasoning_effort_auto, }, &turn_id_for_mailbox, + prompt_context.goal_objective.as_deref(), ) .await; let tool_catalog_for_event = Some(surface.catalog.clone()); diff --git a/crates/tui/src/core/engine/preview.rs b/crates/tui/src/core/engine/preview.rs index 31820aba9c..f5cdff08ad 100644 --- a/crates/tui/src/core/engine/preview.rs +++ b/crates/tui/src/core/engine/preview.rs @@ -312,6 +312,7 @@ impl Engine { McpAccess::PassiveSnapshot, route_context.clone(), "", + prompt_context.goal_objective.as_deref(), ) .await; diff --git a/crates/tui/src/core/engine/preview/tests.rs b/crates/tui/src/core/engine/preview/tests.rs index 51cdef2eff..91aed71cb8 100644 --- a/crates/tui/src/core/engine/preview/tests.rs +++ b/crates/tui/src/core/engine/preview/tests.rs @@ -633,6 +633,7 @@ async fn planned_route_builds_subagent_catalog_without_installed_client() { reasoning_effort_auto: planned.auto_controls_reasoning, }, "", + None, ) .await; assert!( @@ -2186,6 +2187,7 @@ async fn preview_tool_snapshot_has_no_mcp_or_event_side_effects() { reasoning_effort_auto: false, }, "", + None, ) .await; diff --git a/crates/tui/src/core/engine/tests.rs b/crates/tui/src/core/engine/tests.rs index 5d6d71ca9e..2fe5058a5d 100644 --- a/crates/tui/src/core/engine/tests.rs +++ b/crates/tui/src/core/engine/tests.rs @@ -10552,28 +10552,33 @@ fn approval_stamp_scenario() { #[test] fn core_primitives_and_todo_write_default_to_eager() { let always_load = HashSet::new(); - for core in ["read", "write", "edit", "bash", "agent", "todo_write"] { + for core in ["read", "write", "edit", "bash", "todo_write"] { assert!(!should_default_defer_tool(core, &always_load)); } - for searchable in ["File", "Bash", "Git", "Run", "tasks", "git_blame"] { + // Token diet: agent+workflow (21KB of schema) and get/update_goal + // (meaningless without an active goal) stay searchable by default. + for searchable in [ + "File", + "Bash", + "Git", + "Run", + "tasks", + "git_blame", + "agent", + "workflow", + "get_goal", + "update_goal", + ] { assert!(should_default_defer_tool(searchable, &always_load)); } } #[test] fn default_active_contract_keeps_discovery_and_core_tools_eager() { - const EXPECTED_NATIVE: [&str; 10] = [ - "read", - "write", - "edit", - "bash", - "agent", - "workflow", - "todo_write", - "create_goal", - "get_goal", - "update_goal", - ]; + // Token diet: agent+workflow and get/update_goal are searchable by + // default (see `core_primitives_and_todo_write_default_to_eager`). + const EXPECTED_NATIVE: [&str; 6] = + ["read", "write", "edit", "bash", "todo_write", "create_goal"]; assert_eq!( default_active_native_tool_names(), EXPECTED_NATIVE.as_slice() @@ -10612,7 +10617,7 @@ fn default_active_contract_keeps_discovery_and_core_tools_eager() { #[test] fn non_yolo_mode_retains_default_defer_policy() { let always_load = HashSet::new(); - for core in ["read", "write", "edit", "bash", "agent", "todo_write"] { + for core in ["read", "write", "edit", "bash", "todo_write"] { assert!(!should_default_defer_tool(core, &always_load)); } for searchable in [ @@ -10697,9 +10702,11 @@ fn model_tool_catalog_applies_native_and_mcp_deferral() { .and_then(|tool| tool.defer_loading) }; - for core in ["read", "write", "edit", "bash", "agent"] { + for core in ["read", "write", "edit", "bash"] { assert_eq!(defer_loading(core), Some(false)); } + // Token diet: agent stays searchable even on wide surfaces. + assert_eq!(defer_loading("agent"), Some(true)); assert_eq!(defer_loading("Git"), Some(true)); assert_eq!(defer_loading("Run"), Some(true)); assert_eq!(defer_loading("remember"), Some(true)); @@ -10761,9 +10768,11 @@ fn capability_compact_surface_defers_nonessential_core_tools() { .and_then(|tool| tool.defer_loading) }; - for core in ["read", "write", "edit", "bash", "agent"] { + for core in ["read", "write", "edit", "bash"] { assert_eq!(defer_loading(core), Some(false)); } + // Token diet: agent stays searchable even on wide surfaces. + assert_eq!(defer_loading("agent"), Some(true)); assert_eq!(defer_loading("Git"), Some(true)); assert_eq!(defer_loading("update_plan"), Some(true)); assert_eq!(defer_loading(TOOL_SEARCH_NAME), Some(false)); @@ -10791,7 +10800,7 @@ fn capability_full_surface_preserves_small_default_head() { crate::model_profile::ToolSurfaceBudget::Full, ); - for name in ["read", "write", "edit", "bash", "agent"] { + for name in ["read", "write", "edit", "bash"] { assert_eq!( catalog .iter() @@ -10801,6 +10810,14 @@ fn capability_full_surface_preserves_small_default_head() { "{name} should stay eager on full tool surfaces" ); } + // Token diet: agent stays searchable even on full surfaces. + assert_eq!( + catalog + .iter() + .find(|tool| tool.name == "agent") + .and_then(|tool| tool.defer_loading), + Some(true), + ); assert_eq!( catalog .iter() @@ -11065,6 +11082,7 @@ async fn measure_production_mode_tool_catalogs() -> serde_json::Value { McpAccess::PassiveSnapshot, route, "", + None, ) .await; let active = build.surface.active.clone().unwrap_or_default(); @@ -11101,17 +11119,16 @@ fn metric_tool_names<'a>( #[allow(clippy::await_holding_lock)] async fn runtime_contract_tool_metric_uses_canonical_mode_surfaces() { let payload = measure_production_mode_tool_catalogs().await; + // Token diet: the request head carries 7 eager tools; agent+workflow + // and get/update_goal stay searchable (the "full" surface below still + // lists them as catalog members). let expected_active = HashSet::from([ - "agent", "bash", "create_goal", - "get_goal", - "update_goal", "edit", "read", "todo_write", "tool_search", - "workflow", "write", ]); @@ -11137,7 +11154,7 @@ async fn runtime_contract_tool_metric_uses_canonical_mode_surfaces() { assert_eq!( metric_tool_names(&payload, mode, "active"), expected_active, - "{mode} must keep the same request head including goal controls" + "{mode} must keep the same request head" ); } @@ -14138,7 +14155,8 @@ fn plan_mode_toggle_preserves_catalog_byte_stability() { .map(|t| t.name.as_str()) .collect(); - let expected_head = ["agent", "bash", "edit", "read", "tool_search", "write"]; + // Token diet: agent is searchable, not head. + let expected_head = ["bash", "edit", "read", "tool_search", "write"]; assert_eq!(plan_names, expected_head); assert_eq!(agent_names, expected_head); @@ -22303,6 +22321,7 @@ readline.createInterface({ input: process.stdin }).on('line', async line => { McpAccess::Connect, route, "", + None, ); tokio::pin!(build); std::future::poll_fn(|cx| { diff --git a/crates/tui/src/core/engine/tool_catalog.rs b/crates/tui/src/core/engine/tool_catalog.rs index b1d80701f8..bbbe275730 100644 --- a/crates/tui/src/core/engine/tool_catalog.rs +++ b/crates/tui/src/core/engine/tool_catalog.rs @@ -50,10 +50,16 @@ pub(crate) fn is_tool_search_tool(name: &str) -> bool { #[rustfmt::skip] pub(crate) const DEFAULT_ACTIVE_NATIVE_TOOLS: &[&str] = &[ // Core work controls are eager; specialized tools stay searchable. - "read", "write", "edit", "bash", "agent", "workflow", "todo_write", - // Continuation instructions require these controls. Hiding them behind - // discovery leaves a model unable to stop the work it was asked to run. - "create_goal", "get_goal", "update_goal", + // Token diet: agent+workflow are deferred despite being core-adjacent — + // their schemas cost 21KB/turn and BM25 discovery (`tool_search`) finds + // them reliably when delegation is actually wanted (cued-delegation + // probe: discovered and executed; uncued sessions behave identically). + "read", "write", "edit", "bash", "todo_write", + // create_goal stays eager: it is the entry point. get/update_goal cannot + // succeed without an active goal, so they ride `always_load` only while + // one exists (see the catalog build in `engine.rs`) instead of taxing + // every goalless turn. + "create_goal", ]; const CORE_ACTION_TOOL_FALLBACKS: &[CoreActionToolFallback] = &[ @@ -912,7 +918,12 @@ fn discover_tools_with_regex( let regex = compile_user_regex(query) .map_err(|err| ToolError::invalid_input(format!("Invalid regex query: {err}")))?; - let mut matches = Vec::new(); + // Name matches outrank haystack matches: a regex that hits the tool's + // own name is the strongest possible signal, and catalog (alphabetical) + // order otherwise lets an incidental mention evict the literal target + // from the bounded activation cache. + let mut name_matches = Vec::new(); + let mut hay_matches = Vec::new(); let mut scratch = ToolSearchScratch::default(); for tool in catalog { // tool_search loads definitions omitted from the current request. An @@ -922,15 +933,19 @@ fn discover_tools_with_regex( if !tool.defer_loading.unwrap_or(false) || is_tool_search_tool(&tool.name) { continue; } + // Lowercased, like the haystack path below: `Agent` must hit `agent`. + if regex.is_match(&tool.name.to_lowercase()) { + name_matches.push(tool.name.clone()); + continue; + } scratch.load(tool); if regex.is_match(&scratch.hay) { - matches.push(tool.name.clone()); - } - if matches.len() >= max_results { - break; + hay_matches.push(tool.name.clone()); } } - Ok(matches) + name_matches.extend(hay_matches); + name_matches.truncate(max_results); + Ok(name_matches) } fn discover_tools_with_bm25_like(catalog: &[Tool], query: &str, max_results: usize) -> Vec { @@ -1520,7 +1535,7 @@ fn execute_tool_search_inner( json!({ "type": "unavailable_tool_reference", "tool_name": name, - "reason": "The tool schema exceeds the bounded conversation toolbox (8 cached tools, 16KiB of added serialized schemas). Narrow the search or use a smaller matching tool." + "reason": "The tool schema exceeds the bounded conversation toolbox (8 cached tools, 24KiB of added serialized schemas). Narrow the search or use a smaller matching tool." }) })); diff --git a/crates/tui/src/core/engine/tool_catalog/tests.rs b/crates/tui/src/core/engine/tool_catalog/tests.rs index e5de098398..14fce9e9a7 100644 --- a/crates/tui/src/core/engine/tool_catalog/tests.rs +++ b/crates/tui/src/core/engine/tool_catalog/tests.rs @@ -70,32 +70,20 @@ fn published_synthetic_names_agree_with_the_synthetic_predicate() { #[test] fn first_turn_surface_is_stable_across_plan_work_and_operate() { + // Token diet: agent+workflow (21KB of schema) and get/update_goal + // (meaningless without an active goal) stay deferred; discovery via + // tool_search restores them on demand. assert_eq!( DEFAULT_ACTIVE_NATIVE_TOOLS, - &[ - "read", - "write", - "edit", - "bash", - "agent", - "workflow", - "todo_write", - "create_goal", - "get_goal", - "update_goal" - ] + &["read", "write", "edit", "bash", "todo_write", "create_goal"] ); let expected = [ - "agent", "bash", "create_goal", - "get_goal", - "update_goal", "edit", "read", "todo_write", "tool_search", - "workflow", "write", ] .into_iter() @@ -238,7 +226,9 @@ fn successful_cached_execution_updates_lru_without_granting_uncached_names() { remove_evicted_cache_activations(&catalog, &mut active, delta.evicted); active.extend(delta.admitted); assert!(cache.names().any(|name| name == "deferred-0")); - assert!(!cache.names().any(|name| name == "deferred-1")); + // Priority order: deferred-7 was the lowest-priority entry of the + // first batch and was never re-touched, so it is the victim. + assert!(!cache.names().any(|name| name == "deferred-7")); assert!(!touch_cached_tool_after_execution( &catalog, @@ -320,7 +310,7 @@ fn unknown_and_wildcard_allowlists_keep_mcp_startup() { } #[test] -fn compact_surface_keeps_agent_and_workflow_eager() { +fn compact_surface_keeps_agent_and_workflow_deferred() { let catalog = build_model_tool_catalog_with_surface( [ "read", @@ -346,12 +336,30 @@ fn compact_surface_keeps_agent_and_workflow_eager() { .iter() .find(|definition| definition.name == name) .and_then(|definition| definition.defer_loading), - Some(false), + Some(true), "{name}" ); } } +#[test] +fn deferred_core_tools_restore_through_always_load() { + // The diet's escape hatch: discovery/activation and the goal-aware + // catalog build in `engine.rs` re-eager exactly these names. + let mut catalog = ["agent", "workflow", "get_goal", "update_goal", "read"] + .into_iter() + .map(tool) + .collect::>(); + let always_load = ["agent", "workflow", "get_goal", "update_goal"] + .into_iter() + .map(str::to_string) + .collect::>(); + apply_native_tool_deferral(&mut catalog, &always_load); + for definition in &catalog { + assert_eq!(definition.defer_loading, Some(false), "{}", definition.name); + } +} + /// The per-tool Registry paragraph is gone; the catalog builder must leave the /// shell tool's description exactly as the registry produced it, so the KV /// prefix stays byte-stable and there is one Registry authority (the prompt). diff --git a/crates/tui/src/core/session.rs b/crates/tui/src/core/session.rs index 3424e72054..3341082a2f 100644 --- a/crates/tui/src/core/session.rs +++ b/crates/tui/src/core/session.rs @@ -12,11 +12,15 @@ use std::collections::{HashSet, VecDeque}; use std::path::PathBuf; /// Maximum number of deferred schemas a conversation may keep in its active -/// toolbox. The permanent `read`/`write`/`edit`/`bash`/`agent`/`tool_search` -/// router surface is not counted here. +/// toolbox. The permanent eager head is not counted here. pub(crate) const TOOL_ACTIVATION_CACHE_MAX_NAMES: usize = 8; /// Maximum serialized bytes added to requests by cached deferred schemas. -pub(crate) const TOOL_ACTIVATION_CACHE_MAX_SCHEMA_BYTES: usize = 16 * 1024; +/// Sized to co-activate the delegation pair (`agent` 11.7KB + `workflow` +/// 9.7KB) with room for small companions: at 16KB the two thrashed each +/// other out of the cache every discovery round. The prefix diet removed +/// 28KB from the head; this returns 8KB of activation headroom, still a +/// deep net cut. +pub(crate) const TOOL_ACTIVATION_CACHE_MAX_SCHEMA_BYTES: usize = 24 * 1024; /// Bounded, process-local conversation cache for tools activated by /// `tool_search`. @@ -96,6 +100,14 @@ impl ToolActivationCache { /// Touch requested deferred tools in search-result order. An oversized /// schema is rejected; otherwise least-recently-used entries are evicted /// until both bounds hold. + /// + /// Push order is the REVERSE of the request order: eviction pops from + /// the front, so pushing best-match-last keeps the highest-priority + /// tool (which the caller ranked first) alive the longest. Pushing in + /// request order stranded large best matches — `agent` (11.7KB) was + /// discovered, then immediately evicted by its own small-fry company + /// whenever the set overflowed the byte bound. Cross-turn LRU is unaffected: + /// every batch entry is still newer than every older entry. pub(crate) fn activate( &mut self, catalog: &[codewhale_models::Tool], @@ -106,7 +118,9 @@ impl ToolActivationCache { ..ToolActivationDelta::default() }; let mut seen = HashSet::new(); - for name in requested { + let mut ordered: Vec<&String> = requested.iter().collect(); + ordered.reverse(); + for name in ordered { if !seen.insert(name.clone()) { continue; } @@ -469,15 +483,41 @@ mod tests { let delta = cache.activate(&catalog, &requested); assert_eq!(cache.names().count(), TOOL_ACTIVATION_CACHE_MAX_NAMES); + // Request order is priority (search ranked best-first), so the + // TAIL of the request is evicted first — the retained set is the + // eight highest-priority tools. assert_eq!( cache.names().collect::>(), vec![ - "tool_2", "tool_3", "tool_4", "tool_5", "tool_6", "tool_7", "tool_8", "tool_9" + "tool_7", "tool_6", "tool_5", "tool_4", "tool_3", "tool_2", "tool_1", "tool_0" ] ); assert_eq!(delta.admitted.len(), TOOL_ACTIVATION_CACHE_MAX_NAMES); - assert!(delta.evicted.contains(&"tool_0".to_string())); - assert!(delta.evicted.contains(&"tool_1".to_string())); + assert!(delta.evicted.contains(&"tool_8".to_string())); + assert!(delta.evicted.contains(&"tool_9".to_string())); + } + + #[test] + fn best_match_survives_byte_overflow_eviction() { + // Priority-inversion regression: `agent` (11.7KB, best match) was + // discovered, then immediately evicted by its own small-fry company + // whenever the set overflowed the byte bound. + let mut catalog = vec![deferred_tool("big_best", 20_000)]; + for index in 0..8 { + catalog.push(deferred_tool(&format!("small_{index}"), 1_000)); + } + let requested = catalog + .iter() + .map(|tool| tool.name.clone()) + .collect::>(); + let mut cache = ToolActivationCache::default(); + let delta = cache.activate(&catalog, &requested); + + assert!( + delta.admitted.contains(&"big_best".to_string()), + "best match must survive its own discovery set: {delta:?}" + ); + assert!(delta.admitted.len() < requested.len()); } #[test] @@ -496,7 +536,9 @@ mod tests { let names = cache.names().collect::>(); assert!(names.contains(&"tool_0")); - assert!(!names.contains(&"tool_1")); + // tool_7 was the lowest-priority entry of the first batch and was + // never re-touched, so it is the eviction victim — not tool_1. + assert!(!names.contains(&"tool_7")); assert_eq!(names.last().copied(), Some("tool_8")); } diff --git a/crates/tui/src/hooks/executor.rs b/crates/tui/src/hooks/executor.rs index d589718627..e13f816409 100644 --- a/crates/tui/src/hooks/executor.rs +++ b/crates/tui/src/hooks/executor.rs @@ -5868,20 +5868,17 @@ command = "echo project" // here would never fire. This table is checked in both directions, so // a rename fails on the missing entry and a classifier change fails on // the mismatched category. + // Token diet: agent+workflow and get/update_goal are deferred by + // default (discoverable via tool_search), so they left the eager + // table; their classifications are pinned with the other + // deferred tools below instead. const EXPECTED: &[(&str, &str)] = &[ ("read", "safe"), ("write", "file_write"), ("edit", "file_write"), ("bash", "shell"), - // The router itself touches nothing a hook needs to gate. - ("agent", "other"), - ("workflow", "other"), ("todo_write", "safe"), - // Goal controls retain their existing hook classification when - // promoted from deferred discovery to the eager catalog. ("create_goal", "other"), - ("get_goal", "other"), - ("update_goal", "other"), ]; for name in crate::core::engine::tool_catalog::DEFAULT_ACTIVE_NATIVE_TOOLS { let expected = EXPECTED.iter().find(|(n, _)| n == name).map(|(_, c)| *c); @@ -5902,6 +5899,14 @@ command = "echo project" ); } + // Deferred by default, still classified: the router touches + // nothing a hook needs to gate, and the goal controls keep + // their classification when promoted to eager. + assert_eq!(tool_category_for("agent", None), "other"); + assert_eq!(tool_category_for("workflow", None), "other"); + assert_eq!(tool_category_for("get_goal", None), "other"); + assert_eq!(tool_category_for("update_goal", None), "other"); + // The shell surface. assert_eq!(tool_category_for("Bash", None), "shell"); // Retained: shell.rs stamps this for the shell_env event. diff --git a/crates/tui/src/skills/mod.rs b/crates/tui/src/skills/mod.rs index 85c9c96198..af5ddc72e0 100644 --- a/crates/tui/src/skills/mod.rs +++ b/crates/tui/src/skills/mod.rs @@ -1320,7 +1320,7 @@ pub fn render_available_skills_context_for_workspace_with_mode_and_plugins( } /// Progressive-disclosure contract: the model sees a bounded page of skill -/// names, descriptions, and paths, then uses `load_skill` for the complete +/// names and descriptions, then uses `load_skill` for the complete /// catalogue or a specific `SKILL.md` body. /// /// Test-only single-directory variant. Production callers scan the complete @@ -1450,65 +1450,10 @@ fn replace_prompt_path_root(text: &str, root: &str, replacement: &str) -> String out } -/// Render a skill path without leaking private absolute paths into the -/// system-prompt prefix (#4632): workspace skills become workspace-relative, -/// home-dir skills become `~/…`, and anything else is reduced to its trailing -/// components so the prefix stays free of user-identifying absolute paths. -/// Skill paths in the prompt are consumed by the model as text, not by the -/// platform's shell, so normalize Windows separators to forward slashes: -/// the catalog renders identically on every platform (#5473). -fn prompt_display(path: &Path) -> String { - path.display() - .to_string() - .replace(std::path::MAIN_SEPARATOR, "/") -} - -fn privacy_safe_skill_path(path: &Path, workspace: &Path) -> String { - if let Ok(rel) = path.strip_prefix(workspace) { - return prompt_display(rel); - } - if let Some(home) = crate::config::effective_home_dir() - && let Ok(rel) = path.strip_prefix(&home) - { - return format!("~/{}", prompt_display(rel)); - } - match (path.parent().and_then(Path::file_name), path.file_name()) { - (Some(dir), Some(file)) => { - format!("…/{}/{}", dir.to_string_lossy(), file.to_string_lossy()) - } - _ => path - .file_name() - .map(|file| file.to_string_lossy().into_owned()) - .unwrap_or_else(|| "SKILL.md".to_string()), - } -} - -fn path_is_within_root(path: &Path, root: &Path) -> bool { - if path.starts_with(root) { - return true; - } - let Some(canonical_path) = fs::canonicalize(path).ok() else { - return false; - }; - let Some(canonical_root) = fs::canonicalize(root).ok() else { - return false; - }; - canonical_path.starts_with(canonical_root) -} - -fn prompt_skill_path( - path: &Path, - workspace: &Path, - configured_skills_root: Option<&Path>, -) -> Option { - if let Some(root) = configured_skills_root - && path_is_within_root(path, root) - { - return None; - } - Some(privacy_safe_skill_path(path, workspace)) -} - +// Token diet: the ambient index carries names + descriptions only, never +// on-disk paths — `load_skill` resolves names through the registry. (The +// former privacy-safe path renderers lived here; deleting them also deletes +// the #4632/#5473 surface they existed to defend.) #[cfg(test)] fn render_skills_block(registry: &SkillRegistry, locale: &str, workspace: &Path) -> Option { render_skills_block_with_configured_root( @@ -1656,17 +1601,16 @@ Skills are optional instruction packs. This index exposes routing metadata; bodi // instructions or consuming prompt budget. .filter(|skill| skill.invocation != SkillInvocation::ExplicitOnly) .map(|skill| { - // Native skills expose the real on-disk path captured at discovery. - // Plugin skills expose only their reviewed snapshot identity so the - // model cannot bypass the content-bound trust receipt via a mutable - // source path. Paths render privacy-safe (workspace-relative or - // ~/…) so the prompt prefix never embeds absolute user paths - // (#4632). A caller-provided skills root omits its physical path - // because that root may change per session; load_skill still - // resolves the stable skill name through the internal registry. - let display_path = prompt_skill_path(&skill.path, workspace, configured_skills_root); + // Token diet: native skills expose no on-disk path. `load_skill` + // resolves the stable skill name through the internal registry, + // so the path only invited the slower read-the-file bypass the + // tool description already steers away from — at ~50 bytes a row, + // every turn, in the cache-pinned prefix. Plugin skills keep + // their reviewed snapshot identity: the trust receipt must stay + // visible so the model cannot mistake a snapshot for a mutable + // source path. let source = match &skill.source { - SkillSource::Native => display_path.map(|path| format!("file: {path}")), + SkillSource::Native => None, SkillSource::Plugin { plugin_id, plugin_name, diff --git a/crates/tui/src/skills/tests.rs b/crates/tui/src/skills/tests.rs index ba5655f7ed..9cb6478c2f 100644 --- a/crates/tui/src/skills/tests.rs +++ b/crates/tui/src/skills/tests.rs @@ -111,7 +111,7 @@ fn prompt_warning_sanitizer_handles_non_utf8_configured_roots() { } #[test] -fn render_available_skills_context_lists_paths_and_usage() { +fn render_available_skills_context_lists_names_and_usage() { let tmpdir = TempDir::new().unwrap(); create_skill_dir( &tmpdir, @@ -122,17 +122,16 @@ fn render_available_skills_context_lists_paths_and_usage() { let rendered = crate::skills::render_available_skills_context(&tmpdir.path().join("skills")) .expect("skill context"); - // #4632: paths render relative to the skills base dir (privacy-safe), - // so the assertion checks the workspace-relative form. - let expected_path = super::prompt_display(&std::path::Path::new("test-skill").join("SKILL.md")); - + // Token diet: names + descriptions only. `load_skill` resolves the + // name through the registry, so no on-disk path is rendered at all + // (#4632's surface is gone with the paths). assert!(rendered.contains("## Skills")); assert!(rendered.contains("- test-skill: A test skill")); assert!(rendered.contains("load the exact skill before use")); assert!(rendered.contains("do not expand tool, approval, or trust authority")); assert!( - rendered.contains(&expected_path), - "expected path {expected_path:?} not in rendered output" + !rendered.contains("test-skill/SKILL.md"), + "no on-disk paths in the index" ); assert!(!rendered.contains(tmpdir.path().to_str().unwrap_or("/nonexistent"))); assert!(rendered.contains("### Usage")); @@ -181,12 +180,12 @@ fn workspace_prompt_omits_disabled_skills_without_configured_directory() { } #[test] -fn render_available_skills_context_uses_real_dir_name_not_frontmatter_name() { - // Regression: when a community-installed or manually-placed skill +fn render_available_skills_context_resolves_by_name_without_invented_paths() { + // Regression (pathless form): when a community-installed skill // lives in a directory whose name differs from its frontmatter - // `name`, the rendered prompt must point to the real on-disk file - // path, not //SKILL.md (which does - // not exist). + // `name`, the row carries the registry name and no path at all — + // `load_skill` resolves the name to the real on-disk file, so no + // caller can reconstruct a stale /SKILL.md path. let tmpdir = TempDir::new().unwrap(); create_skill_dir( &tmpdir, @@ -197,18 +196,17 @@ fn render_available_skills_context_uses_real_dir_name_not_frontmatter_name() { let rendered = crate::skills::render_available_skills_context(&tmpdir.path().join("skills")) .expect("skill context"); - // #4632: rendered relative to the skills base dir; the regression - // intent (real dir name, not frontmatter name) is unchanged. - let real_path = super::prompt_display(&std::path::Path::new("weird-dir-name").join("SKILL.md")); - let stale_path = super::prompt_display(&std::path::Path::new("friendly-name").join("SKILL.md")); - assert!( - rendered.contains(&real_path), - "expected real on-disk path {real_path:?} in rendered output, got:\n{rendered}" + rendered.contains("- friendly-name: drift case"), + "registry-name row present, got:\n{rendered}" + ); + assert!( + !rendered.contains("weird-dir-name/SKILL.md"), + "no dir-derived path:\n{rendered}" ); assert!( - !rendered.contains(&stale_path), - "rendered output must not invent a path under the frontmatter name:\n{rendered}" + !rendered.contains("friendly-name/SKILL.md"), + "no frontmatter-derived path:\n{rendered}" ); } @@ -1885,9 +1883,7 @@ fn configured_skill_prompt_uses_a_stable_root_in_entries_and_warnings() { .expect("configured skill context"); assert!(rendered.contains("- visual-design: Design assets\n")); - assert!(rendered.contains( - "- workspace-skill: Workspace skill (file: .claude/skills/workspace-skill/SKILL.md)" - )); + assert!(rendered.contains("- workspace-skill: Workspace skill\n")); assert!( rendered .contains("in /visual-design/SKILL.md is not a safe command name") @@ -1897,7 +1893,7 @@ fn configured_skill_prompt_uses_a_stable_root_in_entries_and_warnings() { } #[test] -fn default_workspace_skill_prompt_preserves_its_discoverable_path() { +fn default_workspace_skill_prompt_omits_paths_in_favour_of_load_skill() { let _env_lock = crate::test_support::lock_test_env(); super::clear_skill_discovery_cache(); let tmpdir = TempDir::new().unwrap(); @@ -1927,9 +1923,7 @@ fn default_workspace_skill_prompt_preserves_its_discoverable_path() { ) .expect("workspace skill context"); - assert!(rendered.contains( - "- workspace-skill: Workspace skill (file: .agents/skills/workspace-skill/SKILL.md)" - )); + assert!(rendered.contains("- workspace-skill: Workspace skill\n")); } #[test] diff --git a/crates/tui/src/tools/echolocation.rs b/crates/tui/src/tools/echolocation.rs new file mode 100644 index 0000000000..d03436c322 --- /dev/null +++ b/crates/tui/src/tools/echolocation.rs @@ -0,0 +1,2588 @@ +//! Echolocation: stateless navigation soundings for tool results. +//! +//! No index, no background work, no new dependencies. Like a whale sounding +//! its pod, every read, grep, file search, and project map echoes its +//! surroundings fresh from source — sibling podmates, symbols, import +//! roots, resolved file links — rendered deterministically (sorted, +//! capped, no timestamps) so a repeated call is byte-identical and +//! cache-friendly. Anything the scanners cannot resolve is omitted, +//! never guessed. Sounding languages: Rust, Python, JavaScript/ +//! TypeScript, Go (symbols everywhere; links for Rust/Python/JS; +//! anything else renders podmates only). +//! +//! KV-cache effect: tool-result history only. Soundings never enter the +//! session-pinned prefix. +//! +//! Tuning: `CODEWHALE_ECHO` gates sections for ablation trials +//! (comma-separated `off:` entries, e.g. `off:links,off:mates`; +//! section names: mates, symbols, imports, links, callers, pods, +//! soundings). Unset or malformed means everything on. +//! +//! Terminal buzz: the read footer deepens when the model is closing in. +//! Two distinct files read in a pod (see +//! `ToolContext::pod_visit_count`) put the next read there into buzz +//! mode, which adds the callers section — podmates whose resolved +//! links include this file. Sparse clicks while searching, +//! high-resolution echoes at close range. Deterministic for a fixed +//! history, like everything else here. +//! +//! Known lookalikes (line scanners, not parsers): comment markers +//! inside string literals can mute following lines until the closer +//! (`"/*"` swallows until `"*/"`); Python doc fences toggle on every +//! `"""`/`'''` occurrence, so mixed kinds on one line desync the +//! span. Keywords match at line start only, so the common failure +//! mode is a missed symbol; forging one needs a keyword at a line +//! start inside a multi-line string. Links need that AND an +//! on-disk target, so they forge only when both coincide. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; + +use ignore::{Match as IgnoreMatch, gitignore::Gitignore}; + +/// Hard budget for the rendered footer, in bytes. +pub const POD_FOOTER_MAX_BYTES: usize = 1200; + +/// Max podmate names listed before "+N more". +const MAX_PODMATES: usize = 12; +/// Max symbols listed before "+N more". +const MAX_SYMBOLS: usize = 24; +/// Max import roots listed before "+N more". +const MAX_IMPORT_ROOTS: usize = 12; +/// Max characters kept per sounded name. +const MAX_NAME_LEN: usize = 64; +/// Max matched dirs charted per grep result before `pods_omitted` counts. +const MAX_GREP_PODS: usize = 8; +/// Max mates listed per grep pod. +const MAX_GREP_MATES: usize = 8; +/// Max key files sounded per project map. +const MAX_MAP_SOUNDINGS: usize = 8; +/// Files bigger than this are never sounded. +const MAX_SOUNDING_BYTES: u64 = 8 * 1024 * 1024; +/// Max resolved file links listed before "+N more". +const MAX_LINKS: usize = 8; +/// Max callers (reverse links) listed before "+N more". +const MAX_CALLERS: usize = 8; +/// Mate files bigger than this are not scanned for callers. +const MAX_CALLER_SCAN_BYTES: u64 = 256 * 1024; +/// Tracked pod reads before the next read there enters the buzz. +pub(crate) const BUZZ_VISIT_THRESHOLD: usize = 2; + +/// Ablation gate: true when `CODEWHALE_ECHO` disables `section`. +/// Parsed once per process; unset or malformed means everything on. +fn echo_off(section: &str) -> bool { + static GATES: OnceLock> = OnceLock::new(); + GATES + .get_or_init(|| { + std::env::var("CODEWHALE_ECHO") + .map(|raw| parse_echo_gates(&raw)) + .unwrap_or_default() + }) + .contains(section) +} + +/// Parse `off:,…` entries; anything else is ignored (fail-open: +/// a typo never silences soundings in production). +fn parse_echo_gates(raw: &str) -> BTreeSet { + raw.split(',') + .filter_map(|entry| entry.trim().strip_prefix("off:")) + .map(|name| name.trim().to_string()) + .filter(|name| !name.is_empty()) + .collect() +} + +/// A sounded chart for one file. Empty sections render as nothing; a chart +/// with nothing to say renders as the empty string. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct EchoChart { + pod_dir: Option, + podmates: Vec, + podmate_total: usize, + symbols: Vec, + symbol_total: usize, + import_roots: Vec, + import_total: usize, + links: Vec, + link_total: usize, + callers: Vec, + caller_total: usize, +} + +/// Sound an echolocation chart for a file that was just read. +/// `workspace` anchors link display (workspace-relative paths); +/// `file_path` is the resolved path (for the sibling sounding); `text` +/// is the full decoded buffer (for symbols, imports, and links); `buzz` +/// adds the callers section when the model is closing in on this pod. +/// Never fails: anything unreadable is omitted from the chart rather +/// than failing the read. +pub fn sound_echolocation(workspace: &Path, file_path: &Path, text: &str, buzz: bool) -> EchoChart { + let (pod_dir, mut podmates) = sound_podmates(file_path); + let podmate_total = podmates.len(); + podmates.truncate(MAX_PODMATES); + let (mut symbols, mut import_roots, mut links, link_total, mut callers, caller_total) = + if let Some(lang) = detect_lang(file_path) { + let (links, link_total) = sound_links(workspace, file_path, text, lang); + let (callers, caller_total) = if buzz { + sound_callers(workspace, file_path) + } else { + (Vec::new(), 0) + }; + ( + sound_symbols(text, lang), + sound_import_roots(text, lang) + .into_iter() + .collect::>(), + links, + link_total, + callers, + caller_total, + ) + } else { + (Vec::new(), Vec::new(), Vec::new(), 0, Vec::new(), 0) + }; + let symbol_total = symbols.len(); + symbols.truncate(MAX_SYMBOLS); + let import_total = import_roots.len(); + import_roots.truncate(MAX_IMPORT_ROOTS); + links.truncate(MAX_LINKS); + callers.truncate(MAX_CALLERS); + EchoChart { + pod_dir, + podmates, + podmate_total, + symbols, + symbol_total, + import_roots, + import_total, + links, + link_total, + callers, + caller_total, + } +} + +/// Sound a miss echo for a read that failed with NotFound: the parent +/// pod's mates, so a typo'd path costs one correction instead of a +/// list-dir round trip. `None` when the parent cannot be listed or the +/// mates gate is off. Callers must only invoke this for genuine +/// NotFound failures — never for denylist/permission errors, where an +/// echo would answer the probe it refused. +pub fn sound_miss_echo(file_path: &Path) -> Option { + if echo_off("mates") { + return None; + } + let (pod_dir, mut mates) = sound_podmates(file_path); + if mates.is_empty() { + return None; + } + const MAX_MISS_MATES: usize = 8; + let total = mates.len(); + mates.truncate(MAX_MISS_MATES); + let dir = pod_dir.as_deref().unwrap_or("."); + Some(format!( + "\n\n[Miss echo: {dir} holds: {}]", + render_capped(&mates, total) + )) +} + +impl EchoChart { + /// Render the footer. Deterministic for a fixed file and directory: + /// sorted entries, fixed caps, byte budget, no timestamps. + pub fn render_footer(&self) -> String { + let mut out = String::new(); + if !echo_off("mates") && self.pod_dir.is_some() && self.podmate_total > 0 { + let dir = self.pod_dir.as_deref().unwrap_or("."); + out.push_str(&format!( + "\n\n[Pod: {} ({}): {}]", + dir, + count_noun(self.podmate_total, "entry", "entries"), + render_capped(&self.podmates, self.podmate_total) + )); + } + let mut sounding = String::new(); + if !echo_off("symbols") && self.symbol_total > 0 { + sounding.push_str(&format!( + "{}: {}", + count_noun(self.symbol_total, "symbol", "symbols"), + render_capped(&self.symbols, self.symbol_total) + )); + } + if !echo_off("imports") && self.import_total > 0 { + if !sounding.is_empty() { + sounding.push_str(" | "); + } + sounding.push_str(&format!( + "imports: {}", + render_capped(&self.import_roots, self.import_total) + )); + } + if !echo_off("links") && self.link_total > 0 { + if !sounding.is_empty() { + sounding.push_str(" | "); + } + sounding.push_str(&format!( + "links: {}", + render_capped(&self.links, self.link_total) + )); + } + if !echo_off("callers") && self.caller_total > 0 { + if !sounding.is_empty() { + sounding.push_str(" | "); + } + sounding.push_str(&format!( + "heard by: {}", + render_capped(&self.callers, self.caller_total) + )); + } + if !sounding.is_empty() { + out.push_str(&format!("\n\n[Sounding: {sounding}]")); + } + truncate_bytes(&out, POD_FOOTER_MAX_BYTES) + } +} + +/// Languages with line-scanner soundings. Everything else is prose: +/// podmates only, no symbols/imports/links. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SoundLang { + Rust, + Python, + JavaScript, + Go, +} + +/// Sounding language from file extension. `None` renders podmates only. +fn detect_lang(file_path: &Path) -> Option { + let ext = file_path.extension()?.to_str()?; + match ext { + "rs" => Some(SoundLang::Rust), + "py" | "pyi" => Some(SoundLang::Python), + "js" | "jsx" | "mjs" | "cjs" | "ts" | "tsx" | "mts" | "cts" => Some(SoundLang::JavaScript), + "go" => Some(SoundLang::Go), + _ => None, + } +} + +/// Sibling entries of the file's directory: sorted display names (current +/// file marked, directories suffixed) plus the short directory display. +/// A directory that cannot be listed degrades to an empty pod, silently. +fn sound_podmates(file_path: &Path) -> (Option, Vec) { + let parent = match file_path.parent() { + Some(parent) => parent, + None => return (None, Vec::new()), + }; + let mates = list_dir_entries(parent).unwrap_or_default(); + let current = file_path + .file_name() + .map(|name| name.to_string_lossy().into_owned()); + let display = mates + .into_iter() + .map(|(raw, mut shown)| { + if Some(&raw) == current.as_ref() { + shown.push_str(" (this file)"); + } + shown + }) + .collect(); + (Some(short_dir_display(parent)), display) +} + +/// Visibility gates a pod listing must honor: the calling search hides +/// these names, so pods must not enumerate them either. The denylist +/// always applies; glob/extension/gitignore gates apply when set. +/// `glob_root` anchors glob matching and gitignore stacking, exactly +/// like the calling walk's root. +pub(crate) struct PodVisibility<'a> { + pub include: &'a [String], + pub exclude: &'a [String], + pub extensions: &'a [String], + pub glob_root: &'a Path, + pub gitignore: bool, +} + +impl PodVisibility<'_> { + /// Directed-read visibility: no search gates, denylist still on. + fn directed(glob_root: &Path) -> PodVisibility<'_> { + PodVisibility { + include: &[], + exclude: &[], + extensions: &[], + glob_root, + gitignore: false, + } + } +} + +/// Sorted `(raw, display)` entries of a dir: dotfiles skipped, dirs +/// suffixed with `/`, visibility gates applied. `None` when the dir +/// cannot be listed at all (distinct from an empty dir). +fn list_dir_entries(dir: &Path) -> Option> { + list_dir_entries_filtered(dir, &PodVisibility::directed(dir)) +} + +/// [`list_dir_entries`] plus the caller's search gates: denylist, +/// excludes, includes/extensions (files only, like the walks), and +/// optional gitignore stacking. +fn list_dir_entries_filtered(dir: &Path, vis: &PodVisibility<'_>) -> Option> { + let ignores = vis + .gitignore + .then(|| IgnoreStack::for_dir(vis.glob_root, dir)); + let mut mates = Vec::new(); + for entry in std::fs::read_dir(dir).ok()?.flatten() { + let raw = entry.file_name().to_string_lossy().into_owned(); + if raw.starts_with('.') { + continue; + } + let abs = entry.path(); + let is_dir = entry.file_type().is_ok_and(|kind| kind.is_dir()); + // Denylist first: a pod listing is enumeration, not a directed + // read — denied names stay hidden even as siblings. + if crate::sandbox::read_guard::active().check(&abs).is_err() { + continue; + } + let rel = abs + .strip_prefix(vis.glob_root) + .unwrap_or(&abs) + .to_string_lossy() + .replace('\\', "/"); + if vis + .exclude + .iter() + .any(|pat| super::search::matches_glob(&rel, pat)) + { + continue; + } + if !is_dir { + if !vis.include.is_empty() + && !vis + .include + .iter() + .any(|pat| super::search::matches_glob(&rel, pat)) + { + continue; + } + if !vis.extensions.is_empty() + && !super::file_search::extension_matches(&abs, vis.extensions) + { + continue; + } + } + if ignores + .as_ref() + .is_some_and(|stack| stack.is_ignored(&abs, is_dir)) + { + continue; + } + let mut display = raw.clone(); + if is_dir { + display.push('/'); + } + mates.push((raw, display)); + } + mates.sort(); + Some(mates) +} + +/// Global gitignore + `.gitignore` + `.ignore` matchers stacked +/// shallow → deep from `glob_root` down to the listed dir. Last +/// decisive match wins, mirroring git precedence (repo rules beat the +/// global file). Known residual: `.git/info/exclude` is not stacked, +/// so a name hidden only there still charts — the calling walk is the +/// only other surface that reads it, and directed reads bypass ignore +/// rules entirely, so no boundary is crossed. +struct IgnoreStack { + matchers: Vec, +} + +impl IgnoreStack { + fn for_dir(glob_root: &Path, dir: &Path) -> Self { + // Global first (lowest precedence); a missing/unreadable global + // file yields an empty matcher, never an error. + let mut matchers = vec![Gitignore::global().0]; + let mut chain = Vec::new(); + let mut current = Some(dir); + // Never climb above the glob root (a stray absolute dir must not + // inherit ignore rules from unrelated ancestors). + while let Some(layer) = current.filter(|layer| layer.starts_with(glob_root)) { + chain.push(layer); + if layer == glob_root { + break; + } + current = layer.parent(); + } + chain.reverse(); + for layer in chain { + for file in [".gitignore", ".ignore"] { + let candidate = layer.join(file); + if candidate.is_file() { + matchers.push(Gitignore::new(&candidate).0); + } + } + } + Self { matchers } + } + + fn is_ignored(&self, path: &Path, is_dir: bool) -> bool { + let mut ignored = false; + for matcher in &self.matchers { + match matcher.matched(path, is_dir) { + IgnoreMatch::Ignore(_) => ignored = true, + IgnoreMatch::Whitelist(_) => ignored = false, + IgnoreMatch::None => {} + } + } + ignored + } +} + +/// One sounded pod for a grep result: the matched directory and its +/// entries, matched files marked, capped with an honest total. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct MatchPod { + pub dir: String, + pub mates: Vec, + pub mate_total: usize, +} + +/// Sound pods for search matches: distinct matched dirs, sorted and +/// capped, each with its entries (matched files marked ` (match)`). +/// Entries honor `vis` — the calling search's own gates — so pods +/// never enumerate names the search hides. Returns the pods plus the +/// count of dirs beyond the cap. A dir that cannot be listed still +/// charts its matched files (already search-gated), so a pod is never +/// silently empty. Deterministic for a fixed tree. +pub(crate) fn sound_match_pods( + resolve_root: &Path, + files: &[String], + vis: &PodVisibility<'_>, +) -> (Vec, usize) { + if echo_off("pods") { + return (Vec::new(), 0); + } + let mut by_dir: BTreeMap> = BTreeMap::new(); + for file in files { + if file.is_empty() { + continue; + } + let (dir, name) = split_match_path(file); + by_dir.entry(dir).or_default().insert(name); + } + let mut pods = Vec::new(); + let mut omitted = 0; + for (dir, matched) in &by_dir { + if pods.len() >= MAX_GREP_PODS { + omitted += 1; + continue; + } + pods.push(sound_one_match_pod(resolve_root, dir, matched, vis)); + } + (pods, omitted) +} + +/// Split a match path into (dir display, file name); `.` for the root. +fn split_match_path(file: &str) -> (String, String) { + let path = Path::new(file); + let name = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + let dir = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .map(|parent| parent.to_string_lossy().into_owned()) + .unwrap_or_else(|| ".".to_string()); + (dir, name) +} + +fn sound_one_match_pod( + resolve_root: &Path, + dir: &str, + matched: &BTreeSet, + vis: &PodVisibility<'_>, +) -> MatchPod { + let full = Path::new(dir); + let full = if full.is_absolute() { + full.to_path_buf() + } else { + resolve_root.join(full) + }; + let mut mates: Vec = match list_dir_entries_filtered(&full, vis) { + Some(entries) => entries + .into_iter() + .map(|(raw, mut shown)| { + if matched.contains(&raw) { + shown.push_str(" (match)"); + } + shown + }) + .collect(), + // Dotfiles stay skipped here too, exactly like the listed + // path: the match itself is still in the result set, so no + // information is lost, only the redundant mark. + None => matched + .iter() + .filter(|name| !name.starts_with('.')) + .map(|name| format!("{name} (match)")) + .collect(), + }; + let mate_total = mates.len(); + mates.truncate(MAX_GREP_MATES); + if mate_total > mates.len() { + mates.push(format!("+{} more", mate_total - mates.len())); + } + MatchPod { + dir: dir.to_string(), + mates, + mate_total, + } +} + +/// Sound one line of symbols for each of the first key files of a +/// project map (sorted, capped). Returns the soundings plus the count +/// of key files beyond the cap. Unreadable, oversized, non-Rust, or +/// symbol-free files are skipped silently. +pub(crate) fn sound_key_files( + root: &Path, + key_files: &[String], +) -> (BTreeMap, usize) { + if echo_off("soundings") { + return (BTreeMap::new(), 0); + } + let mut sorted: Vec<&str> = key_files.iter().map(String::as_str).collect(); + sorted.sort(); + let omitted = sorted.len().saturating_sub(MAX_MAP_SOUNDINGS); + let mut soundings = BTreeMap::new(); + for name in sorted.into_iter().take(MAX_MAP_SOUNDINGS) { + if let Some(sounding) = sound_file_symbols(&root.join(name)) { + soundings.insert(name.to_string(), sounding); + } + } + (soundings, omitted) +} + +/// Sound one line of symbols for a single file: `N symbols: a, b, …`. +/// `None` when unreadable, oversized, non-Rust, or symbol-free. +fn sound_file_symbols(path: &Path) -> Option { + let lang = detect_lang(path)?; + if std::fs::metadata(path).is_ok_and(|meta| meta.len() > MAX_SOUNDING_BYTES) { + return None; + } + let text = std::fs::read_to_string(path).ok()?; + let mut symbols = sound_symbols(&text, lang); + let total = symbols.len(); + if total == 0 { + return None; + } + symbols.truncate(MAX_SYMBOLS); + Some(format!( + "{}: {}", + count_noun(total, "symbol", "symbols"), + render_capped(&symbols, total) + )) +} + +/// Last two normal components of a directory, for a compact pod label. +fn short_dir_display(parent: &Path) -> String { + let parts: Vec = parent + .components() + .filter_map(|component| match component { + std::path::Component::Normal(part) => Some(part.to_string_lossy().into_owned()), + _ => None, + }) + .collect(); + if parts.is_empty() { + return ".".to_string(); + } + let start = parts.len().saturating_sub(2); + format!("{}/", parts[start..].join("/")) +} + +/// Item lines in file order, per language. +fn sound_symbols(text: &str, lang: SoundLang) -> Vec { + // Symbols carry 1-based source lines (`fn main:12`) so the model can + // page straight at them — the sounding already sees past truncation, + // and now it can point there too. + let numbered = |n: usize, symbol: String| format!("{symbol}:{n}"); + match lang { + SoundLang::Rust => code_lines(text) + .filter_map(|(n, line)| parse_item_line(line).map(|s| numbered(n, s))) + .collect(), + SoundLang::Python => py_code_lines(text) + .filter_map(|(n, line)| parse_py_item(line).map(|s| numbered(n, s))) + .collect(), + SoundLang::JavaScript => code_lines(text) + .filter_map(|(n, line)| parse_js_item(line).map(|s| numbered(n, s))) + .collect(), + SoundLang::Go => code_lines(text) + .filter_map(|(n, line)| parse_go_item(line).map(|s| numbered(n, s))) + .collect(), + } +} + +/// Distinct import roots, sorted, per language. +fn sound_import_roots(text: &str, lang: SoundLang) -> BTreeSet { + match lang { + SoundLang::Rust => code_lines(text) + .filter_map(|(_, line)| parse_use_root(line)) + .collect(), + SoundLang::Python => py_code_lines(text) + .flat_map(|(_, line)| parse_py_import_roots(line)) + .collect(), + SoundLang::JavaScript => code_lines(text) + .filter_map(|(_, line)| parse_js_import_root(line)) + .collect(), + SoundLang::Go => code_lines(text) + .filter_map(|(_, line)| parse_go_import_root(line)) + .collect(), + } +} + +/// Source lines with `//` lines and `/* */` spans removed. Crude on +/// purpose: string literals containing comment markers are a documented +/// lookalike class (see module docs), not silently misread structure. +fn code_lines(text: &str) -> impl Iterator { + let mut in_block = false; + text.lines().enumerate().filter_map(move |(idx, raw)| { + let mut line = raw.trim_start(); + if in_block { + match line.find("*/") { + Some(end) => { + in_block = false; + line = line[end + 2..].trim_start(); + } + None => return None, + } + } + if line.starts_with("//") { + return None; + } + if let Some(start) = line.find("/*") { + let after_open = &line[start + 2..]; + match after_open.find("*/") { + Some(end) => { + let head = line[..start].trim_end(); + let tail = after_open[end + 2..].trim_start(); + line = if head.is_empty() { tail } else { head }; + if line.is_empty() { + return None; + } + } + None => { + in_block = true; + line = line[..start].trim_end(); + if line.is_empty() { + return None; + } + } + } + } + Some((idx + 1, line)) + }) +} + +/// Parse one `fn|struct|enum|trait|type|mod|static|const|impl|union|macro` +/// line into `kind name`. Qualifiers (`pub`, `pub(..)`, `async`, `const`, +/// `unsafe`, `extern "ABI"`) are stripped first. +fn parse_item_line(mut line: &str) -> Option { + loop { + line = line.trim_start(); + if let Some(rest) = strip_keyword(line, "pub") { + line = strip_visibility_group(rest.trim_start())?; + continue; + } + if let Some(rest) = strip_keyword(line, "extern") { + line = strip_abi_string(rest.trim_start())?; + continue; + } + let mut stripped = false; + for qualifier in ["async", "unsafe"] { + if let Some(rest) = strip_keyword(line, qualifier) { + line = rest; + stripped = true; + break; + } + } + if stripped { + continue; + } + // `const fn` (qualifier) vs `const NAME` (item): strip only when + // `fn` follows, otherwise the keyword table below owns it. + if let Some(rest) = strip_keyword(line, "const") + && strip_keyword(rest.trim_start(), "fn").is_some() + { + line = rest; + continue; + } + break; + } + for (keyword, label) in [ + ("fn", "fn"), + ("struct", "struct"), + ("enum", "enum"), + ("trait", "trait"), + ("type", "type"), + ("mod", "mod"), + ("static", "static"), + ("const", "const"), + ("union", "union"), + ("impl", "impl"), + ("macro_rules", "macro"), + ("macro", "macro"), + ] { + if let Some(rest) = strip_keyword(line, keyword) { + let rest = if keyword == "macro_rules" { + rest.strip_prefix('!').unwrap_or(rest) + } else { + rest + }; + let name = item_head(rest, keyword == "impl")?; + return Some(format!("{label} {name}")); + } + } + None +} + +/// Strip `keyword` only on a word boundary (so `fn` never matches `fnx`). +fn strip_keyword<'a>(line: &'a str, keyword: &str) -> Option<&'a str> { + let rest = line.strip_prefix(keyword)?; + match rest.chars().next() { + None => Some(rest), + Some(next) if next.is_alphanumeric() || next == '_' => None, + _ => Some(rest), + } +} + +/// Skip a `pub(..)` visibility group; bare `pub` falls through unchanged. +fn strip_visibility_group(rest: &str) -> Option<&str> { + match rest.strip_prefix('(') { + Some(after) => Some(&after[after.find(')')? + 1..]), + None => Some(rest), + } +} + +/// Skip an `extern "..."` ABI string; bare `extern` falls through unchanged. +fn strip_abi_string(rest: &str) -> Option<&str> { + match rest.strip_prefix('"') { + Some(after) => Some(&after[after.find('"')? + 1..]), + None => Some(rest), + } +} + +/// The sounded name: first identifier for items, the whole head (bounded) +/// for `impl` blocks (`Foo`, `Trait for Foo`, ` Foo`). +fn item_head(rest: &str, is_impl: bool) -> Option { + let rest = rest.trim_start(); + if is_impl { + let mut head = rest.split(['{', ';']).next()?.trim(); + if let Some(where_at) = head.find(" where ") { + head = head[..where_at].trim(); + } + if head.is_empty() { + return None; + } + return Some(truncate_name(head)); + } + // Capped before collecting: a minified line can run megabytes, + // and the sounding must never allocate the file to name it. + let name: String = rest + .chars() + .take_while(|next| next.is_alphanumeric() || *next == '_') + .take(MAX_NAME_LEN + 1) + .collect(); + if name.is_empty() || name.starts_with(|c: char| c.is_ascii_digit()) { + return None; + } + Some(truncate_name(&name)) +} + +/// First `::` segment of a `use` path (`serde`, `crate`, `super`). +fn parse_use_root(line: &str) -> Option { + let mut line = line.trim_start(); + if let Some(rest) = strip_keyword(line, "pub") { + line = strip_visibility_group(rest.trim_start())?.trim_start(); + } + let rest = strip_keyword(line, "use")?.trim_start(); + let head = rest.split([';', '{', ' ', '\t']).next()?.trim(); + let root = head.split("::").next()?.trim(); + valid_root(root) +} + +/// Resolve `use`/`mod` edges to workspace-relative files that exist on +/// disk (sorted, with an honest total; the caller caps). Only module +/// files (`x.rs` / `x/mod.rs`) resolve — single items living directly +/// in a root file, external crates, and `#[path]` remaps are left to +/// the import roots rather than guessed. +fn sound_links( + workspace: &Path, + file_path: &Path, + text: &str, + lang: SoundLang, +) -> (Vec, usize) { + let mut links = BTreeSet::new(); + match lang { + SoundLang::Rust => { + for (_, line) in code_lines(text) { + if let Some(segments) = parse_use_segments(line) { + if let Some(link) = resolve_use_link(workspace, file_path, &segments) { + links.insert(link); + } + } else if let Some(child) = parse_mod_decl(line) + && let Some(link) = resolve_child_module(workspace, file_path, &child) + { + links.insert(link); + } + } + } + SoundLang::Python => { + for (_, line) in py_code_lines(text) { + for link in resolve_py_links(workspace, file_path, line) { + links.insert(link); + } + } + } + SoundLang::JavaScript => { + for (_, line) in code_lines(text) { + if let Some(link) = resolve_js_link(workspace, file_path, line) { + links.insert(link); + } + } + } + // Go imports name packages (dirs), not files: nothing to link. + SoundLang::Go => {} + } + if let Some(own) = workspace_relative(workspace, file_path) { + links.remove(&own); + } + let total = links.len(); + (links.into_iter().collect(), total) +} + +/// Full `use` segments (`crate::config::Config` → all three). Groups, +/// globs, and `as` renames cut at the module prefix. +fn parse_use_segments(line: &str) -> Option> { + let mut line = line.trim_start(); + if let Some(rest) = strip_keyword(line, "pub") { + line = strip_visibility_group(rest.trim_start())?.trim_start(); + } + let rest = strip_keyword(line, "use")?.trim_start(); + let head = rest.split([';', '{', '*']).next()?.trim(); + let head = head.split(" as ").next()?.trim(); + let segments: Vec = head + .split("::") + .map(str::trim) + .filter(|part| !part.is_empty()) + .map(str::to_string) + .collect(); + if segments.is_empty() + || segments + .iter() + .any(|part| part.len() > MAX_NAME_LEN || !is_ident(part)) + { + return None; + } + Some(segments) +} + +/// `mod name;` child decl. Inline `mod name {` is a symbol, not an edge. +fn parse_mod_decl(line: &str) -> Option { + let mut line = line.trim_start(); + if let Some(rest) = strip_keyword(line, "pub") { + line = strip_visibility_group(rest.trim_start())?.trim_start(); + } + let rest = strip_keyword(line, "mod")?.trim_start(); + let name: String = rest + .chars() + .take_while(|next| next.is_alphanumeric() || *next == '_') + .take(MAX_NAME_LEN + 1) + .collect(); + if name.is_empty() || name.len() > MAX_NAME_LEN || !is_ident(&name) { + return None; + } + match rest[name.len()..].trim_start().chars().next() { + Some(';') => Some(name), + _ => None, + } +} + +fn is_ident(part: &str) -> bool { + // No leading digit: not an identifier in any sounded language, so a + // digit-led token is line noise, never a name. + !part.is_empty() + && part + .chars() + .next() + .is_some_and(|first| !first.is_ascii_digit()) + && part.chars().all(|c| c.is_alphanumeric() || c == '_') +} + +fn resolve_use_link(workspace: &Path, file_path: &Path, segments: &[String]) -> Option { + let first = segments.first()?.as_str(); + match first { + "crate" => { + let root = find_crate_root(workspace, file_path)?; + resolve_module_file(workspace, &root, &segments[1..]) + } + "super" => { + let depth = segments + .iter() + .take_while(|part| part.as_str() == "super") + .count(); + let mut base = super_base(file_path)?; + for _ in 1..depth { + base = base.parent()?.to_path_buf(); + } + resolve_module_file(workspace, &base, &segments[depth..]) + } + "self" => { + let base = self_base(file_path)?; + resolve_module_file(workspace, &base, &segments[1..]) + } + _ => None, + } +} + +/// `mod child;` resolves under the current module: the file's own dir +/// for `mod.rs`, a child dir named for the file stem otherwise. +fn resolve_child_module(workspace: &Path, file_path: &Path, child: &str) -> Option { + let base = self_base(file_path)?; + resolve_module_file(workspace, &base, std::slice::from_ref(&child.to_string())) +} + +/// Longest-prefix module-file match: `a::b::Item` tries `a/b.rs`, +/// `a/b/mod.rs`, then `a.rs`, `a/mod.rs`. First hit on disk wins. +fn resolve_module_file(workspace: &Path, base: &Path, rest: &[String]) -> Option { + for width in (1..=rest.len()).rev() { + let joined = rest[..width].join("/"); + for candidate in [ + base.join(format!("{joined}.rs")), + base.join(&joined).join("mod.rs"), + ] { + if candidate.is_file() { + return workspace_relative(workspace, &candidate); + } + } + } + None +} + +/// `self::` / `mod x;` base: own dir for `mod.rs` and the crate roots +/// (`lib.rs`/`main.rs`, whose children live beside them, not under a +/// `/` dir), else `/`. +fn self_base(file_path: &Path) -> Option { + let dir = file_path.parent()?; + if file_path + .file_name() + .is_some_and(|name| name == "mod.rs" || name == "lib.rs" || name == "main.rs") + { + return Some(dir.to_path_buf()); + } + let stem = file_path.file_stem()?; + Some(dir.join(stem)) +} + +/// First-level `super::` base: parent dir for `mod.rs` (whose own dir +/// is the current module), else the file's own dir. +fn super_base(file_path: &Path) -> Option { + let dir = file_path.parent()?; + if file_path.file_name().is_some_and(|name| name == "mod.rs") { + dir.parent().map(Path::to_path_buf) + } else { + Some(dir.to_path_buf()) + } +} + +/// Crate root: nearest ancestor (inclusive of the file's dir) holding +/// `main.rs` or `lib.rs`, never walking past the workspace. +fn find_crate_root(workspace: &Path, file_path: &Path) -> Option { + let mut current = file_path.parent()?.to_path_buf(); + loop { + if current.join("main.rs").is_file() || current.join("lib.rs").is_file() { + return Some(current); + } + if current == workspace { + return None; + } + current = current.parent()?.to_path_buf(); + } +} + +fn workspace_relative(workspace: &Path, path: &Path) -> Option { + path.strip_prefix(workspace) + .ok() + .map(|rel| rel.to_string_lossy().into_owned()) +} + +/// Python source lines: `#` comments cut, triple-quote doc spans +/// skipped. Crude on purpose: a `#` inside a string literal mangles +/// the tail, but item/import keywords match at line start, so the +/// mangling cannot forge a symbol. Doc-span tracking toggles on every +/// `"""` / `'''` occurrence — one-liner docstrings toggle twice (no +/// net change); mixed kinds on one line are a documented lookalike. +fn py_code_lines(text: &str) -> impl Iterator { + let mut in_doc = false; + text.lines().enumerate().filter_map(move |(idx, raw)| { + let mut line = raw.trim(); + if in_doc { + let fences = line.matches("\"\"\"").count() + line.matches("'''").count(); + if fences % 2 == 1 { + in_doc = false; + } + return None; + } + if let Some(hash) = line.find('#') { + line = line[..hash].trim_end(); + } + let fences = line.matches("\"\"\"").count() + line.matches("'''").count(); + if fences % 2 == 1 { + in_doc = true; + return None; + } + if fences > 0 { + // One-liner docstring as the whole logical line: nothing + // else on it can be a symbol. + let stripped = line.replace("\"\"\"", "").replace("'''", ""); + if stripped.trim().is_empty() { + return None; + } + } + if line.is_empty() { + return None; + } + Some((idx + 1, line)) + }) +} + +/// `def name` / `class Name` (any indent — methods included), +/// `async def` unwrapped. +fn parse_py_item(line: &str) -> Option { + let line = line.trim_start(); + let line = strip_keyword(line, "async").unwrap_or(line).trim_start(); + for (keyword, label) in [("def", "def"), ("class", "class")] { + if let Some(rest) = strip_keyword(line, keyword) { + let name: String = rest + .trim_start() + .chars() + .take_while(|next| next.is_alphanumeric() || *next == '_') + .take(MAX_NAME_LEN + 1) + .collect(); + if !name.is_empty() + && name.len() <= MAX_NAME_LEN + && !name.starts_with(|c: char| c.is_ascii_digit()) + { + return Some(format!("{label} {name}")); + } + return None; + } + } + None +} + +/// Import roots of one line: every `import a, b.c` member, or the +/// `from` module (`a.b` → `a`; pure-relative `from . import x` → `.` +/// for depth, honest). Semicolons split statements. +fn parse_py_import_roots(line: &str) -> Vec { + let line = line.trim_start(); + let mut out = Vec::new(); + if let Some(rest) = strip_keyword(line, "import") { + for stmt in rest.split(';') { + for part in stmt.split(',') { + let part = part.split(" as ").next().unwrap_or("").trim(); + let root = part.split('.').next().unwrap_or("").trim(); + if let Some(root) = valid_root(root) { + out.push(root); + } + } + } + return out; + } + if let Some(rest) = strip_keyword(line, "from") { + let module = rest.split(" import ").next().unwrap_or("").trim(); + let root = py_module_root(module); + if !root.is_empty() { + out.push(root); + } + } + out +} + +/// Root of a Python module path: leading dots collapse to their depth +/// when no named segment leads (`from . import x` → `.`). Named roots +/// validate as identifiers; dots pass through as depth markers. +fn py_module_root(module: &str) -> String { + let dots = module.chars().take_while(|c| *c == '.').count(); + let named = module[dots..].split('.').next().unwrap_or("").trim(); + if named.is_empty() { + ".".repeat(dots.max(1)) + } else { + valid_root(named).unwrap_or_default() + } +} + +fn valid_root(root: &str) -> Option { + if root.is_empty() || root.len() > MAX_NAME_LEN || !is_ident(root) { + return None; + } + Some(root.to_string()) +} + +/// Resolve one Python import line to existing files. Relative imports +/// walk up from the file's dir; absolute ones try the file's dir (flat +/// scripts) then the workspace root (packages). Both `mod.py` and +/// `pkg/__init__.py` shapes. Existence is the honesty guard. +fn resolve_py_links(workspace: &Path, file_path: &Path, line: &str) -> Vec { + let line = line.trim_start(); + let mut out = Vec::new(); + if let Some(rest) = strip_keyword(line, "import") { + // `import a, b.c as d` (semicolons split statements first). + for stmt in rest.split(';') { + for part in stmt.split(',') { + let part = part.split(" as ").next().unwrap_or("").trim(); + let segments: Vec<&str> = part.split('.').map(str::trim).collect(); + if segments.iter().all(|s| is_ident(s)) { + out.extend(resolve_py_module(workspace, file_path, 0, &segments)); + } + } + } + return out; + } + if let Some(rest) = strip_keyword(line, "from") { + // `from [..]mod import names` — parenthesized/continued name + // lists don't matter; only the module part resolves. + let module = rest.split(" import ").next().unwrap_or("").trim(); + let dots = module.chars().take_while(|c| *c == '.').count(); + let segments: Vec<&str> = module[dots..] + .split('.') + .map(str::trim) + .filter(|s| !s.is_empty()) + .collect(); + if segments.iter().all(|s| is_ident(s)) { + // `from . import sib` names the leaf in the import list, + // not the module: resolve each imported name as a sibling. + if dots > 0 && segments.is_empty() { + if let Some(names) = rest.split(" import ").nth(1) { + for name in names + .replace(['(', ')'], " ") + .split(',') + .map(|n| n.split(" as ").next().unwrap_or("").trim()) + .filter(|n| is_ident(n)) + { + out.extend(resolve_py_module( + workspace, + file_path, + dots, + std::slice::from_ref(&name), + )); + } + } + return out; + } + out.extend(resolve_py_module(workspace, file_path, dots, &segments)); + } + return out; + } + out +} + +/// Resolve Python module segments: `dots` levels up from the file's dir +/// (0 = absolute: same dir first, workspace root as fallback), then +/// `x.py` or `x/__init__.py`. Same-dir wins outright — no double links. +fn resolve_py_module( + workspace: &Path, + file_path: &Path, + dots: usize, + segments: &[&str], +) -> Vec { + if segments.is_empty() { + return Vec::new(); + } + if dots == 0 { + if let Some(dir) = file_path.parent() { + let hits = py_candidates(workspace, dir, segments); + if !hits.is_empty() { + return hits; + } + } + return py_candidates(workspace, workspace, segments); + } + let mut base = match file_path.parent() { + Some(dir) => dir.to_path_buf(), + None => return Vec::new(), + }; + for _ in 1..dots { + match base.parent() { + Some(parent) => base = parent.to_path_buf(), + None => return Vec::new(), + } + } + py_candidates(workspace, &base, segments) +} + +/// `x.py` / `x/__init__.py` under one base, workspace-relative. +fn py_candidates(workspace: &Path, base: &Path, segments: &[&str]) -> Vec { + let joined = segments.join("/"); + [ + base.join(format!("{joined}.py")), + base.join(&joined).join("__init__.py"), + ] + .into_iter() + .filter(|candidate| candidate.is_file()) + .filter_map(|candidate| workspace_relative(workspace, &candidate)) + .collect() +} + +/// JS/TS item lines: `function`/`class`, `const`/`let`/`var` arrows, +/// `type`/`interface`/`enum`. `export`/`default`/`async`/`declare`/ +/// `abstract` unwrapped. Method shorthand skipped (indistinguishable +/// from calls at line-scan resolution — documented omission). +fn parse_js_item(line: &str) -> Option { + let mut line = line.trim_start(); + for qualifier in ["export", "default", "declare", "abstract", "async"] { + if let Some(rest) = strip_keyword(line, qualifier) { + line = rest.trim_start(); + } + } + for (keyword, label) in [ + ("function", "function"), + ("class", "class"), + ("interface", "interface"), + ("enum", "enum"), + ] { + if let Some(rest) = strip_keyword(line, keyword) { + let rest = rest + .trim_start() + .strip_prefix('*') + .unwrap_or(rest) + .trim_start(); + let name = ident_head(rest)?; + return Some(format!("{label} {name}")); + } + } + if let Some(rest) = strip_keyword(line, "type") { + let name = ident_head(rest.trim_start())?; + return Some(format!("type {name}")); + } + for qualifier in ["const", "let", "var"] { + if let Some(rest) = strip_keyword(line, qualifier) { + let rest = rest.trim_start(); + let name = ident_head(rest)?; + let after = rest[name.len()..].trim_start(); + let body = after.strip_prefix('=').unwrap_or("").trim_start(); + // Arrow/function value, not data: `= (`, `= async`, `= function`, `=>`. + if body.starts_with('(') + || strip_keyword(body, "async").is_some() + || strip_keyword(body, "function").is_some() + || body.contains("=>") + { + return Some(format!("function {name}")); + } + return None; + } + } + None +} + +fn ident_head(rest: &str) -> Option { + let name: String = rest + .chars() + .take_while(|next| next.is_alphanumeric() || *next == '_' || *next == '$') + .take(MAX_NAME_LEN + 1) + .collect(); + if name.is_empty() + || name.len() > MAX_NAME_LEN + || name.starts_with(|c: char| c.is_ascii_digit()) + { + return None; + } + Some(name) +} + +/// JS/TS import roots: relative → `.`; scoped `@s/n` kept whole; +/// otherwise the first path segment. Covers `import`, `export-from`, +/// `require`, and dynamic `import()`. +fn parse_js_import_root(line: &str) -> Option { + let spec = js_import_spec(line)?; + if spec.starts_with('.') { + return Some(".".to_string()); + } + if let Some(scoped) = spec.strip_prefix('@') { + let mut parts = scoped.split('/'); + let (scope, name) = (parts.next()?, parts.next()?); + if scope.is_empty() || name.is_empty() { + return None; + } + return Some(format!("@{scope}/{name}")); + } + let root = spec.split('/').next()?.trim(); + valid_root(root) +} + +/// The module specifier of an import-ish line, unquoted. +fn js_import_spec(line: &str) -> Option { + let line = line.trim_start(); + // Side-effect and from-forms: `import 'x'`, `import … from 'x'`, `export … from 'x'`. + if strip_keyword(line, "import").is_some() || strip_keyword(line, "export").is_some() { + return quoted_tail(line); + } + // `require('x')`, `import('x')` anywhere a call starts the line. + // The opener needs a boundary behind it: `myrequire(` and a call + // spelled inside a string literal (`"require('x')"`) must not forge + // imports. + for opener in ["require(", "import("] { + // Every occurrence: a `myrequire(` earlier on the line must not + // shadow a real call later on it. + let mut start = 0; + while let Some(rel) = line[start..].find(opener) { + let at = start + rel; + let bounded = line[..at].chars().next_back().is_none_or(|prev| { + !(prev.is_alphanumeric() + || prev == '_' + || prev == '$' + || prev == '\'' + || prev == '"' + || prev == '`') + }); + start = at + opener.len(); + if !bounded { + continue; + } + let after = line[start..].trim_start(); + // A computed argument (`require(name)`) is not a specifier; + // keep scanning rather than bailing on the whole line. + let Some(quote) = after.chars().next() else { + continue; + }; + if quote == '\'' || quote == '"' || quote == '`' { + return after[1..].split(quote).next().map(str::to_string); + } + } + } + None +} + +/// The module specifier: first quoted string after `from` when present +/// (`import … from 'x'`), else the first quoted string on the line +/// (side-effect `import 'x'`). Scoping to `from` keeps a trailing +/// string literal (`log("hi")`) from shadowing the specifier, and the +/// earliest opening quote of any kind wins, so a later apostrophe in +/// a trailing expression can never shadow the specifier either. +fn quoted_tail(line: &str) -> Option { + // Both separators are 6 bytes; the max rfind wins either way. + let tail = [" from ", " from\t"] + .iter() + .filter_map(|sep| line.rfind(sep)) + .max() + .map(|at| &line[at + 6..]) + .unwrap_or(line); + let (pos, quote) = ['\'', '"', '`'] + .iter() + .filter_map(|quote| tail.find(*quote).map(|pos| (pos, *quote))) + .min()?; + // `pos` is the byte index of a one-byte quote, so `pos + 1` is a + // boundary. + tail[pos + 1..] + .split(quote) + .next() + .filter(|spec| !spec.is_empty()) + .map(str::to_string) +} + +/// JS/TS extension search order for relative links. +const JS_LINK_EXTENSIONS: [&str; 8] = ["js", "ts", "jsx", "tsx", "mjs", "cjs", "mts", "cts"]; + +/// Resolve relative JS/TS specifiers (`./x`, `../x`) against the file's +/// dir: bare, +extensions, +`/index`+extensions. Bare package names +/// never resolve (node_modules is not walked). Query/hash suffixes cut. +fn resolve_js_link(workspace: &Path, file_path: &Path, line: &str) -> Option { + let mut spec = js_import_spec(line)?; + if !spec.starts_with('.') { + return None; + } + spec = spec.split(['?', '#']).next()?.to_string(); + let dir = file_path.parent()?; + let target = dir.join(&spec); + let mut candidates = vec![target.clone()]; + for ext in JS_LINK_EXTENSIONS { + candidates.push(target.with_extension(ext)); + } + for ext in JS_LINK_EXTENSIONS { + candidates.push(target.join(format!("index.{ext}"))); + } + for candidate in candidates { + if candidate.is_file() { + return workspace_relative(workspace, &candidate).map(|rel| dot_clean(&rel)); + } + } + None +} + +/// Lexically collapse `a/../b` (no FS touch — `canonicalize` would +/// resolve symlinks like `/tmp` and break the workspace prefix). +fn dot_clean(rel: &str) -> String { + let mut parts: Vec<&str> = Vec::new(); + for part in rel.split('/') { + if part == ".." { + parts.pop(); + } else if !part.is_empty() && part != "." { + parts.push(part); + } + } + parts.join("/") +} + +/// Go item lines: `func Name`, methods as `func Type.Method`, +/// `type Name …`. +fn parse_go_item(line: &str) -> Option { + let line = line.trim_start(); + if let Some(rest) = strip_keyword(line, "func") { + let rest = rest.trim_start(); + if let Some(after) = rest.strip_prefix('(') { + // Method: receiver `(name Type)` or `(Type)`. + let receiver = after.split(')').next()?.trim(); + let typ = receiver.split_whitespace().last()?; + let typ = typ.trim_start_matches('*').trim_start_matches("[]"); + let after = after.split(')').nth(1)?.trim_start(); + let name = ident_head(after)?; + if typ.is_empty() || !is_ident(typ) { + return None; + } + return Some(format!("func {typ}.{name}")); + } + let name = ident_head(rest)?; + return Some(format!("func {name}")); + } + if let Some(rest) = strip_keyword(line, "type") { + let rest = rest.trim_start(); + // Single decl only; `type ( … )` blocks list names bare — + // resolving those needs block tracking, documented omission. + if rest.starts_with('(') { + return None; + } + let name = ident_head(rest)?; + return Some(format!("type {name}")); + } + None +} + +/// Go import roots: last path segment (package name by convention). +/// Covers `import` lines (with optional alias) and the bare quoted +/// entries of `import ( … )` blocks. Anything else starting with an +/// identifier (`return "x"`) is not an import — the alias shape is +/// indistinguishable at line resolution, so keyword-less aliased +/// entries are a documented miss. +fn parse_go_import_root(line: &str) -> Option { + let line = line.trim_start(); + let rest = match strip_keyword(line, "import") { + Some(rest) => { + let rest = rest.trim_start(); + // Optional alias: `name "…"`, `_ "…"`, `. "…"`. + if rest.starts_with('"') || rest.starts_with('`') { + rest + } else { + rest.split_whitespace().nth(1)?.trim_start() + } + } + None => { + if line.starts_with('"') || line.starts_with('`') { + line + } else { + return None; + } + } + }; + let quote = rest.chars().next()?; + if quote != '"' && quote != '`' { + return None; + } + let path = rest[1..].split(quote).next()?; + let root = path.split('/').next_back()?.trim(); + valid_root(root) +} + +/// Reverse links: podmates whose own resolved links include this file. +/// The echo coming back. Exact by construction (reuses the link +/// resolver, no name guessing) and bounded (same-dir mates only, +/// oversized mates skipped, visibility-filtered like the pod itself). +fn sound_callers(workspace: &Path, file_path: &Path) -> (Vec, usize) { + let own = match workspace_relative(workspace, file_path) { + Some(own) => own, + None => return (Vec::new(), 0), + }; + let parent = match file_path.parent() { + Some(parent) => parent, + None => return (Vec::new(), 0), + }; + let mates = list_dir_entries(parent).unwrap_or_default(); + let mut callers = BTreeSet::new(); + for (raw, _) in mates { + let mate_path = parent.join(&raw); + let Some(mate_lang) = detect_lang(&mate_path) else { + continue; + }; + if mate_path.as_path() == file_path { + continue; + } + if std::fs::metadata(&mate_path).is_ok_and(|meta| meta.len() > MAX_CALLER_SCAN_BYTES) { + continue; + } + let Ok(text) = std::fs::read_to_string(&mate_path) else { + continue; + }; + let (links, _) = sound_links(workspace, &mate_path, &text, mate_lang); + if links.iter().any(|link| link == &own) + && let Some(rel) = workspace_relative(workspace, &mate_path) + { + callers.insert(rel); + } + } + let total = callers.len(); + (callers.into_iter().collect(), total) +} + +fn truncate_name(head: &str) -> String { + if head.chars().count() <= MAX_NAME_LEN { + return head.to_string(); + } + let kept: String = head.chars().take(MAX_NAME_LEN).collect(); + format!("{kept}…") +} + +fn count_noun(count: usize, singular: &str, plural: &str) -> String { + if count == 1 { + format!("1 {singular}") + } else { + format!("{count} {plural}") + } +} + +fn render_capped(items: &[String], total: usize) -> String { + let mut out = items.join(", "); + if total > items.len() { + if !out.is_empty() { + out.push_str(", "); + } + out.push_str(&format!("+{} more", total - items.len())); + } + out +} + +/// Hard byte cap at a char boundary, with an ellipsis marker when cut. +fn truncate_bytes(text: &str, max_bytes: usize) -> String { + if text.len() <= max_bytes { + return text.to_string(); + } + let mut end = max_bytes.saturating_sub("…".len()); + while !text.is_char_boundary(end) { + end = end.saturating_sub(1); + } + format!("{}…", &text[..end]) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use std::time::{Duration, Instant}; + + fn open_visibility(glob_root: &Path) -> PodVisibility<'_> { + PodVisibility { + include: &[], + exclude: &[], + extensions: &[], + glob_root, + gitignore: false, + } + } + + #[test] + fn miss_echo_names_parent_mates_for_typoed_paths() { + let dir = tempfile::tempdir().expect("tempdir"); + for name in ["alpha.rs", "beta.rs", "gamma.rs"] { + fs::write(dir.path().join(name), "x").expect("fixture"); + } + let echo = sound_miss_echo(&dir.path().join("alpah.rs")).expect("echo"); + assert!(echo.starts_with("\n\n[Miss echo: "), "{echo}"); + assert!(echo.contains("alpha.rs"), "{echo}"); + assert!(echo.contains("beta.rs"), "{echo}"); + assert!(echo.contains("gamma.rs"), "{echo}"); + assert!(!echo.contains("alpah.rs"), "{echo}"); + } + + #[test] + fn miss_echo_caps_mates_and_reports_the_rest() { + let dir = tempfile::tempdir().expect("tempdir"); + for i in 0..12 { + fs::write(dir.path().join(format!("f{i:02}.rs")), "x").expect("fixture"); + } + let echo = sound_miss_echo(&dir.path().join("nope.rs")).expect("echo"); + assert!(echo.contains("+4 more"), "{echo}"); + assert!(echo.len() < 600, "miss echo stays small: {echo}"); + } + + #[test] + fn miss_echo_is_none_when_parent_is_unlistable() { + let dir = tempfile::tempdir().expect("tempdir"); + let missing = dir.path().join("no-such-dir").join("nope.rs"); + assert_eq!(sound_miss_echo(&missing), None); + } + + const SAMPLE: &str = r#"use std::collections::BTreeSet; +use std::path::Path; +use crate::tools::echolocation::sound_echolocation; +use super::MAX_SYMBOLS; +pub use serde_json::Value as JsonValue; +use tokio::fs::{ + self, + File, +}; + +// a comment mentioning fn ghost +/// doc comment: struct Phantom +/* block comment: enum Shade */ +#[derive(Debug)] +pub struct Whale { + name: String, +} + +pub(crate) async fn swim() {} +unsafe extern "C" fn breach() {} +const DEPTH: u32 = 11_000; +static CURRENT: &str = "cold"; +type Pod = Vec; +trait Navigable {} +impl Navigable for Whale {} +impl Whale { + fn dive(&self) {} +} +mod abyss; +macro_rules! song { + () => {}; +} +fnx not_an_item; +define nothing; +"#; + + #[test] + fn items_sound_in_file_order() { + let symbols = sound_symbols(SAMPLE, SoundLang::Rust); + let names: Vec<&str> = symbols.iter().map(String::as_str).collect(); + assert_eq!( + names, + [ + "struct Whale:15", + "fn swim:19", + "fn breach:20", + "const DEPTH:21", + "static CURRENT:22", + "type Pod:23", + "trait Navigable:24", + "impl Navigable for Whale:25", + "impl Whale:26", + "fn dive:27", + "mod abyss:29", + "macro song:30", + ] + ); + } + + #[test] + fn import_roots_dedupe_and_sort() { + let roots: Vec = sound_import_roots(SAMPLE, SoundLang::Rust) + .into_iter() + .collect(); + assert_eq!(roots, ["crate", "serde_json", "std", "super", "tokio"]); + } + + #[test] + fn block_comments_stay_silent_across_lines() { + let text = "/*\nfn hidden_one() {}\nuse hidden::root;\n*/\nfn visible() {}\n"; + assert_eq!(sound_symbols(text, SoundLang::Rust), ["fn visible:5"]); + assert!(sound_import_roots(text, SoundLang::Rust).is_empty()); + } + + #[test] + fn code_after_a_closed_comment_still_sounds() { + assert_eq!( + sound_symbols("/* note */ fn after() {}\n", SoundLang::Rust), + ["fn after:1"] + ); + assert_eq!( + sound_symbols("let x = 1; /* fn nope() {} */\n", SoundLang::Rust), + Vec::::new() + ); + } + + #[test] + fn podmates_sort_mark_and_cap() { + let dir = tempfile::tempdir().expect("tempdir"); + for name in ["zebra.rs", "alpha.rs", "main.rs", ".hidden", "beta"] { + fs::write(dir.path().join(name), "x").expect("fixture"); + } + fs::create_dir(dir.path().join("shoal")).expect("subdir"); + let chart = sound_echolocation( + dir.path(), + &dir.path().join("main.rs"), + "fn main() {}\n", + false, + ); + assert_eq!(chart.podmate_total, 5); + assert_eq!( + chart.podmates, + [ + "alpha.rs", + "beta", + "main.rs (this file)", + "shoal/", + "zebra.rs", + ] + ); + } + + #[test] + fn long_pods_truncate_with_a_count() { + let dir = tempfile::tempdir().expect("tempdir"); + for index in 0..20 { + fs::write(dir.path().join(format!("file-{index:02}.rs")), "x").expect("fixture"); + } + let chart = sound_echolocation(dir.path(), &dir.path().join("file-00.rs"), "x\n", false); + assert_eq!(chart.podmate_total, 20); + assert_eq!(chart.podmates.len(), MAX_PODMATES); + let footer = chart.render_footer(); + assert!(footer.contains("+8 more"), "{footer}"); + assert!(footer.len() <= POD_FOOTER_MAX_BYTES); + } + + #[test] + fn footer_is_deterministic_and_budgeted() { + let dir = tempfile::tempdir().expect("tempdir"); + let long = "n".repeat(200); + let mut text = String::new(); + for index in 0..100 { + text.push_str(&format!("fn {long}_{index}() {{}}\n")); + } + fs::write(dir.path().join("big.rs"), &text).expect("fixture"); + let path = dir.path().join("big.rs"); + let first = sound_echolocation(dir.path(), &path, &text, false).render_footer(); + let second = sound_echolocation(dir.path(), &path, &text, false).render_footer(); + assert_eq!(first, second); + assert!(first.len() <= POD_FOOTER_MAX_BYTES, "{}", first.len()); + assert!(first.starts_with("\n\n[Pod:"), "{first}"); + } + + #[test] + fn prose_files_get_podmates_but_no_sounding() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("notes.txt"), "fn not_rust() {}\n").expect("fixture"); + let footer = sound_echolocation( + dir.path(), + &dir.path().join("notes.txt"), + "fn not_rust() {}\n", + false, + ) + .render_footer(); + assert!(footer.contains("[Pod:"), "{footer}"); + assert!(!footer.contains("[Sounding:"), "{footer}"); + } + + #[test] + fn unlistable_non_source_renders_empty() { + let chart = sound_echolocation(Path::new("."), Path::new("lonely"), "hello\n", false); + assert_eq!(chart.render_footer(), ""); + } + + #[test] + fn whole_footer_shape() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("lib.rs"), SAMPLE).expect("fixture"); + fs::write(dir.path().join("main.rs"), "fn main() {}\n").expect("fixture"); + let footer = sound_echolocation(dir.path(), &dir.path().join("lib.rs"), SAMPLE, false) + .render_footer(); + assert!(footer.contains("12 symbols"), "{footer}"); + assert!(footer.contains("fn swim"), "{footer}"); + assert!( + footer.contains("imports: crate, serde_json, std, super, tokio"), + "{footer}" + ); + assert!(footer.contains("lib.rs (this file), main.rs"), "{footer}"); + } + + #[test] + fn match_pods_group_mark_and_sort() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::create_dir(dir.path().join("net")).expect("subdir"); + fs::create_dir(dir.path().join("store")).expect("subdir"); + for name in [ + "net/server.rs", + "net/client.rs", + "store/cache.rs", + "main.rs", + ] { + fs::write(dir.path().join(name), "x").expect("fixture"); + } + let vis = open_visibility(dir.path()); + let (pods, omitted) = sound_match_pods( + dir.path(), + &["store/cache.rs".to_string(), "net/server.rs".to_string()], + &vis, + ); + assert_eq!(omitted, 0); + assert_eq!(pods.len(), 2); + assert_eq!(pods[0].dir, "net"); + assert_eq!(pods[0].mates, ["client.rs", "server.rs (match)"]); + assert_eq!(pods[1].dir, "store"); + assert_eq!(pods[1].mates, ["cache.rs (match)"]); + } + + #[test] + fn match_pods_cap_dirs_and_mates() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut files = Vec::new(); + for index in 0..12 { + let sub = format!("pod-{index:02}"); + fs::create_dir(dir.path().join(&sub)).expect("subdir"); + for mate in 0..12 { + fs::write(dir.path().join(&sub).join(format!("m{mate}.rs")), "x").expect("fixture"); + } + files.push(format!("{sub}/m0.rs")); + } + let vis = open_visibility(dir.path()); + let (pods, omitted) = sound_match_pods(dir.path(), &files, &vis); + assert_eq!(pods.len(), MAX_GREP_PODS); + assert_eq!(omitted, 12 - MAX_GREP_PODS); + assert_eq!(pods[0].mate_total, 12); + assert!(pods[0].mates.last().expect("cap").starts_with("+4 more")); + } + + #[test] + fn match_pods_chart_matched_files_when_dir_unlistable() { + let dir = tempfile::tempdir().expect("tempdir"); + let vis = open_visibility(dir.path()); + let (pods, omitted) = sound_match_pods( + dir.path(), + &["gone/deep.rs".to_string(), "".to_string()], + &vis, + ); + assert_eq!(omitted, 0); + assert_eq!(pods.len(), 1); + assert_eq!(pods[0].dir, "gone"); + assert_eq!(pods[0].mates, ["deep.rs (match)"]); + } + + #[test] + fn match_pods_honor_excludes_and_includes() { + let dir = tempfile::tempdir().expect("tempdir"); + for name in ["keep.rs", "skip.log", "notes.md"] { + fs::write(dir.path().join(name), "x").expect("fixture"); + } + let open = open_visibility(dir.path()); + let exclude = ["*.log".to_string()]; + let vis = PodVisibility { + exclude: &exclude, + ..open + }; + let (pods, _) = sound_match_pods(dir.path(), &["keep.rs".to_string()], &vis); + assert_eq!(pods[0].mates, ["keep.rs (match)", "notes.md"]); + let include = ["*.rs".to_string()]; + let vis = PodVisibility { + include: &include, + ..open + }; + let (pods, _) = sound_match_pods(dir.path(), &["keep.rs".to_string()], &vis); + assert_eq!(pods[0].mates, ["keep.rs (match)"]); + } + + #[test] + fn match_pods_honor_gitignore_stacking() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join(".gitignore"), "ignored.txt\n").expect("fixture"); + for name in ["ignored.txt", "keep.txt"] { + fs::write(dir.path().join(name), "x").expect("fixture"); + } + let open = open_visibility(dir.path()); + let vis = PodVisibility { + gitignore: true, + ..open + }; + let (pods, _) = sound_match_pods(dir.path(), &["keep.txt".to_string()], &vis); + assert_eq!(pods[0].mates, ["keep.txt (match)"]); + } + + #[test] + fn match_pods_hide_denied_siblings() { + let dir = tempfile::tempdir().expect("tempdir"); + let secret = dir.path().join("secret.txt"); + for name in ["open.txt", "secret.txt"] { + fs::write(dir.path().join(name), "x").expect("fixture"); + } + let prior = crate::sandbox::read_guard::active(); + crate::sandbox::read_guard::set_active(crate::sandbox::read_guard::ReadDenylist::build( + true, + &[secret], + &[], + )); + let vis = open_visibility(dir.path()); + let (pods, _) = sound_match_pods(dir.path(), &["open.txt".to_string()], &vis); + crate::sandbox::read_guard::set_active((*prior).clone()); + assert_eq!(pods[0].mates, ["open.txt (match)"]); + } + + #[test] + fn key_files_sound_rust_and_skip_the_rest() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("main.rs"), "fn main() {}\nfn serve() {}\n").expect("fixture"); + fs::write(dir.path().join("notes.md"), "# hi\n").expect("fixture"); + let (soundings, omitted) = + sound_key_files(dir.path(), &["notes.md".to_string(), "main.rs".to_string()]); + assert_eq!(omitted, 0); + assert_eq!(soundings.len(), 1); + assert_eq!( + soundings.get("main.rs").map(String::as_str), + Some("2 symbols: fn main:1, fn serve:2") + ); + } + + #[test] + fn links_resolve_crate_super_self_and_mod_edges() { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path(); + for path in [ + "src/main.rs", + "src/config.rs", + "src/net/mod.rs", + "src/net/server.rs", + "src/net/http.rs", + ] { + let full = root.join(path); + fs::create_dir_all(full.parent().expect("parent")).expect("mkdir"); + fs::write(&full, "x\n").expect("fixture"); + } + // crate:: + super:: + external (skipped) + missing (skipped). + let text = "use crate::config::Config;\n\ + use crate::net::http::Handler;\n\ + use super::server::serve;\n\ + use tokio::fs::File;\n\ + use crate::missing::Thing;\n"; + let (links, total) = + sound_links(root, &root.join("src/net/http.rs"), text, SoundLang::Rust); + // crate::net::http resolves to the file itself: dropped as a self-link. + assert_eq!(total, 2); + assert_eq!(links, ["src/config.rs", "src/net/server.rs"]); + // self:: + mod decl from a mod.rs. + let (links, _) = sound_links( + root, + &root.join("src/net/mod.rs"), + "use self::http::Handler;\nmod server;\nmod absent;\n", + SoundLang::Rust, + ); + assert_eq!(links, ["src/net/http.rs", "src/net/server.rs"]); + // mod decl from a plain file resolves under /. + fs::create_dir_all(root.join("src/net/http")).expect("mkdir"); + fs::write(root.join("src/net/http/router.rs"), "x\n").expect("fixture"); + let (links, _) = sound_links( + root, + &root.join("src/net/http.rs"), + "mod router;\nmod nope;\n", + SoundLang::Rust, + ); + assert_eq!(links, ["src/net/http/router.rs"]); + } + + #[test] + fn links_skip_self_and_inline_mods() { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path(); + fs::create_dir_all(root.join("src")).expect("mkdir"); + fs::write(root.join("src/main.rs"), "fn main() {}\n").expect("fixture"); + fs::write(root.join("src/config.rs"), "pub struct Config;\n").expect("fixture"); + // `use crate::config` from config.rs itself: self-link dropped. + // Inline `mod tests {` is a symbol, never an edge. + let (links, total) = sound_links( + root, + &root.join("src/config.rs"), + "use crate::config::Config;\nmod tests {\n}\n", + SoundLang::Rust, + ); + assert_eq!(total, 0); + assert!(links.is_empty()); + } + + #[test] + fn links_render_in_footer() { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path(); + fs::create_dir_all(root.join("src")).expect("mkdir"); + fs::write(root.join("src/main.rs"), "fn main() {}\n").expect("fixture"); + let footer = sound_echolocation( + root, + &root.join("src/main.rs"), + "use crate::config::Config;\nfn main() {}\n", + false, + ) + .render_footer(); + // config.rs does not exist: imports show, links stay silent. + assert!(footer.contains("imports: crate"), "{footer}"); + assert!(!footer.contains("links:"), "{footer}"); + fs::write(root.join("src/config.rs"), "pub struct Config;\n").expect("fixture"); + let footer = sound_echolocation( + root, + &root.join("src/main.rs"), + "use crate::config::Config;\nfn main() {}\n", + false, + ) + .render_footer(); + assert!(footer.contains("links: src/config.rs"), "{footer}"); + } + + #[test] + fn key_files_cap_with_omitted_count() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut keys = Vec::new(); + for index in 0..12 { + let name = format!("k{index:02}.rs"); + fs::write(dir.path().join(&name), "fn f() {}\n").expect("fixture"); + keys.push(name); + } + let (soundings, omitted) = sound_key_files(dir.path(), &keys); + assert_eq!(soundings.len(), MAX_MAP_SOUNDINGS); + assert_eq!(omitted, 12 - MAX_MAP_SOUNDINGS); + let again = sound_key_files(dir.path(), &keys); + assert_eq!(again, (soundings, omitted)); + } + + #[test] + fn callers_are_exact_reverse_links() { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path(); + fs::create_dir_all(root.join("src/net")).expect("mkdir"); + fs::write(root.join("src/main.rs"), "fn main() {}\n").expect("fixture"); + fs::write( + root.join("src/net/mod.rs"), + "pub mod server;\npub mod client;\n", + ) + .expect("fixture"); + fs::write( + root.join("src/net/server.rs"), + "use crate::config::Config;\nfn serve() {}\n", + ) + .expect("fixture"); + fs::write( + root.join("src/net/client.rs"), + "use super::server::serve;\nfn connect() {}\n", + ) + .expect("fixture"); + let (callers, total) = sound_callers(root, &root.join("src/net/server.rs")); + assert_eq!(total, 2); + assert_eq!(callers, ["src/net/client.rs", "src/net/mod.rs"]); + } + + #[test] + fn callers_stay_inside_the_pod() { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path(); + fs::create_dir_all(root.join("src/net")).expect("mkdir"); + fs::write(root.join("src/main.rs"), "fn main() {}\n").expect("fixture"); + fs::write(root.join("src/config.rs"), "pub struct Config;\n").expect("fixture"); + fs::write( + root.join("src/net/server.rs"), + "use crate::config::Config;\nfn serve() {}\n", + ) + .expect("fixture"); + // server.rs links to config.rs, but lives in another pod: + // same-pod callers stay silent rather than half-answer. + let (callers, total) = sound_callers(root, &root.join("src/config.rs")); + assert_eq!(total, 0); + assert!(callers.is_empty()); + } + + #[test] + fn buzz_adds_heard_by_and_stays_deterministic() { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path(); + fs::create_dir_all(root.join("src/net")).expect("mkdir"); + fs::write(root.join("src/main.rs"), "fn main() {}\n").expect("fixture"); + fs::write(root.join("src/net/mod.rs"), "pub mod server;\n").expect("fixture"); + let text = "fn serve() {}\n"; + fs::write(root.join("src/net/server.rs"), text).expect("fixture"); + let plain = + sound_echolocation(root, &root.join("src/net/server.rs"), text, false).render_footer(); + assert!(!plain.contains("heard by"), "{plain}"); + let first = + sound_echolocation(root, &root.join("src/net/server.rs"), text, true).render_footer(); + let second = + sound_echolocation(root, &root.join("src/net/server.rs"), text, true).render_footer(); + assert_eq!(first, second); + assert!(first.contains("heard by: src/net/mod.rs"), "{first}"); + } + + #[test] + fn echo_gates_parse_off_entries_and_ignore_noise() { + assert!(parse_echo_gates("").is_empty()); + assert!(parse_echo_gates("mates").is_empty()); + let gates = parse_echo_gates("off:links, off:mates ,bogus,off:,off:links"); + assert_eq!( + gates.into_iter().collect::>(), + ["links".to_string(), "mates".to_string()] + ); + } + + const PY_SAMPLE: &str = r#"import os, sys as system +from pkg.sub import thing +from . import sibling +from ..parent import helper +import localmod + +"""Module docstring mentioning def ghost.""" + +class Whale: + """Docstring with class Phantom inside.""" + + def swim(self): + pass + + async def dive(self): + pass + +def surface(): + # def sunk: + return 1 + +define_not_a_keyword = True +"#; + + #[test] + fn python_sounds_defs_classes_and_imports() { + let symbols = sound_symbols(PY_SAMPLE, SoundLang::Python); + assert_eq!( + symbols, + [ + "class Whale:9", + "def swim:12", + "def dive:15", + "def surface:18", + ] + ); + let roots: Vec = sound_import_roots(PY_SAMPLE, SoundLang::Python) + .into_iter() + .collect(); + assert_eq!(roots, [".", "localmod", "os", "parent", "pkg", "sys"]); + } + + #[test] + fn python_links_resolve_relative_and_absolute() { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path(); + for path in [ + "app.py", + "util.py", + "pkg/__init__.py", + "pkg/mod.py", + "pkg/sub/__init__.py", + "pkg/sub/deep.py", + ] { + let full = root.join(path); + fs::create_dir_all(full.parent().expect("parent")).expect("mkdir"); + fs::write(&full, "x\n").expect("fixture"); + } + // Absolute: same dir wins; package shape at root. + let (links, _) = sound_links( + root, + &root.join("app.py"), + "import util\nfrom pkg.sub import thing\nimport os\n", + SoundLang::Python, + ); + assert_eq!(links, ["pkg/sub/__init__.py", "util.py"]); + // Relative: `from . import sib` + `from .. import x`. + let (links, _) = sound_links( + root, + &root.join("pkg/sub/deep.py"), + "from . import sib\nfrom .. import mod\nfrom ... import nowhere\n", + SoundLang::Python, + ); + assert_eq!(links, ["pkg/mod.py"]); + } + + const JS_SAMPLE: &str = r#"import React from 'react'; +import { serve } from "./server"; +import '@scope/pkg/sub'; +import './setup'; +const fs = require("fs"); +const dynamic = await import("./chunk"); + +export async function swim() {} +export default class Whale {} +const dive = (x) => x; +let data = 5; +type Alias = string; +interface Shape {} +enum Color { Red } + +if (x) { y(); } +"#; + + #[test] + fn js_sounds_functions_classes_and_imports() { + let symbols = sound_symbols(JS_SAMPLE, SoundLang::JavaScript); + assert!( + symbols.contains(&"function swim:8".to_string()), + "{symbols:?}" + ); + assert!( + symbols.contains(&"class Whale:9".to_string()), + "{symbols:?}" + ); + assert!( + symbols.contains(&"function dive:10".to_string()), + "{symbols:?}" + ); + assert!( + symbols.contains(&"type Alias:12".to_string()), + "{symbols:?}" + ); + assert!( + symbols.contains(&"interface Shape:13".to_string()), + "{symbols:?}" + ); + assert!( + symbols.contains(&"enum Color:14".to_string()), + "{symbols:?}" + ); + assert!(!symbols.iter().any(|s| s.contains("data")), "{symbols:?}"); + let roots: Vec = sound_import_roots(JS_SAMPLE, SoundLang::JavaScript) + .into_iter() + .collect(); + assert_eq!(roots, [".", "@scope/pkg", "fs", "react"]); + } + + #[test] + fn js_links_resolve_relative_only() { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path(); + fs::create_dir_all(root.join("src/ui")).expect("mkdir"); + for path in ["src/app.ts", "src/server.ts", "src/ui/index.tsx"] { + fs::write(root.join(path), "x\n").expect("fixture"); + } + let (links, _) = sound_links( + root, + &root.join("src/app.ts"), + "import { s } from './server';\nimport { U } from './ui';\nimport _ from 'lodash';\n", + SoundLang::JavaScript, + ); + assert_eq!(links, ["src/server.ts", "src/ui/index.tsx"]); + } + + const GO_SAMPLE: &str = r#"package net + +import ( + "fmt" +) + +import "os" +import alias "example.com/foo/bar" +import _ "example.com/foo/side" + +func Serve() {} +func (s *Server) Handle() {} +func (s Server) Close() {} +type Server struct{} +type ( + Grouped int +) + +func main() { + fmt.Println("hi") + return "literal" +} +"#; + + #[test] + fn go_sounds_funcs_methods_types_and_imports() { + let symbols = sound_symbols(GO_SAMPLE, SoundLang::Go); + assert!( + symbols.contains(&"func Serve:11".to_string()), + "{symbols:?}" + ); + assert!( + symbols.contains(&"func Server.Handle:12".to_string()), + "{symbols:?}" + ); + assert!( + symbols.contains(&"func Server.Close:13".to_string()), + "{symbols:?}" + ); + assert!( + symbols.contains(&"type Server:14".to_string()), + "{symbols:?}" + ); + assert!( + !symbols.iter().any(|s| s.contains("Grouped")), + "{symbols:?}" + ); + let roots: Vec = sound_import_roots(GO_SAMPLE, SoundLang::Go) + .into_iter() + .collect(); + assert_eq!(roots, ["bar", "fmt", "os", "side"]); + // Go names packages, not files: no links by design. + let dir = tempfile::tempdir().expect("tempdir"); + let (links, total) = sound_links( + dir.path(), + &dir.path().join("x.go"), + GO_SAMPLE, + SoundLang::Go, + ); + assert_eq!(total, 0); + assert!(links.is_empty()); + } + + #[test] + fn detect_lang_routes_by_extension() { + assert_eq!(detect_lang(Path::new("a.rs")), Some(SoundLang::Rust)); + assert_eq!(detect_lang(Path::new("a.py")), Some(SoundLang::Python)); + assert_eq!(detect_lang(Path::new("a.pyi")), Some(SoundLang::Python)); + assert_eq!(detect_lang(Path::new("a.ts")), Some(SoundLang::JavaScript)); + assert_eq!(detect_lang(Path::new("a.tsx")), Some(SoundLang::JavaScript)); + assert_eq!(detect_lang(Path::new("a.jsx")), Some(SoundLang::JavaScript)); + assert_eq!(detect_lang(Path::new("a.mjs")), Some(SoundLang::JavaScript)); + assert_eq!(detect_lang(Path::new("a.go")), Some(SoundLang::Go)); + assert_eq!(detect_lang(Path::new("a.txt")), None); + assert_eq!(detect_lang(Path::new("Makefile")), None); + } + + #[test] + fn scanners_survive_adversarial_text() { + let long_line = "x".repeat(1_000_000); + let nasties = [ + "\0\0\0", + "fn \u{feff}\u{200b}\u{202e}evil() {}", + "/* unterminated", + "\"\"\"unterminated doc", + "def f(:\n pass", + "use \u{1f980}::x;", + "import \u{1f980}", + "func (((((", + "/* /* /* nested", + "'''\"\"\"'\"\" mixed", + "#\u{0}def fake():", + "\r\ndef crlf():\r\n", + "type (", + "export default", + "from", + "import", + long_line.as_str(), + ]; + for lang in [ + SoundLang::Rust, + SoundLang::Python, + SoundLang::JavaScript, + SoundLang::Go, + ] { + for nasty in nasties { + let first = sound_symbols(nasty, lang); + let second = sound_symbols(nasty, lang); + assert_eq!(first, second, "{lang:?} {nasty:?}"); + let _ = sound_import_roots(nasty, lang); + } + } + } + + #[test] + fn sounding_stays_bounded_and_fast_on_huge_files() { + let dir = tempfile::tempdir().expect("tempdir"); + let big = "fn f() {}\n".repeat(200_000); + let path = dir.path().join("big.rs"); + fs::write(&path, &big).expect("fixture"); + let start = Instant::now(); + let first = sound_echolocation(dir.path(), &path, &big, true).render_footer(); + let second = sound_echolocation(dir.path(), &path, &big, true).render_footer(); + assert_eq!(first, second); + assert!(first.len() <= POD_FOOTER_MAX_BYTES, "{}", first.len()); + // Absurdly generous: catches hangs and super-linear blowups, + // not machine speed (measured separately for the thesis). + assert!( + start.elapsed() < Duration::from_secs(10), + "{:?}", + start.elapsed() + ); + } + + #[test] + fn sounding_is_stable_on_real_repo_files() { + // Byte-stability on the real thing (not just fixtures), with + // wall-time reported for cost analysis (`--nocapture`). + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let mut sounded = 0; + for rel in [ + "src/core/engine/turn_loop.rs", + "src/tools/spec.rs", + "src/tools/echolocation.rs", + ] { + let path = root.join(rel); + let text = fs::read_to_string(&path).expect("repo file"); + let first = sound_echolocation(root, &path, &text, true).render_footer(); + let second = sound_echolocation(root, &path, &text, true).render_footer(); + assert_eq!(first, second, "{rel}"); + assert!(first.len() <= POD_FOOTER_MAX_BYTES, "{rel}"); + sounded += 1; + } + assert_eq!(sounded, 3); + } + + #[test] + fn non_rust_footer_renders_full_sounding() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("swim.py"), PY_SAMPLE).expect("fixture"); + fs::write(dir.path().join("other.py"), "x = 1\n").expect("fixture"); + let footer = sound_echolocation(dir.path(), &dir.path().join("swim.py"), PY_SAMPLE, false) + .render_footer(); + assert!(footer.contains("[Pod:"), "{footer}"); + assert!(footer.contains("def swim"), "{footer}"); + assert!(footer.contains("imports:"), "{footer}"); + } + + #[test] + fn crate_root_self_and_mod_edges_resolve_beside_the_root() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("lib.rs"), + "mod pod;\nuse self::song::Sing;\n", + ) + .expect("fixture"); + fs::write(dir.path().join("pod.rs"), "fn swim() {}\n").expect("fixture"); + fs::create_dir(dir.path().join("song")).expect("mkdir"); + fs::write(dir.path().join("song/mod.rs"), "fn sing() {}\n").expect("fixture"); + let text = fs::read_to_string(dir.path().join("lib.rs")).expect("read"); + let (links, _) = sound_links( + dir.path(), + &dir.path().join("lib.rs"), + &text, + SoundLang::Rust, + ); + assert!(links.contains(&"pod.rs".to_string()), "{links:?}"); + assert!(links.contains(&"song/mod.rs".to_string()), "{links:?}"); + } + + #[test] + fn earliest_quote_wins_the_specifier() { + assert_eq!( + js_import_spec("import x from \"mod\"; foo('bar')"), + Some("mod".to_string()) + ); + assert_eq!( + js_import_spec("import x from 'mod'; foo(\"bar\")"), + Some("mod".to_string()) + ); + assert_eq!( + js_import_spec("import 'side-effect'"), + Some("side-effect".to_string()) + ); + } + + #[test] + fn require_opener_needs_a_boundary() { + // `myrequire(` forges nothing … + assert_eq!(js_import_spec("const x = myrequire('a')"), None); + // … nor does a call spelled inside a string … + assert_eq!(js_import_spec("const s = \"require('a')\""), None); + // … but a real call later on the same line still sounds … + assert_eq!( + js_import_spec("myrequire('a'); const x = require('b')"), + Some("b".to_string()) + ); + // … as does a computed argument followed by a real one. + assert_eq!( + js_import_spec("require(name); require('c')"), + Some("c".to_string()) + ); + } + + #[test] + fn digit_led_tokens_are_never_names() { + assert!(sound_symbols("fn 1x() {}\nmod 2y;\n", SoundLang::Rust).is_empty()); + assert!(sound_symbols("def 2y():\n", SoundLang::Python).is_empty()); + assert!(sound_import_roots("use 123;\n", SoundLang::Rust).is_empty()); + assert_eq!( + sound_symbols("fn fine() {}\n", SoundLang::Rust), + ["fn fine:1"] + ); + } + + #[test] + fn minified_runs_never_blow_up_a_symbol() { + let run = "a".repeat(10_000); + let symbols = sound_symbols(&format!("fn {run}() {{}}\n"), SoundLang::Rust); + assert_eq!(symbols.len(), 1); + assert!(symbols[0].len() <= MAX_NAME_LEN + 8, "{}", symbols[0].len()); + let js = sound_symbols(&format!("function {run}() {{}}\n"), SoundLang::JavaScript); + assert!(js.is_empty(), "{js:?}"); + } + + #[test] + fn buzz_read_of_a_crowded_pod_stays_fast() { + // Worst-case shape: 200 mates × 50KB each, all re-scanned for + // callers on every buzz read. The 5s bound is orders past the + // expected tens of milliseconds — it trips only on pathological + // regression, never on a loaded machine. + let dir = tempfile::tempdir().expect("tempdir"); + let body = "fn swim() {}\n".repeat(4_000); + for i in 0..200 { + fs::write(dir.path().join(format!("mate{i:03}.rs")), &body).expect("fixture"); + } + let target = dir.path().join("target.rs"); + fs::write(&target, "fn target() {}\n").expect("fixture"); + let text = fs::read_to_string(&target).expect("read"); + let start = Instant::now(); + let chart = sound_echolocation(dir.path(), &target, &text, true); + let footer = chart.render_footer(); + let elapsed = start.elapsed(); + assert!(footer.len() <= POD_FOOTER_MAX_BYTES); + assert!( + elapsed < Duration::from_secs(5), + "buzz read took {elapsed:?}" + ); + } + + #[test] + #[cfg(unix)] + fn pod_listing_skips_symlinks_into_denied_trees() { + // The denylist filter is distinct from the dotfile skip: a + // non-dot symlink into a denied tree must not enumerate. Needs a + // real ~/.ssh to resolve against; honest skip otherwise (the + // read-level twin is + // `read_file_refuses_a_workspace_symlink_pointing_at_a_denied_tree`). + let _env_lock = crate::test_support::lock_test_env(); + let Some(home) = dirs::home_dir() else { + return; + }; + if !home.join(".ssh").is_dir() { + return; + } + let dir = tempfile::tempdir().expect("tempdir"); + std::os::unix::fs::symlink(home.join(".ssh"), dir.path().join("vault_link")) + .expect("symlink"); + fs::write(dir.path().join("app.rs"), "fn main() {}\n").expect("fixture"); + let entries = list_dir_entries(dir.path()).expect("listed"); + let raws: Vec<&str> = entries.iter().map(|(raw, _)| raw.as_str()).collect(); + assert!(raws.contains(&"app.rs"), "{raws:?}"); + assert!(!raws.contains(&"vault_link"), "{raws:?}"); + } + + #[test] + fn unlistable_pod_fallback_skips_dotfiles_like_the_listed_path() { + let dir = tempfile::tempdir().expect("tempdir"); + let (pods, _) = sound_match_pods( + dir.path(), + &["nodir/.env".to_string(), "nodir/app.rs".to_string()], + &open_visibility(dir.path()), + ); + assert_eq!(pods.len(), 1); + assert!( + pods[0].mates.contains(&"app.rs (match)".to_string()), + "{:?}", + pods[0].mates + ); + assert!( + !pods[0].mates.iter().any(|mate| mate.contains(".env")), + "{:?}", + pods[0].mates + ); + } +} diff --git a/crates/tui/src/tools/file.rs b/crates/tui/src/tools/file.rs index c2b10548b2..0eed80dbab 100644 --- a/crates/tui/src/tools/file.rs +++ b/crates/tui/src/tools/file.rs @@ -774,7 +774,16 @@ impl ReadFileTool { enforce_read_denylist(&file_path, "read")?; check_file_operation_cancelled(context)?; let bytes = tokio::fs::read(&file_path).await.map_err(|error| { - ToolError::execution_failed(format!("Failed to read {}: {error}", file_path.display())) + let mut message = format!("Failed to read {}: {error}", file_path.display()); + // Miss echo: only for genuine NotFound — never for denylist or + // permission failures, where listing the parent would answer a + // refused probe. + if error.kind() == std::io::ErrorKind::NotFound + && let Some(echo) = crate::tools::echolocation::sound_miss_echo(&file_path) + { + message.push_str(&echo); + } + ToolError::execution_failed(message) })?; // #6283: every read response carries the file's byte size, line // count, and truncation flag so the caller can page deliberately @@ -863,6 +872,24 @@ impl ReadFileTool { } } + // Echolocation: stateless kin footer sounded fresh from source + // (no index, no new deps). Deterministic and hard-budgeted, so a + // repeated read is byte-identical. Two distinct files read in + // this pod put the next read there into the terminal buzz + // (callers included). + let buzz = file_path.parent().is_some_and(|parent| { + context.pod_visit_count(parent) >= crate::tools::echolocation::BUZZ_VISIT_THRESHOLD + }); + output.push_str( + &crate::tools::echolocation::sound_echolocation( + &context.workspace, + &file_path, + &text, + buzz, + ) + .render_footer(), + ); + // This internal observation keeps hidden legacy edit replay working, // but no hash or read-before-edit ceremony reaches the lowercase // schema or result. diff --git a/crates/tui/src/tools/file/tests.rs b/crates/tui/src/tools/file/tests.rs index 9d52d57b64..9d6bb14df7 100644 --- a/crates/tui/src/tools/file/tests.rs +++ b/crates/tui/src/tools/file/tests.rs @@ -78,7 +78,10 @@ async fn contract_read_returns_a_file_of_more_than_two_thousand_short_lines_whol let result = ReadFileTool::execute_contract_read(json!({"path": "many.txt"}), &context) .await .expect("read result"); - assert_eq!(result.content, content); + assert!( + result.content.starts_with(&content), + "whole file first, pod footer after" + ); assert!( !result.content.contains("[Showing lines"), "no truncation footer" @@ -86,7 +89,7 @@ async fn contract_read_returns_a_file_of_more_than_two_thousand_short_lines_whol } #[tokio::test] -async fn contract_read_returns_an_ordinary_source_file_whole_without_a_footer() { +async fn contract_read_appends_a_pod_footer_to_a_whole_source_file() { let temporary = tempfile::tempdir().expect("tempdir"); let content = "fn main() {\n println!(\"hi\");\n}\n"; std::fs::write(temporary.path().join("main.rs"), content).expect("fixture"); @@ -94,7 +97,66 @@ async fn contract_read_returns_an_ordinary_source_file_whole_without_a_footer() let result = ReadFileTool::execute_contract_read(json!({"path": "main.rs"}), &context) .await .expect("read result"); - assert_eq!(result.content, content); + assert!( + result.content.starts_with(content), + "whole file first, pod footer after" + ); + assert!( + result.content.contains("[Pod:"), + "pod footer present: {}", + result.content + ); + assert!( + result.content.contains("[Sounding: 1 symbol: fn main:1]"), + "sounded symbol present: {}", + result.content + ); +} + +#[tokio::test] +async fn contract_read_miss_echoes_parent_mates_on_not_found() { + let temporary = tempfile::tempdir().expect("tempdir"); + std::fs::write(temporary.path().join("alpha.rs"), "fn a() {}\n").expect("fixture"); + std::fs::write(temporary.path().join("beta.rs"), "fn b() {}\n").expect("fixture"); + let context = ToolContext::new(temporary.path()); + let error = ReadFileTool::execute_contract_read(json!({"path": "alpah.rs"}), &context) + .await + .expect_err("typo'd path must fail"); + match error { + ToolError::ExecutionFailed { message } => { + assert!(message.contains("Failed to read"), "{message}"); + assert!(message.contains("[Miss echo: "), "{message}"); + assert!(message.contains("alpha.rs"), "{message}"); + assert!(message.contains("beta.rs"), "{message}"); + } + other => panic!("expected ordinary read failure, got {other:?}"), + } +} + +#[tokio::test] +async fn contract_read_refused_probe_gets_no_miss_echo() { + // Security gate: a denylisted path must fail WITHOUT the miss echo — + // listing the parent would answer the refused probe. The denylist + // check runs before the IO read, so the refusal wins whether or not + // the file exists. Mirrors the home-dir fixture pattern in + // file/tests/tools.rs (the guard snapshots HOME process-wide). + let _env_lock = crate::test_support::lock_test_env(); + let Some(home) = dirs::home_dir() else { + return; + }; + let temporary = tempfile::tempdir().expect("tempdir"); + let context = ToolContext::new(temporary.path()); + let probe = home.join(".ssh").join("id_ed25519_no_such_key"); + let error = + ReadFileTool::execute_contract_read(json!({"path": probe.to_string_lossy()}), &context) + .await + .expect_err("denylisted path must fail"); + let message = error.to_string(); + assert!(message.contains("deny-list"), "{message}"); + assert!( + !message.contains("[Miss echo: "), + "refused probe must not echo: {message}" + ); } #[test] @@ -173,7 +235,10 @@ async fn contract_read_max_bytes_raises_the_budget_for_one_call() { ) .await .expect("raised budget read"); - assert_eq!(raised.content, content); + assert!( + raised.content.starts_with(&content), + "whole file first, pod footer after" + ); // Above the hard maximum clamps down; the file still fits, so it is whole. let clamped = ReadFileTool::execute_contract_read( @@ -182,7 +247,10 @@ async fn contract_read_max_bytes_raises_the_budget_for_one_call() { ) .await .expect("clamped budget read"); - assert_eq!(clamped.content, content); + assert!( + clamped.content.starts_with(&content), + "whole file first, pod footer after" + ); } #[tokio::test] @@ -201,17 +269,14 @@ async fn contract_read_paginates_an_oversized_file_with_an_honest_budget_footer( let first = ReadFileTool::execute_contract_read(json!({"path": "big.txt"}), &context) .await .expect("first page"); - let footer = first - .content - .rsplit_once("\n\n") - .expect("footer present") - .1 - .to_string(); - assert_eq!( - footer, - "[Showing lines 1-100 of 2000 (1.9MB total, 100000-byte output budget). Use offset=101 to continue, or max_bytes up to 500000 to read more per call.]" + assert!( + first.content.contains( + "[Showing lines 1-100 of 2000 (1.9MB total, 100000-byte output budget). Use offset=101 to continue, or max_bytes up to 500000 to read more per call.]" + ), + "paging footer exact: {}", + first.content ); - let shown = first.content.rsplit_once("\n\n").expect("body").0; + let shown = first.content.split("\n\n[").next().expect("body"); assert_eq!(shown.lines().count(), 100); // The named continuation offset is exact: page two starts on line 101. @@ -307,8 +372,8 @@ async fn contract_read_pages_stay_bounded_and_cover_the_whole_file() { ); let body = result .content - .rsplit_once("\n\n[") - .map(|(body, _)| body) + .split("\n\n[") + .next() .unwrap_or(&result.content); seen.extend(body.lines().map(str::to_string)); let truncated = metadata["truncated"].as_bool().expect("truncated flag"); @@ -323,7 +388,7 @@ async fn contract_read_pages_stay_bounded_and_cover_the_whole_file() { } /// #6283: ordinary whole reads carry the same paging metadata (with -/// truncated=false) and keep their footer-free shape. +/// truncated=false); the pod footer follows the whole file. #[tokio::test] async fn contract_read_metadata_for_ordinary_whole_read() { let temporary = tempfile::tempdir().expect("tempdir"); @@ -334,7 +399,15 @@ async fn contract_read_metadata_for_ordinary_whole_read() { let result = ReadFileTool::execute_contract_read(json!({"path": "small.txt"}), &context) .await .expect("read result"); - assert_eq!(result.content, content); + assert!( + result.content.starts_with(content), + "whole file first, pod footer after" + ); + assert!( + result.content.contains("[Pod:"), + "pod footer present: {}", + result.content + ); let metadata = result.metadata.clone().expect("paging metadata"); assert_eq!(metadata["size"], content.len() as u64); assert_eq!(metadata["truncated"], false); @@ -399,9 +472,12 @@ async fn contract_read_reports_huge_first_line_with_exact_bash_fallback() { let result = ReadFileTool::execute_contract_read(json!({"path": "huge.txt"}), &context) .await .expect("read result"); - assert_eq!( - result.content, - "[Line 1 is 97.7KB, exceeds the 100000-byte output budget for this call. Use bash: sed -n '1p' huge.txt | head -c 100000]" + assert!( + result.content.starts_with( + "[Line 1 is 97.7KB, exceeds the 100000-byte output budget for this call. Use bash: sed -n '1p' huge.txt | head -c 100000]" + ), + "exact bash fallback first, pod footer after: {}", + result.content ); } @@ -417,9 +493,12 @@ async fn contract_read_offset_oob_and_limit_continuation_match_contract() { ) .await .expect("limited read"); - assert_eq!( - limited.content, - "two\n\n[1 more lines in file (13B total). Use offset=3 to continue.]" + assert!( + limited + .content + .starts_with("two\n\n[1 more lines in file (13B total). Use offset=3 to continue.]"), + "page plus paging footer first, pod footer after: {}", + limited.content ); let error = @@ -451,7 +530,11 @@ async fn contract_read_uses_magic_not_extension_for_images() { let text = ReadFileTool::execute_contract_read(json!({"path": "plain.png"}), &context) .await .expect("fake extension remains text"); - assert_eq!(text.content, "ordinary text"); + assert!( + text.content.starts_with("ordinary text"), + "text first, pod footer after: {}", + text.content + ); let image = ReadFileTool::execute_contract_read(json!({"path": "renamed.data"}), &context) .await .expect("real image uses typed transport"); @@ -611,3 +694,39 @@ async fn contract_edit_rejects_read_only_target_before_atomic_replace() { assert!(error.to_string().contains("readable and writable")); assert_eq!(std::fs::read_to_string(path).expect("unchanged"), "alpha\n"); } + +#[tokio::test] +async fn contract_read_enters_terminal_buzz_on_third_pod_visit() { + let temporary = tempfile::tempdir().expect("tempdir"); + let root = temporary.path(); + std::fs::create_dir_all(root.join("src/net")).expect("mkdir"); + std::fs::write(root.join("src/main.rs"), "fn main() {}\n").expect("fixture"); + std::fs::write(root.join("src/net/mod.rs"), "pub mod server;\n").expect("fixture"); + std::fs::write(root.join("src/net/server.rs"), "fn serve() {}\n").expect("fixture"); + std::fs::write(root.join("src/net/client.rs"), "fn connect() {}\n").expect("fixture"); + let context = ToolContext::new(root); + + // First two pod reads: full footer, no callers yet. + for path in ["src/net/mod.rs", "src/net/client.rs"] { + let result = ReadFileTool::execute_contract_read(json!({"path": path}), &context) + .await + .expect("read result"); + assert!(!result.content.contains("heard by"), "{path}"); + } + // Third distinct pod read: the buzz adds who links here. + let result = + ReadFileTool::execute_contract_read(json!({"path": "src/net/server.rs"}), &context) + .await + .expect("read result"); + assert!( + result.content.contains("heard by: src/net/mod.rs"), + "{}", + result.content + ); + // Another pod stays sparse: the buzz is per-pod, not global. + std::fs::write(root.join("src/other.rs"), "fn other() {}\n").expect("fixture"); + let result = ReadFileTool::execute_contract_read(json!({"path": "src/other.rs"}), &context) + .await + .expect("read result"); + assert!(!result.content.contains("heard by"), "{}", result.content); +} diff --git a/crates/tui/src/tools/file_search.rs b/crates/tui/src/tools/file_search.rs index f8756dcf70..cfdb5485a9 100644 --- a/crates/tui/src/tools/file_search.rs +++ b/crates/tui/src/tools/file_search.rs @@ -110,6 +110,9 @@ impl ToolSpec for FileSearchTool { let extensions = parse_extensions(&input); let exclude_patterns = parse_exclude_patterns(&input); + let pod_root = base_path.clone(); + let pod_excludes = exclude_patterns.clone(); + let pod_extensions = extensions.clone(); let matches = search_files_async( query.to_string(), base_path, @@ -121,7 +124,34 @@ impl ToolSpec for FileSearchTool { context.follow_symlinks, ) .await?; - ToolResult::json(&matches).map_err(|e| ToolError::execution_failed(e.to_string())) + // Echolocation sounding: chart the matched dirs (sorted, capped) + // so each candidate arrives with its neighborhood. Dirs resolve + // against the search root, which is the workspace by default. + let paths: Vec = matches.iter().map(|item| item.path.clone()).collect(); + let vis = super::echolocation::PodVisibility { + include: &[], + exclude: &pod_excludes, + extensions: &pod_extensions, + glob_root: &pod_root, + gitignore: true, + }; + let (pods, pods_omitted) = super::echolocation::sound_match_pods(&pod_root, &paths, &vis); + let pods_json: Vec = pods + .iter() + .map(|pod| { + json!({ + "dir": pod.dir, + "mates": pod.mates, + "mate_total": pod.mate_total, + }) + }) + .collect(); + ToolResult::json(&json!({ + "matches": matches, + "pods": pods_json, + "pods_omitted": pods_omitted, + })) + .map_err(|e| ToolError::execution_failed(e.to_string())) } } @@ -346,7 +376,7 @@ fn should_exclude(rel_path: &str, exclude_patterns: &[String]) -> bool { .any(|pattern| matches_glob(rel_path, pattern)) } -fn extension_matches(path: &Path, extensions: &[String]) -> bool { +pub(crate) fn extension_matches(path: &Path, extensions: &[String]) -> bool { let Some(ext) = path.extension().and_then(|e| e.to_str()) else { return false; }; @@ -463,6 +493,36 @@ mod tests { assert!(result.content.contains("main.rs")); } + #[tokio::test] + async fn test_file_search_charts_match_pods() { + let tmp = tempdir().expect("tempdir"); + let root = tmp.path(); + std::fs::create_dir_all(root.join("src/net")).expect("mkdir"); + std::fs::write(root.join("src/net/server.rs"), "fn serve() {}\n").expect("write"); + std::fs::write(root.join("src/net/client.rs"), "fn connect() {}\n").expect("write"); + + let ctx = ToolContext::new(root.to_path_buf()); + let result = FileSearchTool + .execute(json!({"query": "server", "limit": 5}), &ctx) + .await + .expect("execute"); + + assert!(result.success); + let body: Value = serde_json::from_str(&result.content).expect("json result"); + assert!(!body["matches"].as_array().expect("matches").is_empty()); + let pods = body["pods"].as_array().expect("pods array"); + assert_eq!(pods.len(), 1); + assert_eq!(pods[0]["dir"], "src/net"); + let mates: Vec<&str> = pods[0]["mates"] + .as_array() + .expect("mates") + .iter() + .filter_map(Value::as_str) + .collect(); + assert!(mates.contains(&"server.rs (match)"), "{mates:?}"); + assert!(mates.contains(&"client.rs"), "{mates:?}"); + } + #[tokio::test] async fn test_file_search_respects_gitignore() { let tmp = tempdir().expect("tempdir"); @@ -518,9 +578,9 @@ mod tests { .expect("execute"); assert!(result.success); - let matches: Value = serde_json::from_str(&result.content).expect("search json"); + let body: Value = serde_json::from_str(&result.content).expect("search json"); assert!( - matches + body["matches"] .as_array() .expect("matches") .iter() @@ -545,9 +605,9 @@ mod tests { .expect("execute"); assert!(result.success); - let matches: Value = serde_json::from_str(&result.content).expect("search json"); + let body: Value = serde_json::from_str(&result.content).expect("search json"); assert!( - matches + body["matches"] .as_array() .expect("matches") .iter() diff --git a/crates/tui/src/tools/mod.rs b/crates/tui/src/tools/mod.rs index ee5c469e78..31be522a63 100644 --- a/crates/tui/src/tools/mod.rs +++ b/crates/tui/src/tools/mod.rs @@ -24,6 +24,7 @@ pub mod file; pub mod file_search; pub mod finance; +pub mod echolocation; pub mod fetch_url; pub mod file_tool; pub mod fim; diff --git a/crates/tui/src/tools/project.rs b/crates/tui/src/tools/project.rs index d29328742e..67e3b4a14b 100644 --- a/crates/tui/src/tools/project.rs +++ b/crates/tui/src/tools/project.rs @@ -17,6 +17,8 @@ struct ProjectMap { tree: String, summary: String, key_files: Vec, + key_file_soundings: std::collections::BTreeMap, + key_files_omitted: usize, } #[async_trait] @@ -84,9 +86,41 @@ fn generate_project_map( } } + // Echolocation sounding: one line of symbols per key file (sorted, + // capped), so the map says what lives where, not just what exists. + let (key_file_soundings, key_files_omitted) = + super::echolocation::sound_key_files(root, &key_files); + Ok(ProjectMap { tree, summary, key_files, + key_file_soundings, + key_files_omitted, }) } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn project_map_sounds_key_file_symbols() { + let dir = tempfile::tempdir().expect("tempdir"); + std::fs::write(dir.path().join("main.rs"), "fn main() {}\nfn serve() {}\n") + .expect("fixture"); + std::fs::write(dir.path().join("Cargo.toml"), "[package]\n").expect("fixture"); + let context = ToolContext::new(dir.path().to_path_buf()); + let result = ProjectMapTool + .execute(json!({}), &context) + .await + .expect("execute"); + assert!(result.success); + let body: Value = serde_json::from_str(&result.content).expect("json result"); + assert_eq!( + body["key_file_soundings"]["main.rs"], + "2 symbols: fn main:1, fn serve:2" + ); + assert!(body["key_file_soundings"].get("Cargo.toml").is_none()); + } +} diff --git a/crates/tui/src/tools/search.rs b/crates/tui/src/tools/search.rs index 0f8b65ba7b..90b752d03d 100644 --- a/crates/tui/src/tools/search.rs +++ b/crates/tui/src/tools/search.rs @@ -32,13 +32,17 @@ const MAX_FILE_SIZE: u64 = 10 * 1024 * 1024; // 10MB /// tree. Mirrors the file_search tool so both blocking searches behave the same. const GREP_FILES_TIMEOUT: Duration = Duration::from_secs(30); -/// Result of a grep match +/// Result of a grep match. Empty context arrays are omitted on the +/// wire (models request context in ~0% of calls); `default` keeps old +/// payloads parsing. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GrepMatch { pub file: String, pub line_number: usize, pub line: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] pub context_before: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] pub context_after: Vec, } @@ -250,6 +254,31 @@ impl ToolSpec for GrepFilesTool { .map(|item| grep_match_to_json(item, context_lines)) .collect(); + // Echolocation sounding: chart the matched dirs (sorted, capped) + // so the model sees each match's podmates without a follow-up + // list. Computed here, inside the blocking worker, with the + // workspace already at hand. + let matched_files: Vec = results.iter().map(|item| item.file.clone()).collect(); + let vis = crate::tools::echolocation::PodVisibility { + include: &include_patterns, + exclude: &exclude_patterns, + extensions: &[], + glob_root: &search_path, + gitignore: false, + }; + let (pods, pods_omitted) = + crate::tools::echolocation::sound_match_pods(&workspace, &matched_files, &vis); + let pods_json: Vec = pods + .iter() + .map(|pod| { + json!({ + "dir": pod.dir, + "mates": pod.mates, + "mate_total": pod.mate_total, + }) + }) + .collect(); + // Build result. When context_lines == 1, return the single context // line as a string instead of a one-item array. That keeps the common // "show just the adjacent line" case easy for model callers to read. @@ -258,6 +287,8 @@ impl ToolSpec for GrepFilesTool { "total_matches": total_matches, "files_searched": files_searched, "truncated": total_matches > max_results, + "pods": pods_json, + "pods_omitted": pods_omitted, })) }) .await?; diff --git a/crates/tui/src/tools/search/tests.rs b/crates/tui/src/tools/search/tests.rs index 1a87c28320..e0f4c5d58a 100644 --- a/crates/tui/src/tools/search/tests.rs +++ b/crates/tui/src/tools/search/tests.rs @@ -99,6 +99,35 @@ fn test_matches_glob_unicode_filename() { assert!(!matches_glob(filename, "nonexistent*")); } +#[tokio::test] +async fn grep_pods_chart_matched_dirs_with_marks() { + let tmp = tempdir().expect("tempdir"); + let ctx = ToolContext::new(tmp.path().to_path_buf()); + fs::create_dir(tmp.path().join("net")).expect("subdir"); + fs::write(tmp.path().join("net/server.rs"), "fn serve() {}\n").expect("write"); + fs::write(tmp.path().join("net/client.rs"), "fn connect() {}\n").expect("write"); + + let result = GrepFilesTool + .execute(json!({"pattern": "serve"}), &ctx) + .await + .expect("execute"); + assert!(result.success); + let body: Value = serde_json::from_str(&result.content).expect("json result"); + assert_eq!(body["total_matches"], 1); + let pods = body["pods"].as_array().expect("pods array"); + assert_eq!(pods.len(), 1); + assert_eq!(pods[0]["dir"], "net"); + let mates: Vec<&str> = pods[0]["mates"] + .as_array() + .expect("mates") + .iter() + .filter_map(Value::as_str) + .collect(); + assert!(mates.contains(&"client.rs"), "{mates:?}"); + assert!(mates.contains(&"server.rs (match)"), "{mates:?}"); + assert_eq!(body["pods_omitted"], 0); +} + #[tokio::test] async fn test_grep_files_basic() { let tmp = tempdir().expect("tempdir"); @@ -400,12 +429,13 @@ async fn test_grep_files_ring_buffer_context_matches_full_read() { let parsed: Value = serde_json::from_str(&result.content).unwrap(); let matches = parsed["matches"].as_array().unwrap(); assert_eq!(matches.len(), 3); - assert_eq!(matches[0]["context_before"], json!([])); + // Token diet: empty context arrays are omitted on the wire. + assert!(matches[0].get("context_before").is_none()); assert_eq!(matches[0]["context_after"], json!(["b1", "b2"])); assert_eq!(matches[1]["context_before"], json!(["b2", "b3"])); assert_eq!(matches[1]["context_after"], json!(["a1", "a2"])); assert_eq!(matches[2]["context_before"], json!(["a2", "a3"])); - assert_eq!(matches[2]["context_after"], json!([])); + assert!(matches[2].get("context_after").is_none()); assert_eq!(matches[2]["line_number"].as_u64().unwrap(), 9); } diff --git a/crates/tui/src/tools/spec.rs b/crates/tui/src/tools/spec.rs index 9348645a07..916374b341 100644 --- a/crates/tui/src/tools/spec.rs +++ b/crates/tui/src/tools/spec.rs @@ -982,6 +982,23 @@ impl ToolContext { tracker.reads.insert(path.to_path_buf(), snapshot); } + /// Distinct files read under `dir` so far this session. Echolocation's + /// hunt-phase sensor: 2+ tracked reads means the model is working this + /// pod and the next read there enters the terminal buzz. Lock + /// contention degrades to zero (sparse sounding), never an error. + pub(crate) fn pod_visit_count(&self, dir: &Path) -> usize { + self.file_read_tracker + .lock() + .map(|tracker| { + tracker + .reads + .keys() + .filter(|path| path.parent() == Some(dir)) + .count() + }) + .unwrap_or(0) + } + /// Require a successful, still-fresh `read_file` snapshot before a narrow /// in-place edit. This catches model edits made against guessed or stale /// content while leaving transactional patch preflight separate. diff --git a/crates/tui/src/tools/subagent/mod.rs b/crates/tui/src/tools/subagent/mod.rs index bb4e6834af..1a17db1e2d 100644 --- a/crates/tui/src/tools/subagent/mod.rs +++ b/crates/tui/src/tools/subagent/mod.rs @@ -17845,7 +17845,7 @@ impl SubAgentToolRegistry { fn deferred_catalog_for_model(&self, agent_type: &FleetRole) -> Vec { // Every allowed tool remains searchable. Native deferral then leaves the - // fixed lowercase primitives plus agent/tool_search active; Web, MCP, + // fixed lowercase primitives plus tool_search active; agent, Web, MCP, // plugins, and other tools stay discoverable rather than eager. let mut catalog = self.tools_for_model(agent_type); catalog.retain(|tool| !is_tool_search_tool(&tool.name)); diff --git a/crates/tui/src/tools/subagent/tests.rs b/crates/tui/src/tools/subagent/tests.rs index 57abf56b7a..d1e2f0f41c 100644 --- a/crates/tui/src/tools/subagent/tests.rs +++ b/crates/tui/src/tools/subagent/tests.rs @@ -7732,6 +7732,76 @@ fn enabled_agent_surface_options() -> AgentToolSurfaceOptions { /// /// Request-boundary tests use this fixture so Moonshot compatibility coverage /// cannot drift into a hand-maintained approximation of the child surface. +#[test] +fn deferred_agent_and_workflow_are_discoverable_by_name() { + // Diet regression: deferral must never strand the tools. The real + // registry (not fixtures) carries them deferred, and both match + // algorithms surface them for the obvious queries. + use crate::core::engine::tool_catalog::{ + TOOL_SEARCH_NAME, apply_native_tool_deferral, execute_tool_search_with_cache, + }; + use crate::core::session::ToolActivationCache; + use crate::tools::registry::ToolRegistryBuilder; + use crate::tools::{ToolContext, todo}; + use std::collections::HashSet; + + let tmp = tempdir().expect("tempdir"); + let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 4); + let registry = ToolRegistryBuilder::new() + .with_agent_runtime_surface( + None, + "test-model".to_string(), + enabled_agent_surface_options(), + todo::new_shared_todo_list(), + crate::tools::plan::new_shared_plan_state(), + ) + .with_subagent_tools(manager, stub_runtime()) + .build(ToolContext::new(tmp.path().to_path_buf())); + let mut catalog = registry.to_api_tools(); + let always_load = HashSet::new(); + apply_native_tool_deferral(&mut catalog, &always_load); + for name in ["agent", "workflow"] { + assert_eq!( + catalog + .iter() + .find(|tool| tool.name == name) + .and_then(|tool| tool.defer_loading), + Some(true), + "{name} is registered and deferred" + ); + } + + for (query, match_kind, wanted) in [ + ("agent", "regex", "agent"), + ("workflow", "regex", "workflow"), + ("spawn subagent worker", "bm25", "agent"), + ("spawn subagent worker", "bm25", "workflow"), + ] { + let mut active = HashSet::new(); + let mut cache = ToolActivationCache::default(); + let found = execute_tool_search_with_cache( + TOOL_SEARCH_NAME, + &json!({"query": query, "match": match_kind}), + &catalog, + &mut active, + &mut cache, + ) + .expect("tool_search runs") + .metadata + .expect("metadata")["tool_references"] + .as_array() + .expect("refs") + .iter() + .filter_map(|v| v.as_str()) + .map(str::to_string) + .collect::>(); + assert!( + found.contains(&wanted.to_string()), + "{match_kind} {query:?} must surface {wanted}; got {found:?}" + ); + } +} + pub(crate) fn kimi_general_child_request_tools_fixture() -> Vec { let tmp = tempdir().expect("tempdir"); let mut runtime = @@ -7855,27 +7925,27 @@ async fn execute_surface_tool( } #[test] -fn small_surface_starts_with_core_tools_and_read_only_goal_control() { +fn small_surface_starts_with_core_tools_and_discoverable_goal_control() { let registry = small_surface_registry(FleetRole::Builder); let catalog = registry.deferred_catalog_for_model(&FleetRole::Builder); let mut surface = SubAgentToolSurface::new(catalog, &[]); + // Token diet: agent+workflow+get_goal ride discovery, not the head. assert_eq!( model_tool_names(model_request_tools(&mut surface)), - [ - "agent", - "bash", - "edit", - "get_goal", - "read", - "todo_write", - "tool_search", - "workflow", - "write", - ] - .into_iter() - .map(str::to_string) - .collect() + ["bash", "edit", "read", "todo_write", "tool_search", "write",] + .into_iter() + .map(str::to_string) + .collect() + ); + // The goal control still exists — deferred, not deleted. + assert_eq!( + surface + .catalog + .iter() + .find(|tool| tool.name == "get_goal") + .and_then(|tool| tool.defer_loading), + Some(true) ); let strict = surface.request_tools(surface.catalog.clone(), true); assert_eq!( @@ -8065,19 +8135,20 @@ fn small_surface_caches_are_independent_bounded_and_revalidated() { let mut first = SubAgentToolSurface::new(catalog.clone(), &warm); let mut second = SubAgentToolSurface::new(catalog, &[]); let first_names = model_tool_names(model_request_tools(&mut first)); - assert!(!first_names.contains("deferred_0")); - assert!(first_names.contains("deferred_8")); - assert!(!model_tool_names(model_request_tools(&mut second)).contains("deferred_8")); + // Priority order: the warm tail (lowest priority) is evicted first. + assert!(!first_names.contains("deferred_8")); + assert!(first_names.contains("deferred_0")); + assert!(!model_tool_names(model_request_tools(&mut second)).contains("deferred_0")); - first.catalog.retain(|tool| tool.name != "deferred_8"); - assert!(!model_tool_names(model_request_tools(&mut first)).contains("deferred_8")); + first.catalog.retain(|tool| tool.name != "deferred_0"); + assert!(!model_tool_names(model_request_tools(&mut first)).contains("deferred_0")); first .catalog - .push(synthetic_deferred_tool("oversized", 17 * 1024)); + .push(synthetic_deferred_tool("oversized", 25 * 1024)); assert!(first.hydrate("oversized").is_err()); let mut byte_catalog = (0..3) - .map(|index| synthetic_deferred_tool(&format!("bytes_{index}"), 6 * 1024)) + .map(|index| synthetic_deferred_tool(&format!("bytes_{index}"), 9 * 1024)) .collect::>(); ensure_advanced_tooling( &mut byte_catalog, @@ -8091,8 +8162,8 @@ fn small_surface_caches_are_independent_bounded_and_revalidated() { .collect::>(); let mut byte_surface = SubAgentToolSurface::new(byte_catalog, &byte_warm); let byte_names = model_tool_names(model_request_tools(&mut byte_surface)); - assert!(!byte_names.contains("bytes_0")); - assert!(byte_names.contains("bytes_1") && byte_names.contains("bytes_2")); + assert!(!byte_names.contains("bytes_2")); + assert!(byte_names.contains("bytes_0") && byte_names.contains("bytes_1")); } #[tokio::test] @@ -8139,21 +8210,14 @@ fn small_surface_depth_cap_removes_only_agent() { registry.deferred_catalog_for_model(&FleetRole::Builder), &[], ); + // Token diet: the depth cap still removes only agent; get_goal and + // workflow are absent from the head by deferral, not by depth. assert_eq!( model_tool_names(model_request_tools(&mut surface)), - [ - "bash", - "edit", - "get_goal", - "read", - "todo_write", - "tool_search", - "workflow", - "write" - ] - .into_iter() - .map(str::to_string) - .collect() + ["bash", "edit", "read", "todo_write", "tool_search", "write",] + .into_iter() + .map(str::to_string) + .collect() ); } @@ -22952,7 +23016,11 @@ fn tool_search_cannot_return_a_retired_agents_tool() { let tmp = tempdir().expect("tempdir"); let registry = subagent_registry_for_catalog(tmp.path()); let mut catalog = registry.to_api_tools(); - apply_native_tool_deferral(&mut catalog, &HashSet::new()); + // Token diet: agent is searchable by default; this test's subject is + // retired-tool exclusion, so its precondition re-eagers agent through + // the same always-load path discovery uses. + let always_load = HashSet::from(["agent".to_string()]); + apply_native_tool_deferral(&mut catalog, &always_load); assert!( catalog .iter() diff --git a/crates/tui/src/tools/workflow/mod.rs b/crates/tui/src/tools/workflow/mod.rs index 82c21ab9c3..e8c5529b0a 100644 --- a/crates/tui/src/tools/workflow/mod.rs +++ b/crates/tui/src/tools/workflow/mod.rs @@ -2381,9 +2381,10 @@ impl RunPayloadBounds { } } -/// Build the model-facing view of a run record (#2974). The JSON shape is -/// identical to the full record (panel hydration and history cards keep -/// working unchanged), but the unbounded parts are clipped: +/// Build the model-facing view of a run record (#2974). Panel hydration +/// and history cards keep working unchanged (they read keys this view +/// preserves), but the unbounded parts are clipped and each event is +/// projected to its model-essential keys: /// /// - `events`: newest `WORKFLOW_RESULT_EVENTS_TAIL` entries. /// - `progress`: newest `WORKFLOW_RESULT_PROGRESS_TAIL` lines. @@ -2393,9 +2394,16 @@ impl RunPayloadBounds { /// when the serialized value exceeds `WORKFLOW_RESULT_VALUE_MAX_CHARS`. /// - `execution.leaf_results[*].output`: per-leaf preview capped at /// `WORKFLOW_RESULT_LEAF_OUTPUT_MAX_CHARS`. +/// - event objects: exact top-level duplicates (`owner_session_id`, +/// `workflow_goal`, `workflow_id`, `source_path`, `token_budget`) are +/// removed; distinct `workspace` values hoist to top-level +/// `event_workspaces` once instead of repeating per event; routing, +/// isolation, and payload keys stay. /// /// Full detail remains available in `.codewhale/workflow-runs.jsonl`; every -/// clip adds an explicit note/pointer so the model can fetch more on demand. +/// lossy clip adds an explicit note/pointer so the model can fetch more on +/// demand. The key projection is lossless by construction (duplicates live +/// on at top level; workspaces hoist), so it carries no pointer. fn bounded_run_record_value( record: &WorkflowRunRecord, journal_path: &Path, @@ -2416,14 +2424,53 @@ fn bounded_run_record_value( ); obj.insert("progress_count".to_string(), json!(record.progress_count)); + let mut workspaces = std::collections::BTreeSet::new(); if let Some(events) = obj.get_mut("events").and_then(Value::as_array_mut) { if events.len() > WORKFLOW_RESULT_EVENTS_TAIL { let omitted = events.len() - WORKFLOW_RESULT_EVENTS_TAIL; events.drain(..omitted); bounds.events_omitted = omitted; } + // Token diet: project each event to model-essential keys. Dropped + // keys are exact top-level duplicates (`RunStarted` clones the + // record's goal/id/source/budget verbatim; `owner_session_id` + // repeats on every event). `workspace` is hoisted instead of + // dropped: the distinct set rides top-level once, since a + // worktree path names an isolation dir the model may need. + // `git_branch` stays per event — short run context that can vary + // per child. Routing keys (profile, resolved_*, route_source), + // worktree (isolation signal), and message/title/status/error/ + // usage stay: asserted contract (`run_record_events_project_to_ + // model_essential_keys`), and the model reads them. Full rows + // stay in the journal either way. + for event in events.iter_mut() { + let Some(fields) = event.as_object_mut() else { + continue; + }; + if let Some(workspace) = fields.remove("workspace") + && let Some(path) = workspace.as_str() + && !path.is_empty() + { + workspaces.insert(path.to_string()); + } + for key in [ + "owner_session_id", + "workflow_goal", + "workflow_id", + "source_path", + "token_budget", + ] { + fields.remove(key); + } + } bounds.events_returned = events.len(); } + if !workspaces.is_empty() { + obj.insert( + "event_workspaces".to_string(), + Value::Array(workspaces.into_iter().map(Value::String).collect()), + ); + } if bounds.events_omitted > 0 { obj.insert( "events_note".to_string(), @@ -11554,6 +11601,99 @@ FINAL RECEIPT ); } + #[test] + fn run_record_events_project_to_model_essential_keys() { + fn task_started(task_id: &str, workspace: &str) -> WorkflowUiEvent { + WorkflowUiEvent::at( + 2, + "session-test", + WorkflowUiEventKind::TaskStarted(Box::new(WorkflowTaskStartedEvent { + task_id: task_id.to_string(), + label: Some("dig".to_string()), + role: None, + profile: Some("deep".to_string()), + model: None, + strength: None, + thinking: None, + requested_reasoning: None, + effective_reasoning: None, + resolved_role: None, + resolved_profile: None, + resolved_provider: "zai".to_string(), + resolved_model: "glm-5".to_string(), + route_source: "fleet".to_string(), + child_route: None, + worktree: true, + workspace: Some(PathBuf::from(workspace)), + git_branch: Some("feature/x".to_string()), + parent_task_id: None, + depth: 0, + workflow_run_id: None, + workflow_phase_id: None, + workflow_task_label: None, + workflow_child_index: None, + fleet_receipt: None, + })), + ) + } + + let tmp = tempfile::tempdir().expect("tempdir"); + let mut record = WorkflowRunRecord::new( + "workflow_projection_run".to_string(), + Some("session-test".to_string()), + Some(PathBuf::from("/ws/flow.json")), + Some(1_000), + None, + ); + record.workflow_id = Some("flow-1".to_string()); + record.workflow_goal = Some("ship it".to_string()); + record.push_event(WorkflowUiEvent::at( + 1, + "session-test", + WorkflowUiEventKind::RunStarted { + workflow_id: Some("flow-1".to_string()), + workflow_goal: Some("ship it".to_string()), + source_path: Some(PathBuf::from("/ws/flow.json")), + token_budget: Some(1_000), + }, + )); + // Two dispatches, one shared workspace: the hoist dedupes. + record.push_event(task_started("t1", "/tmp/wt-1")); + record.push_event(task_started("t2", "/tmp/wt-1")); + + let journal = tmp.path().join("workflow-runs.jsonl"); + let (payload, bounds) = bounded_run_record_value(&record, &journal); + assert!(!bounds.truncated()); + let events = payload["events"].as_array().expect("events"); + + // Duplicates are gone from every event … + for event in events { + for key in [ + "owner_session_id", + "workflow_goal", + "workflow_id", + "source_path", + "token_budget", + "workspace", + ] { + assert!(event.get(key).is_none(), "dropped {key}: {event}"); + } + } + // … but live on at top level (duplicates) or hoisted (workspaces). + assert_eq!(payload["workflow_goal"], "ship it"); + assert_eq!(payload["token_budget"], 1_000); + assert_eq!(payload["event_workspaces"], json!(["/tmp/wt-1"])); + + // Routing, isolation, and payload keys stay on the dispatch event. + let dispatch = &events[1]; + assert_eq!(dispatch["task_id"], "t1"); + assert_eq!(dispatch["route_source"], "fleet"); + assert_eq!(dispatch["resolved_provider"], "zai"); + assert_eq!(dispatch["resolved_model"], "glm-5"); + assert_eq!(dispatch["worktree"], true); + assert_eq!(dispatch["git_branch"], "feature/x"); + } + #[test] fn workflow_status_payload_keeps_small_records_intact() { let tmp = tempfile::tempdir().expect("tempdir"); diff --git a/crates/workflow/src/lib.rs b/crates/workflow/src/lib.rs index 6ef38fa021..43277f09ef 100644 --- a/crates/workflow/src/lib.rs +++ b/crates/workflow/src/lib.rs @@ -639,7 +639,7 @@ pub struct BranchResult { pub status: WorkflowRunStatus, #[serde(default)] pub usage: WorkflowUsage, - #[serde(default)] + #[serde(default, skip_serializing_if = "WorkflowMemoUsage::is_zero")] pub memo_usage: WorkflowMemoUsage, #[serde(default)] pub artifacts: Vec, @@ -660,7 +660,7 @@ pub struct LeafResult { pub status: WorkflowRunStatus, #[serde(default)] pub usage: WorkflowUsage, - #[serde(default)] + #[serde(default, skip_serializing_if = "WorkflowMemoUsage::is_zero")] pub memo_usage: WorkflowMemoUsage, #[serde(default)] pub output: Option, @@ -718,6 +718,15 @@ pub struct WorkflowMemoUsage { } impl WorkflowMemoUsage { + /// True when every counter is zero: 136 bytes of nothing on the wire. + pub fn is_zero(&self) -> bool { + self.armh_hits == 0 + && self.armh_misses == 0 + && self.armh_saved_estimated_tokens == 0 + && self.provider_prompt_cache_hits == 0 + && self.provider_prompt_cache_misses == 0 + } + pub(crate) fn add_assign(&mut self, other: Self) { self.armh_hits = self.armh_hits.saturating_add(other.armh_hits); self.armh_misses = self.armh_misses.saturating_add(other.armh_misses); @@ -775,7 +784,7 @@ pub struct WorkflowExecution { pub status: WorkflowRunStatus, #[serde(default)] pub usage: WorkflowUsage, - #[serde(default)] + #[serde(default, skip_serializing_if = "WorkflowMemoUsage::is_zero")] pub memo_usage: WorkflowMemoUsage, #[serde(default)] pub leaf_results: Vec, @@ -828,7 +837,7 @@ pub struct MockLeafOutcome { pub status: WorkflowRunStatus, #[serde(default)] pub usage: WorkflowUsage, - #[serde(default)] + #[serde(default, skip_serializing_if = "WorkflowMemoUsage::is_zero")] pub memo_usage: WorkflowMemoUsage, #[serde(default)] pub output: Option, From 3c0d20f55be0d6d7733a3f41b96e6f459da38b5a Mon Sep 17 00:00:00 2001 From: AdityaVG13 Date: Sun, 20 Sep 2026 23:41:41 -0400 Subject: [PATCH 02/26] docs(surface): sync public tool-surface contracts with the diet DEFAULT_ACTIVE_NATIVE_TOOLS is six names now (agent, workflow, get/update_goal deferred); the web contract mirror, tool docs, and mode docs still claimed ten eager + eleven with tool_search. Activation cache bound 16 -> 24 KiB in facts, docs, and web copy. Found by the fork staging PR's Lint & Type Check gate. --- docs/MODES.md | 2 +- docs/TOOL_SURFACE.md | 48 ++++++++++++------------- docs/public-surface-facts.json | 8 ++--- docs/zh_hans/MODES.md | 2 +- web/lib/content/tools.ts | 6 ++-- web/lib/public-surface-contract.test.ts | 4 +-- 6 files changed, 33 insertions(+), 37 deletions(-) diff --git a/docs/MODES.md b/docs/MODES.md index c6057f9e99..da76c67f15 100644 --- a/docs/MODES.md +++ b/docs/MODES.md @@ -33,7 +33,7 @@ Run `/mode` to open the mode picker, or switch directly with `/mode work`, `/mode plan`, or `/mode operate`. - **Plan**: design-first prompting. The stable primitive names remain familiar, but the runtime centrally refuses file mutation and shell execution. Read-only inspection and policy-allowed research, including deferred Web search/fetch, remain available. -- **Work** (internally `agent`): ordinary multi-step execution. The first-turn toolbox includes `read`, `write`, `edit`, `bash`, `agent`, `workflow`, and `todo_write`, plus `create_goal`, `get_goal`, and `update_goal` so goal controls are available without discovery. Goals still require an explicit user request; approval, sandbox, repository law, and managed policy decide what may execute. +- **Work** (internally `agent`): ordinary multi-step execution. The first-turn toolbox includes `read`, `write`, `edit`, `bash`, `todo_write`, and `create_goal`, plus synthetic `tool_search`; `agent`, `workflow`, `get_goal`, and `update_goal` load through discovery (`get_goal`/`update_goal` ride eager while a goal is active). Goals still require an explicit user request; approval, sandbox, repository law, and managed policy decide what may execute. - **Operate**: manage a goal through planned steps and verified results. Fleet configures the same sub-agents and roles that execute those steps. It has the same primitive identities and execution authority as Work. Goals are model-decided: the agent calls `create_goal` when a request is a durable objective, and `/goal` always works as the direct user control — the host never infers a goal from wording. Once a goal exists, the transcript shows `◆ goal set · Operate keeps working until it is verified · /goal to edit`. An explicit `/goal` declaration always wins, `/goal` still edits it, and an existing goal is never replaced. The parent session is the **operator**: handle small or tightly coupled tasks directly. Before multi-step delegation, state a compact plan with named steps, dependencies, bounded file scopes and a completion check — or skip the ceremony when the delegation is a single bounded child — then run it through the existing Workflow tool. Parallelize independent steps; each phase receives the previous phase's results, and a dependent step cannot start when a required result is missing. A single bounded independent task can use a direct `agent` call. Reuse a worker with followup for corrections and report completed, blocked and next steps. **Dispatch is not completion** — write-capable children must return real verification evidence. The first Operate turn of a session appends this contract once as a user-role runtime message (append-only history, never the pinned system prompt), so Plan, Work, and Operate keep one shared prompt prefix. `Act` and `/mode act` remain compatibility aliases for Work. Saved settings diff --git a/docs/TOOL_SURFACE.md b/docs/TOOL_SURFACE.md index d04a01c4bc..aa4d116984 100644 --- a/docs/TOOL_SURFACE.md +++ b/docs/TOOL_SURFACE.md @@ -17,27 +17,26 @@ Implementation sources: ## Default-active contract -New turns start with ten eager native names plus synthetic `tool_search`: +New turns start with six eager native names plus synthetic `tool_search`: 1. `read` 2. `write` 3. `edit` 4. `bash` -5. `agent` -6. `workflow` -7. `todo_write` -8. `create_goal` -9. `get_goal` -10. `update_goal` -11. `tool_search` (synthetic, always active) - -The ten native names are `DEFAULT_ACTIVE_NATIVE_TOOLS` in +5. `todo_write` +6. `create_goal` +7. `tool_search` (synthetic, always active) + +The six native names are `DEFAULT_ACTIVE_NATIVE_TOOLS` in `crates/tui/src/core/engine/tool_catalog.rs`, pinned by -`default_active_contract_keeps_discovery_and_core_tools_eager`. An authority -boundary may remove `agent` at the maximum child depth, but route size alone -must not change this core vocabulary. +`default_active_contract_keeps_discovery_and_core_tools_eager`. `agent`, +`workflow`, `get_goal`, and `update_goal` are deferred by default and load +via `tool_search` (`get_goal`/`update_goal` ride eager only while a goal is +active); an authority boundary may narrow discovery further, but route size +alone must not change this core vocabulary. -The direct schemas deliberately stay small: +The core schemas deliberately stay small (deferred rows load via +`tool_search` rather than riding every turn): | Tool | Input | Purpose | |---|---|---| @@ -45,12 +44,12 @@ The direct schemas deliberately stay small: | `write` | `path`, `content` | Create or replace a file. | | `edit` | `path`, `edits` | Apply one or more unambiguous text replacements against one original snapshot. | | `bash` | `command`, optional `timeout` | Run one cancellable foreground shell command and return a bounded tail. | -| `agent` | delegated task and optional scope/context controls | Start or inspect focused child work. | -| `workflow` | plan/script/source_path plus run controls | Coordinate multi-agent phases with dependencies and completion checks. | +| `agent` (deferred) | delegated task and optional scope/context controls | Start or inspect focused child work. | +| `workflow` (deferred) | plan/script/source_path plus run controls | Coordinate multi-agent phases with dependencies and completion checks. | | `todo_write` | complete replacement list of `{content, status}` items | Keep optional, agent-owned progress notes for genuinely multi-step work. | | `create_goal` | objective plus optional budget | Start the session goal the turn works toward. | -| `get_goal` | none | Read the active goal and its progress. | -| `update_goal` | terminal status | Mark the goal complete or blocked. | +| `get_goal` (deferred unless a goal is active) | none | Read the active goal and its progress. | +| `update_goal` (deferred unless a goal is active) | terminal status | Mark the goal complete or blocked. | | `tool_search` | `query`, optional matching controls | Discover policy-allowed deferred tools and add selected schemas to this conversation's toolbox. | Mode is an authority decision, not a synonym system. Plan, Work, and Operate @@ -90,11 +89,12 @@ from Plan mode and refused under a worker authority envelope. ### Conversation toolbox cache A successful search activation is remembered by name for the current -conversation. The cache holds at most eight deferred names and 16 KiB of -serialized schemas, evicts least-recently-used entries, and revalidates every -entry against the current catalog and policy before advertising it again. A -session sync clears it. The cache cannot resurrect a removed, denied, or -newly-eager tool. +conversation. The cache holds at most eight deferred names and 24 KiB of +added serialized schemas, evicts least-recently-used entries (best match +last, so the highest-priority discovery survives its own set), and +revalidates every entry against the current catalog and policy before +advertising it again. A session sync clears it. The cache cannot resurrect +a removed, denied, or newly-eager tool. Each subagent gets its own policy-filtered deferred catalog, always-present `tool_search`, and bounded activation cache. Forked messages and instructions @@ -273,7 +273,7 @@ exits 0 with "0 passed; N filtered out" when a filter matches nothing, so a misspelled filter is indistinguishable from a pass. (Three filters printed here before v0.9.4 named tests that did not exist.) -The provider-free receipt must report the eleven default-active names listed +The provider-free receipt must report the seven default-active names listed above. A separate repository-wide tool count may include deferred, dynamic, feature-gated, and compatibility-only registrations; it is not the number of tools placed in the first-turn model catalog. diff --git a/docs/public-surface-facts.json b/docs/public-surface-facts.json index 40ffd063a9..7bfb6709f4 100644 --- a/docs/public-surface-facts.json +++ b/docs/public-surface-facts.json @@ -104,12 +104,8 @@ "write", "edit", "bash", - "agent", - "workflow", "todo_write", - "create_goal", - "get_goal", - "update_goal" + "create_goal" ], "schemas": { "read": [ @@ -144,7 +140,7 @@ }, "activationCache": { "maximumNames": 8, - "maximumSchemaBytes": 16384, + "maximumSchemaBytes": 24576, "scope": "per conversation; each subagent owns an independent policy-filtered cache" }, "agentConcurrency": { diff --git a/docs/zh_hans/MODES.md b/docs/zh_hans/MODES.md index a71bcb5961..a0202cbff8 100644 --- a/docs/zh_hans/MODES.md +++ b/docs/zh_hans/MODES.md @@ -19,7 +19,7 @@ Workflow 通过同一个子智能体运行时执行命名步骤；Fleet 管理 按 `Tab` 补全 composer 菜单，或在 composer 为空时循环切换可见模式:**Plan → Work → Operate → Plan**。`Tab` 从不发送或排队 composer 文本；用 `Enter` 发送或排队。按 `Shift+Tab` 循环切换权限姿态(Ask → Auto-Review → Full Access)。按 `Ctrl+T` 循环切换推理强度。运行 `/mode` 打开模式选择器，或直接用 `/mode work`、`/mode plan`、`/mode operate` 切换。 - **Plan**:设计优先的提示方式。稳定的原语名称保持熟悉，但运行时集中拒绝文件修改和 shell 执行。只读检查与策略允许的研究(包括延迟的 Web 搜索/抓取)仍然可用。 -- **Work**(内部为 `agent`):普通的多步执行。第一回合的工具箱包含 `read`、`write`、`edit`、`bash`、`agent` 和 `todo_write`，以及无需搜索即可使用的目标控制工具 `create_goal`、`get_goal` 和 `update_goal`。创建目标仍须用户明确要求；审批、沙箱、仓库法和托管策略决定什么可以执行。 +- **Work**(内部为 `agent`):普通的多步执行。第一回合的工具箱包含 `read`、`write`、`edit`、`bash`、`todo_write` 和 `create_goal`，以及合成工具 `tool_search`；`agent`、`workflow`、`get_goal` 和 `update_goal` 通过搜索发现加载（`get_goal`/`update_goal` 在目标激活时直接可用）。创建目标仍须用户明确要求；审批、沙箱、仓库法和托管策略决定什么可以执行。 - **Operate**:通过计划中的步骤和已验证的结果推进目标。它与 Work 使用相同的工具和执行权限，Fleet 管理执行这些步骤的子智能体及其角色。小任务或紧密耦合的工作由父会话直接完成。多步骤委派先列出步骤、依赖、明确的文件范围和完成检查——如果只是单个有限的子任务，可跳过这一仪式——再通过现有 Workflow 执行。独立步骤可以并行；下一阶段接收上一阶段的结果，缺少必要结果时不会启动依赖它的工作。单个独立任务可以直接调用 `agent`；修改应通过 followup 复用已有子智能体。进度按已完成、受阻和下一步汇报。**派发不等于完成** — 有写权限的子智能体必须返回真实的验证证据。 `Act` 和 `/mode act` 仍然是 Work 的兼容别名。保存的设置仍然规范化为内部值 `agent`。 diff --git a/web/lib/content/tools.ts b/web/lib/content/tools.ts index cb90c0078b..b41f2cbac6 100644 --- a/web/lib/content/tools.ts +++ b/web/lib/content/tools.ts @@ -11,12 +11,12 @@ export const TOOLS_COPY = { { name: "write", detail: { en: "path · content", zh: "path · content" } }, { name: "edit", detail: { en: "path · edits", zh: "path · edits" } }, { name: "bash", detail: { en: "command · timeout?", zh: "command · timeout?" } }, - { name: "agent · workflow", detail: { en: "Delegate work and coordinate dependent phases.", zh: "委派工作并协调有依赖关系的阶段。" } }, + { name: "agent · workflow", detail: { en: "Loaded on demand via tool_search. Delegate work and coordinate dependent phases.", zh: "通过 tool_search 按需加载。委派工作并协调有依赖关系的阶段。" } }, { name: "todo_write", detail: { en: "Replace the task list with content and status entries.", zh: "用包含内容与状态的条目替换任务列表。" } }, - { name: "create_goal · get_goal · update_goal", detail: { en: "Create, inspect and update the active goal.", zh: "创建、查看与更新当前目标。" } }, + { name: "create_goal · get_goal · update_goal", detail: { en: "create_goal is available without discovery; get_goal/update_goal load on demand while a goal is active.", zh: "create_goal 无需搜索即可使用；get_goal/update_goal 仅在目标激活时按需加载。" } }, { name: "tool_search", detail: { en: "Discover policy-allowed tools. Each child has its own search and activation cache.", zh: "发现策略允许的工具。每个子 Agent 拥有独立的搜索与激活缓存。" } }, { name: "Git · Run · tasks · remember · Web · MCP · plugins", detail: { en: "Deferred: loaded by tool_search only when policy permits.", zh: "延迟加载：仅在策略允许时由 tool_search 加载。" } }, - { name: "8 names / 16 KiB", detail: { en: "Conversation toolbox cache; independent per child and revalidated every turn.", zh: "会话工具箱缓存；每个子 Agent 独立，每轮重新校验。" } }, + { name: "8 names / 24 KiB", detail: { en: "Conversation toolbox cache; independent per child and revalidated every turn.", zh: "会话工具箱缓存；每个子 Agent 独立，每轮重新校验。" } }, { name: "Web search/fetch", detail: { en: "Scout and Reviewer children can discover these without gaining mutation or arbitrary network authority.", zh: "侦察与审查子 Agent 可发现这些工具，但不会因此获得写入或任意网络权限。" } }, { name: "MCP", detail: { en: "mcp__ — registered from ~/.codewhale/mcp.json.", zh: "mcp__——从 ~/.codewhale/mcp.json 注册。" } }, ], diff --git a/web/lib/public-surface-contract.test.ts b/web/lib/public-surface-contract.test.ts index 74e7c35913..f64eb94968 100644 --- a/web/lib/public-surface-contract.test.ts +++ b/web/lib/public-surface-contract.test.ts @@ -440,7 +440,7 @@ done }); expect(matrix.toolSurface.activationCache).toEqual({ maximumNames: 8, - maximumSchemaBytes: 16_384, + maximumSchemaBytes: 24_576, scope: "per conversation; each subagent owns an independent policy-filtered cache", }); expect(matrix.toolSurface.agentConcurrency).toEqual({ @@ -469,7 +469,7 @@ done expect(registry).toContain("Arc::new(LowercaseBashTool)"); expect(registry).toContain('FileTool::new("File")'); expect(registry).toContain('BashTool::new("Bash")'); - expect(toolsPage).toContain("8 names / 16 KiB"); + expect(toolsPage).toContain("8 names / 24 KiB"); expect(toolsPage).toContain("Web search/fetch"); expect(toolsPage).not.toContain("docs/TOOL_LIFECYCLE.md"); }); From 9ef7047d93098e58115b8146bfd2d777cdac0382 Mon Sep 17 00:00:00 2001 From: AdityaVG13 Date: Mon, 21 Sep 2026 01:43:48 -0400 Subject: [PATCH 03/26] feat(echo): squeeze soundings, edit echoes, grep/skill clamps Footer squeeze (all lossless): same-pod links/callers render bare, import roots a link already names drop by file stem or package path, framing tightens to [Pod (N)] / [Sound], and sounded paths are /-separated on every OS. The squeeze follows CODEWHALE_ECHO gates so an off:links trial measures links alone. Edit echoes name touched symbols plus impacted callers on write/edit; the diff rides metadata for the TUI while the receipt stays one line. Grep max_results is clamped with honest truncation, skill listings are capped, JS arrows sound as their source token, and ECHOLOCATION.md documents the surfaces. Signed-off-by: AdityaVG13 --- crates/tui/src/session_manager.rs | 10 +- crates/tui/src/tools/diff_format.rs | 15 +- crates/tui/src/tools/echolocation.rs | 412 +++++++++++++++++++++++++-- crates/tui/src/tools/file.rs | 17 +- crates/tui/src/tools/file/tests.rs | 58 +++- crates/tui/src/tools/search.rs | 45 ++- crates/tui/src/tools/search/tests.rs | 49 ++++ crates/tui/src/tools/skill.rs | 81 +++++- docs/ECHOLOCATION.md | 64 +++++ 9 files changed, 685 insertions(+), 66 deletions(-) create mode 100644 docs/ECHOLOCATION.md diff --git a/crates/tui/src/session_manager.rs b/crates/tui/src/session_manager.rs index 40978facf6..ce2e5656a4 100644 --- a/crates/tui/src/session_manager.rs +++ b/crates/tui/src/session_manager.rs @@ -3829,13 +3829,9 @@ mod tests { ); let journal = unstamped.journal.as_ref().expect("journal"); assert!( - journal - .entries - .iter() - .all(|entry| { - entry.created_at >= before_save - && entry.created_at <= unstamped.metadata.created_at - }), + journal.entries.iter().all(|entry| { + entry.created_at >= before_save && entry.created_at <= unstamped.metadata.created_at + }), "unstamped entries are created during save, before snapshot metadata" ); } diff --git a/crates/tui/src/tools/diff_format.rs b/crates/tui/src/tools/diff_format.rs index bb3549b26b..9d83e84f58 100644 --- a/crates/tui/src/tools/diff_format.rs +++ b/crates/tui/src/tools/diff_format.rs @@ -1,13 +1,12 @@ -//! Build unified-diff strings for tool results. +//! Build unified-diff strings for tool-result metadata. //! //! `edit_file` and `write_file` capture the file contents before and after -//! the mutation and emit a unified diff at the head of their `ToolResult` -//! output. The TUI's `output_looks_like_diff` detector then routes the -//! payload through `diff_render::render_diff`, which renders it with line -//! numbers and coloured `+`/`-` gutters (#505). -//! -//! The diff is also a strict UX upgrade for the model — it sees exactly -//! which lines changed instead of a one-line summary. +//! the mutation and stash a unified diff in the result `metadata` (under +//! `mutation.diff`) for the TUI, which routes it through +//! `diff_render::render_diff` with line numbers and coloured `+`/`-` +//! gutters (#505). The model-facing `content` stays a one-line receipt — +//! the model learns what changed from the edit echo (touched symbols and +//! impacted callers), not from the diff. use similar::TextDiff; diff --git a/crates/tui/src/tools/echolocation.rs b/crates/tui/src/tools/echolocation.rs index d03436c322..9e59b757c4 100644 --- a/crates/tui/src/tools/echolocation.rs +++ b/crates/tui/src/tools/echolocation.rs @@ -68,6 +68,19 @@ const MAX_CALLERS: usize = 8; const MAX_CALLER_SCAN_BYTES: u64 = 256 * 1024; /// Tracked pod reads before the next read there enters the buzz. pub(crate) const BUZZ_VISIT_THRESHOLD: usize = 2; +/// Max touched symbols named per edit echo before "+N more". +const MAX_EDIT_TOUCHED: usize = 6; +/// Max impacted callers named per edit echo before "+N more". +const MAX_EDIT_CALLERS: usize = 4; +/// Hard byte budget for one edit echo line. +const EDIT_ECHO_MAX_BYTES: usize = 400; +/// Changed spans attributed per edit; past this the edit is a rewrite +/// and the first spans already name its symbols. +const MAX_EDIT_SPANS: usize = 32; +/// Time budget for the touched-symbol diff. Myers is quadratic in the +/// worst case; past this `similar` approximates instead of blocking +/// the edit path, and the echo degrades to first-span attribution. +const EDIT_DIFF_TIMEOUT_MS: u64 = 100; /// Ablation gate: true when `CODEWHALE_ECHO` disables `section`. /// Parsed once per process; unset or malformed means everything on. @@ -96,7 +109,6 @@ fn parse_echo_gates(raw: &str) -> BTreeSet { /// with nothing to say renders as the empty string. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct EchoChart { - pod_dir: Option, podmates: Vec, podmate_total: usize, symbols: Vec, @@ -109,6 +121,31 @@ pub(crate) struct EchoChart { caller_total: usize, } +/// Shorten workspace-relative links to bare names when they live in +/// the same pod (directory) as the sounded file: the pod frame makes +/// them unambiguous, and the mates list shows the same names. +/// Cross-pod links keep their relative path. Never fails closed the +/// wrong way — on any doubt the full path stays. +fn pod_relative_names(workspace: &Path, file_path: &Path, links: Vec) -> Vec { + let Some(home) = file_path.parent() else { + return links; + }; + links + .into_iter() + .map(|link| { + let short = workspace.join(&link).parent() == Some(home); + if short { + Path::new(&link) + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or(link) + } else { + link + } + }) + .collect() +} + /// Sound an echolocation chart for a file that was just read. /// `workspace` anchors link display (workspace-relative paths); /// `file_path` is the resolved path (for the sibling sounding); `text` @@ -117,7 +154,7 @@ pub(crate) struct EchoChart { /// Never fails: anything unreadable is omitted from the chart rather /// than failing the read. pub fn sound_echolocation(workspace: &Path, file_path: &Path, text: &str, buzz: bool) -> EchoChart { - let (pod_dir, mut podmates) = sound_podmates(file_path); + let (_, mut podmates) = sound_podmates(file_path); let podmate_total = podmates.len(); podmates.truncate(MAX_PODMATES); let (mut symbols, mut import_roots, mut links, link_total, mut callers, caller_total) = @@ -143,12 +180,38 @@ pub fn sound_echolocation(workspace: &Path, file_path: &Path, text: &str, buzz: }; let symbol_total = symbols.len(); symbols.truncate(MAX_SYMBOLS); + // Squeeze, all lossless: same-pod links and callers render bare + // (the pod frame disambiguates), and import roots a shown link + // already names drop — the link says the same thing with a path + // attached. Totals count what renders, so "+N more" stays honest. + // Each transform follows its section gate: an `off:links` trial + // measures links alone, never roots too. + if !echo_off("links") { + links = pod_relative_names(workspace, file_path, links); + } + if !echo_off("callers") { + callers = pod_relative_names(workspace, file_path, callers); + } + if !echo_off("links") && !echo_off("imports") { + let link_stems: Vec<&str> = links + .iter() + .filter_map(|link| Path::new(link).file_stem()?.to_str()) + .collect(); + import_roots.retain(|root| { + if link_stems.contains(&root.as_str()) { + return false; + } + // Package roots resolve under their own dir + // (`pkg` under `pkg/__init__.py`). + let prefix = format!("{root}/"); + !links.iter().any(|link| link.starts_with(&prefix)) + }); + } let import_total = import_roots.len(); import_roots.truncate(MAX_IMPORT_ROOTS); links.truncate(MAX_LINKS); callers.truncate(MAX_CALLERS); EchoChart { - pod_dir, podmates, podmate_total, symbols, @@ -191,12 +254,13 @@ impl EchoChart { /// sorted entries, fixed caps, byte budget, no timestamps. pub fn render_footer(&self) -> String { let mut out = String::new(); - if !echo_off("mates") && self.pod_dir.is_some() && self.podmate_total > 0 { - let dir = self.pod_dir.as_deref().unwrap_or("."); + // No dir: the footer trails the read of this exact file, so the + // pod frame is already known (miss echoes and grep pods keep + // theirs — those orient across paths, not within one). + if !echo_off("mates") && self.podmate_total > 0 { out.push_str(&format!( - "\n\n[Pod: {} ({}): {}]", - dir, - count_noun(self.podmate_total, "entry", "entries"), + "\n\n[Pod ({}): {}]", + self.podmate_total, render_capped(&self.podmates, self.podmate_total) )); } @@ -210,7 +274,7 @@ impl EchoChart { } if !echo_off("imports") && self.import_total > 0 { if !sounding.is_empty() { - sounding.push_str(" | "); + sounding.push_str("; "); } sounding.push_str(&format!( "imports: {}", @@ -219,7 +283,7 @@ impl EchoChart { } if !echo_off("links") && self.link_total > 0 { if !sounding.is_empty() { - sounding.push_str(" | "); + sounding.push_str("; "); } sounding.push_str(&format!( "links: {}", @@ -228,7 +292,7 @@ impl EchoChart { } if !echo_off("callers") && self.caller_total > 0 { if !sounding.is_empty() { - sounding.push_str(" | "); + sounding.push_str("; "); } sounding.push_str(&format!( "heard by: {}", @@ -236,7 +300,7 @@ impl EchoChart { )); } if !sounding.is_empty() { - out.push_str(&format!("\n\n[Sounding: {sounding}]")); + out.push_str(&format!("\n\n[Sound: {sounding}]")); } truncate_bytes(&out, POD_FOOTER_MAX_BYTES) } @@ -490,7 +554,7 @@ fn split_match_path(file: &str) -> (String, String) { let dir = path .parent() .filter(|parent| !parent.as_os_str().is_empty()) - .map(|parent| parent.to_string_lossy().into_owned()) + .map(slash_path) .unwrap_or_else(|| ".".to_string()); (dir, name) } @@ -603,23 +667,135 @@ fn sound_symbols(text: &str, lang: SoundLang) -> Vec { // Symbols carry 1-based source lines (`fn main:12`) so the model can // page straight at them — the sounding already sees past truncation, // and now it can point there too. - let numbered = |n: usize, symbol: String| format!("{symbol}:{n}"); + sound_numbered_symbols(text, lang) + .into_iter() + .map(|(n, symbol)| format!("{symbol}:{n}")) + .collect() +} + +/// Structured symbol sounding: 1-based line + bare `kind name`, in file +/// order. The edit echo attributes changed spans against these lines. +fn sound_numbered_symbols(text: &str, lang: SoundLang) -> Vec<(usize, String)> { match lang { SoundLang::Rust => code_lines(text) - .filter_map(|(n, line)| parse_item_line(line).map(|s| numbered(n, s))) + .filter_map(|(n, line)| parse_item_line(line).map(|s| (n, s))) .collect(), SoundLang::Python => py_code_lines(text) - .filter_map(|(n, line)| parse_py_item(line).map(|s| numbered(n, s))) + .filter_map(|(n, line)| parse_py_item(line).map(|s| (n, s))) .collect(), SoundLang::JavaScript => code_lines(text) - .filter_map(|(n, line)| parse_js_item(line).map(|s| numbered(n, s))) + .filter_map(|(n, line)| parse_js_item(line).map(|s| (n, s))) .collect(), SoundLang::Go => code_lines(text) - .filter_map(|(n, line)| parse_go_item(line).map(|s| numbered(n, s))) + .filter_map(|(n, line)| parse_go_item(line).map(|s| (n, s))) .collect(), } } +/// Changed new-side spans as (1-based start, new-line count). Pure +/// deletions contribute their insertion point with length 0. Reuses +/// the `similar` diff the unified-diff builder already depends on, +/// under a hard timeout. +fn changed_spans(before: &str, after: &str) -> Vec<(usize, usize)> { + if before == after { + return Vec::new(); + } + similar::TextDiff::configure() + .timeout(std::time::Duration::from_millis(EDIT_DIFF_TIMEOUT_MS)) + .diff_lines(before, after) + .ops() + .iter() + .filter_map(|op| match op { + similar::DiffOp::Equal { .. } => None, + similar::DiffOp::Delete { new_index, .. } => Some((new_index + 1, 0)), + similar::DiffOp::Insert { + new_index, new_len, .. + } => Some((new_index + 1, *new_len)), + similar::DiffOp::Replace { + new_index, new_len, .. + } => Some((new_index + 1, *new_len)), + }) + .take(MAX_EDIT_SPANS) + .collect() +} + +/// Symbols the mutation touched, in file order: declarations born +/// inside a changed span, else the nearest declaration above it (the +/// edit landed inside that symbol's body). Returns the shown names +/// plus the distinct total for the "+N more" tail. +fn touched_symbols(file_path: &Path, before: &str, after: &str) -> (Vec, usize) { + let empty = (Vec::new(), 0); + let Some(lang) = detect_lang(file_path) else { + return empty; + }; + let symbols = sound_numbered_symbols(after, lang); + if symbols.is_empty() { + return empty; + } + let mut touched: Vec = Vec::new(); + let mut touched_lines: Vec = Vec::new(); + for (start, len) in changed_spans(before, after) { + let mut born_inside = false; + for (line, name) in &symbols { + if *line >= start && line.saturating_sub(start) < len { + born_inside = true; + if !touched_lines.contains(line) { + touched_lines.push(*line); + touched.push(format!("{name}:{line}")); + } + } + } + if !born_inside + && let Some((line, name)) = symbols.iter().rev().find(|(line, _)| *line <= start) + && !touched_lines.contains(line) + { + touched_lines.push(*line); + touched.push(format!("{name}:{line}")); + } + } + let total = touched.len(); + touched.truncate(MAX_EDIT_TOUCHED); + (touched, total) +} + +/// Model-visible edit echo: which symbols the write/edit touched and +/// which podmates link this file (impacted callers). The model-facing +/// mutation receipt is one line and the diff rides metadata for the +/// TUI, so without this the model never learns what its edit hit. +/// `None` when there is nothing to say. Gated by the existing +/// `symbols`/`callers` sections; budgeted hard. +pub fn sound_edit_echo( + workspace: &Path, + file_path: &Path, + before: &str, + after: &str, +) -> Option { + if before == after { + return None; + } + let mut halves = Vec::new(); + if !echo_off("symbols") { + let (shown, total) = touched_symbols(file_path, before, after); + if total > 0 { + halves.push(format!("touched {}", render_capped(&shown, total))); + } + } + if !echo_off("callers") { + let (callers, total) = sound_callers(workspace, file_path); + if total > 0 { + let shown = &callers[..callers.len().min(MAX_EDIT_CALLERS)]; + halves.push(format!("heard by {}", render_capped(shown, total))); + } + } + if halves.is_empty() { + return None; + } + Some(truncate_bytes( + &format!("[Edit echo: {}]", halves.join("; ")), + EDIT_ECHO_MAX_BYTES, + )) +} + /// Distinct import roots, sorted, per language. fn sound_import_roots(text: &str, lang: SoundLang) -> BTreeSet { match lang { @@ -1007,10 +1183,20 @@ fn find_crate_root(workspace: &Path, file_path: &Path) -> Option { } } +/// Render a path with `/` separators on every platform, so sounded +/// paths are byte-identical on Windows, macOS, and Linux (portable +/// tests, shared prompt caches). Components are re-joined rather than +/// string-replaced so a literal `\` in a file name survives, and `.` +/// / `..` segments pass through untouched for [`dot_clean`]. +fn slash_path(path: &Path) -> String { + path.components() + .map(|component| component.as_os_str().to_string_lossy()) + .collect::>() + .join("/") +} + fn workspace_relative(workspace: &Path, path: &Path) -> Option { - path.strip_prefix(workspace) - .ok() - .map(|rel| rel.to_string_lossy().into_owned()) + path.strip_prefix(workspace).ok().map(slash_path) } /// Python source lines: `#` comments cut, triple-quote doc spans @@ -1274,7 +1460,7 @@ fn parse_js_item(line: &str) -> Option { || strip_keyword(body, "function").is_some() || body.contains("=>") { - return Some(format!("function {name}")); + return Some(format!("{qualifier} {name}")); } return None; } @@ -1766,7 +1952,7 @@ define nothing; let second = sound_echolocation(dir.path(), &path, &text, false).render_footer(); assert_eq!(first, second); assert!(first.len() <= POD_FOOTER_MAX_BYTES, "{}", first.len()); - assert!(first.starts_with("\n\n[Pod:"), "{first}"); + assert!(first.starts_with("\n\n[Pod ("), "{first}"); } #[test] @@ -1780,8 +1966,8 @@ define nothing; false, ) .render_footer(); - assert!(footer.contains("[Pod:"), "{footer}"); - assert!(!footer.contains("[Sounding:"), "{footer}"); + assert!(footer.contains("[Pod ("), "{footer}"); + assert!(!footer.contains("[Sound:"), "{footer}"); } #[test] @@ -2030,7 +2216,26 @@ define nothing; false, ) .render_footer(); - assert!(footer.contains("links: src/config.rs"), "{footer}"); + assert!(footer.contains("links: config.rs"), "{footer}"); + } + + #[test] + fn shadowed_import_roots_drop_by_stem_and_package_path() { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path(); + let text = "import a\nimport pkg\n"; + fs::write(root.join("main.py"), text).expect("fixture"); + fs::write(root.join("a.py"), "x = 1\n").expect("fixture"); + fs::create_dir(root.join("pkg")).expect("mkdir"); + fs::write(root.join("pkg/__init__.py"), "y = 2\n").expect("fixture"); + let footer = sound_echolocation(root, &root.join("main.py"), text, false).render_footer(); + // Both roots are named by their links: file stem (`a` under + // `a.py`) and package path (`pkg` under `pkg/__init__.py`). + // Sounded paths are `/`-separated on every OS; vacuous on + // unix, load-bearing on Windows CI. + assert!(!footer.contains("imports:"), "{footer}"); + assert!(footer.contains("links: a.py, pkg/__init__.py"), "{footer}"); + assert!(!footer.contains('\\'), "{footer}"); } #[test] @@ -2111,7 +2316,7 @@ define nothing; let second = sound_echolocation(root, &root.join("src/net/server.rs"), text, true).render_footer(); assert_eq!(first, second); - assert!(first.contains("heard by: src/net/mod.rs"), "{first}"); + assert!(first.contains("heard by: mod.rs"), "{first}"); } #[test] @@ -2231,7 +2436,7 @@ if (x) { y(); } "{symbols:?}" ); assert!( - symbols.contains(&"function dive:10".to_string()), + symbols.contains(&"const dive:10".to_string()), "{symbols:?}" ); assert!( @@ -2433,7 +2638,7 @@ func main() { fs::write(dir.path().join("other.py"), "x = 1\n").expect("fixture"); let footer = sound_echolocation(dir.path(), &dir.path().join("swim.py"), PY_SAMPLE, false) .render_footer(); - assert!(footer.contains("[Pod:"), "{footer}"); + assert!(footer.contains("[Pod ("), "{footer}"); assert!(footer.contains("def swim"), "{footer}"); assert!(footer.contains("imports:"), "{footer}"); } @@ -2565,6 +2770,155 @@ func main() { assert!(!raws.contains(&"vault_link"), "{raws:?}"); } + #[test] + fn tmp_measure_footer_breakdown() { + // TEMPORARY squeeze probe; removed after the numbers land. + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let samples = [ + "crates/tui/src/tools/echolocation.rs", + "crates/tui/src/tools/file.rs", + "crates/tui/src/tools/search.rs", + "crates/tui/src/tools/diff_format.rs", + "crates/tui/src/tools/skill.rs", + "crates/tui/src/core/engine/tool_catalog.rs", + "crates/tui/src/skills/mod.rs", + "crates/workflow/src/lib.rs", + "web/lib/content/tools.ts", + ]; + for rel in samples { + let path = root.join(rel); + let Ok(text) = fs::read_to_string(&path) else { + continue; + }; + let chart = sound_echolocation(&root, &path, &text, true); + let footer = chart.render_footer(); + let mates: usize = chart.podmates.iter().map(|m| m.len() + 2).sum(); + let syms: usize = chart.symbols.iter().map(|m| m.len() + 2).sum(); + let imps: usize = chart.import_roots.iter().map(|m| m.len() + 2).sum(); + let links: usize = chart.links.iter().map(|m| m.len() + 2).sum(); + let calls: usize = chart.callers.iter().map(|m| m.len() + 2).sum(); + // Import roots shadowed by a resolved link (same stem). + let link_stems: Vec<&str> = chart + .links + .iter() + .filter_map(|l| l.rsplit('/').next()?.split('.').next()) + .collect(); + let shadowed = chart + .import_roots + .iter() + .filter(|r| link_stems.contains(&r.as_str())) + .count(); + eprintln!( + "BREAKDOWN {rel} file={}B footer={}B mates={}B({}/{}) syms={}B({}/{}) imps={}B({}/{},shadow={}) links={}B({}/{}) calls={}B({}/{})", + text.len(), + footer.len(), + mates, + chart.podmates.len(), + chart.podmate_total, + syms, + chart.symbols.len(), + chart.symbol_total, + imps, + chart.import_roots.len(), + chart.import_total, + shadowed, + links, + chart.links.len(), + chart.link_total, + calls, + chart.callers.len(), + chart.caller_total, + ); + } + } + + #[test] + fn block_comments_mute_and_release_across_lines() { + let text = "/*\nfn fake() {}\n*/\nfn real() {}\n"; + let symbols = sound_symbols(text, SoundLang::Rust); + assert_eq!(symbols, ["fn real:4"]); + // A comment opener inside a string still mutes until the closer: + // the failure mode is a miss, never a forge. + let text = "let s = \"/*\";\nfn muted() {}\nlet t = \"*/\";\nfn kept() {}\n"; + let symbols = sound_symbols(text, SoundLang::Rust); + assert_eq!(symbols, ["fn kept:4"]); + } + + #[test] + fn py_decorators_methods_and_comment_fences() { + let text = "@route(\"/x\")\ndef handler():\n pass\nclass Pod:\n def swim(self):\n pass\n# a \"\"\" in a comment toggles nothing\ndef after():\n"; + let symbols = sound_symbols(text, SoundLang::Python); + assert!( + symbols.contains(&"def handler:2".to_string()), + "{symbols:?}" + ); + assert!(symbols.contains(&"class Pod:4".to_string()), "{symbols:?}"); + assert!(symbols.contains(&"def swim:5".to_string()), "{symbols:?}"); + assert!(symbols.contains(&"def after:8".to_string()), "{symbols:?}"); + } + + #[test] + fn js_anonymous_and_arrow_shapes() { + // Anonymous default export: no name to sound, no phantom. + assert!(sound_symbols("export default function() {}\n", SoundLang::JavaScript).is_empty()); + let symbols = sound_symbols( + "const swim = () => {};\nlet dive = async () => {};\nvar data = 42;\n", + SoundLang::JavaScript, + ); + assert!(symbols.contains(&"const swim:1".to_string()), "{symbols:?}"); + assert!(symbols.contains(&"let dive:2".to_string()), "{symbols:?}"); + assert!(!symbols.iter().any(|s| s.contains("data")), "{symbols:?}"); + } + + #[test] + fn edit_echo_names_touched_symbol_and_callers() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("a.py"), "import b\nprint(b.x)\n").expect("fixture"); + let before = "x = 1\ndef swim():\n return x\n"; + let after = "x = 1\ndef swim():\n return x + 1\n"; + fs::write(dir.path().join("b.py"), after).expect("fixture"); + let echo = + sound_edit_echo(dir.path(), &dir.path().join("b.py"), before, after).expect("echo"); + assert!(echo.contains("touched def swim:2"), "{echo}"); + assert!(echo.contains("heard by a.py"), "{echo}"); + } + + #[test] + fn edit_echo_attributes_body_edits_to_enclosing_symbol() { + let dir = tempfile::tempdir().expect("tempdir"); + // No mates: touched only, no heard-by half. + let before = "fn main() {\n let x = 1;\n println!(\"{x}\");\n}\nfn other() {}\n"; + let after = "fn main() {\n let x = 2;\n println!(\"{x}\");\n}\nfn other() {}\n"; + fs::write(dir.path().join("solo.rs"), after).expect("fixture"); + let echo = + sound_edit_echo(dir.path(), &dir.path().join("solo.rs"), before, after).expect("echo"); + assert_eq!(echo, "[Edit echo: touched fn main:1]"); + } + + #[test] + fn edit_echo_silent_when_nothing_to_say() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("note.txt"), "hi\n").expect("fixture"); + // Identical: no mutation, no echo. + assert!( + sound_edit_echo(dir.path(), &dir.path().join("note.txt"), "hi\n", "hi\n").is_none() + ); + // Unknown language, no mates: neither half can sound. + assert!( + sound_edit_echo(dir.path(), &dir.path().join("note.txt"), "hi\n", "yo\n").is_none() + ); + } + + #[test] + fn edit_echo_new_file_names_first_symbols() { + let dir = tempfile::tempdir().expect("tempdir"); + let after = "fn alpha() {}\nfn beta() {}\n"; + fs::write(dir.path().join("fresh.rs"), after).expect("fixture"); + let echo = + sound_edit_echo(dir.path(), &dir.path().join("fresh.rs"), "", after).expect("echo"); + assert!(echo.contains("touched fn alpha:1, fn beta:2"), "{echo}"); + } + #[test] fn unlistable_pod_fallback_skips_dotfiles_like_the_listed_path() { let dir = tempfile::tempdir().expect("tempdir"); diff --git a/crates/tui/src/tools/file.rs b/crates/tui/src/tools/file.rs index 0eed80dbab..db75bd3f28 100644 --- a/crates/tui/src/tools/file.rs +++ b/crates/tui/src/tools/file.rs @@ -623,6 +623,17 @@ async fn contract_mutation_result( ) -> ToolResult { let paths = [file_path.to_path_buf()]; let diagnostics = lsp_diagnostics_for_paths(context, &paths).await; + // Echolocation: the model-facing receipt is one line while the diff + // rides metadata for the TUI, so the echo tells the model what its + // edit hit (touched symbols) and what hears it (impacted callers). + // Deterministic, hard-budgeted, None when there is nothing to say. + let mut summary = summary; + if let Some(echo) = + crate::tools::echolocation::sound_edit_echo(&context.workspace, file_path, before, after) + { + summary.push_str("\n\n"); + summary.push_str(&echo); + } ToolResult::success(summary).with_metadata(json!({ "event": "file.mutation", "lsp_diagnostics": diagnostics, @@ -1560,7 +1571,9 @@ impl WriteFileTool { drop(mutation_guard); let outcome = if existed_before { "updated" } else { "created" }; - let utf16_units = written.encode_utf16().count(); + // Bytes are bytes: the receipt reports what was written to disk, + // not UTF-16 code units (which differ on non-ASCII content). + let bytes_written = written.len(); Ok(contract_mutation_result( context, &file_path, @@ -1568,7 +1581,7 @@ impl WriteFileTool { prior_contents.as_ref(), &written, outcome, - format!("Successfully wrote {utf16_units} bytes to {path_str}"), + format!("Successfully wrote {bytes_written} bytes to {path_str}"), ) .await) } diff --git a/crates/tui/src/tools/file/tests.rs b/crates/tui/src/tools/file/tests.rs index 9d6bb14df7..e7052d219a 100644 --- a/crates/tui/src/tools/file/tests.rs +++ b/crates/tui/src/tools/file/tests.rs @@ -102,12 +102,12 @@ async fn contract_read_appends_a_pod_footer_to_a_whole_source_file() { "whole file first, pod footer after" ); assert!( - result.content.contains("[Pod:"), + result.content.contains("[Pod ("), "pod footer present: {}", result.content ); assert!( - result.content.contains("[Sounding: 1 symbol: fn main:1]"), + result.content.contains("[Sound: 1 symbol: fn main:1]"), "sounded symbol present: {}", result.content ); @@ -404,7 +404,7 @@ async fn contract_read_metadata_for_ordinary_whole_read() { "whole file first, pod footer after" ); assert!( - result.content.contains("[Pod:"), + result.content.contains("[Pod ("), "pod footer present: {}", result.content ); @@ -647,6 +647,56 @@ async fn queued_parallel_contract_edits_preserve_both_changes() { ); } +#[tokio::test] +async fn contract_edit_echoes_touched_symbols_and_callers() { + let temporary = tempfile::tempdir().expect("tempdir"); + let context = ToolContext::new(temporary.path()); + std::fs::write(temporary.path().join("a.py"), "import b\nprint(b.swim())\n").expect("fixture"); + std::fs::write(temporary.path().join("b.py"), "def swim():\n return 1\n").expect("fixture"); + let result = EditFileTool::execute_contract_edits( + json!({ + "path": "b.py", + "edits": [{"oldText": "return 1", "newText": "return 2"}] + }), + &context, + ) + .await + .expect("edit runs"); + assert!( + result.content.contains("Successfully replaced"), + "{}", + result.content + ); + assert!( + result.content.contains("[Edit echo: touched def swim:1"), + "{}", + result.content + ); + assert!( + result.content.contains("heard by a.py"), + "{}", + result.content + ); +} + +#[tokio::test] +async fn contract_write_receipt_counts_bytes_not_utf16_units() { + // "héllo\n" is 7 bytes but 6 UTF-16 units; the receipt must say 7. + let temporary = tempfile::tempdir().expect("tempdir"); + let context = ToolContext::new(temporary.path()); + let result = WriteFileTool::execute_contract_write( + json!({"path": "note.txt", "content": "héllo\n"}), + &context, + ) + .await + .expect("write runs"); + assert!( + result.content.contains("Successfully wrote 7 bytes"), + "{}", + result.content + ); +} + #[tokio::test] async fn cancelled_queued_pi_write_never_starts() { let temporary = tempfile::tempdir().expect("tempdir"); @@ -719,7 +769,7 @@ async fn contract_read_enters_terminal_buzz_on_third_pod_visit() { .await .expect("read result"); assert!( - result.content.contains("heard by: src/net/mod.rs"), + result.content.contains("heard by: mod.rs"), "{}", result.content ); diff --git a/crates/tui/src/tools/search.rs b/crates/tui/src/tools/search.rs index 90b752d03d..eb5de8d322 100644 --- a/crates/tui/src/tools/search.rs +++ b/crates/tui/src/tools/search.rs @@ -95,7 +95,7 @@ impl ToolSpec for GrepFilesTool { }, "max_results": { "type": "integer", - "description": "Maximum number of results to return (default: 100)" + "description": "Maximum number of results to return (default: 100, max: 1000)" } }, "required": ["pattern"] @@ -122,8 +122,12 @@ impl ToolSpec for GrepFilesTool { .unwrap_or(usize::MAX) .min(1000); let case_insensitive = optional_bool(&input, "case_insensitive", false)?; + // Bounded like file_search's limit: an explicit value is the + // model's choice, but a runaway (or typo) must not mint an + // unbounded JSON array. let max_results = usize::try_from(optional_u64(&input, "max_results", MAX_RESULTS as u64)?) - .unwrap_or(MAX_RESULTS); + .unwrap_or(MAX_RESULTS) + .clamp(1, 1000); // Parse include patterns let include_patterns: Vec = input @@ -202,6 +206,10 @@ impl ToolSpec for GrepFilesTool { let mut results: Vec = Vec::new(); let mut files_searched = 0; let mut total_matches = 0; + // Proven cut, not inferred: set only when a match exists past + // what we return (the old `total > max` could never fire — + // totals were budget-capped before the comparison). + let mut truncated = false; visit_files( &search_path, @@ -210,9 +218,6 @@ impl ToolSpec for GrepFilesTool { cancel_token, follow_symlinks, &mut |file_path| { - if results.len() >= max_results { - return Ok(WalkControl::Stop); - } check_cancelled(cancel_token)?; // Skip files that are too large @@ -229,8 +234,8 @@ impl ToolSpec for GrepFilesTool { .to_string_lossy() .to_string(); - let budget = max_results - results.len(); - let Some(file_matches) = search_file_streaming( + let budget = max_results.saturating_sub(results.len()); + let Some((file_matches, file_had_more)) = search_file_streaming( file_path, &relative_path, ®ex, @@ -245,6 +250,12 @@ impl ToolSpec for GrepFilesTool { files_searched += 1; total_matches += file_matches.len(); results.extend(file_matches); + if file_had_more { + // A match exists past the returned set: stop and say + // so. Exactly-max walks complete with `false`. + truncated = true; + return Ok(WalkControl::Stop); + } Ok(WalkControl::Continue) }, )?; @@ -286,7 +297,7 @@ impl ToolSpec for GrepFilesTool { "matches": matches_json, "total_matches": total_matches, "files_searched": files_searched, - "truncated": total_matches > max_results, + "truncated": truncated, "pods": pods_json, "pods_omitted": pods_omitted, })) @@ -372,7 +383,7 @@ fn search_file_streaming( context_lines: usize, budget: usize, cancel_token: Option<&CancellationToken>, -) -> Result>, ToolError> { +) -> Result, bool)>, ToolError> { let Ok(file) = fs::File::open(path) else { return Ok(None); }; @@ -380,6 +391,9 @@ fn search_file_streaming( let mut raw: Vec = Vec::new(); let mut before: VecDeque = VecDeque::new(); let mut matches: Vec = Vec::new(); + // True when a match exists past the collected budget, so the caller + // can report truncation honestly instead of inferring it. + let mut file_had_more = false; // Matches still waiting for after-context lines: (index into `matches`, // lines still needed). Entries complete in FIFO order. let mut pending: VecDeque<(usize, usize)> = VecDeque::new(); @@ -420,7 +434,11 @@ fn search_file_streaming( pending.pop_front(); } - if matches.len() < budget && regex.is_match(line) { + // Past the budget we keep testing but stop collecting; the + // first surplus match proves the cut. With nothing pending (no + // after-context owed) we can stop scanning immediately. + let is_match = regex.is_match(line); + if matches.len() < budget && is_match { matches.push(GrepMatch { file: relative_path.to_string(), line_number: line_idx + 1, @@ -431,6 +449,11 @@ fn search_file_streaming( if context_lines > 0 { pending.push_back((matches.len() - 1, context_lines)); } + } else if is_match { + file_had_more = true; + if pending.is_empty() { + break; + } } if context_lines > 0 { @@ -442,7 +465,7 @@ fn search_file_streaming( line_idx += 1; } - Ok(Some(matches)) + Ok(Some((matches, file_had_more))) } /// Flow control for the streaming file walk. diff --git a/crates/tui/src/tools/search/tests.rs b/crates/tui/src/tools/search/tests.rs index e0f4c5d58a..2a7fcde2dd 100644 --- a/crates/tui/src/tools/search/tests.rs +++ b/crates/tui/src/tools/search/tests.rs @@ -8,6 +8,55 @@ use crate::tools::spec::{ApprovalRequirement, ToolContext, ToolSpec}; use super::{GrepFilesTool, matches_glob}; +#[tokio::test] +async fn grep_max_results_is_clamped_not_trusted() { + let tmp = tempdir().expect("tempdir"); + let body = "needle here\n".repeat(1_200); + fs::write(tmp.path().join("hay.txt"), &body).expect("write"); + let ctx = ToolContext::new(tmp.path()); + let result = GrepFilesTool + .execute(json!({"pattern": "needle", "max_results": 1_000_000}), &ctx) + .await + .expect("grep runs"); + let payload: Value = serde_json::from_str(&result.content).expect("json"); + assert_eq!(payload["matches"].as_array().expect("matches").len(), 1000); + assert_eq!(payload["truncated"], true); + assert_eq!(payload["total_matches"], 1000); +} + +#[tokio::test] +async fn grep_truncated_fires_only_on_a_proven_cut() { + let tmp = tempdir().expect("tempdir"); + // Exactly max: the walk completes, nothing cut, flag stays false. + fs::write(tmp.path().join("exact.txt"), "needle\n".repeat(3)).expect("write"); + // Over max across files: the second file proves the cut. + fs::write(tmp.path().join("a.txt"), "needle\n".repeat(2)).expect("write"); + fs::write(tmp.path().join("b.txt"), "needle\n".repeat(5)).expect("write"); + let ctx = ToolContext::new(tmp.path()); + + let result = GrepFilesTool + .execute( + json!({"pattern": "needle", "path": "exact.txt", "max_results": 3}), + &ctx, + ) + .await + .expect("grep runs"); + let payload: Value = serde_json::from_str(&result.content).expect("json"); + assert_eq!(payload["matches"].as_array().expect("matches").len(), 3); + assert_eq!(payload["truncated"], false); + + let result = GrepFilesTool + .execute( + json!({"pattern": "needle", "path": "a.txt", "max_results": 1}), + &ctx, + ) + .await + .expect("grep runs"); + let payload: Value = serde_json::from_str(&result.content).expect("json"); + assert_eq!(payload["matches"].as_array().expect("matches").len(), 1); + assert_eq!(payload["truncated"], true); +} + #[test] fn grep_description_matches_default_exclusion_behavior() { let description = GrepFilesTool.description(); diff --git a/crates/tui/src/tools/skill.rs b/crates/tui/src/tools/skill.rs index fddbb8e11d..700c5c45df 100644 --- a/crates/tui/src/tools/skill.rs +++ b/crates/tui/src/tools/skill.rs @@ -18,11 +18,17 @@ use async_trait::async_trait; use serde_json::{Value, json}; use crate::skills::{ - Skill, SkillDiscoveryMode, SkillSource, discover_for_workspace_and_dir_with_mode_and_plugins, + MAX_SKILL_DESCRIPTION_CHARS, Skill, SkillDiscoveryMode, SkillSource, + discover_for_workspace_and_dir_with_mode_and_plugins, discover_in_workspace_with_mode_and_plugins, skill_directories_for_workspace_and_dir, skills_directories_for_mode, }; +/// Max rows in a `load_skill` listing. Past this an unbounded dump costs +/// more than it routes; the overflow line keeps the count honest and +/// every skill stays loadable by exact name. +const LOAD_SKILL_LIST_MAX_ROWS: usize = 100; + use super::spec::{ ApprovalRequirement, ToolCapability, ToolContext, ToolError, ToolResult, ToolSpec, }; @@ -99,20 +105,38 @@ impl ToolSpec for LoadSkillTool { } .into_enabled(); - // Listing mode: empty name, "*", or "list" returns the full registry (#4651). + // Listing mode: empty name, "*", or "list" returns the registry (#4651), + // capped at LOAD_SKILL_LIST_MAX_ROWS with descriptions shortened to + // the same ceiling the prompt index uses. if name.is_empty() || name == "*" || name == "list" { let skills = registry.list(); if skills.is_empty() { return Ok(ToolResult::success("No skills installed.")); } let mut listing = format!("Available skills ({}):\n", skills.len()); - for skill in skills { - if skill.description.trim().is_empty() { + for skill in skills.iter().take(LOAD_SKILL_LIST_MAX_ROWS) { + let description = skill.description.trim(); + if description.is_empty() { listing.push_str(&format!(" - {}\n", skill.name)); } else { - listing.push_str(&format!(" - {} — {}\n", skill.name, skill.description)); + let capped: String = description + .chars() + .take(MAX_SKILL_DESCRIPTION_CHARS) + .collect(); + let marker = if capped.len() < description.len() { + "…" + } else { + "" + }; + listing.push_str(&format!(" - {} — {capped}{marker}\n", skill.name)); } } + if skills.len() > LOAD_SKILL_LIST_MAX_ROWS { + listing.push_str(&format!( + " ... and {} more installed; load any by exact name.\n", + skills.len() - LOAD_SKILL_LIST_MAX_ROWS + )); + } return Ok(ToolResult::success(listing)); } @@ -601,6 +625,53 @@ mod tests { } } + #[tokio::test] + async fn execute_listing_caps_rows_and_descriptions_honestly() { + let _lock = crate::test_support::lock_test_env(); + let tmp = tempdir().unwrap(); + let _home = crate::test_support::EnvVarGuard::set("HOME", tmp.path().join("home")); + let _cw_home = + crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", tmp.path().join("cw-home")); + let workspace = tmp.path().to_path_buf(); + let skills_dir = workspace.join(".codewhale").join("skills"); + let long = "d".repeat(500); + for index in 0..105 { + let description = if index == 0 { long.as_str() } else { "Short." }; + write_skill( + &skills_dir, + &format!("skill-{index:03}"), + description, + "Body.", + ); + } + + let context = ToolContext::new(workspace); + let result = LoadSkillTool + .execute(json!({"name": "list"}), &context) + .await + .expect("listing should succeed"); + assert!( + result.content.contains("Available skills (105)"), + "{}", + result.content + ); + assert!( + result + .content + .contains("... and 5 more installed; load any by exact name."), + "{}", + result.content + ); + assert!(!result.content.contains("skill-104"), "{}", result.content); + let capped: String = "d".repeat(400); + assert!( + result.content.contains(&format!("skill-000 — {capped}…")), + "{}", + result.content + ); + assert!(!result.content.contains(&long), "description must shorten"); + } + #[tokio::test] async fn execute_listing_reports_empty_registry_plainly() { let _lock = crate::test_support::lock_test_env(); diff --git a/docs/ECHOLOCATION.md b/docs/ECHOLOCATION.md new file mode 100644 index 0000000000..5b34830adf --- /dev/null +++ b/docs/ECHOLOCATION.md @@ -0,0 +1,64 @@ +# Echolocation + +Stateless, index-free navigation soundings. File tools echo their +surroundings fresh from source — no index to build, no background work, +no new dependencies. Every sounding is deterministic (sorted, capped, +hard-budgeted, `/`-separated paths on every OS), so a repeated call is +byte-identical and cache-friendly, and soundings never enter the +session-pinned prefix: they ride tool results (append-only history) +only. + +## What each surface emits + +- **read** appends a footer: `[Pod (N): …]` sibling mates (bare + names — the dir is the file just read, so it goes unstated) plus + `[Sound: N symbols: …; imports: …; links: …]` (line-numbered + symbols, import roots, cross-pod file links; same-pod links render + bare and import roots a link already names (file stem or package + path) are dropped). Budget: 1200B. +- **Terminal buzz**: after two distinct files are read in one pod, the + next read there adds `heard by …` — podmates whose resolved links + include this file. Sparse clicks while searching, high-resolution + echoes at close range. +- **read miss** (`NotFound` only — never on denylist/permission refusals) + appends `[Miss echo: ]`: podmates of the missing file's parent, + so a typo still orients the next call. +- **grep / file_search** chart each matched dir: mates with `(match)` + marks, under the calling search's own visibility gates (denylist, + excludes, extensions, gitignore). Pods never enumerate names the + calling search hides. +- **project_map** sounds each key file: one line of `name:line` symbols + per file, so the map says what lives where. +- **write / edit** append `[Edit echo: touched …; heard by …]`: the + symbols the mutation touched (nearest declaration at or above each + changed span, via the same `similar` diff that builds the unified + diff, under a 100ms timeout) and the podmates linking the file. + The model-facing receipt is one line and the diff rides metadata + for the TUI, so the echo is how the model learns what its edit hit. + +## Languages + +Symbol, import, and link sounding covers Rust, Python, JavaScript / +TypeScript, and Go via deliberately small line scanners (see +`crates/tui/src/tools/echolocation.rs` for the documented lookalike +classes). Anything unresolvable is omitted, never guessed; links +require an on-disk target. + +## Tuning and ablation + +`CODEWHALE_ECHO` gates sections for trials +(`off:`, comma-separated; names: `mates`, `symbols`, `imports`, +`links`, `callers`, `pods`, `soundings`). Unset or malformed means +everything on. The gates are process-pinned and read-only; there is no +per-session or per-call control, by design. The squeeze follows the +same gates, so an `off:links` trial measures links alone. + +## Non-goals + +- No tool description teaches the footer format: the brackets are + self-describing, and every description byte on an eager tool taxes + the session prefix. The diet stops where behavior would degrade. +- No cross-pod callers: reverse links scan same-dir mates only, so the + cost of a sounding is bounded by one directory listing plus a + bounded re-read of small mates. +- No `patch` (multi-file) echoes yet; single-file write/edit only. From 3abb70f6dbcf0805c2ff24aad17f7147925cc119 Mon Sep 17 00:00:00 2001 From: AdityaVG13 Date: Mon, 21 Sep 2026 02:39:03 -0400 Subject: [PATCH 04/26] feat(echo): recover relative-import leaf names, drop temp probe Relative imports now sound their leaf instead of a bare dot (from . import sib -> sib, ./server -> server), which composes with stem-shadowing when the link resolves and names the module when it does not. Removes the temporary footer-breakdown probe (its numbers landed in the PR; the eprintln tripped the print_stderr deny and flipped Lint red). Signed-off-by: AdityaVG13 --- crates/tui/src/tools/echolocation.rs | 117 +++++++++------------------ 1 file changed, 37 insertions(+), 80 deletions(-) diff --git a/crates/tui/src/tools/echolocation.rs b/crates/tui/src/tools/echolocation.rs index 9e59b757c4..cc4b01f447 100644 --- a/crates/tui/src/tools/echolocation.rs +++ b/crates/tui/src/tools/echolocation.rs @@ -1264,9 +1264,10 @@ fn parse_py_item(line: &str) -> Option { None } -/// Import roots of one line: every `import a, b.c` member, or the -/// `from` module (`a.b` → `a`; pure-relative `from . import x` → `.` -/// for depth, honest). Semicolons split statements. +/// Import roots of one line: every `import a, b.c` member, the +/// `from` module (`a.b` → `a`), or the import-list leaves for +/// pure-relative modules (`from . import sib` → `sib`). Semicolons +/// split statements. fn parse_py_import_roots(line: &str) -> Vec { let line = line.trim_start(); let mut out = Vec::new(); @@ -1283,7 +1284,26 @@ fn parse_py_import_roots(line: &str) -> Vec { return out; } if let Some(rest) = strip_keyword(line, "from") { - let module = rest.split(" import ").next().unwrap_or("").trim(); + let mut parts = rest.split(" import "); + let module = parts.next().unwrap_or("").trim(); + if module.trim_start_matches('.').is_empty() { + // Pure-relative: the module names no package, so leaf names + // come from the import list (`from . import sib` → `sib`), + // which stem-shadowing then drops when the link resolves. + let list = parts.next().unwrap_or(""); + for part in list.split(',') { + let name = part + .split(" as ") + .next() + .unwrap_or("") + .trim_matches(['(', ')', ' ', '\t']); + let leaf = name.split('.').next().unwrap_or("").trim(); + if let Some(root) = valid_root(leaf) { + out.push(root); + } + } + return out; + } let root = py_module_root(module); if !root.is_empty() { out.push(root); @@ -1292,17 +1312,13 @@ fn parse_py_import_roots(line: &str) -> Vec { out } -/// Root of a Python module path: leading dots collapse to their depth -/// when no named segment leads (`from . import x` → `.`). Named roots -/// validate as identifiers; dots pass through as depth markers. +/// Root of a Python module path: first named segment after any +/// leading dots (`..parent.x` → `parent`). Pure-relative modules never +/// reach here — the caller takes leaf names from the import list. fn py_module_root(module: &str) -> String { let dots = module.chars().take_while(|c| *c == '.').count(); let named = module[dots..].split('.').next().unwrap_or("").trim(); - if named.is_empty() { - ".".repeat(dots.max(1)) - } else { - valid_root(named).unwrap_or_default() - } + valid_root(named).unwrap_or_default() } fn valid_root(root: &str) -> Option { @@ -1483,13 +1499,16 @@ fn ident_head(rest: &str) -> Option { Some(name) } -/// JS/TS import roots: relative → `.`; scoped `@s/n` kept whole; -/// otherwise the first path segment. Covers `import`, `export-from`, -/// `require`, and dynamic `import()`. +/// JS/TS import roots: relative specs resolve to the first named +/// segment past the dots (`./server` → `server`); scoped `@s/n` kept +/// whole; otherwise the first path segment. Covers `import`, +/// `export-from`, `require`, and dynamic `import()`. fn parse_js_import_root(line: &str) -> Option { let spec = js_import_spec(line)?; if spec.starts_with('.') { - return Some(".".to_string()); + let tail = spec.trim_start_matches('.').trim_start_matches('/'); + let first = tail.split('/').next().unwrap_or("").trim(); + return valid_root(first); } if let Some(scoped) = spec.strip_prefix('@') { let mut parts = scoped.split('/'); @@ -2369,7 +2388,7 @@ define_not_a_keyword = True let roots: Vec = sound_import_roots(PY_SAMPLE, SoundLang::Python) .into_iter() .collect(); - assert_eq!(roots, [".", "localmod", "os", "parent", "pkg", "sys"]); + assert_eq!(roots, ["localmod", "os", "parent", "pkg", "sibling", "sys"]); } #[test] @@ -2455,7 +2474,7 @@ if (x) { y(); } let roots: Vec = sound_import_roots(JS_SAMPLE, SoundLang::JavaScript) .into_iter() .collect(); - assert_eq!(roots, [".", "@scope/pkg", "fs", "react"]); + assert_eq!(roots, ["@scope/pkg", "chunk", "fs", "react", "server", "setup"]); } #[test] @@ -2770,68 +2789,6 @@ func main() { assert!(!raws.contains(&"vault_link"), "{raws:?}"); } - #[test] - fn tmp_measure_footer_breakdown() { - // TEMPORARY squeeze probe; removed after the numbers land. - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); - let samples = [ - "crates/tui/src/tools/echolocation.rs", - "crates/tui/src/tools/file.rs", - "crates/tui/src/tools/search.rs", - "crates/tui/src/tools/diff_format.rs", - "crates/tui/src/tools/skill.rs", - "crates/tui/src/core/engine/tool_catalog.rs", - "crates/tui/src/skills/mod.rs", - "crates/workflow/src/lib.rs", - "web/lib/content/tools.ts", - ]; - for rel in samples { - let path = root.join(rel); - let Ok(text) = fs::read_to_string(&path) else { - continue; - }; - let chart = sound_echolocation(&root, &path, &text, true); - let footer = chart.render_footer(); - let mates: usize = chart.podmates.iter().map(|m| m.len() + 2).sum(); - let syms: usize = chart.symbols.iter().map(|m| m.len() + 2).sum(); - let imps: usize = chart.import_roots.iter().map(|m| m.len() + 2).sum(); - let links: usize = chart.links.iter().map(|m| m.len() + 2).sum(); - let calls: usize = chart.callers.iter().map(|m| m.len() + 2).sum(); - // Import roots shadowed by a resolved link (same stem). - let link_stems: Vec<&str> = chart - .links - .iter() - .filter_map(|l| l.rsplit('/').next()?.split('.').next()) - .collect(); - let shadowed = chart - .import_roots - .iter() - .filter(|r| link_stems.contains(&r.as_str())) - .count(); - eprintln!( - "BREAKDOWN {rel} file={}B footer={}B mates={}B({}/{}) syms={}B({}/{}) imps={}B({}/{},shadow={}) links={}B({}/{}) calls={}B({}/{})", - text.len(), - footer.len(), - mates, - chart.podmates.len(), - chart.podmate_total, - syms, - chart.symbols.len(), - chart.symbol_total, - imps, - chart.import_roots.len(), - chart.import_total, - shadowed, - links, - chart.links.len(), - chart.link_total, - calls, - chart.callers.len(), - chart.caller_total, - ); - } - } - #[test] fn block_comments_mute_and_release_across_lines() { let text = "/*\nfn fake() {}\n*/\nfn real() {}\n"; From 0368c82b2b18c8adb6519f79a71a119b76235842 Mon Sep 17 00:00:00 2001 From: AdityaVG13 Date: Mon, 21 Sep 2026 03:32:04 -0400 Subject: [PATCH 05/26] Fork-prefix cache inheritance for subagents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Child on the same route/model/tools inherits the parent's cached prompt prefix (system prompt + compacted history) as its frozen LivePrefix base, warming from the warm cache instead of going cold. Resolution is a pure deterministic gate (resolve_fork_inherit) — same route, same model, same or subset tools, child config digest must descend from the parent chain — so narrowed roles and grandchildren correctly fall back cold. No human-language compression anywhere; the child extends the parent's verbatim prefix with its own delegation message (append-only history per docs/CACHE.md), keeping prompt-cache prefix-matching intact. --- crates/tui/src/core/engine.rs | 9 + crates/tui/src/core/engine/tests.rs | 2 + crates/tui/src/core/engine/turn_loop.rs | 35 ++++ crates/tui/src/prompt_zones.rs | 261 ++++++++++++++++++++++++ crates/tui/src/tools/echolocation.rs | 5 +- crates/tui/src/tools/subagent/mod.rs | 117 ++++++++++- crates/tui/src/tools/subagent/tests.rs | 48 +++++ 7 files changed, 469 insertions(+), 8 deletions(-) diff --git a/crates/tui/src/core/engine.rs b/crates/tui/src/core/engine.rs index 42c5e5f13b..d955455ec3 100644 --- a/crates/tui/src/core/engine.rs +++ b/crates/tui/src/core/engine.rs @@ -884,6 +884,13 @@ pub struct Engine { /// Additive exact provider id. `None` preserves the legacy root-literal /// custom route across snapshots and config reloads. api_provider_id: Option, + /// Live header cell for fork-prefix inheritance: the turn loop writes + /// the latest request's exact (system, tools, route, hit-flag) here + /// per model call, and fork contexts clone the handle so a + /// same-route child can extend the parent's cached prefix. Safe as + /// a session-lifetime cell: turns run strictly one at a time, so it + /// always holds the latest request of the running turn. + live_header: crate::prompt_zones::SharedLiveHeader, active_route_limits: Option, active_route_capabilities: codewhale_config::route::RouteCapabilities, rx_op: mpsc::Receiver, @@ -1739,6 +1746,7 @@ impl Engine { api_provider, api_provider_identity, api_provider_id, + live_header: crate::prompt_zones::new_live_header_cell(), active_route_limits, active_route_capabilities: codewhale_config::route::RouteCapabilities::default(), rx_op, @@ -4543,6 +4551,7 @@ impl Engine { .await; Some(SubAgentForkContext { messages: self.messages_with_turn_metadata(), + live_header: Arc::clone(&self.live_header), structured_state_block: state.to_system_block(), // Resolve at spawn time so a todo_write earlier in this turn // reaches the child rather than freezing turn-start state. diff --git a/crates/tui/src/core/engine/tests.rs b/crates/tui/src/core/engine/tests.rs index 2fe5058a5d..630a83b15e 100644 --- a/crates/tui/src/core/engine/tests.rs +++ b/crates/tui/src/core/engine/tests.rs @@ -15417,6 +15417,7 @@ async fn fork_state_block_reuses_the_snapshot_body() { }; let fork_context = crate::tools::subagent::SubAgentForkContext { messages: engine.messages_with_turn_metadata(), + live_header: crate::prompt_zones::new_live_header_cell(), structured_state_block: state.to_system_block(), work_source: Some(engine.todo_source()), }; @@ -15633,6 +15634,7 @@ async fn same_turn_fork_carries_the_updated_todo() { .to_system_block(); let fork_context = crate::tools::subagent::SubAgentForkContext { messages: engine.messages_with_turn_metadata(), + live_header: crate::prompt_zones::new_live_header_cell(), structured_state_block: stable_block.clone(), work_source: Some(engine.todo_source()), }; diff --git a/crates/tui/src/core/engine/turn_loop.rs b/crates/tui/src/core/engine/turn_loop.rs index 643b1b7046..5b010defff 100644 --- a/crates/tui/src/core/engine/turn_loop.rs +++ b/crates/tui/src/core/engine/turn_loop.rs @@ -1499,6 +1499,32 @@ impl Engine { // first call) so we can resend it on a transparent retry below // when the wire dies before any content was streamed (#103). let stream_request = request; + // Fork-prefix inheritance: capture this final request's exact + // header for fork children (cheap clones next to a network + // call; skipped entirely when the trial gate is off). The + // pending-route block below touches billing/events only, so + // these bytes are what the provider caches. + if crate::prompt_zones::fork_inherit_enabled() + && let Some(tools_json) = crate::prompt_zones::ordered_tool_catalog_json( + stream_request.tools.as_deref().unwrap_or(&[]), + ) + { + *self.live_header.lock() = Some(crate::prompt_zones::LiveHeaderSnapshot { + system: stream_request.system.clone(), + active_names: stream_request + .tools + .as_deref() + .unwrap_or(&[]) + .iter() + .map(|tool| tool.name.clone()) + .collect(), + tools_json, + model: stream_request.model.clone(), + provider: self.api_provider, + provider_identity: self.api_provider_identity.clone(), + last_hit_tokens: None, + }); + } let _ = self .tx_event .send(Event::ToolRequestSnapshot { @@ -1665,6 +1691,15 @@ impl Engine { // transport error is still a billed, incomplete response; it must // not be discarded and re-issued. turn.add_parent_usage(&usage); + // Fork-prefix inheritance: mark the snapshotted header hot or + // cold from this response's usage. Sequential loop order + // guarantees this usage answers the request captured above. + if crate::prompt_zones::fork_inherit_enabled() { + let mut guard = self.live_header.lock(); + if let Some(cell) = guard.as_mut() { + cell.last_hit_tokens = usage.prompt_cache_hit_tokens; + } + } turn.note_parent_prompt_len(self.session.messages.len()); self.session.latest_parent_input_tokens = turn.latest_parent_input_tokens; if usage_reported { diff --git a/crates/tui/src/prompt_zones.rs b/crates/tui/src/prompt_zones.rs index 14e38d1076..7bead51601 100644 --- a/crates/tui/src/prompt_zones.rs +++ b/crates/tui/src/prompt_zones.rs @@ -182,6 +182,138 @@ impl std::fmt::Display for PrefixDrift { } } +// ── LiveHeader (fork-prefix inheritance) ───────────────────────────────── + +/// Exact header of the parent turn's latest model request, shared with +/// fork children so a same-route child can extend the parent's live +/// cached prefix instead of starting cold. +/// +/// Written once per model request by the turn loop (after the request +/// is final), read once per fork spawn. `None` until the first request +/// of the session goes out. The history half of the prefix rides +/// `SubAgentForkContext.messages` (turn-start snapshot); the system and +/// tools here must byte-match a cached request or inheritance silently +/// costs more than a cold start — every gate below fails closed. +#[derive(Debug, Clone)] +pub struct LiveHeaderSnapshot { + /// Exact system prompt, shape-preserving (`Text` stays `Text`). + pub system: Option, + /// Wire-order canonical tools JSON (see [`ordered_tool_catalog_json`]). + pub tools_json: String, + /// Wire-order tool names, for warming the child's activation cache + /// in admission order so its rebuilt wire block can byte-match. + pub active_names: Vec, + /// Route the request was built for (caches are per route). + pub model: String, + pub provider: crate::config::ApiProvider, + pub provider_identity: String, + /// Cache-hit tokens on the response to the snapshotted request. + /// `None` until usage lands; inheritance requires `Some(>0)` — a + /// provably hot prefix, not a hopefully warm one. + pub last_hit_tokens: Option, +} + +/// Cloneable handle to the turn's live-header cell. Created once per +/// Engine (turns run strictly one at a time); fork contexts clone the +/// `Arc` and read it at spawn. +pub type SharedLiveHeader = Arc>>; + +/// Empty live-header cell: nothing cached yet, every gate reads cold. +#[must_use] +pub fn new_live_header_cell() -> SharedLiveHeader { + Arc::new(parking_lot::Mutex::new(None)) +} + +/// Order-sensitive canonical tools serialization: per-tool JSON in +/// wire order, joined by `\n`. Unlike [`tool_catalog_digest`] (sorted +/// for hashing), this must byte-match across independently built +/// catalogs — order is part of the wire bytes. `None` if any tool +/// fails to serialize; the inherit gate treats that as a mismatch. +#[must_use] +pub fn ordered_tool_catalog_json(tools: &[Tool]) -> Option { + let mut out = String::new(); + for (index, tool) in tools.iter().enumerate() { + if index > 0 { + out.push('\n'); + } + out.push_str(&serde_json::to_string(tool).ok()?); + } + Some(out) +} + +/// Trial gate: `CODEWHALE_FORK_INHERIT=off` disables prefix inheritance +/// (same-binary A/B against the default-on arm). Process-pinned. +#[must_use] +pub fn fork_inherit_enabled() -> bool { + static GATE: std::sync::OnceLock = std::sync::OnceLock::new(); + *GATE.get_or_init(|| { + std::env::var("CODEWHALE_FORK_INHERIT") + .map(|raw| raw.trim() != "off") + .unwrap_or(true) + }) +} + +/// Fork-inherit decision with a trial-log reason. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ForkInheritDecision { + pub inherit: bool, + pub reason: &'static str, +} + +/// Pure inherit gate. `child_tools_json` is the child's own rebuilt +/// wire block (warmed with the snapshot's names) — equality here is +/// what makes the provider see one continuous prefix. The grant +/// boundary enforces itself through this comparison: names outside +/// the child's grant cannot warm, so narrowed roles fall back to +/// cold without a role table. `gate_on` is [`fork_inherit_enabled`] +/// read by the caller, so every arm stays unit-testable. +#[must_use] +pub fn resolve_fork_inherit( + gate_on: bool, + snapshot: Option<&LiveHeaderSnapshot>, + child_model: &str, + child_provider: crate::config::ApiProvider, + child_provider_identity: &str, + child_tools_json: Option<&str>, + history_len: usize, +) -> ForkInheritDecision { + let cold = |reason: &'static str| ForkInheritDecision { + inherit: false, + reason, + }; + if !gate_on { + return cold("gate_off"); + } + let Some(snapshot) = snapshot else { + return cold("no_snapshot"); + }; + if snapshot.system.is_none() { + return cold("no_parent_system"); + } + if snapshot.model != child_model { + return cold("model_mismatch"); + } + if snapshot.provider != child_provider { + return cold("provider_mismatch"); + } + if snapshot.provider_identity != child_provider_identity { + return cold("route_mismatch"); + } + if snapshot.last_hit_tokens.unwrap_or(0) == 0 { + return cold("prefix_cold"); + } + if history_len == 0 { + return cold("empty_history"); + } + if child_tools_json != Some(snapshot.tools_json.as_str()) { + return cold("tools_mismatch"); + } + ForkInheritDecision { + inherit: true, + reason: "inherited", + } +} + // ── AppendLog ────────────────────────────────────────────────────────── /// Append-only conversation history. Derefs to `&[Message]` via @@ -733,4 +865,133 @@ mod tests { panic!("expected Text content block"); } } + + fn inherit_snapshot() -> LiveHeaderSnapshot { + LiveHeaderSnapshot { + system: Some(SystemPrompt::Text("parent system".to_string())), + tools_json: "{\"name\":\"read\"}".to_string(), + active_names: vec!["read".to_string()], + model: "deepseek-chat".to_string(), + provider: crate::config::ApiProvider::Deepseek, + provider_identity: "deepseek".to_string(), + last_hit_tokens: Some(1200), + } + } + + #[test] + fn ordered_tool_json_is_order_sensitive_and_stable() { + let pair = [make_tool("a"), make_tool("b")]; + let swapped = [make_tool("b"), make_tool("a")]; + let forward = ordered_tool_catalog_json(&pair).expect("serializes"); + let backward = ordered_tool_catalog_json(&swapped).expect("serializes"); + assert_ne!(forward, backward, "wire order is part of the bytes"); + assert_eq!( + forward, + ordered_tool_catalog_json(&pair).expect("stable"), + "same order serializes identically" + ); + assert_eq!(ordered_tool_catalog_json(&[]).expect("empty"), ""); + } + + #[test] + fn fork_inherit_requires_every_gate() { + let snapshot = inherit_snapshot(); + let decide = |snapshot: Option<&LiveHeaderSnapshot>, model: &str, tools: Option<&str>| { + resolve_fork_inherit( + true, + snapshot, + model, + crate::config::ApiProvider::Deepseek, + "deepseek", + tools, + 4, + ) + }; + let ok = decide( + Some(&snapshot), + "deepseek-chat", + Some("{\"name\":\"read\"}"), + ); + assert!(ok.inherit, "{ok:?}"); + assert_eq!(ok.reason, "inherited"); + + assert_eq!( + decide(None, "deepseek-chat", Some("x")).reason, + "no_snapshot" + ); + let mut no_system = snapshot.clone(); + no_system.system = None; + assert_eq!( + decide(Some(&no_system), "deepseek-chat", Some("x")).reason, + "no_parent_system" + ); + assert_eq!( + decide(Some(&snapshot), "other-model", Some("x")).reason, + "model_mismatch" + ); + let provider_mismatch = resolve_fork_inherit( + true, + Some(&snapshot), + "deepseek-chat", + crate::config::ApiProvider::Openai, + "deepseek", + Some("x"), + 4, + ); + assert_eq!(provider_mismatch.reason, "provider_mismatch"); + assert!(!provider_mismatch.inherit); + let route_mismatch = resolve_fork_inherit( + true, + Some(&snapshot), + "deepseek-chat", + crate::config::ApiProvider::Deepseek, + "custom-mirror", + Some("x"), + 4, + ); + assert_eq!(route_mismatch.reason, "route_mismatch"); + for cold in [None, Some(0)] { + let mut snapshot = snapshot.clone(); + snapshot.last_hit_tokens = cold; + assert_eq!( + decide(Some(&snapshot), "deepseek-chat", Some("x")).reason, + "prefix_cold", + "cold={cold:?}" + ); + } + let empty_history = resolve_fork_inherit( + true, + Some(&snapshot), + "deepseek-chat", + crate::config::ApiProvider::Deepseek, + "deepseek", + Some("{\"name\":\"read\"}"), + 0, + ); + assert_eq!(empty_history.reason, "empty_history"); + assert_eq!( + decide( + Some(&snapshot), + "deepseek-chat", + Some("{\"name\":\"other\"}") + ) + .reason, + "tools_mismatch" + ); + assert_eq!( + decide(Some(&snapshot), "deepseek-chat", None).reason, + "tools_mismatch" + ); + let gated_off = resolve_fork_inherit( + false, + Some(&snapshot), + "deepseek-chat", + crate::config::ApiProvider::Deepseek, + "deepseek", + Some("{\"name\":\"read\"}"), + 4, + ); + assert_eq!(gated_off.reason, "gate_off"); + assert!(!gated_off.inherit); + } } diff --git a/crates/tui/src/tools/echolocation.rs b/crates/tui/src/tools/echolocation.rs index cc4b01f447..bcd0474cff 100644 --- a/crates/tui/src/tools/echolocation.rs +++ b/crates/tui/src/tools/echolocation.rs @@ -2474,7 +2474,10 @@ if (x) { y(); } let roots: Vec = sound_import_roots(JS_SAMPLE, SoundLang::JavaScript) .into_iter() .collect(); - assert_eq!(roots, ["@scope/pkg", "chunk", "fs", "react", "server", "setup"]); + assert_eq!( + roots, + ["@scope/pkg", "chunk", "fs", "react", "server", "setup"] + ); } #[test] diff --git a/crates/tui/src/tools/subagent/mod.rs b/crates/tui/src/tools/subagent/mod.rs index 1a17db1e2d..d313444dfb 100644 --- a/crates/tui/src/tools/subagent/mod.rs +++ b/crates/tui/src/tools/subagent/mod.rs @@ -2327,6 +2327,11 @@ fn terminal_mailbox_message(result: &SubAgentResult, report_ref: Option<&str>) - #[derive(Clone, Debug)] pub struct SubAgentForkContext { pub messages: Vec, + /// Handle to the parent turn's live-header cell (exact system, tools, + /// route, and hot-flag of its latest request). A same-route fork child + /// reads it at spawn to extend the parent's cached prefix instead of + /// starting cold; empty cells (resume, grandchildren) read cold. + pub live_header: crate::prompt_zones::SharedLiveHeader, /// Stable, To-do-free parent state captured once at turn start. History /// semantics stay exactly as they were: this text is not re-derived per /// spawn. @@ -2366,6 +2371,7 @@ impl SubAgentForkContext { Self { messages: self.messages.clone(), + live_header: Arc::clone(&self.live_header), structured_state_block, work_source: self.work_source.clone(), } @@ -10616,6 +10622,9 @@ async fn spawn_subagent_from_input( .unwrap_or(checkpoint_messages); let resume_ctx = SubAgentForkContext { messages, + // Resume forks a settled agent: nothing is live in the + // provider cache, so inheritance stays off by construction. + live_header: crate::prompt_zones::new_live_header_cell(), structured_state_block: None, work_source: None, }; @@ -12942,7 +12951,6 @@ async fn run_subagent( let fork_context = fork_context_enabled .then_some(runtime.fork_context.as_ref()) .flatten(); - let request_system = subagent_request_system_prompt(&system_prompt); // Refresh only the Work portion of the inherited state, now, at the fork // seam (#3983). The parent's captured transcript and its stable state text // are untouched. @@ -13019,6 +13027,10 @@ async fn run_subagent( &agent_id, SubAgentForkContext { messages: messages.clone(), + // v1 is single-level: grandchildren fork cold. Recursive + // inheritance (propagating a child's own live header) is a + // follow-up once single-level trials report. + live_header: crate::prompt_zones::new_live_header_cell(), structured_state_block: None, // A grandchild forks *this* agent, so it inherits this agent's own // list, resolved when that spawn actually happens. @@ -13051,7 +13063,92 @@ async fn run_subagent( )); } let tool_catalog = tool_registry.deferred_catalog_for_model(&agent_type); - let mut tool_surface = SubAgentToolSurface::new(tool_catalog, &[]); + // Fork-prefix inheritance: when this fork can extend the parent's + // live cached prefix (same route, hot prefix, byte-equal wire + // tools), the child's system is the parent's exact bytes and its + // first request prices the shared history as cache hits. Every + // fallback below keeps today's cold start byte-identical. + let gate_on = crate::prompt_zones::fork_inherit_enabled(); + let snapshot = if gate_on && fork_context_enabled { + refreshed_fork_context + .as_ref() + .and_then(|context| context.live_header.lock().clone()) + } else { + None + }; + let parent_warm_names: &[String] = snapshot + .as_ref() + .map(|snapshot| snapshot.active_names.as_slice()) + .unwrap_or(&[]); + let strict_tool_mode = runtime + .api_config + .as_ref() + .and_then(|config| config.strict_tool_mode) + .unwrap_or(false); + let mut tool_surface = SubAgentToolSurface::new(tool_catalog, parent_warm_names); + let first_wire_tools = tool_surface.request_tools( + tool_registry.deferred_catalog_for_model(&agent_type), + strict_tool_mode, + ); + let inherit_route = runtime + .client + .effective_route_envelope(&runtime.model, chrono::Utc::now()); + let decision = crate::prompt_zones::resolve_fork_inherit( + gate_on, + snapshot.as_ref(), + &inherit_route.model, + inherit_route.provider, + &inherit_route.provider_identity, + crate::prompt_zones::ordered_tool_catalog_json(&first_wire_tools).as_deref(), + messages.len(), + ); + let inherited_system = if decision.inherit { + snapshot + .as_ref() + .and_then(|snapshot| snapshot.system.clone()) + } else { + None + }; + let inherited = inherited_system.is_some(); + let request_system = match inherited_system { + Some(system) => system, + None => { + if !parent_warm_names.is_empty() { + // Cold fallback keeps today's exact behavior: rebuild the + // surface without the parent names tried for the gate. + tool_surface = SubAgentToolSurface::new( + tool_registry.deferred_catalog_for_model(&agent_type), + &[], + ); + } + subagent_request_system_prompt(&system_prompt) + } + }; + if inherited && let Some(snapshot) = snapshot.as_ref() { + let system_bytes = serde_json::to_string(&snapshot.system) + .map(|json| json.len() as u64) + .unwrap_or(0); + let history_bytes = serde_json::to_string(&messages) + .map(|json| json.len() as u64) + .unwrap_or(0); + tracing::info!( + target: "subagent", + agent_id = agent_id.as_str(), + bytes = system_bytes + .saturating_add(snapshot.tools_json.len() as u64) + .saturating_add(history_bytes), + model = runtime.model.as_str(), + "fork child inheriting parent cached prefix" + ); + } else if fork_context_enabled { + tracing::info!( + target: "subagent", + agent_id = agent_id.as_str(), + reason = decision.reason, + "fork child cold start" + ); + } + let mut inherit_first_hit_logged = false; let mut steps = 0; let mut final_result: Option = None; let mut pending_inputs: VecDeque = VecDeque::new(); @@ -13228,11 +13325,7 @@ async fn run_subagent( let tools = tool_surface.request_tools( tool_registry.deferred_catalog_for_model(&agent_type), - runtime - .api_config - .as_ref() - .and_then(|config| config.strict_tool_mode) - .unwrap_or(false), + strict_tool_mode, ); let request_active_tool_names = tool_surface.active_names.clone(); let has_tools = !tools.is_empty(); @@ -13499,6 +13592,16 @@ async fn run_subagent( .await; tokens_used = tokens_used.saturating_add(usage_total_tokens(&response.usage)); + if inherited && !inherit_first_hit_logged { + inherit_first_hit_logged = true; + tracing::info!( + target: "subagent", + agent_id = agent_id.as_str(), + hit_tokens = response.usage.prompt_cache_hit_tokens.unwrap_or(0), + input_tokens = response.usage.input_tokens, + "fork child first response cache hits" + ); + } // #6194 item 7: one over-bound step lands the run through the normal // budget-death path (digest + hand-back + preservation note) instead diff --git a/crates/tui/src/tools/subagent/tests.rs b/crates/tui/src/tools/subagent/tests.rs index d1e2f0f41c..7f695c1a2e 100644 --- a/crates/tui/src/tools/subagent/tests.rs +++ b/crates/tui/src/tools/subagent/tests.rs @@ -5376,6 +5376,7 @@ fn forked_subagent_messages_preserve_parent_prefix_then_append_task() { }; let fork_context = SubAgentForkContext { messages: vec![parent_message.clone()], + live_header: crate::prompt_zones::new_live_header_cell(), structured_state_block: Some("## Fork State\n- Mode: `AGENT`".to_string()), work_source: None, }; @@ -8005,6 +8006,7 @@ async fn small_surface_read_only_child_discovers_web_deferred() { async fn small_surface_fork_context_survives_fresh_child_discovery() { let registry = small_surface_registry(FleetRole::Builder); let context = SubAgentForkContext { + live_header: crate::prompt_zones::new_live_header_cell(), messages: vec![ Message { role: Role::Assistant, @@ -8103,6 +8105,48 @@ async fn small_surface_denied_warm_tool_is_not_resurrected() { assert!(surface.hydrate("Web").is_err()); } +fn synthetic_eager_tool(name: &str) -> Tool { + Tool { + tool_type: Some("function".to_string()), + name: name.to_string(), + description: "eager".to_string(), + input_schema: json!({"type": "object", "properties": {}}), + allowed_callers: None, + defer_loading: Some(false), + input_examples: None, + strict: None, + cache_control: None, + } +} + +#[test] +fn warmed_surfaces_rebuild_byte_identical_wire_tools() { + // The fork-inherit gate compares the child's rebuilt wire block + // against the parent's snapshot: same catalog plus same warm names + // must serialize identically, and warming must change the bytes. + let catalog = vec![ + synthetic_eager_tool("read"), + synthetic_deferred_tool("agent", 8), + synthetic_eager_tool(TOOL_SEARCH_NAME), + ]; + let warm = ["agent".to_string()]; + let mut first = SubAgentToolSurface::new(catalog.clone(), &warm); + let mut second = SubAgentToolSurface::new(catalog.clone(), &warm); + let first_wire = model_request_tools(&mut first); + let second_wire = model_request_tools(&mut second); + let first_json = + crate::prompt_zones::ordered_tool_catalog_json(&first_wire).expect("serializes"); + let second_json = + crate::prompt_zones::ordered_tool_catalog_json(&second_wire).expect("serializes"); + assert_eq!(first_json, second_json, "same catalog plus same names"); + assert!(first_wire.iter().any(|tool| tool.name == "agent")); + let mut cold = SubAgentToolSurface::new(catalog, &[]); + let cold_wire = model_request_tools(&mut cold); + let cold_json = crate::prompt_zones::ordered_tool_catalog_json(&cold_wire).expect("serializes"); + assert_ne!(cold_json, first_json, "warming must change the wire block"); + assert!(!cold_wire.iter().any(|tool| tool.name == "agent")); +} + fn synthetic_deferred_tool(name: &str, description_bytes: usize) -> Tool { Tool { tool_type: Some("function".to_string()), @@ -11517,6 +11561,7 @@ fn fresh_forked_and_nested_subagents_share_authority_bound_skill_catalogs() { ); let fork_context = SubAgentForkContext { + live_header: crate::prompt_zones::new_live_header_cell(), messages: vec![Message { role: Role::User, content: vec![ContentBlock::Text { @@ -11548,6 +11593,7 @@ fn fresh_forked_and_nested_subagents_share_authority_bound_skill_catalogs() { &direct_child, "agent_parent", SubAgentForkContext { + live_header: crate::prompt_zones::new_live_header_cell(), messages: Vec::new(), structured_state_block: None, work_source: None, @@ -12261,6 +12307,7 @@ async fn parent_todo_state_still_reaches_children_as_immutable_fork_context() { let mut parent = stub_runtime(); write_todos_as(&parent, &["parent step one"]).await; parent.fork_context = Some(SubAgentForkContext { + live_header: crate::prompt_zones::new_live_header_cell(), messages: Vec::new(), structured_state_block: Some( "## Fork State\n\n### Work\n\nTo-do (0% settled)\n- [ ] #1 parent step one\n" @@ -16310,6 +16357,7 @@ fn nested_tool_runtime_routes_child_completions_to_local_inbox() { let direct_child_runtime = runtime_with_depth(1, Some(root_tx)); let fork_context = SubAgentForkContext { messages: Vec::new(), + live_header: crate::prompt_zones::new_live_header_cell(), structured_state_block: None, work_source: None, }; From 04e473218d24da8076682b0fbcc604bb06828eb4 Mon Sep 17 00:00:00 2001 From: AdityaVG13 Date: Mon, 21 Sep 2026 04:50:04 -0400 Subject: [PATCH 06/26] Prefix-tier fork inheritance with live-trial benchmarks Replace whole-block tool equality (which never held: parent wire blocks grow by deferred discovery while child grants are fixed and narrower) with longest byte-equal head resolution. The child adopts the parent's exact system bytes and sends the matched head on its first request; the rest of its grant stays deferred and activates on demand, so truncation costs cache bytes, never capability. Full head (inherited_full) prices shared history as hits; partial head (inherited_prefix) still prices system plus matched tools. Both sides now canonicalize through tool_to_api_json (the wire shape), so internal-only flag skew (defer_loading, allowed_callers) can't false-negative the comparison. The hot proof carries forward across append-only requests, since spawns execute before the current request's usage lands. Fresh spawns, grandchildren, resumes, route skew, and cold prefixes stay cold with distinct reasons. Trial instrumentation (CODEWHALE_FORK_TRIAL=1, default off) mirrors the decision, prefix span, history-divergence point, and child first-response cache split to stderr as JSON lines for same-binary A/B measurement; headless exec installs no tracing subscriber, so the tracing trial logs are invisible there. Measured (same-binary A/B, DeepSeek, unique histories, child first-response): true cold 0/9421 hits; primed cold 4736/9389; prefix k=3 5504/9941; full k=6 8576/13251. Inherited children reported correct results in every run. DeepSeek counts hits in 64-token blocks. Follow-ups: fork transcripts freeze at turn 1 (the live tail never rides), framing sits ahead of where the tail would (breaking the head match), grandchildren carry ~40KB all-miss transcripts (single-level v1). --- crates/core/src/prefix_cache.rs | 6 +- crates/tui/src/core/engine/turn_loop.rs | 19 +- crates/tui/src/prompt_zones.rs | 324 ++++++++++++++++++++---- crates/tui/src/tools/subagent/mod.rs | 226 ++++++++++++++--- crates/tui/src/tools/subagent/tests.rs | 12 +- 5 files changed, 478 insertions(+), 109 deletions(-) diff --git a/crates/core/src/prefix_cache.rs b/crates/core/src/prefix_cache.rs index 4dc3bfc9f4..6f25e163e8 100644 --- a/crates/core/src/prefix_cache.rs +++ b/crates/core/src/prefix_cache.rs @@ -712,7 +712,11 @@ impl PrefixStabilityManager { /// Serialize a tool to the same JSON shape the chat API receives, /// excluding internal-only fields like `allowed_callers`, `defer_loading`, /// `input_examples`, and `cache_control` that are never sent to DeepSeek. -fn tool_to_api_json(tool: &Tool) -> Option { +/// +/// Shared by the fingerprint manager and fork-prefix inheritance: both +/// need byte-equality against provider-cached prefixes, and raw `Tool` +/// serialization would false-negative on internal-only flag skew. +pub fn tool_to_api_json(tool: &Tool) -> Option { let mut value = serde_json::json!({ "type": "function", "function": { diff --git a/crates/tui/src/core/engine/turn_loop.rs b/crates/tui/src/core/engine/turn_loop.rs index 5b010defff..14e1ad1adc 100644 --- a/crates/tui/src/core/engine/turn_loop.rs +++ b/crates/tui/src/core/engine/turn_loop.rs @@ -1503,13 +1503,23 @@ impl Engine { // header for fork children (cheap clones next to a network // call; skipped entirely when the trial gate is off). The // pending-route block below touches billing/events only, so - // these bytes are what the provider caches. + // these bytes are what the provider caches. The hot proof + // carries forward while the session grows append-only: tools + // execute before this request's usage lands, so without the + // carry every mid-turn spawn would read cold. if crate::prompt_zones::fork_inherit_enabled() - && let Some(tools_json) = crate::prompt_zones::ordered_tool_catalog_json( + && let Some(tools_json) = crate::prompt_zones::wire_tool_catalog_json( stream_request.tools.as_deref().unwrap_or(&[]), ) { - *self.live_header.lock() = Some(crate::prompt_zones::LiveHeaderSnapshot { + let mut guard = self.live_header.lock(); + let carried = crate::prompt_zones::carry_hot_forward( + guard.as_ref(), + &stream_request.system, + &tools_json, + &stream_request.messages, + ); + *guard = Some(crate::prompt_zones::LiveHeaderSnapshot { system: stream_request.system.clone(), active_names: stream_request .tools @@ -1519,10 +1529,11 @@ impl Engine { .map(|tool| tool.name.clone()) .collect(), tools_json, + history: stream_request.messages.clone(), model: stream_request.model.clone(), provider: self.api_provider, provider_identity: self.api_provider_identity.clone(), - last_hit_tokens: None, + last_hit_tokens: carried, }); } let _ = self diff --git a/crates/tui/src/prompt_zones.rs b/crates/tui/src/prompt_zones.rs index 7bead51601..bb610117fa 100644 --- a/crates/tui/src/prompt_zones.rs +++ b/crates/tui/src/prompt_zones.rs @@ -198,7 +198,7 @@ impl std::fmt::Display for PrefixDrift { pub struct LiveHeaderSnapshot { /// Exact system prompt, shape-preserving (`Text` stays `Text`). pub system: Option, - /// Wire-order canonical tools JSON (see [`ordered_tool_catalog_json`]). + /// Wire-order canonical tools JSON (see [`wire_tool_catalog_json`]). pub tools_json: String, /// Wire-order tool names, for warming the child's activation cache /// in admission order so its rebuilt wire block can byte-match. @@ -207,8 +207,15 @@ pub struct LiveHeaderSnapshot { pub model: String, pub provider: crate::config::ApiProvider, pub provider_identity: String, - /// Cache-hit tokens on the response to the snapshotted request. - /// `None` until usage lands; inheritance requires `Some(>0)` — a + /// Request messages the hot proof covers. Each new request must + /// start with these byte-for-byte (element-wise) to keep the + /// proof; compaction or any rewrite resets to cold. + pub history: Vec, + /// Cache-hit tokens proving the snapshotted head hot. Set from the + /// response's usage, then carried forward across append-only + /// requests (see [`carry_hot_forward`]) so a fork mid-turn reads + /// the still-valid proof instead of the current request's + /// not-yet-known usage. Inheritance requires `Some(>0)` — a /// provably hot prefix, not a hopefully warm one. pub last_hit_tokens: Option, } @@ -224,19 +231,22 @@ pub fn new_live_header_cell() -> SharedLiveHeader { Arc::new(parking_lot::Mutex::new(None)) } -/// Order-sensitive canonical tools serialization: per-tool JSON in -/// wire order, joined by `\n`. Unlike [`tool_catalog_digest`] (sorted -/// for hashing), this must byte-match across independently built -/// catalogs — order is part of the wire bytes. `None` if any tool -/// fails to serialize; the inherit gate treats that as a mismatch. +/// Wire-canonical tools serialization: per-tool chat-API JSON in wire +/// order, joined by `\n`. Each segment goes through +/// [`codewhale_core::prefix_cache::tool_to_api_json`], so internal-only +/// fields (`defer_loading`, `allowed_callers`, …) that never reach the +/// provider cannot false-negative a byte comparison. Unlike +/// [`tool_catalog_digest`] (sorted for hashing), order is part of the +/// wire bytes. `None` if any tool fails to serialize; the inherit gate +/// treats that as a mismatch. #[must_use] -pub fn ordered_tool_catalog_json(tools: &[Tool]) -> Option { +pub fn wire_tool_catalog_json(tools: &[Tool]) -> Option { let mut out = String::new(); for (index, tool) in tools.iter().enumerate() { if index > 0 { out.push('\n'); } - out.push_str(&serde_json::to_string(tool).ok()?); + out.push_str(&codewhale_core::prefix_cache::tool_to_api_json(tool)?); } Some(out) } @@ -253,33 +263,97 @@ pub fn fork_inherit_enabled() -> bool { }) } -/// Fork-inherit decision with a trial-log reason. -#[derive(Debug, Clone, PartialEq, Eq)] +/// Trial diagnostics: `CODEWHALE_FORK_TRIAL=1` mirrors fork-inherit trial +/// events (decision + child first-response cache split) to stderr as +/// single-line JSON. Headless `exec` installs no tracing subscriber, so +/// the `tracing` trial logs are invisible there; this is the metered +/// readout for same-binary A/B trials. Off unless explicitly enabled. +#[must_use] +pub fn fork_trial_diag_enabled() -> bool { + static GATE: std::sync::OnceLock = std::sync::OnceLock::new(); + *GATE.get_or_init(|| { + std::env::var("CODEWHALE_FORK_TRIAL") + .map(|raw| raw.trim() == "1") + .unwrap_or(false) + }) +} + +/// Hot-proof carry-over for the per-request snapshot write. +/// +/// Tool calls (including fork spawns) execute before the current +/// request's usage lands, so a snapshot that reset its proof on every +/// write would read cold at every spawn. The new request keeps the old +/// proof only when it extends the old head byte-for-byte: same system, +/// same wire tools, messages grown append-only. Drift, compaction, or +/// any rewrite resets to `None`. Fails closed throughout. +#[must_use] +pub fn carry_hot_forward( + old: Option<&LiveHeaderSnapshot>, + new_system: &Option, + new_tools_json: &str, + new_history: &[Message], +) -> Option { + let old = old?; + let hits = old.last_hit_tokens.filter(|hits| *hits > 0)?; + if &old.system != new_system { + return None; + } + if old.tools_json != new_tools_json { + return None; + } + if !new_history.starts_with(&old.history) { + return None; + } + Some(hits) +} + +/// Fork-inherit decision with a trial-log reason and, when inheriting, +/// the head of the parent's wire block the child reproduces byte-for-byte. +#[derive(Debug, Clone, PartialEq)] pub struct ForkInheritDecision { pub inherit: bool, pub reason: &'static str, + /// Longest byte-equal head of the parent's wire tools, resolved from + /// the child's own catalog (post strict-transform). The child's first + /// request sends exactly these; the rest of its grant stays deferred + /// and activates on demand, so truncation costs cache bytes, never + /// capability. Empty unless `inherit` is true. + pub prefix_tools: Vec, } -/// Pure inherit gate. `child_tools_json` is the child's own rebuilt -/// wire block (warmed with the snapshot's names) — equality here is -/// what makes the provider see one continuous prefix. The grant -/// boundary enforces itself through this comparison: names outside -/// the child's grant cannot warm, so narrowed roles fall back to -/// cold without a role table. `gate_on` is [`fork_inherit_enabled`] -/// read by the caller, so every arm stays unit-testable. +/// Pure inherit gate with graceful prefix tiers. +/// +/// Full-block equality almost never holds in production: the parent's +/// wire block grows by deferred discovery while the child's grant is +/// fixed and usually narrower. Instead the gate resolves the longest +/// head of the parent's wire block the child reproduces byte-for-byte +/// (same route, hot proof, append-only history all still required): +/// a full head (`inherited_full`) prices the shared history as hits; +/// a partial head (`inherited_prefix`) still prices the system plus +/// the matched tools as hits. No common head (`tools_mismatch`) and +/// every other failure stay cold. +/// +/// The grant boundary enforces itself through catalog lookup: names +/// outside the child's grant end the run, so narrowed roles inherit a +/// shorter head instead of cold-starting — no role table. `gate_on` is +/// [`fork_inherit_enabled`] read by the caller, so every arm stays +/// unit-testable. #[must_use] +#[allow(clippy::too_many_arguments)] pub fn resolve_fork_inherit( gate_on: bool, snapshot: Option<&LiveHeaderSnapshot>, child_model: &str, child_provider: crate::config::ApiProvider, child_provider_identity: &str, - child_tools_json: Option<&str>, + child_catalog: &[Tool], + child_strict: bool, history_len: usize, ) -> ForkInheritDecision { let cold = |reason: &'static str| ForkInheritDecision { inherit: false, reason, + prefix_tools: Vec::new(), }; if !gate_on { return cold("gate_off"); @@ -305,12 +379,42 @@ pub fn resolve_fork_inherit( if history_len == 0 { return cold("empty_history"); } - if child_tools_json != Some(snapshot.tools_json.as_str()) { + if snapshot.active_names.is_empty() { + return cold("no_parent_tools"); + } + let parent_wire: Vec<&str> = snapshot.tools_json.split('\n').collect(); + let mut prefix = Vec::new(); + for (index, name) in snapshot.active_names.iter().enumerate() { + let Some(parent_json) = parent_wire.get(index) else { + break; + }; + let Some(tool) = child_catalog.iter().find(|tool| &tool.name == name) else { + break; + }; + let mut tool = tool.clone(); + if child_strict { + crate::tools::schema_sanitize::prepare_tools_for_strict_mode(std::slice::from_mut( + &mut tool, + )); + } + let wire = codewhale_core::prefix_cache::tool_to_api_json(&tool); + if wire.as_deref() != Some(*parent_json) { + break; + } + prefix.push(tool); + } + if prefix.is_empty() { return cold("tools_mismatch"); } + let full = prefix.len() == snapshot.active_names.len(); ForkInheritDecision { inherit: true, - reason: "inherited", + reason: if full { + "inherited_full" + } else { + "inherited_prefix" + }, + prefix_tools: prefix, } } @@ -867,66 +971,105 @@ mod tests { } fn inherit_snapshot() -> LiveHeaderSnapshot { + let tools = vec![make_tool("read"), make_tool("agent")]; LiveHeaderSnapshot { system: Some(SystemPrompt::Text("parent system".to_string())), - tools_json: "{\"name\":\"read\"}".to_string(), - active_names: vec!["read".to_string()], + tools_json: wire_tool_catalog_json(&tools).expect("serializes"), + active_names: tools.iter().map(|tool| tool.name.clone()).collect(), model: "deepseek-chat".to_string(), provider: crate::config::ApiProvider::Deepseek, provider_identity: "deepseek".to_string(), + history: vec![make_message("user", "hi")], last_hit_tokens: Some(1200), } } #[test] - fn ordered_tool_json_is_order_sensitive_and_stable() { + fn wire_tool_json_is_order_sensitive_and_stable() { let pair = [make_tool("a"), make_tool("b")]; let swapped = [make_tool("b"), make_tool("a")]; - let forward = ordered_tool_catalog_json(&pair).expect("serializes"); - let backward = ordered_tool_catalog_json(&swapped).expect("serializes"); + let forward = wire_tool_catalog_json(&pair).expect("serializes"); + let backward = wire_tool_catalog_json(&swapped).expect("serializes"); assert_ne!(forward, backward, "wire order is part of the bytes"); assert_eq!( forward, - ordered_tool_catalog_json(&pair).expect("stable"), + wire_tool_catalog_json(&pair).expect("stable"), "same order serializes identically" ); - assert_eq!(ordered_tool_catalog_json(&[]).expect("empty"), ""); + assert_eq!(wire_tool_catalog_json(&[]).expect("empty"), ""); + // Internal-only flags never reach the wire bytes. + let mut flagged = make_tool("a"); + flagged.defer_loading = Some(true); + flagged.allowed_callers = Some(vec!["x".to_string()]); + assert_eq!( + wire_tool_catalog_json(&[flagged]).expect("serializes"), + wire_tool_catalog_json(&[make_tool("a")]).expect("serializes"), + ); } #[test] fn fork_inherit_requires_every_gate() { let snapshot = inherit_snapshot(); - let decide = |snapshot: Option<&LiveHeaderSnapshot>, model: &str, tools: Option<&str>| { + let full_catalog = vec![make_tool("read"), make_tool("agent")]; + let decide = |snapshot: Option<&LiveHeaderSnapshot>, model: &str, catalog: &[Tool]| { resolve_fork_inherit( true, snapshot, model, crate::config::ApiProvider::Deepseek, "deepseek", - tools, + catalog, + false, 4, ) }; - let ok = decide( - Some(&snapshot), - "deepseek-chat", - Some("{\"name\":\"read\"}"), - ); + let ok = decide(Some(&snapshot), "deepseek-chat", &full_catalog); assert!(ok.inherit, "{ok:?}"); - assert_eq!(ok.reason, "inherited"); + assert_eq!(ok.reason, "inherited_full"); + assert_eq!(ok.prefix_tools.len(), 2); + + // Narrowed grant: head matches, tail grant-gap ends the run. + let narrowed = vec![make_tool("read")]; + let prefix = decide(Some(&snapshot), "deepseek-chat", &narrowed); + assert!(prefix.inherit, "{prefix:?}"); + assert_eq!(prefix.reason, "inherited_prefix"); + assert_eq!(prefix.prefix_tools.len(), 1); + + // Grant gap at the head: no common head, stays cold. + let gap_at_head = vec![make_tool("agent")]; + assert_eq!( + decide(Some(&snapshot), "deepseek-chat", &gap_at_head).reason, + "tools_mismatch" + ); + // Schema skew on the head tool also ends the run at zero. + let mut skewed = make_tool("read"); + skewed.description = "different".to_string(); + assert_eq!( + decide(Some(&snapshot), "deepseek-chat", &[skewed]).reason, + "tools_mismatch" + ); + // Internal-only flag skew must NOT end the run: those bytes + // never reach the provider. + let mut flagged = make_tool("read"); + flagged.defer_loading = Some(true); + flagged.allowed_callers = Some(vec!["parent-only".to_string()]); + let flagged_catalog = vec![flagged, make_tool("agent")]; + let flagged_ok = decide(Some(&snapshot), "deepseek-chat", &flagged_catalog); + assert!(flagged_ok.inherit, "{flagged_ok:?}"); + assert_eq!(flagged_ok.reason, "inherited_full"); assert_eq!( - decide(None, "deepseek-chat", Some("x")).reason, + decide(None, "deepseek-chat", &full_catalog).reason, "no_snapshot" ); let mut no_system = snapshot.clone(); no_system.system = None; assert_eq!( - decide(Some(&no_system), "deepseek-chat", Some("x")).reason, + decide(Some(&no_system), "deepseek-chat", &full_catalog).reason, "no_parent_system" ); assert_eq!( - decide(Some(&snapshot), "other-model", Some("x")).reason, + decide(Some(&snapshot), "other-model", &full_catalog).reason, "model_mismatch" ); let provider_mismatch = resolve_fork_inherit( @@ -935,7 +1078,8 @@ mod tests { "deepseek-chat", crate::config::ApiProvider::Openai, "deepseek", - Some("x"), + &full_catalog, + false, 4, ); assert_eq!(provider_mismatch.reason, "provider_mismatch"); @@ -946,7 +1090,8 @@ mod tests { "deepseek-chat", crate::config::ApiProvider::Deepseek, "custom-mirror", - Some("x"), + &full_catalog, + false, 4, ); assert_eq!(route_mismatch.reason, "route_mismatch"); @@ -954,7 +1099,7 @@ mod tests { let mut snapshot = snapshot.clone(); snapshot.last_hit_tokens = cold; assert_eq!( - decide(Some(&snapshot), "deepseek-chat", Some("x")).reason, + decide(Some(&snapshot), "deepseek-chat", &full_catalog).reason, "prefix_cold", "cold={cold:?}" ); @@ -965,22 +1110,16 @@ mod tests { "deepseek-chat", crate::config::ApiProvider::Deepseek, "deepseek", - Some("{\"name\":\"read\"}"), + &full_catalog, + false, 0, ); assert_eq!(empty_history.reason, "empty_history"); + let mut no_tools = snapshot.clone(); + no_tools.active_names.clear(); assert_eq!( - decide( - Some(&snapshot), - "deepseek-chat", - Some("{\"name\":\"other\"}") - ) - .reason, - "tools_mismatch" - ); - assert_eq!( - decide(Some(&snapshot), "deepseek-chat", None).reason, - "tools_mismatch" + decide(Some(&no_tools), "deepseek-chat", &full_catalog).reason, + "no_parent_tools" ); let gated_off = resolve_fork_inherit( false, @@ -988,10 +1127,83 @@ mod tests { "deepseek-chat", crate::config::ApiProvider::Deepseek, "deepseek", - Some("{\"name\":\"read\"}"), + &full_catalog, + false, 4, ); assert_eq!(gated_off.reason, "gate_off"); assert!(!gated_off.inherit); } + + #[test] + fn carry_hot_forward_keeps_append_only_and_resets_rewrites() { + let snapshot = inherit_snapshot(); + let grown = vec![ + make_message("user", "hi"), + make_message("assistant", "hello"), + ]; + assert_eq!( + carry_hot_forward( + Some(&snapshot), + &snapshot.system, + &snapshot.tools_json, + &grown, + ), + Some(1200), + "append-only growth keeps the proof" + ); + // Identical re-send (transparent retry) also extends. + assert_eq!( + carry_hot_forward( + Some(&snapshot), + &snapshot.system, + &snapshot.tools_json, + &snapshot.history, + ), + Some(1200) + ); + let edited = vec![make_message("user", "edited")]; + assert_eq!( + carry_hot_forward( + Some(&snapshot), + &snapshot.system, + &snapshot.tools_json, + &edited, + ), + None, + "in-place rewrite resets" + ); + assert_eq!( + carry_hot_forward(Some(&snapshot), &snapshot.system, &snapshot.tools_json, &[]), + None, + "compaction shrink resets" + ); + let other_system = Some(SystemPrompt::Text("other".to_string())); + assert_eq!( + carry_hot_forward(Some(&snapshot), &other_system, &snapshot.tools_json, &grown), + None, + "system drift resets" + ); + assert_eq!( + carry_hot_forward( + Some(&snapshot), + &snapshot.system, + "{\"name\":\"other\"}", + &grown + ), + None, + "tool drift resets" + ); + assert_eq!( + carry_hot_forward(None, &snapshot.system, &snapshot.tools_json, &grown), + None + ); + let mut cold = snapshot.clone(); + cold.last_hit_tokens = Some(0); + assert_eq!( + carry_hot_forward(Some(&cold), &cold.system, &cold.tools_json, &grown), + None, + "zero hits never carry" + ); + } } diff --git a/crates/tui/src/tools/subagent/mod.rs b/crates/tui/src/tools/subagent/mod.rs index d313444dfb..d6e73b5e94 100644 --- a/crates/tui/src/tools/subagent/mod.rs +++ b/crates/tui/src/tools/subagent/mod.rs @@ -13064,9 +13064,11 @@ async fn run_subagent( } let tool_catalog = tool_registry.deferred_catalog_for_model(&agent_type); // Fork-prefix inheritance: when this fork can extend the parent's - // live cached prefix (same route, hot prefix, byte-equal wire - // tools), the child's system is the parent's exact bytes and its - // first request prices the shared history as cache hits. Every + // live cached prefix (same route, hot proof, append-only history), + // the child adopts the parent's exact system bytes and sends the + // longest byte-equal head of the parent's wire block on its first + // request. The rest of its grant stays deferred and activates on + // demand, so truncation costs cache bytes, never capability. Every // fallback below keeps today's cold start byte-identical. let gate_on = crate::prompt_zones::fork_inherit_enabled(); let snapshot = if gate_on && fork_context_enabled { @@ -13076,32 +13078,36 @@ async fn run_subagent( } else { None }; - let parent_warm_names: &[String] = snapshot - .as_ref() - .map(|snapshot| snapshot.active_names.as_slice()) - .unwrap_or(&[]); let strict_tool_mode = runtime .api_config .as_ref() .and_then(|config| config.strict_tool_mode) .unwrap_or(false); - let mut tool_surface = SubAgentToolSurface::new(tool_catalog, parent_warm_names); - let first_wire_tools = tool_surface.request_tools( - tool_registry.deferred_catalog_for_model(&agent_type), - strict_tool_mode, - ); let inherit_route = runtime .client .effective_route_envelope(&runtime.model, chrono::Utc::now()); - let decision = crate::prompt_zones::resolve_fork_inherit( - gate_on, - snapshot.as_ref(), - &inherit_route.model, - inherit_route.provider, - &inherit_route.provider_identity, - crate::prompt_zones::ordered_tool_catalog_json(&first_wire_tools).as_deref(), - messages.len(), - ); + // Fresh spawns never consult the gate: without the fork framing + // blocks the parent's system would overwrite the role system, so + // they keep today's cold start (and report `fresh_spawn`, distinct + // from a fork whose snapshot cell was empty). + let decision = if fork_context_enabled { + crate::prompt_zones::resolve_fork_inherit( + gate_on, + snapshot.as_ref(), + &inherit_route.model, + inherit_route.provider, + &inherit_route.provider_identity, + &tool_catalog, + strict_tool_mode, + messages.len(), + ) + } else { + crate::prompt_zones::ForkInheritDecision { + inherit: false, + reason: "fresh_spawn", + prefix_tools: Vec::new(), + } + }; let inherited_system = if decision.inherit { snapshot .as_ref() @@ -13112,18 +13118,22 @@ async fn run_subagent( let inherited = inherited_system.is_some(); let request_system = match inherited_system { Some(system) => system, - None => { - if !parent_warm_names.is_empty() { - // Cold fallback keeps today's exact behavior: rebuild the - // surface without the parent names tried for the gate. - tool_surface = SubAgentToolSurface::new( - tool_registry.deferred_catalog_for_model(&agent_type), - &[], - ); - } - subagent_request_system_prompt(&system_prompt) - } + None => subagent_request_system_prompt(&system_prompt), + }; + // The first request sends the inherited head verbatim; prefix-warm + // the surface so turns 2+ keep it instead of reverting to the bare + // eager set. Empty when cold: today's exact behavior. + let inherited_prefix: Option> = if inherited { + Some(decision.prefix_tools.clone()) + } else { + None }; + let prefix_names: Vec = decision + .prefix_tools + .iter() + .map(|tool| tool.name.clone()) + .collect(); + let mut tool_surface = SubAgentToolSurface::new(tool_catalog, &prefix_names); if inherited && let Some(snapshot) = snapshot.as_ref() { let system_bytes = serde_json::to_string(&snapshot.system) .map(|json| json.len() as u64) @@ -13134,6 +13144,8 @@ async fn run_subagent( tracing::info!( target: "subagent", agent_id = agent_id.as_str(), + reason = decision.reason, + prefix_tools = decision.prefix_tools.len(), bytes = system_bytes .saturating_add(snapshot.tools_json.len() as u64) .saturating_add(history_bytes), @@ -13148,6 +13160,100 @@ async fn run_subagent( "fork child cold start" ); } + if crate::prompt_zones::fork_trial_diag_enabled() { + // History-divergence probe: the child's history should start + // with the snapshotted request's messages byte-for-byte; the + // first divergence (Work refresh, thinking blocks, edits) is + // where the provider's prefix match ends and misses begin. + let mut div_detail = String::new(); + if decision.inherit + && let Some(snapshot) = snapshot.as_ref() + { + let mut div = String::from("\"extends\""); + for (index, old) in snapshot.history.iter().enumerate() { + let Some(new) = messages.get(index) else { + div = format!("\"child-shorter@{index}\""); + break; + }; + if old != new { + let head = |message: &Message| { + format!("{:?}", message.content) + .replace('"', "'") + .chars() + .take(150) + .collect::() + }; + div = format!( + "\"msg{index} {:?}-vs-{:?} old:{} new:{}\"", + old.role, + new.role, + head(old), + head(new) + ); + break; + } + } + div_detail = format!(",\"history_div\":{div}"); + } + let prefix_names: Vec<&str> = decision + .prefix_tools + .iter() + .map(|tool| tool.name.as_str()) + .collect(); + let parent_names: &[String] = snapshot + .as_ref() + .map(|snapshot| snapshot.active_names.as_slice()) + .unwrap_or(&[]); + // On a partial head, show where the run ended and both sides + // (truncated) so trials can tell grant-gaps from schema skew. + // Rebuilds the catalog: diag-gated, never on the hot path. + let mut break_detail = String::new(); + if decision.inherit + && let Some(name) = parent_names.get(prefix_names.len()) + { + let parent_json = snapshot + .as_ref() + .and_then(|snapshot| snapshot.tools_json.split('\n').nth(prefix_names.len())) + .unwrap_or(""); + let catalog = tool_registry.deferred_catalog_for_model(&agent_type); + let child_json = catalog + .iter() + .find(|tool| &tool.name == name) + .and_then(|tool| { + let mut tool = tool.clone(); + if strict_tool_mode { + crate::tools::schema_sanitize::prepare_tools_for_strict_mode( + std::slice::from_mut(&mut tool), + ); + } + codewhale_core::prefix_cache::tool_to_api_json(&tool) + }) + .unwrap_or_else(|| "".to_string()); + let trunc = |text: &str| { + text.replace('"', "'") + .chars() + .take(20000) + .collect::() + }; + break_detail = format!( + ",\"break_at\":\"{name}\",\"parent_json\":\"{}\",\"child_json\":\"{}\"", + trunc(parent_json), + trunc(&child_json), + ); + } + eprintln!( + "{{\"codewhale_fork_trial\":\"decision\",\"agent_id\":\"{}\",\"inherited\":{},\"reason\":\"{}\",\"prefix\":{},\"parent_n\":{},\"parent_tools\":{},\"prefix_tools\":{}{}{}}}", + agent_id.as_str(), + inherited, + decision.reason, + prefix_names.len(), + parent_names.len(), + serde_json::to_string(parent_names).unwrap_or_default(), + serde_json::to_string(&prefix_names).unwrap_or_default(), + break_detail, + div_detail, + ); + } let mut inherit_first_hit_logged = false; let mut steps = 0; let mut final_result: Option = None; @@ -13323,10 +13429,20 @@ async fn run_subagent( budget_pacing_notice_sent = true; } - let tools = tool_surface.request_tools( + let mut tools = tool_surface.request_tools( tool_registry.deferred_catalog_for_model(&agent_type), strict_tool_mode, ); + // Steps count from 1: the first request sends the inherited + // head verbatim (see the gate above), and the surface's active + // set is restricted to match so dispatch, tool_search, and + // hydrate agree with the wire block. Later steps grow normally. + if steps == 1 + && let Some(prefix) = inherited_prefix.as_ref() + { + tools = prefix.clone(); + tool_surface.active_names = prefix.iter().map(|tool| tool.name.clone()).collect(); + } let request_active_tool_names = tool_surface.active_names.clone(); let has_tools = !tools.is_empty(); // The report-only response keeps the pinned tool catalog but asks the @@ -13592,15 +13708,43 @@ async fn run_subagent( .await; tokens_used = tokens_used.saturating_add(usage_total_tokens(&response.usage)); - if inherited && !inherit_first_hit_logged { + if !inherit_first_hit_logged { inherit_first_hit_logged = true; - tracing::info!( - target: "subagent", - agent_id = agent_id.as_str(), - hit_tokens = response.usage.prompt_cache_hit_tokens.unwrap_or(0), - input_tokens = response.usage.input_tokens, - "fork child first response cache hits" - ); + if inherited { + tracing::info!( + target: "subagent", + agent_id = agent_id.as_str(), + hit_tokens = response.usage.prompt_cache_hit_tokens.unwrap_or(0), + input_tokens = response.usage.input_tokens, + "fork child first response cache hits" + ); + } + if crate::prompt_zones::fork_trial_diag_enabled() { + // Component wire sizes pin down HOW FAR into the request + // the provider's prefix match reached: hits <= system + // means tools diverged; hits <= system+tools means the + // history diverged; beyond that the history is hitting. + let system_bytes = serde_json::to_string(&request_system) + .map(|json| json.len()) + .unwrap_or(0); + let tools_bytes = serde_json::to_string(&tools) + .map(|json| json.len()) + .unwrap_or(0); + let history_bytes = serde_json::to_string(&messages) + .map(|json| json.len()) + .unwrap_or(0); + eprintln!( + "{{\"codewhale_fork_trial\":\"first_response\",\"agent_id\":\"{}\",\"inherited\":{},\"input\":{},\"hit\":{},\"miss\":{},\"system_bytes\":{},\"tools_bytes\":{},\"history_bytes\":{}}}", + agent_id.as_str(), + inherited, + response.usage.input_tokens, + response.usage.prompt_cache_hit_tokens.unwrap_or(0), + response.usage.prompt_cache_miss_tokens.unwrap_or(0), + system_bytes, + tools_bytes, + history_bytes, + ); + } } // #6194 item 7: one over-bound step lands the run through the normal diff --git a/crates/tui/src/tools/subagent/tests.rs b/crates/tui/src/tools/subagent/tests.rs index 7f695c1a2e..b359fd7ee4 100644 --- a/crates/tui/src/tools/subagent/tests.rs +++ b/crates/tui/src/tools/subagent/tests.rs @@ -8121,9 +8121,8 @@ fn synthetic_eager_tool(name: &str) -> Tool { #[test] fn warmed_surfaces_rebuild_byte_identical_wire_tools() { - // The fork-inherit gate compares the child's rebuilt wire block - // against the parent's snapshot: same catalog plus same warm names - // must serialize identically, and warming must change the bytes. + // Same catalog plus same warm names must serialize identically, + // and warming must change the wire block. let catalog = vec![ synthetic_eager_tool("read"), synthetic_deferred_tool("agent", 8), @@ -8134,15 +8133,14 @@ fn warmed_surfaces_rebuild_byte_identical_wire_tools() { let mut second = SubAgentToolSurface::new(catalog.clone(), &warm); let first_wire = model_request_tools(&mut first); let second_wire = model_request_tools(&mut second); - let first_json = - crate::prompt_zones::ordered_tool_catalog_json(&first_wire).expect("serializes"); + let first_json = crate::prompt_zones::wire_tool_catalog_json(&first_wire).expect("serializes"); let second_json = - crate::prompt_zones::ordered_tool_catalog_json(&second_wire).expect("serializes"); + crate::prompt_zones::wire_tool_catalog_json(&second_wire).expect("serializes"); assert_eq!(first_json, second_json, "same catalog plus same names"); assert!(first_wire.iter().any(|tool| tool.name == "agent")); let mut cold = SubAgentToolSurface::new(catalog, &[]); let cold_wire = model_request_tools(&mut cold); - let cold_json = crate::prompt_zones::ordered_tool_catalog_json(&cold_wire).expect("serializes"); + let cold_json = crate::prompt_zones::wire_tool_catalog_json(&cold_wire).expect("serializes"); assert_ne!(cold_json, first_json, "warming must change the wire block"); assert!(!cold_wire.iter().any(|tool| tool.name == "agent")); } From 8693d6451a631735977647aa2353c3e96e2cdb5f Mon Sep 17 00:00:00 2001 From: AdityaVG13 Date: Mon, 21 Sep 2026 05:07:42 -0400 Subject: [PATCH 07/26] Allow trial stderr sink; document fork inheritance The CODEWHALE_FORK_TRIAL readout intentionally writes JSON lines to stderr (headless exec installs no tracing subscriber, so the tracing trial logs are invisible there). Narrowly allow clippy::print_stderr on the two env-gated diagnostic blocks with justification, following the existing in-repo precedent. Documents the fork-prefix contract, measured benchmarks, trial procedure, and known follow-ups in docs/CACHE.md. --- crates/tui/src/tools/subagent/mod.rs | 6 +++++ docs/CACHE.md | 40 ++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/crates/tui/src/tools/subagent/mod.rs b/crates/tui/src/tools/subagent/mod.rs index d6e73b5e94..faf058c6e9 100644 --- a/crates/tui/src/tools/subagent/mod.rs +++ b/crates/tui/src/tools/subagent/mod.rs @@ -13160,6 +13160,10 @@ async fn run_subagent( "fork child cold start" ); } + // Trial-diagnostic stderr sink: headless `exec` installs no + // tracing subscriber, so `eprintln!` is the A/B readout channel by + // design. Env-gated, JSON lines, default off. + #[allow(clippy::print_stderr)] if crate::prompt_zones::fork_trial_diag_enabled() { // History-divergence probe: the child's history should start // with the snapshotted request's messages byte-for-byte; the @@ -13719,6 +13723,8 @@ async fn run_subagent( "fork child first response cache hits" ); } + // Same trial sink as the decision line above. + #[allow(clippy::print_stderr)] if crate::prompt_zones::fork_trial_diag_enabled() { // Component wire sizes pin down HOW FAR into the request // the provider's prefix match reached: hits <= system diff --git a/docs/CACHE.md b/docs/CACHE.md index b7bdd96efe..91e979f361 100644 --- a/docs/CACHE.md +++ b/docs/CACHE.md @@ -99,6 +99,46 @@ change, a project-file change) into the prefix — append it as a user-role message instead. A later request must be `previous ⊕ suffix` unless a logged header change or a history reset explains the difference. +## Fork-prefix inheritance (subagents) + +A forked subagent (`fork_context=true`) on the same route/model extends the +parent's cached prefix instead of cold-starting: it adopts the parent's +exact system bytes and sends the longest byte-equal head of the parent's +wire tool block on its first request. The rest of its grant stays deferred +and activates on demand, so truncation costs cache bytes, never capability. + +Contract details: + +- **Hot proof required.** The turn loop snapshots each final request's + header plus a hit count carried forward across append-only requests; + the gate (`resolve_fork_inherit`, pure) admits only a provably hot + (`>0` hits), same-route head. Spawns execute before the current + request's usage lands, so without the carry every mid-turn spawn + would read cold. +- **Wire-canonical comparison.** Both sides canonicalize through + `tool_to_api_json` (the chat-API shape); internal-only flags + (`defer_loading`, `allowed_callers`) never reach the wire and cannot + false-negative the match. +- **Graceful tiers.** `inherited_full` (whole head) prices shared + history as hits; `inherited_prefix` (partial head) still prices the + system plus matched tools. Fresh spawns, grandchildren, resumes, + route skew, and cold prefixes stay cold with distinct reasons. +- **KV-cache effect:** frozen-prefix extension. The child's first + request is `parent-head ⊕ suffix` (framing + task); nothing volatile + is spliced into the prefix. + +Measured child first-response (same-binary A/B, DeepSeek, unique +histories; DeepSeek counts hits in 64-token blocks): true cold 0/9421 +hits; primed cold 4736/9389; prefix k=3 5504/9941; full k=6 +8576/13251. Trial readout: `CODEWHALE_FORK_TRIAL=1` mirrors the +decision, prefix span, history-divergence point, and cache split to +stderr as JSON lines; `CODEWHALE_FORK_INHERIT=off` is the control arm. + +Known follow-ups (measured, not implemented): fork transcripts freeze +at turn 1 (the live tail never rides, capping wins at system+tools); +framing blocks sit ahead of where the tail would; v1 is single-level +(grandchildren carry ~40KB all-miss transcripts today). + ## Deferred: full reconstructability (Layer 3) DeepSeek Harness derives every request from an append-only session log via a From 48df7b925a4e67d2a2e63ce0154bd37e4c1e07a9 Mon Sep 17 00:00:00 2001 From: AdityaVG13 Date: Mon, 21 Sep 2026 17:51:06 -0400 Subject: [PATCH 08/26] Fix echo paths on symlinked workspaces (macOS, Windows) The tool layer hands echolocation canonical file paths while the workspace may be non-canonical (macOS /var to /private/var tempdirs, Windows verbatim paths). Three spots compared them raw and silently missed everywhere but Linux: - workspace_relative now canonicalizes both sides before giving up, restoring caller and link matches. - pod_relative_names compares in workspace-relative space instead of joining the raw workspace, restoring same-pod shortening. - Grep match paths fall back through the same canonical strip instead of rendering absolute. Verified against the three CI failures locally on macOS; Linux suites are unaffected (fast paths unchanged). --- crates/tui/src/tools/echolocation.rs | 18 ++++++++++++++++-- crates/tui/src/tools/search.rs | 21 +++++++++++++++++---- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/crates/tui/src/tools/echolocation.rs b/crates/tui/src/tools/echolocation.rs index bcd0474cff..32fa7c3b06 100644 --- a/crates/tui/src/tools/echolocation.rs +++ b/crates/tui/src/tools/echolocation.rs @@ -130,10 +130,15 @@ fn pod_relative_names(workspace: &Path, file_path: &Path, links: Vec) -> let Some(home) = file_path.parent() else { return links; }; + // Compare in workspace-relative space: joining the workspace here + // breaks when it is non-canonical but the file path is canonical + // (macOS /var, Windows \\?\ verbatim paths). + let home_rel = workspace_relative(workspace, home); links .into_iter() .map(|link| { - let short = workspace.join(&link).parent() == Some(home); + let short = + Path::new(&link).parent().and_then(|parent| parent.to_str()) == home_rel.as_deref(); if short { Path::new(&link) .file_name() @@ -1196,7 +1201,16 @@ fn slash_path(path: &Path) -> String { } fn workspace_relative(workspace: &Path, path: &Path) -> Option { - path.strip_prefix(workspace).ok().map(slash_path) + if let Some(rel) = path.strip_prefix(workspace).ok().map(slash_path) { + return Some(rel); + } + // The tool layer hands us canonical file paths while the workspace + // may be non-canonical (macOS /var -> /private/var tempdirs, + // Windows \\?\ verbatim paths). Canonicalize both sides before + // giving up, or every caller/link match silently misses there. + let workspace = workspace.canonicalize().ok()?; + let path = path.canonicalize().ok()?; + path.strip_prefix(&workspace).ok().map(slash_path) } /// Python source lines: `#` comments cut, triple-quote doc spans diff --git a/crates/tui/src/tools/search.rs b/crates/tui/src/tools/search.rs index eb5de8d322..6c38905a1e 100644 --- a/crates/tui/src/tools/search.rs +++ b/crates/tui/src/tools/search.rs @@ -227,12 +227,25 @@ impl ToolSpec for GrepFilesTool { return Ok(WalkControl::Continue); } - // Get relative path from workspace + // Relative path from workspace. The walk root comes + // back canonical while the workspace may not be + // (macOS /var, Windows \\?\ verbatim paths), so + // canonicalize both sides before falling back to + // the absolute path. let relative_path = file_path .strip_prefix(&workspace) - .unwrap_or(file_path) - .to_string_lossy() - .to_string(); + .ok() + .map(|rel| rel.to_string_lossy().to_string()) + .or_else(|| { + let ws = workspace.canonicalize().ok(); + let fp = file_path.canonicalize().ok(); + ws.zip(fp).and_then(|(ws, fp)| { + fp.strip_prefix(&ws) + .ok() + .map(|rel| rel.to_string_lossy().to_string()) + }) + }) + .unwrap_or_else(|| file_path.to_string_lossy().to_string()); let budget = max_results.saturating_sub(results.len()); let Some((file_matches, file_had_more)) = search_file_streaming( From 83d01c1e04ba68d77bda744f70e09009c9d3a181 Mon Sep 17 00:00:00 2001 From: CodeWhale Bot Date: Wed, 14 Oct 2026 03:12:55 -0700 Subject: [PATCH 09/26] docs(credits): include AdityaVG13 in candidate website credits Refs #6393 The first own-head Version drift job failed its contributor-credit gate because the website list omitted @AdityaVG13, whose original source-navigation and fork-prefix work is preserved in this branch. Add their literal handle beside the existing unreleased contributor without changing product source or the credit gate. Frozen combined canonical verification: npm 540 Node plus 772 Vitest passed; 0 failed; 7 skipped. npm run check:web, fmt and diff checks pass. This exact file matches the verified source. The original failed job log is retained; fresh exact-head hosted CI, reviews and Windows owner-recovery proof remain required. No upload, release, provider call or Windows distribution acceptance occurred. --- web/lib/release-credits.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/web/lib/release-credits.ts b/web/lib/release-credits.ts index a4fbfc34ec..2f905925ce 100644 --- a/web/lib/release-credits.ts +++ b/web/lib/release-credits.ts @@ -34,7 +34,10 @@ export const RELEASE_CONTRIBUTORS: string[] = [ * one. scripts/check-contributor-credit.py requires them here now; the release * cut moves them into RELEASE_CONTRIBUTORS with that release's changelog block. */ -export const UNRELEASED_CONTRIBUTORS: string[] = []; +export const UNRELEASED_CONTRIBUTORS: string[] = [ + "@aboimpinto", + "@AdityaVG13", +]; /** * Contributors who helped with reports, reproductions, and verification. From 63188386ad73242a3f8d558beab821ca3a0b0d4e Mon Sep 17 00:00:00 2001 From: CodeWhale Bot Date: Wed, 14 Oct 2026 21:26:32 -0700 Subject: [PATCH 10/26] test(file): expect the pod footer on ranged and empty-range reads Preserve @AdityaVG13's source-echo work. #6393 intentionally appends the pod footer after the paging footer (see contract_read_appends_a_pod_footer and the "page plus paging footer first, pod footer after" case); three assertions in two older main tests still encoded the pre-echo output and failed on every OS. The macOS-only grep_pods failure was a grep path bug on main, fixed there in c6bcf78c01 and merged in here. Evidence (macOS, merged tree): nextest tools::file + tools::search + echolocation + budget_handback: 273 passed, 0 failed; cargo fmt --check clean. Hosted CI on this head is the full-suite gate. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NBKbhx19BRbZtvWK8rVKjz --- crates/tui/src/tools/file/tests.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/tui/src/tools/file/tests.rs b/crates/tui/src/tools/file/tests.rs index 62d26f6571..49b854e319 100644 --- a/crates/tui/src/tools/file/tests.rs +++ b/crates/tui/src/tools/file/tests.rs @@ -528,7 +528,7 @@ async fn contract_read_accepts_whole_number_floats_and_still_refuses_fractions() .expect("float-typed ranged read"); assert_eq!( ranged.content, - "two\n\n[1 more lines in file (13B total). Use offset=3 to continue.]" + "two\n\n[1 more lines in file (13B total). Use offset=3 to continue.]\n\n[Pod (1): lines.txt (this file)]" ); for (key, bad) in [ @@ -1155,7 +1155,7 @@ async fn contract_read_newline_dense_ranges_preserve_trailing_and_empty_lines() ) .await .unwrap(); - assert_eq!(result.content, "\n"); + assert_eq!(result.content, "\n\n\n[Pod (1): dense.txt (this file)]"); let metadata = result.metadata.as_ref().unwrap(); assert_eq!(metadata["size"], text.len()); assert_eq!(metadata["line_count"], text.len() + 1); @@ -1168,7 +1168,7 @@ async fn contract_read_newline_dense_ranges_preserve_trailing_and_empty_lines() .unwrap(); assert_eq!( empty.content, - "\n\n[1 more lines in file (2.0MB total). Use offset=2097153 to continue.]" + "\n\n[1 more lines in file (2.0MB total). Use offset=2097153 to continue.]\n\n[Pod (1): dense.txt (this file)]" ); } From c00c4ea0ae8e0fb2b459ed2783d68fe73b7170fc Mon Sep 17 00:00:00 2001 From: CodeWhale Bot Date: Thu, 15 Oct 2026 07:59:55 -0700 Subject: [PATCH 11/26] test(subagent): allow durable setup before timing out the report fixture PR #6393 own-head Ubuntu CI 38684332452 attempt 2 reported 19173 passed, 1 failed and 34 skipped: the existing failure/timeout fixture observed one request instead of two. Its 100ms step timeout also bounded the ordinary request and pre-report durable prompt persistence. Use the fixture's existing two-second timeout case and require the report-start notification before asserting the terminal result. Preserve the exact two-request, no-retry, usage, missing-coverage and deliverable assertions; include the mode and result in the request-count diagnostic. No production deadline, receipt persistence, or provider behavior changes. No new test functions or code comments. Exact source review and rustfmt passed. This source-only follow-up preserves the original contributor's history and draft status; actual new-head native CI and npm/web execution remain pending before merge. The canonical checkout and peer build inputs are unchanged. Refs #6393 Signed-off-by: CodeWhale Bot --- .../tui/src/tools/subagent/budget_handback_tests.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/tui/src/tools/subagent/budget_handback_tests.rs b/crates/tui/src/tools/subagent/budget_handback_tests.rs index 50929e2205..2a2d8cf8c6 100644 --- a/crates/tui/src/tools/subagent/budget_handback_tests.rs +++ b/crates/tui/src/tools/subagent/budget_handback_tests.rs @@ -174,9 +174,7 @@ async fn fixture(mode: &'static str, first_tokens: u64, max_steps: u32) -> Fixtu runtime.spawn_depth = spec.spawn_depth; runtime.allow_shell = false; runtime.accept_edits = false; - runtime.step_api_timeout = if mode == "timeout" { - Duration::from_millis(100) - } else if mode == "work-timeout" { + runtime.step_api_timeout = if mode == "work-timeout" { // Past the 6s work deadline, and bounding the held hand-back call // no tighter than the 9s hard deadline already does. Duration::from_secs(10) @@ -418,8 +416,15 @@ async fn budget_handback_turn_failure_and_timeout_do_not_add_worker_retries() { ("timeout", "report deadline expired"), ] { let mut fixture = fixture(mode, 15, 1).await; + tokio::time::timeout(Duration::from_secs(30), fixture.report_started.notified()) + .await + .expect("report request must reach the fixture"); let result = fixture.finish().await; - assert_eq!(fixture.requests.lock().unwrap().len(), 2); + assert_eq!( + fixture.requests.lock().unwrap().len(), + 2, + "{mode}: {result:?}" + ); assert_eq!(result.status, SubAgentStatus::BudgetExhausted); assert!( result.result.as_deref().unwrap().contains(reason), From 56877299219d8958fffe0d1e67d78784c77b07a8 Mon Sep 17 00:00:00 2001 From: CodeWhale Bot Date: Thu, 15 Oct 2026 09:43:01 -0700 Subject: [PATCH 12/26] fix(ci): retain runtime identity measurements on mismatch #6393 intentionally removes native skill file paths from the ambient index, but its unchanged runtime budget rejects the first new prompt fingerprint. Keep the complete provider-free measurement in the failing checker output so all changed identities and numeric limits can be reviewed from the exact CI source. Existing validation, budget contents and exit status remain unchanged. Validation: all 22 existing runtime-budget checker tests passed against the exact proposed script; syntax and four-line diff reviewed. Ubuntu 19,174/0/34 skipped and Windows 18,429/0/27 skipped belong to parent d991cffdf2, not this new head. macOS and fresh hosted qualification remain pending. No local native build, provider call or budget relaxation. Canonical peer files untouched. Refs #6393 Signed-off-by: CodeWhale Bot --- scripts/check-runtime-contract-budget.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/check-runtime-contract-budget.py b/scripts/check-runtime-contract-budget.py index 45b04c2e32..26e54d2030 100755 --- a/scripts/check-runtime-contract-budget.py +++ b/scripts/check-runtime-contract-budget.py @@ -693,6 +693,10 @@ def main(argv: Sequence[str] | None = None) -> int: except RuntimeContractError as error: print(f"[runtime-contract-budget] ERROR: {error}", file=sys.stderr) if str(error).startswith("identity changed"): + print( + "[runtime-contract-budget] receipt=" + json.dumps(receipt, sort_keys=True), + file=sys.stderr, + ) print( "If the identity change is intended, land the new budget in this PR:\n" f" {grow_command}", From d9589796bde7a97fa3393e1acc3895430724cf9c Mon Sep 17 00:00:00 2001 From: AdityaVG13 Date: Sun, 20 Sep 2026 21:48:36 -0400 Subject: [PATCH 13/26] feat(echo): stateless index-free echolocation + program-wide token diet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sound every file read, grep, file search, and project map with its surroundings — sibling podmates, line-numbered symbols, import roots, resolved file links, and (in the terminal buzz) reverse-link callers — sounded fresh from source with no index, no background work, and no new dependencies. Deterministic (sorted, capped, budgeted) so repeated calls are byte-identical and cache-friendly. Echolocation (new: tools/echolocation.rs, ~2.5k lines incl. tests): - read footer: [Pod] mates + [Sounding] symbols/imports/links/heard-by, 1200B hard budget; miss echo on NotFound (denylist-gated, never fires on refused probes); terminal buzz adds callers after 2 distinct files read in a pod - grep + file_search chart matched dirs (caller's own visibility gates: denylist, excludes, extensions, gitignore); project_map sounds key files; 4 sounding languages (Rust/Python/JS-TS/Go) Token diet (no compression, behavior-preserving): - prefix 14,241 -> 7,118 tokens/turn (-50.0%): skill path cut (load_skill resolves names), MCP prose 1526->548B (scar-tissue phrases kept), agent+workflow deferral, goal tools conditional-eager keyed off the turn's resolved goal (preview/wire-safe) - activation cache: best-match-last push + regex name-rank + 16->24KiB bound (fixes deferral stranding agent behind its own discovery set) - grep empty-context omit, workflow event projection (lossless: duplicates dropped, workspaces hoisted, branch kept) + zero-memo omit - wire-prefix dump moved to PreparedOutboundRequest (exact-final-body) Evidence: targeted echo 46, workflow 131, file 145 green; full tui+workflow suites green except 4 failures reproducing identically on base c75b23397 (pre-existing, unrelated) + known timing flakes proven solo-pass; clippy zero on owned files. Research: codewhale-research/ ECHOLOCATION-THESIS.md (exp-10/11/12 + hardening audit). --- crates/tui/src/client/chat.rs | 91 +- crates/tui/src/client/prepared.rs | 1 + crates/tui/src/client/test_cases_02.rs | 6 +- crates/tui/src/core/engine/preview.rs | 1 + crates/tui/src/core/engine/preview/tests.rs | 2 + .../src/core/engine/tests/test_cases_09.rs | 17 +- .../src/core/engine/tests/test_cases_10.rs | 12 +- crates/tui/src/core/engine/tool_catalog.rs | 36 +- .../tui/src/core/engine/tool_catalog/tests.rs | 37 +- crates/tui/src/hooks/executor.rs | 19 +- crates/tui/src/skills/mod.rs | 84 +- crates/tui/src/skills/tests.rs | 50 +- crates/tui/src/tools/echolocation.rs | 2588 +++++++++++++++++ crates/tui/src/tools/file.rs | 32 +- crates/tui/src/tools/file/tests.rs | 171 +- crates/tui/src/tools/file_search.rs | 74 +- crates/tui/src/tools/mod.rs | 1 + crates/tui/src/tools/project.rs | 34 + crates/tui/src/tools/search.rs | 33 +- crates/tui/src/tools/search/tests.rs | 34 +- crates/tui/src/tools/spec.rs | 17 + crates/tui/src/tools/subagent/tests.rs | 98 +- crates/tui/src/tools/workflow/mod.rs | 148 +- crates/workflow/src/lib.rs | 17 +- 24 files changed, 3387 insertions(+), 216 deletions(-) create mode 100644 crates/tui/src/tools/echolocation.rs diff --git a/crates/tui/src/client/chat.rs b/crates/tui/src/client/chat.rs index a05a2d8480..9aafae6b33 100644 --- a/crates/tui/src/client/chat.rs +++ b/crates/tui/src/client/chat.rs @@ -1826,7 +1826,6 @@ impl<'a> PromptBuilder<'a> { ), false, ); - dump_system_prompt_if_requested(&messages); if provider == ProviderKind::Arcee { apply_arcee_waf_safe_message_encoding(&mut messages); } @@ -1906,38 +1905,82 @@ const SYSTEM_PROMPT_DUMP_BEGIN: &str = "<<>>"; const SYSTEM_PROMPT_DUMP_END: &str = "<<>>"; const ARCEE_WAF_TEXT_SPLIT_TRIGGERS: &[(&str, &str, &str)] = &[("python -c", "python ", "-c")]; -fn dump_system_prompt_if_requested(messages: &[Value]) { +/// Dump the wire prefix (system region + tools array) when the +/// system-prompt dump flag is set, so prefix-token audits can measure the +/// catalog beside the prompt. Called once from +/// [`PreparedOutboundRequest::new`](super::prepared::PreparedOutboundRequest::new), +/// the single funnel for all three wire dialects, so the bytes are the +/// exact-final-body bytes. Debug-only, zero production impact. +pub(crate) fn dump_wire_prefix_if_requested(body: &Value) { let Ok(flag) = std::env::var(SYSTEM_PROMPT_DUMP_ENV) else { return; }; if !matches!(flag.trim(), "1" | "true" | "TRUE" | "yes" | "YES") { return; } - let Some(prompt) = messages.iter().find_map(system_message_text) else { - return; - }; let mut stderr = std::io::stderr().lock(); - let _ = writeln!(stderr, "{SYSTEM_PROMPT_DUMP_BEGIN}"); - let _ = writeln!(stderr, "{prompt}"); - let _ = writeln!(stderr, "{SYSTEM_PROMPT_DUMP_END}"); -} - -fn system_message_text(message: &Value) -> Option { - if message.get("role").and_then(Value::as_str) != Some("system") { - return None; + if let Some(prompt) = wire_system_text(body) { + let _ = writeln!(stderr, "{SYSTEM_PROMPT_DUMP_BEGIN}"); + let _ = writeln!(stderr, "{prompt}"); + let _ = writeln!(stderr, "{SYSTEM_PROMPT_DUMP_END}"); + } + if let Some(tools) = body.get("tools") { + let _ = writeln!(stderr, "<<>>"); + let _ = writeln!(stderr, "{tools}"); + let _ = writeln!(stderr, "<<>>"); + } +} + +/// Best-effort system-region text across the three wire dialects: +/// Anthropic `system` (string or blocks), Responses `instructions`, and +/// chat `messages[]` with a system/developer role. +fn wire_system_text(body: &Value) -> Option { + if let Some(system) = body.get("system") { + return Some(match system { + Value::String(text) => text.clone(), + Value::Array(parts) => { + let text = parts + .iter() + .filter_map(|part| part.get("text").and_then(Value::as_str)) + .collect::>() + .join("\n"); + if text.is_empty() { + system.to_string() + } else { + text + } + } + _ => system.to_string(), + }); } - match message.get("content")? { - Value::String(text) => Some(text.clone()), - Value::Array(parts) => { - let text = parts - .iter() - .filter_map(|part| part.get("text").and_then(Value::as_str)) - .collect::>() - .join(""); - (!text.is_empty()).then_some(text) - } - _ => None, + if let Some(instructions) = body.get("instructions").and_then(Value::as_str) { + return Some(instructions.to_string()); } + body.get("messages")? + .as_array()? + .iter() + .find_map(|message| { + if !matches!( + message.get("role").and_then(Value::as_str), + Some("system" | "developer") + ) { + return None; + } + match message.get("content")? { + Value::String(text) => Some(text.clone()), + Value::Array(parts) => { + // Same "\n" block join as the Anthropic arm above, so + // the audit text is dialect-consistent. + let text = parts + .iter() + .filter_map(|part| part.get("text").and_then(Value::as_str)) + .collect::>() + .join("\n"); + (!text.is_empty()).then_some(text) + } + _ => None, + } + }) } fn apply_arcee_waf_safe_message_encoding(messages: &mut [Value]) { diff --git a/crates/tui/src/client/prepared.rs b/crates/tui/src/client/prepared.rs index e8357c83be..4eb82cd4b5 100644 --- a/crates/tui/src/client/prepared.rs +++ b/crates/tui/src/client/prepared.rs @@ -305,6 +305,7 @@ impl PreparedOutboundRequest { replay_input_tokens: Option, entrypoint: CallerStreamMode, ) -> Self { + super::chat::dump_wire_prefix_if_requested(&body); let reasoning = ReasoningReceipt::from_body(dialect, &body, requested_effort); Self { dialect, diff --git a/crates/tui/src/client/test_cases_02.rs b/crates/tui/src/client/test_cases_02.rs index cdf2bebef5..a2df66111f 100644 --- a/crates/tui/src/client/test_cases_02.rs +++ b/crates/tui/src/client/test_cases_02.rs @@ -448,14 +448,14 @@ let expected_names = crate::core::engine::default_active_native_tool_names() .iter() .copied() - // Children can inspect the shared goal, but only its owning - // session can create it or change its completion state. + // Children discover goal inspection via search; only the + // owning session can create a goal or change its + // completion state, so those never ride the child surface. .filter(|name| !matches!(*name, "create_goal" | "update_goal")) .chain([crate::core::engine::tool_catalog::TOOL_SEARCH_NAME]) .map(str::to_string) .collect(); assert_eq!(source_names, expected_names); - assert!(source_names.contains("get_goal")); assert!(!source_names.contains("create_goal")); assert!(!source_names.contains("update_goal")); diff --git a/crates/tui/src/core/engine/preview.rs b/crates/tui/src/core/engine/preview.rs index 67fe132f83..257426ebc7 100644 --- a/crates/tui/src/core/engine/preview.rs +++ b/crates/tui/src/core/engine/preview.rs @@ -313,6 +313,7 @@ impl Engine { McpAccess::PassiveSnapshot, route_context.clone(), "", + prompt_context.goal_objective.as_deref(), ) .await; diff --git a/crates/tui/src/core/engine/preview/tests.rs b/crates/tui/src/core/engine/preview/tests.rs index 05e896c93d..7c8945acd0 100644 --- a/crates/tui/src/core/engine/preview/tests.rs +++ b/crates/tui/src/core/engine/preview/tests.rs @@ -641,6 +641,7 @@ async fn planned_route_builds_subagent_catalog_without_installed_client() { reasoning_effort_auto: planned.auto_controls_reasoning, }, "", + None, ) .await; assert!( @@ -2195,6 +2196,7 @@ async fn preview_tool_snapshot_has_no_mcp_or_event_side_effects() { reasoning_effort_auto: false, }, "", + None, ) .await; diff --git a/crates/tui/src/core/engine/tests/test_cases_09.rs b/crates/tui/src/core/engine/tests/test_cases_09.rs index c1e6c79558..26469221b5 100644 --- a/crates/tui/src/core/engine/tests/test_cases_09.rs +++ b/crates/tui/src/core/engine/tests/test_cases_09.rs @@ -1604,10 +1604,23 @@ fn only_the_stamped_approval_note_is_hidden_from_the_person() { #[test] fn core_primitives_and_todo_write_default_to_eager() { let always_load = HashSet::new(); - for core in ["read", "write", "edit", "bash", "agent", "todo_write"] { + for core in ["read", "write", "edit", "bash", "todo_write"] { assert!(!should_default_defer_tool(core, &always_load)); } - for searchable in ["File", "Bash", "Git", "Run", "tasks", "git_blame"] { + // Token diet: agent+workflow (21KB of schema) and get/update_goal + // (meaningless without an active goal) stay searchable by default. + for searchable in [ + "File", + "Bash", + "Git", + "Run", + "tasks", + "git_blame", + "agent", + "workflow", + "get_goal", + "update_goal", + ] { assert!(should_default_defer_tool(searchable, &always_load)); } } diff --git a/crates/tui/src/core/engine/tests/test_cases_10.rs b/crates/tui/src/core/engine/tests/test_cases_10.rs index 79c4f6512f..1925e705e9 100644 --- a/crates/tui/src/core/engine/tests/test_cases_10.rs +++ b/crates/tui/src/core/engine/tests/test_cases_10.rs @@ -1,19 +1,17 @@ #[test] fn default_active_contract_keeps_discovery_and_core_tools_eager() { - const EXPECTED_NATIVE: [&str; 13] = [ + // Token diet: agent+workflow (21KB of schema) and get/update_goal + // (meaningless without an active goal) stay searchable by default. + const EXPECTED_NATIVE: [&str; 9] = [ "read", "write", "edit", "file_search", "bash", - "agent", - "workflow", "todo_write", - "create_goal", - "get_goal", - "update_goal", "task_shell_wait", "load_skill", + "create_goal", ]; assert_eq!( default_active_native_tool_names(), @@ -53,12 +51,12 @@ fn default_active_contract_keeps_discovery_and_core_tools_eager() { #[test] fn non_yolo_mode_retains_default_defer_policy() { let always_load = HashSet::new(); + // Token diet: agent stays searchable even on wide surfaces. for core in [ "read", "write", "edit", "bash", - "agent", "todo_write", "load_skill", ] { diff --git a/crates/tui/src/core/engine/tool_catalog.rs b/crates/tui/src/core/engine/tool_catalog.rs index 95dc2db2db..bb899112d1 100644 --- a/crates/tui/src/core/engine/tool_catalog.rs +++ b/crates/tui/src/core/engine/tool_catalog.rs @@ -51,16 +51,25 @@ pub(crate) fn is_tool_search_tool(name: &str) -> bool { #[rustfmt::skip] pub(crate) const DEFAULT_ACTIVE_NATIVE_TOOLS: &[&str] = &[ // Core work controls are eager; specialized tools stay searchable. - "read", "write", "edit", "file_search", "bash", "agent", "workflow", "todo_write", + "read", "write", "edit", "file_search", "bash", "todo_write", // Continuation instructions require these controls. Hiding them behind // discovery leaves a model unable to stop the work it was asked to run. - "create_goal", "get_goal", "update_goal", "task_shell_wait", + "task_shell_wait", // The pinned `## Skills` index tells the model to call `load_skill`, so // the tool has to be on the wire for that instruction to be true. Behind // `tool_search` it cost a discovery hop plus a `change:tool_surface` // re-pin every time a skill was used, against ~134 pinned bytes to have // it eager beside the index the prefix already carries. "load_skill", + // create_goal is eager: it is the entry point. Token diet: agent and + // workflow stay deferred despite being core-adjacent — their schemas + // cost ~21KB/turn and BM25 discovery (`tool_search`) finds them + // reliably when delegation is actually wanted (cued-delegation probe: + // discovered and executed; uncued sessions behave identically). + // get/update_goal cannot succeed without an active goal, so they ride + // `always_load` only while one exists (see the catalog build in + // `engine.rs`) instead of taxing every goalless turn. + "create_goal", ]; const CORE_ACTION_TOOL_FALLBACKS: &[CoreActionToolFallback] = &[ @@ -950,7 +959,12 @@ fn discover_tools_with_regex( let regex = compile_user_regex(query) .map_err(|err| ToolError::invalid_input(format!("Invalid regex query: {err}")))?; - let mut matches = Vec::new(); + // Name matches outrank haystack matches: a regex that hits the tool's + // own name is the strongest possible signal, and catalog (alphabetical) + // order otherwise lets an incidental mention evict the literal target + // from the bounded activation cache. + let mut name_matches = Vec::new(); + let mut hay_matches = Vec::new(); let mut scratch = ToolSearchScratch::default(); for tool in catalog { // tool_search loads definitions omitted from the current request. An @@ -960,15 +974,19 @@ fn discover_tools_with_regex( if !tool.defer_loading.unwrap_or(false) || is_tool_search_tool(&tool.name) { continue; } + // Lowercased, like the haystack path below: `Agent` must hit `agent`. + if regex.is_match(&tool.name.to_lowercase()) { + name_matches.push(tool.name.clone()); + continue; + } scratch.load(tool); if regex.is_match(&scratch.hay) { - matches.push(tool.name.clone()); - } - if matches.len() >= max_results { - break; + hay_matches.push(tool.name.clone()); } } - Ok(matches) + name_matches.extend(hay_matches); + name_matches.truncate(max_results); + Ok(name_matches) } fn discover_tools_with_bm25_like(catalog: &[Tool], query: &str, max_results: usize) -> Vec { @@ -1592,7 +1610,7 @@ fn execute_tool_search_inner( json!({ "type": "unavailable_tool_reference", "tool_name": name, - "reason": "The tool schema exceeds the bounded conversation toolbox (8 cached tools, 16KiB of added serialized schemas). Narrow the search or use a smaller matching tool." + "reason": "The tool schema exceeds the bounded conversation toolbox (8 cached tools, 24KiB of added serialized schemas). Narrow the search or use a smaller matching tool." }) })); unavailable_references.extend(cache_held.iter().map(|name| { diff --git a/crates/tui/src/core/engine/tool_catalog/tests.rs b/crates/tui/src/core/engine/tool_catalog/tests.rs index 4542d5b0ad..4ad844a2b4 100644 --- a/crates/tui/src/core/engine/tool_catalog/tests.rs +++ b/crates/tui/src/core/engine/tool_catalog/tests.rs @@ -97,6 +97,9 @@ fn published_synthetic_names_agree_with_the_synthetic_predicate() { #[test] fn first_turn_surface_is_stable_across_plan_work_and_operate() { + // Token diet: agent+workflow (21KB of schema) and get/update_goal + // (meaningless without an active goal) stay deferred; discovery via + // tool_search restores them on demand. assert_eq!( DEFAULT_ACTIVE_NATIVE_TOOLS, &[ @@ -105,22 +108,15 @@ fn first_turn_surface_is_stable_across_plan_work_and_operate() { "edit", "file_search", "bash", - "agent", - "workflow", "todo_write", - "create_goal", - "get_goal", - "update_goal", "task_shell_wait", - "load_skill" + "load_skill", + "create_goal" ] ); let expected = [ - "agent", "bash", "create_goal", - "get_goal", - "update_goal", "edit", "file_search", "load_skill", @@ -128,7 +124,6 @@ fn first_turn_surface_is_stable_across_plan_work_and_operate() { "task_shell_wait", "todo_write", "tool_search", - "workflow", "write", ] .into_iter() @@ -347,7 +342,7 @@ fn unknown_and_wildcard_allowlists_keep_mcp_startup() { } #[test] -fn compact_surface_keeps_agent_and_workflow_eager() { +fn compact_surface_keeps_agent_and_workflow_deferred() { let catalog = build_model_tool_catalog_with_surface( [ "read", @@ -373,12 +368,30 @@ fn compact_surface_keeps_agent_and_workflow_eager() { .iter() .find(|definition| definition.name == name) .and_then(|definition| definition.defer_loading), - Some(false), + Some(true), "{name}" ); } } +#[test] +fn deferred_core_tools_restore_through_always_load() { + // The diet's escape hatch: discovery/activation and the goal-aware + // catalog build in `engine.rs` re-eager exactly these names. + let mut catalog = ["agent", "workflow", "get_goal", "update_goal", "read"] + .into_iter() + .map(tool) + .collect::>(); + let always_load = ["agent", "workflow", "get_goal", "update_goal"] + .into_iter() + .map(str::to_string) + .collect::>(); + apply_native_tool_deferral(&mut catalog, &always_load); + for definition in &catalog { + assert_eq!(definition.defer_loading, Some(false), "{}", definition.name); + } +} + /// The per-tool Registry paragraph is gone; the catalog builder must leave the /// shell tool's description exactly as the registry produced it, so the KV /// prefix stays byte-stable and there is one Registry authority (the prompt). diff --git a/crates/tui/src/hooks/executor.rs b/crates/tui/src/hooks/executor.rs index 60656fe727..10c87767ed 100644 --- a/crates/tui/src/hooks/executor.rs +++ b/crates/tui/src/hooks/executor.rs @@ -6786,6 +6786,10 @@ command = "echo project" // here would never fire. This table is checked in both directions, so // a rename fails on the missing entry and a classifier change fails on // the mismatched category. + // Token diet: agent+workflow and get/update_goal are deferred by + // default (discoverable via tool_search), so they left the eager + // table; their classifications are pinned with the other + // deferred tools below instead. const EXPECTED: &[(&str, &str)] = &[ ("read", "safe"), ("write", "file_write"), @@ -6794,15 +6798,8 @@ command = "echo project" // keeps the classification it already had as a deferred tool. ("file_search", "other"), ("bash", "shell"), - // The router itself touches nothing a hook needs to gate. - ("agent", "other"), - ("workflow", "other"), ("todo_write", "safe"), - // Goal controls retain their existing hook classification when - // promoted from deferred discovery to the eager catalog. ("create_goal", "other"), - ("get_goal", "other"), - ("update_goal", "other"), ("task_shell_wait", "other"), // Reads the skill registry, not caller-named paths, so it keeps // the classification it already had as a deferred tool. Making it @@ -6828,6 +6825,14 @@ command = "echo project" ); } + // Deferred by default, still classified: the router touches + // nothing a hook needs to gate, and the goal controls keep + // their classification when promoted to eager. + assert_eq!(tool_category_for("agent", None), "other"); + assert_eq!(tool_category_for("workflow", None), "other"); + assert_eq!(tool_category_for("get_goal", None), "other"); + assert_eq!(tool_category_for("update_goal", None), "other"); + // The shell surface. assert_eq!(tool_category_for("Bash", None), "shell"); // Retained: shell.rs stamps this for the shell_env event. diff --git a/crates/tui/src/skills/mod.rs b/crates/tui/src/skills/mod.rs index bd9dfd6d85..6e2e7a4206 100644 --- a/crates/tui/src/skills/mod.rs +++ b/crates/tui/src/skills/mod.rs @@ -1643,7 +1643,7 @@ pub fn render_available_skills_context_for_workspace_with_mode_and_plugins( } /// Progressive-disclosure contract: the model sees a bounded page of skill -/// names, descriptions, and paths, then uses `load_skill` for the complete +/// names and descriptions, then uses `load_skill` for the complete /// catalogue or a specific `SKILL.md` body. /// /// Test-only single-directory variant. Production callers scan the complete @@ -1776,65 +1776,10 @@ fn replace_prompt_path_root(text: &str, root: &str, replacement: &str) -> String out } -/// Render a skill path without leaking private absolute paths into the -/// system-prompt prefix (#4632): workspace skills become workspace-relative, -/// home-dir skills become `~/…`, and anything else is reduced to its trailing -/// components so the prefix stays free of user-identifying absolute paths. -/// Skill paths in the prompt are consumed by the model as text, not by the -/// platform's shell, so normalize Windows separators to forward slashes: -/// the catalog renders identically on every platform (#5473). -fn prompt_display(path: &Path) -> String { - path.display() - .to_string() - .replace(std::path::MAIN_SEPARATOR, "/") -} - -fn privacy_safe_skill_path(path: &Path, workspace: &Path) -> String { - if let Ok(rel) = path.strip_prefix(workspace) { - return prompt_display(rel); - } - if let Some(home) = crate::config::effective_home_dir() - && let Ok(rel) = path.strip_prefix(&home) - { - return format!("~/{}", prompt_display(rel)); - } - match (path.parent().and_then(Path::file_name), path.file_name()) { - (Some(dir), Some(file)) => { - format!("…/{}/{}", dir.to_string_lossy(), file.to_string_lossy()) - } - _ => path - .file_name() - .map(|file| file.to_string_lossy().into_owned()) - .unwrap_or_else(|| "SKILL.md".to_string()), - } -} - -fn path_is_within_root(path: &Path, root: &Path) -> bool { - if path.starts_with(root) { - return true; - } - let Some(canonical_path) = fs::canonicalize(path).ok() else { - return false; - }; - let Some(canonical_root) = fs::canonicalize(root).ok() else { - return false; - }; - canonical_path.starts_with(canonical_root) -} - -fn prompt_skill_path( - path: &Path, - workspace: &Path, - configured_skills_root: Option<&Path>, -) -> Option { - if let Some(root) = configured_skills_root - && path_is_within_root(path, root) - { - return None; - } - Some(privacy_safe_skill_path(path, workspace)) -} - +// Token diet: the ambient index carries names + descriptions only, never +// on-disk paths — `load_skill` resolves names through the registry. (The +// former privacy-safe path renderers lived here; deleting them also deletes +// the #4632/#5473 surface they existed to defend.) #[cfg(test)] fn render_skills_block(registry: &SkillRegistry, locale: &str, workspace: &Path) -> Option { render_skills_block_with_configured_root( @@ -1984,17 +1929,16 @@ Skills are optional instruction packs. This index exposes routing metadata; bodi // instructions or consuming prompt budget. .filter(|skill| skill.invocation.model_invocable()) .map(|skill| { - // Native skills expose the real on-disk path captured at discovery. - // Plugin skills expose only their reviewed snapshot identity so the - // model cannot bypass the content-bound trust receipt via a mutable - // source path. Paths render privacy-safe (workspace-relative or - // ~/…) so the prompt prefix never embeds absolute user paths - // (#4632). A caller-provided skills root omits its physical path - // because that root may change per session; load_skill still - // resolves the stable skill name through the internal registry. - let display_path = prompt_skill_path(&skill.path, workspace, configured_skills_root); + // Token diet: native skills expose no on-disk path. `load_skill` + // resolves the stable skill name through the internal registry, + // so the path only invited the slower read-the-file bypass the + // tool description already steers away from — at ~50 bytes a row, + // every turn, in the cache-pinned prefix. Plugin skills keep + // their reviewed snapshot identity: the trust receipt must stay + // visible so the model cannot mistake a snapshot for a mutable + // source path. let source = match &skill.source { - SkillSource::Native => display_path.map(|path| format!("file: {path}")), + SkillSource::Native => None, SkillSource::Plugin { plugin_id, plugin_name, diff --git a/crates/tui/src/skills/tests.rs b/crates/tui/src/skills/tests.rs index 7d8a98b542..70190df869 100644 --- a/crates/tui/src/skills/tests.rs +++ b/crates/tui/src/skills/tests.rs @@ -326,7 +326,7 @@ fn prompt_warning_sanitizer_handles_non_utf8_configured_roots() { } #[test] -fn render_available_skills_context_lists_paths_and_usage() { +fn render_available_skills_context_lists_names_and_usage() { let tmpdir = TempDir::new().unwrap(); create_skill_dir( &tmpdir, @@ -337,17 +337,16 @@ fn render_available_skills_context_lists_paths_and_usage() { let rendered = crate::skills::render_available_skills_context(&tmpdir.path().join("skills")) .expect("skill context"); - // #4632: paths render relative to the skills base dir (privacy-safe), - // so the assertion checks the workspace-relative form. - let expected_path = super::prompt_display(&std::path::Path::new("test-skill").join("SKILL.md")); - + // Token diet: names + descriptions only. `load_skill` resolves the + // name through the registry, so no on-disk path is rendered at all + // (#4632's surface is gone with the paths). assert!(rendered.contains("## Skills")); assert!(rendered.contains("- test-skill: A test skill")); assert!(rendered.contains("call `load_skill` with that exact name before starting the work")); assert!(rendered.contains("do not expand tool, approval, or trust authority")); assert!( - rendered.contains(&expected_path), - "expected path {expected_path:?} not in rendered output" + !rendered.contains("test-skill/SKILL.md"), + "no on-disk paths in the index" ); assert!(!rendered.contains(tmpdir.path().to_str().unwrap_or("/nonexistent"))); assert!(rendered.contains("### Usage")); @@ -397,12 +396,12 @@ fn workspace_prompt_omits_disabled_skills_without_configured_directory() { } #[test] -fn render_available_skills_context_uses_real_dir_name_not_frontmatter_name() { - // Regression: when a community-installed or manually-placed skill +fn render_available_skills_context_resolves_by_name_without_invented_paths() { + // Regression (pathless form): when a community-installed skill // lives in a directory whose name differs from its frontmatter - // `name`, the rendered prompt must point to the real on-disk file - // path, not //SKILL.md (which does - // not exist). + // `name`, the row carries the registry name and no path at all — + // `load_skill` resolves the name to the real on-disk file, so no + // caller can reconstruct a stale /SKILL.md path. let tmpdir = TempDir::new().unwrap(); create_skill_dir( &tmpdir, @@ -413,18 +412,17 @@ fn render_available_skills_context_uses_real_dir_name_not_frontmatter_name() { let rendered = crate::skills::render_available_skills_context(&tmpdir.path().join("skills")) .expect("skill context"); - // #4632: rendered relative to the skills base dir; the regression - // intent (real dir name, not frontmatter name) is unchanged. - let real_path = super::prompt_display(&std::path::Path::new("weird-dir-name").join("SKILL.md")); - let stale_path = super::prompt_display(&std::path::Path::new("friendly-name").join("SKILL.md")); - assert!( - rendered.contains(&real_path), - "expected real on-disk path {real_path:?} in rendered output, got:\n{rendered}" + rendered.contains("- friendly-name: drift case"), + "registry-name row present, got:\n{rendered}" + ); + assert!( + !rendered.contains("weird-dir-name/SKILL.md"), + "no dir-derived path:\n{rendered}" ); assert!( - !rendered.contains(&stale_path), - "rendered output must not invent a path under the frontmatter name:\n{rendered}" + !rendered.contains("friendly-name/SKILL.md"), + "no frontmatter-derived path:\n{rendered}" ); } @@ -2162,9 +2160,7 @@ fn configured_skill_prompt_uses_a_stable_root_in_entries_and_warnings() { .expect("configured skill context"); assert!(rendered.contains("- visual-design: Design assets\n")); - assert!(rendered.contains( - "- workspace-skill: Workspace skill (file: .claude/skills/workspace-skill/SKILL.md)" - )); + assert!(rendered.contains("- workspace-skill: Workspace skill\n")); assert!( rendered .contains("in /visual-design/SKILL.md is not a safe command name") @@ -2174,7 +2170,7 @@ fn configured_skill_prompt_uses_a_stable_root_in_entries_and_warnings() { } #[test] -fn default_workspace_skill_prompt_preserves_its_discoverable_path() { +fn default_workspace_skill_prompt_omits_paths_in_favour_of_load_skill() { let _env_lock = crate::test_support::lock_test_env(); super::clear_skill_discovery_cache(); let tmpdir = TempDir::new().unwrap(); @@ -2205,9 +2201,7 @@ fn default_workspace_skill_prompt_preserves_its_discoverable_path() { ) .expect("workspace skill context"); - assert!(rendered.contains( - "- workspace-skill: Workspace skill (file: .agents/skills/workspace-skill/SKILL.md)" - )); + assert!(rendered.contains("- workspace-skill: Workspace skill\n")); } #[test] diff --git a/crates/tui/src/tools/echolocation.rs b/crates/tui/src/tools/echolocation.rs new file mode 100644 index 0000000000..d03436c322 --- /dev/null +++ b/crates/tui/src/tools/echolocation.rs @@ -0,0 +1,2588 @@ +//! Echolocation: stateless navigation soundings for tool results. +//! +//! No index, no background work, no new dependencies. Like a whale sounding +//! its pod, every read, grep, file search, and project map echoes its +//! surroundings fresh from source — sibling podmates, symbols, import +//! roots, resolved file links — rendered deterministically (sorted, +//! capped, no timestamps) so a repeated call is byte-identical and +//! cache-friendly. Anything the scanners cannot resolve is omitted, +//! never guessed. Sounding languages: Rust, Python, JavaScript/ +//! TypeScript, Go (symbols everywhere; links for Rust/Python/JS; +//! anything else renders podmates only). +//! +//! KV-cache effect: tool-result history only. Soundings never enter the +//! session-pinned prefix. +//! +//! Tuning: `CODEWHALE_ECHO` gates sections for ablation trials +//! (comma-separated `off:` entries, e.g. `off:links,off:mates`; +//! section names: mates, symbols, imports, links, callers, pods, +//! soundings). Unset or malformed means everything on. +//! +//! Terminal buzz: the read footer deepens when the model is closing in. +//! Two distinct files read in a pod (see +//! `ToolContext::pod_visit_count`) put the next read there into buzz +//! mode, which adds the callers section — podmates whose resolved +//! links include this file. Sparse clicks while searching, +//! high-resolution echoes at close range. Deterministic for a fixed +//! history, like everything else here. +//! +//! Known lookalikes (line scanners, not parsers): comment markers +//! inside string literals can mute following lines until the closer +//! (`"/*"` swallows until `"*/"`); Python doc fences toggle on every +//! `"""`/`'''` occurrence, so mixed kinds on one line desync the +//! span. Keywords match at line start only, so the common failure +//! mode is a missed symbol; forging one needs a keyword at a line +//! start inside a multi-line string. Links need that AND an +//! on-disk target, so they forge only when both coincide. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; + +use ignore::{Match as IgnoreMatch, gitignore::Gitignore}; + +/// Hard budget for the rendered footer, in bytes. +pub const POD_FOOTER_MAX_BYTES: usize = 1200; + +/// Max podmate names listed before "+N more". +const MAX_PODMATES: usize = 12; +/// Max symbols listed before "+N more". +const MAX_SYMBOLS: usize = 24; +/// Max import roots listed before "+N more". +const MAX_IMPORT_ROOTS: usize = 12; +/// Max characters kept per sounded name. +const MAX_NAME_LEN: usize = 64; +/// Max matched dirs charted per grep result before `pods_omitted` counts. +const MAX_GREP_PODS: usize = 8; +/// Max mates listed per grep pod. +const MAX_GREP_MATES: usize = 8; +/// Max key files sounded per project map. +const MAX_MAP_SOUNDINGS: usize = 8; +/// Files bigger than this are never sounded. +const MAX_SOUNDING_BYTES: u64 = 8 * 1024 * 1024; +/// Max resolved file links listed before "+N more". +const MAX_LINKS: usize = 8; +/// Max callers (reverse links) listed before "+N more". +const MAX_CALLERS: usize = 8; +/// Mate files bigger than this are not scanned for callers. +const MAX_CALLER_SCAN_BYTES: u64 = 256 * 1024; +/// Tracked pod reads before the next read there enters the buzz. +pub(crate) const BUZZ_VISIT_THRESHOLD: usize = 2; + +/// Ablation gate: true when `CODEWHALE_ECHO` disables `section`. +/// Parsed once per process; unset or malformed means everything on. +fn echo_off(section: &str) -> bool { + static GATES: OnceLock> = OnceLock::new(); + GATES + .get_or_init(|| { + std::env::var("CODEWHALE_ECHO") + .map(|raw| parse_echo_gates(&raw)) + .unwrap_or_default() + }) + .contains(section) +} + +/// Parse `off:,…` entries; anything else is ignored (fail-open: +/// a typo never silences soundings in production). +fn parse_echo_gates(raw: &str) -> BTreeSet { + raw.split(',') + .filter_map(|entry| entry.trim().strip_prefix("off:")) + .map(|name| name.trim().to_string()) + .filter(|name| !name.is_empty()) + .collect() +} + +/// A sounded chart for one file. Empty sections render as nothing; a chart +/// with nothing to say renders as the empty string. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct EchoChart { + pod_dir: Option, + podmates: Vec, + podmate_total: usize, + symbols: Vec, + symbol_total: usize, + import_roots: Vec, + import_total: usize, + links: Vec, + link_total: usize, + callers: Vec, + caller_total: usize, +} + +/// Sound an echolocation chart for a file that was just read. +/// `workspace` anchors link display (workspace-relative paths); +/// `file_path` is the resolved path (for the sibling sounding); `text` +/// is the full decoded buffer (for symbols, imports, and links); `buzz` +/// adds the callers section when the model is closing in on this pod. +/// Never fails: anything unreadable is omitted from the chart rather +/// than failing the read. +pub fn sound_echolocation(workspace: &Path, file_path: &Path, text: &str, buzz: bool) -> EchoChart { + let (pod_dir, mut podmates) = sound_podmates(file_path); + let podmate_total = podmates.len(); + podmates.truncate(MAX_PODMATES); + let (mut symbols, mut import_roots, mut links, link_total, mut callers, caller_total) = + if let Some(lang) = detect_lang(file_path) { + let (links, link_total) = sound_links(workspace, file_path, text, lang); + let (callers, caller_total) = if buzz { + sound_callers(workspace, file_path) + } else { + (Vec::new(), 0) + }; + ( + sound_symbols(text, lang), + sound_import_roots(text, lang) + .into_iter() + .collect::>(), + links, + link_total, + callers, + caller_total, + ) + } else { + (Vec::new(), Vec::new(), Vec::new(), 0, Vec::new(), 0) + }; + let symbol_total = symbols.len(); + symbols.truncate(MAX_SYMBOLS); + let import_total = import_roots.len(); + import_roots.truncate(MAX_IMPORT_ROOTS); + links.truncate(MAX_LINKS); + callers.truncate(MAX_CALLERS); + EchoChart { + pod_dir, + podmates, + podmate_total, + symbols, + symbol_total, + import_roots, + import_total, + links, + link_total, + callers, + caller_total, + } +} + +/// Sound a miss echo for a read that failed with NotFound: the parent +/// pod's mates, so a typo'd path costs one correction instead of a +/// list-dir round trip. `None` when the parent cannot be listed or the +/// mates gate is off. Callers must only invoke this for genuine +/// NotFound failures — never for denylist/permission errors, where an +/// echo would answer the probe it refused. +pub fn sound_miss_echo(file_path: &Path) -> Option { + if echo_off("mates") { + return None; + } + let (pod_dir, mut mates) = sound_podmates(file_path); + if mates.is_empty() { + return None; + } + const MAX_MISS_MATES: usize = 8; + let total = mates.len(); + mates.truncate(MAX_MISS_MATES); + let dir = pod_dir.as_deref().unwrap_or("."); + Some(format!( + "\n\n[Miss echo: {dir} holds: {}]", + render_capped(&mates, total) + )) +} + +impl EchoChart { + /// Render the footer. Deterministic for a fixed file and directory: + /// sorted entries, fixed caps, byte budget, no timestamps. + pub fn render_footer(&self) -> String { + let mut out = String::new(); + if !echo_off("mates") && self.pod_dir.is_some() && self.podmate_total > 0 { + let dir = self.pod_dir.as_deref().unwrap_or("."); + out.push_str(&format!( + "\n\n[Pod: {} ({}): {}]", + dir, + count_noun(self.podmate_total, "entry", "entries"), + render_capped(&self.podmates, self.podmate_total) + )); + } + let mut sounding = String::new(); + if !echo_off("symbols") && self.symbol_total > 0 { + sounding.push_str(&format!( + "{}: {}", + count_noun(self.symbol_total, "symbol", "symbols"), + render_capped(&self.symbols, self.symbol_total) + )); + } + if !echo_off("imports") && self.import_total > 0 { + if !sounding.is_empty() { + sounding.push_str(" | "); + } + sounding.push_str(&format!( + "imports: {}", + render_capped(&self.import_roots, self.import_total) + )); + } + if !echo_off("links") && self.link_total > 0 { + if !sounding.is_empty() { + sounding.push_str(" | "); + } + sounding.push_str(&format!( + "links: {}", + render_capped(&self.links, self.link_total) + )); + } + if !echo_off("callers") && self.caller_total > 0 { + if !sounding.is_empty() { + sounding.push_str(" | "); + } + sounding.push_str(&format!( + "heard by: {}", + render_capped(&self.callers, self.caller_total) + )); + } + if !sounding.is_empty() { + out.push_str(&format!("\n\n[Sounding: {sounding}]")); + } + truncate_bytes(&out, POD_FOOTER_MAX_BYTES) + } +} + +/// Languages with line-scanner soundings. Everything else is prose: +/// podmates only, no symbols/imports/links. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SoundLang { + Rust, + Python, + JavaScript, + Go, +} + +/// Sounding language from file extension. `None` renders podmates only. +fn detect_lang(file_path: &Path) -> Option { + let ext = file_path.extension()?.to_str()?; + match ext { + "rs" => Some(SoundLang::Rust), + "py" | "pyi" => Some(SoundLang::Python), + "js" | "jsx" | "mjs" | "cjs" | "ts" | "tsx" | "mts" | "cts" => Some(SoundLang::JavaScript), + "go" => Some(SoundLang::Go), + _ => None, + } +} + +/// Sibling entries of the file's directory: sorted display names (current +/// file marked, directories suffixed) plus the short directory display. +/// A directory that cannot be listed degrades to an empty pod, silently. +fn sound_podmates(file_path: &Path) -> (Option, Vec) { + let parent = match file_path.parent() { + Some(parent) => parent, + None => return (None, Vec::new()), + }; + let mates = list_dir_entries(parent).unwrap_or_default(); + let current = file_path + .file_name() + .map(|name| name.to_string_lossy().into_owned()); + let display = mates + .into_iter() + .map(|(raw, mut shown)| { + if Some(&raw) == current.as_ref() { + shown.push_str(" (this file)"); + } + shown + }) + .collect(); + (Some(short_dir_display(parent)), display) +} + +/// Visibility gates a pod listing must honor: the calling search hides +/// these names, so pods must not enumerate them either. The denylist +/// always applies; glob/extension/gitignore gates apply when set. +/// `glob_root` anchors glob matching and gitignore stacking, exactly +/// like the calling walk's root. +pub(crate) struct PodVisibility<'a> { + pub include: &'a [String], + pub exclude: &'a [String], + pub extensions: &'a [String], + pub glob_root: &'a Path, + pub gitignore: bool, +} + +impl PodVisibility<'_> { + /// Directed-read visibility: no search gates, denylist still on. + fn directed(glob_root: &Path) -> PodVisibility<'_> { + PodVisibility { + include: &[], + exclude: &[], + extensions: &[], + glob_root, + gitignore: false, + } + } +} + +/// Sorted `(raw, display)` entries of a dir: dotfiles skipped, dirs +/// suffixed with `/`, visibility gates applied. `None` when the dir +/// cannot be listed at all (distinct from an empty dir). +fn list_dir_entries(dir: &Path) -> Option> { + list_dir_entries_filtered(dir, &PodVisibility::directed(dir)) +} + +/// [`list_dir_entries`] plus the caller's search gates: denylist, +/// excludes, includes/extensions (files only, like the walks), and +/// optional gitignore stacking. +fn list_dir_entries_filtered(dir: &Path, vis: &PodVisibility<'_>) -> Option> { + let ignores = vis + .gitignore + .then(|| IgnoreStack::for_dir(vis.glob_root, dir)); + let mut mates = Vec::new(); + for entry in std::fs::read_dir(dir).ok()?.flatten() { + let raw = entry.file_name().to_string_lossy().into_owned(); + if raw.starts_with('.') { + continue; + } + let abs = entry.path(); + let is_dir = entry.file_type().is_ok_and(|kind| kind.is_dir()); + // Denylist first: a pod listing is enumeration, not a directed + // read — denied names stay hidden even as siblings. + if crate::sandbox::read_guard::active().check(&abs).is_err() { + continue; + } + let rel = abs + .strip_prefix(vis.glob_root) + .unwrap_or(&abs) + .to_string_lossy() + .replace('\\', "/"); + if vis + .exclude + .iter() + .any(|pat| super::search::matches_glob(&rel, pat)) + { + continue; + } + if !is_dir { + if !vis.include.is_empty() + && !vis + .include + .iter() + .any(|pat| super::search::matches_glob(&rel, pat)) + { + continue; + } + if !vis.extensions.is_empty() + && !super::file_search::extension_matches(&abs, vis.extensions) + { + continue; + } + } + if ignores + .as_ref() + .is_some_and(|stack| stack.is_ignored(&abs, is_dir)) + { + continue; + } + let mut display = raw.clone(); + if is_dir { + display.push('/'); + } + mates.push((raw, display)); + } + mates.sort(); + Some(mates) +} + +/// Global gitignore + `.gitignore` + `.ignore` matchers stacked +/// shallow → deep from `glob_root` down to the listed dir. Last +/// decisive match wins, mirroring git precedence (repo rules beat the +/// global file). Known residual: `.git/info/exclude` is not stacked, +/// so a name hidden only there still charts — the calling walk is the +/// only other surface that reads it, and directed reads bypass ignore +/// rules entirely, so no boundary is crossed. +struct IgnoreStack { + matchers: Vec, +} + +impl IgnoreStack { + fn for_dir(glob_root: &Path, dir: &Path) -> Self { + // Global first (lowest precedence); a missing/unreadable global + // file yields an empty matcher, never an error. + let mut matchers = vec![Gitignore::global().0]; + let mut chain = Vec::new(); + let mut current = Some(dir); + // Never climb above the glob root (a stray absolute dir must not + // inherit ignore rules from unrelated ancestors). + while let Some(layer) = current.filter(|layer| layer.starts_with(glob_root)) { + chain.push(layer); + if layer == glob_root { + break; + } + current = layer.parent(); + } + chain.reverse(); + for layer in chain { + for file in [".gitignore", ".ignore"] { + let candidate = layer.join(file); + if candidate.is_file() { + matchers.push(Gitignore::new(&candidate).0); + } + } + } + Self { matchers } + } + + fn is_ignored(&self, path: &Path, is_dir: bool) -> bool { + let mut ignored = false; + for matcher in &self.matchers { + match matcher.matched(path, is_dir) { + IgnoreMatch::Ignore(_) => ignored = true, + IgnoreMatch::Whitelist(_) => ignored = false, + IgnoreMatch::None => {} + } + } + ignored + } +} + +/// One sounded pod for a grep result: the matched directory and its +/// entries, matched files marked, capped with an honest total. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct MatchPod { + pub dir: String, + pub mates: Vec, + pub mate_total: usize, +} + +/// Sound pods for search matches: distinct matched dirs, sorted and +/// capped, each with its entries (matched files marked ` (match)`). +/// Entries honor `vis` — the calling search's own gates — so pods +/// never enumerate names the search hides. Returns the pods plus the +/// count of dirs beyond the cap. A dir that cannot be listed still +/// charts its matched files (already search-gated), so a pod is never +/// silently empty. Deterministic for a fixed tree. +pub(crate) fn sound_match_pods( + resolve_root: &Path, + files: &[String], + vis: &PodVisibility<'_>, +) -> (Vec, usize) { + if echo_off("pods") { + return (Vec::new(), 0); + } + let mut by_dir: BTreeMap> = BTreeMap::new(); + for file in files { + if file.is_empty() { + continue; + } + let (dir, name) = split_match_path(file); + by_dir.entry(dir).or_default().insert(name); + } + let mut pods = Vec::new(); + let mut omitted = 0; + for (dir, matched) in &by_dir { + if pods.len() >= MAX_GREP_PODS { + omitted += 1; + continue; + } + pods.push(sound_one_match_pod(resolve_root, dir, matched, vis)); + } + (pods, omitted) +} + +/// Split a match path into (dir display, file name); `.` for the root. +fn split_match_path(file: &str) -> (String, String) { + let path = Path::new(file); + let name = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + let dir = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .map(|parent| parent.to_string_lossy().into_owned()) + .unwrap_or_else(|| ".".to_string()); + (dir, name) +} + +fn sound_one_match_pod( + resolve_root: &Path, + dir: &str, + matched: &BTreeSet, + vis: &PodVisibility<'_>, +) -> MatchPod { + let full = Path::new(dir); + let full = if full.is_absolute() { + full.to_path_buf() + } else { + resolve_root.join(full) + }; + let mut mates: Vec = match list_dir_entries_filtered(&full, vis) { + Some(entries) => entries + .into_iter() + .map(|(raw, mut shown)| { + if matched.contains(&raw) { + shown.push_str(" (match)"); + } + shown + }) + .collect(), + // Dotfiles stay skipped here too, exactly like the listed + // path: the match itself is still in the result set, so no + // information is lost, only the redundant mark. + None => matched + .iter() + .filter(|name| !name.starts_with('.')) + .map(|name| format!("{name} (match)")) + .collect(), + }; + let mate_total = mates.len(); + mates.truncate(MAX_GREP_MATES); + if mate_total > mates.len() { + mates.push(format!("+{} more", mate_total - mates.len())); + } + MatchPod { + dir: dir.to_string(), + mates, + mate_total, + } +} + +/// Sound one line of symbols for each of the first key files of a +/// project map (sorted, capped). Returns the soundings plus the count +/// of key files beyond the cap. Unreadable, oversized, non-Rust, or +/// symbol-free files are skipped silently. +pub(crate) fn sound_key_files( + root: &Path, + key_files: &[String], +) -> (BTreeMap, usize) { + if echo_off("soundings") { + return (BTreeMap::new(), 0); + } + let mut sorted: Vec<&str> = key_files.iter().map(String::as_str).collect(); + sorted.sort(); + let omitted = sorted.len().saturating_sub(MAX_MAP_SOUNDINGS); + let mut soundings = BTreeMap::new(); + for name in sorted.into_iter().take(MAX_MAP_SOUNDINGS) { + if let Some(sounding) = sound_file_symbols(&root.join(name)) { + soundings.insert(name.to_string(), sounding); + } + } + (soundings, omitted) +} + +/// Sound one line of symbols for a single file: `N symbols: a, b, …`. +/// `None` when unreadable, oversized, non-Rust, or symbol-free. +fn sound_file_symbols(path: &Path) -> Option { + let lang = detect_lang(path)?; + if std::fs::metadata(path).is_ok_and(|meta| meta.len() > MAX_SOUNDING_BYTES) { + return None; + } + let text = std::fs::read_to_string(path).ok()?; + let mut symbols = sound_symbols(&text, lang); + let total = symbols.len(); + if total == 0 { + return None; + } + symbols.truncate(MAX_SYMBOLS); + Some(format!( + "{}: {}", + count_noun(total, "symbol", "symbols"), + render_capped(&symbols, total) + )) +} + +/// Last two normal components of a directory, for a compact pod label. +fn short_dir_display(parent: &Path) -> String { + let parts: Vec = parent + .components() + .filter_map(|component| match component { + std::path::Component::Normal(part) => Some(part.to_string_lossy().into_owned()), + _ => None, + }) + .collect(); + if parts.is_empty() { + return ".".to_string(); + } + let start = parts.len().saturating_sub(2); + format!("{}/", parts[start..].join("/")) +} + +/// Item lines in file order, per language. +fn sound_symbols(text: &str, lang: SoundLang) -> Vec { + // Symbols carry 1-based source lines (`fn main:12`) so the model can + // page straight at them — the sounding already sees past truncation, + // and now it can point there too. + let numbered = |n: usize, symbol: String| format!("{symbol}:{n}"); + match lang { + SoundLang::Rust => code_lines(text) + .filter_map(|(n, line)| parse_item_line(line).map(|s| numbered(n, s))) + .collect(), + SoundLang::Python => py_code_lines(text) + .filter_map(|(n, line)| parse_py_item(line).map(|s| numbered(n, s))) + .collect(), + SoundLang::JavaScript => code_lines(text) + .filter_map(|(n, line)| parse_js_item(line).map(|s| numbered(n, s))) + .collect(), + SoundLang::Go => code_lines(text) + .filter_map(|(n, line)| parse_go_item(line).map(|s| numbered(n, s))) + .collect(), + } +} + +/// Distinct import roots, sorted, per language. +fn sound_import_roots(text: &str, lang: SoundLang) -> BTreeSet { + match lang { + SoundLang::Rust => code_lines(text) + .filter_map(|(_, line)| parse_use_root(line)) + .collect(), + SoundLang::Python => py_code_lines(text) + .flat_map(|(_, line)| parse_py_import_roots(line)) + .collect(), + SoundLang::JavaScript => code_lines(text) + .filter_map(|(_, line)| parse_js_import_root(line)) + .collect(), + SoundLang::Go => code_lines(text) + .filter_map(|(_, line)| parse_go_import_root(line)) + .collect(), + } +} + +/// Source lines with `//` lines and `/* */` spans removed. Crude on +/// purpose: string literals containing comment markers are a documented +/// lookalike class (see module docs), not silently misread structure. +fn code_lines(text: &str) -> impl Iterator { + let mut in_block = false; + text.lines().enumerate().filter_map(move |(idx, raw)| { + let mut line = raw.trim_start(); + if in_block { + match line.find("*/") { + Some(end) => { + in_block = false; + line = line[end + 2..].trim_start(); + } + None => return None, + } + } + if line.starts_with("//") { + return None; + } + if let Some(start) = line.find("/*") { + let after_open = &line[start + 2..]; + match after_open.find("*/") { + Some(end) => { + let head = line[..start].trim_end(); + let tail = after_open[end + 2..].trim_start(); + line = if head.is_empty() { tail } else { head }; + if line.is_empty() { + return None; + } + } + None => { + in_block = true; + line = line[..start].trim_end(); + if line.is_empty() { + return None; + } + } + } + } + Some((idx + 1, line)) + }) +} + +/// Parse one `fn|struct|enum|trait|type|mod|static|const|impl|union|macro` +/// line into `kind name`. Qualifiers (`pub`, `pub(..)`, `async`, `const`, +/// `unsafe`, `extern "ABI"`) are stripped first. +fn parse_item_line(mut line: &str) -> Option { + loop { + line = line.trim_start(); + if let Some(rest) = strip_keyword(line, "pub") { + line = strip_visibility_group(rest.trim_start())?; + continue; + } + if let Some(rest) = strip_keyword(line, "extern") { + line = strip_abi_string(rest.trim_start())?; + continue; + } + let mut stripped = false; + for qualifier in ["async", "unsafe"] { + if let Some(rest) = strip_keyword(line, qualifier) { + line = rest; + stripped = true; + break; + } + } + if stripped { + continue; + } + // `const fn` (qualifier) vs `const NAME` (item): strip only when + // `fn` follows, otherwise the keyword table below owns it. + if let Some(rest) = strip_keyword(line, "const") + && strip_keyword(rest.trim_start(), "fn").is_some() + { + line = rest; + continue; + } + break; + } + for (keyword, label) in [ + ("fn", "fn"), + ("struct", "struct"), + ("enum", "enum"), + ("trait", "trait"), + ("type", "type"), + ("mod", "mod"), + ("static", "static"), + ("const", "const"), + ("union", "union"), + ("impl", "impl"), + ("macro_rules", "macro"), + ("macro", "macro"), + ] { + if let Some(rest) = strip_keyword(line, keyword) { + let rest = if keyword == "macro_rules" { + rest.strip_prefix('!').unwrap_or(rest) + } else { + rest + }; + let name = item_head(rest, keyword == "impl")?; + return Some(format!("{label} {name}")); + } + } + None +} + +/// Strip `keyword` only on a word boundary (so `fn` never matches `fnx`). +fn strip_keyword<'a>(line: &'a str, keyword: &str) -> Option<&'a str> { + let rest = line.strip_prefix(keyword)?; + match rest.chars().next() { + None => Some(rest), + Some(next) if next.is_alphanumeric() || next == '_' => None, + _ => Some(rest), + } +} + +/// Skip a `pub(..)` visibility group; bare `pub` falls through unchanged. +fn strip_visibility_group(rest: &str) -> Option<&str> { + match rest.strip_prefix('(') { + Some(after) => Some(&after[after.find(')')? + 1..]), + None => Some(rest), + } +} + +/// Skip an `extern "..."` ABI string; bare `extern` falls through unchanged. +fn strip_abi_string(rest: &str) -> Option<&str> { + match rest.strip_prefix('"') { + Some(after) => Some(&after[after.find('"')? + 1..]), + None => Some(rest), + } +} + +/// The sounded name: first identifier for items, the whole head (bounded) +/// for `impl` blocks (`Foo`, `Trait for Foo`, ` Foo`). +fn item_head(rest: &str, is_impl: bool) -> Option { + let rest = rest.trim_start(); + if is_impl { + let mut head = rest.split(['{', ';']).next()?.trim(); + if let Some(where_at) = head.find(" where ") { + head = head[..where_at].trim(); + } + if head.is_empty() { + return None; + } + return Some(truncate_name(head)); + } + // Capped before collecting: a minified line can run megabytes, + // and the sounding must never allocate the file to name it. + let name: String = rest + .chars() + .take_while(|next| next.is_alphanumeric() || *next == '_') + .take(MAX_NAME_LEN + 1) + .collect(); + if name.is_empty() || name.starts_with(|c: char| c.is_ascii_digit()) { + return None; + } + Some(truncate_name(&name)) +} + +/// First `::` segment of a `use` path (`serde`, `crate`, `super`). +fn parse_use_root(line: &str) -> Option { + let mut line = line.trim_start(); + if let Some(rest) = strip_keyword(line, "pub") { + line = strip_visibility_group(rest.trim_start())?.trim_start(); + } + let rest = strip_keyword(line, "use")?.trim_start(); + let head = rest.split([';', '{', ' ', '\t']).next()?.trim(); + let root = head.split("::").next()?.trim(); + valid_root(root) +} + +/// Resolve `use`/`mod` edges to workspace-relative files that exist on +/// disk (sorted, with an honest total; the caller caps). Only module +/// files (`x.rs` / `x/mod.rs`) resolve — single items living directly +/// in a root file, external crates, and `#[path]` remaps are left to +/// the import roots rather than guessed. +fn sound_links( + workspace: &Path, + file_path: &Path, + text: &str, + lang: SoundLang, +) -> (Vec, usize) { + let mut links = BTreeSet::new(); + match lang { + SoundLang::Rust => { + for (_, line) in code_lines(text) { + if let Some(segments) = parse_use_segments(line) { + if let Some(link) = resolve_use_link(workspace, file_path, &segments) { + links.insert(link); + } + } else if let Some(child) = parse_mod_decl(line) + && let Some(link) = resolve_child_module(workspace, file_path, &child) + { + links.insert(link); + } + } + } + SoundLang::Python => { + for (_, line) in py_code_lines(text) { + for link in resolve_py_links(workspace, file_path, line) { + links.insert(link); + } + } + } + SoundLang::JavaScript => { + for (_, line) in code_lines(text) { + if let Some(link) = resolve_js_link(workspace, file_path, line) { + links.insert(link); + } + } + } + // Go imports name packages (dirs), not files: nothing to link. + SoundLang::Go => {} + } + if let Some(own) = workspace_relative(workspace, file_path) { + links.remove(&own); + } + let total = links.len(); + (links.into_iter().collect(), total) +} + +/// Full `use` segments (`crate::config::Config` → all three). Groups, +/// globs, and `as` renames cut at the module prefix. +fn parse_use_segments(line: &str) -> Option> { + let mut line = line.trim_start(); + if let Some(rest) = strip_keyword(line, "pub") { + line = strip_visibility_group(rest.trim_start())?.trim_start(); + } + let rest = strip_keyword(line, "use")?.trim_start(); + let head = rest.split([';', '{', '*']).next()?.trim(); + let head = head.split(" as ").next()?.trim(); + let segments: Vec = head + .split("::") + .map(str::trim) + .filter(|part| !part.is_empty()) + .map(str::to_string) + .collect(); + if segments.is_empty() + || segments + .iter() + .any(|part| part.len() > MAX_NAME_LEN || !is_ident(part)) + { + return None; + } + Some(segments) +} + +/// `mod name;` child decl. Inline `mod name {` is a symbol, not an edge. +fn parse_mod_decl(line: &str) -> Option { + let mut line = line.trim_start(); + if let Some(rest) = strip_keyword(line, "pub") { + line = strip_visibility_group(rest.trim_start())?.trim_start(); + } + let rest = strip_keyword(line, "mod")?.trim_start(); + let name: String = rest + .chars() + .take_while(|next| next.is_alphanumeric() || *next == '_') + .take(MAX_NAME_LEN + 1) + .collect(); + if name.is_empty() || name.len() > MAX_NAME_LEN || !is_ident(&name) { + return None; + } + match rest[name.len()..].trim_start().chars().next() { + Some(';') => Some(name), + _ => None, + } +} + +fn is_ident(part: &str) -> bool { + // No leading digit: not an identifier in any sounded language, so a + // digit-led token is line noise, never a name. + !part.is_empty() + && part + .chars() + .next() + .is_some_and(|first| !first.is_ascii_digit()) + && part.chars().all(|c| c.is_alphanumeric() || c == '_') +} + +fn resolve_use_link(workspace: &Path, file_path: &Path, segments: &[String]) -> Option { + let first = segments.first()?.as_str(); + match first { + "crate" => { + let root = find_crate_root(workspace, file_path)?; + resolve_module_file(workspace, &root, &segments[1..]) + } + "super" => { + let depth = segments + .iter() + .take_while(|part| part.as_str() == "super") + .count(); + let mut base = super_base(file_path)?; + for _ in 1..depth { + base = base.parent()?.to_path_buf(); + } + resolve_module_file(workspace, &base, &segments[depth..]) + } + "self" => { + let base = self_base(file_path)?; + resolve_module_file(workspace, &base, &segments[1..]) + } + _ => None, + } +} + +/// `mod child;` resolves under the current module: the file's own dir +/// for `mod.rs`, a child dir named for the file stem otherwise. +fn resolve_child_module(workspace: &Path, file_path: &Path, child: &str) -> Option { + let base = self_base(file_path)?; + resolve_module_file(workspace, &base, std::slice::from_ref(&child.to_string())) +} + +/// Longest-prefix module-file match: `a::b::Item` tries `a/b.rs`, +/// `a/b/mod.rs`, then `a.rs`, `a/mod.rs`. First hit on disk wins. +fn resolve_module_file(workspace: &Path, base: &Path, rest: &[String]) -> Option { + for width in (1..=rest.len()).rev() { + let joined = rest[..width].join("/"); + for candidate in [ + base.join(format!("{joined}.rs")), + base.join(&joined).join("mod.rs"), + ] { + if candidate.is_file() { + return workspace_relative(workspace, &candidate); + } + } + } + None +} + +/// `self::` / `mod x;` base: own dir for `mod.rs` and the crate roots +/// (`lib.rs`/`main.rs`, whose children live beside them, not under a +/// `/` dir), else `/`. +fn self_base(file_path: &Path) -> Option { + let dir = file_path.parent()?; + if file_path + .file_name() + .is_some_and(|name| name == "mod.rs" || name == "lib.rs" || name == "main.rs") + { + return Some(dir.to_path_buf()); + } + let stem = file_path.file_stem()?; + Some(dir.join(stem)) +} + +/// First-level `super::` base: parent dir for `mod.rs` (whose own dir +/// is the current module), else the file's own dir. +fn super_base(file_path: &Path) -> Option { + let dir = file_path.parent()?; + if file_path.file_name().is_some_and(|name| name == "mod.rs") { + dir.parent().map(Path::to_path_buf) + } else { + Some(dir.to_path_buf()) + } +} + +/// Crate root: nearest ancestor (inclusive of the file's dir) holding +/// `main.rs` or `lib.rs`, never walking past the workspace. +fn find_crate_root(workspace: &Path, file_path: &Path) -> Option { + let mut current = file_path.parent()?.to_path_buf(); + loop { + if current.join("main.rs").is_file() || current.join("lib.rs").is_file() { + return Some(current); + } + if current == workspace { + return None; + } + current = current.parent()?.to_path_buf(); + } +} + +fn workspace_relative(workspace: &Path, path: &Path) -> Option { + path.strip_prefix(workspace) + .ok() + .map(|rel| rel.to_string_lossy().into_owned()) +} + +/// Python source lines: `#` comments cut, triple-quote doc spans +/// skipped. Crude on purpose: a `#` inside a string literal mangles +/// the tail, but item/import keywords match at line start, so the +/// mangling cannot forge a symbol. Doc-span tracking toggles on every +/// `"""` / `'''` occurrence — one-liner docstrings toggle twice (no +/// net change); mixed kinds on one line are a documented lookalike. +fn py_code_lines(text: &str) -> impl Iterator { + let mut in_doc = false; + text.lines().enumerate().filter_map(move |(idx, raw)| { + let mut line = raw.trim(); + if in_doc { + let fences = line.matches("\"\"\"").count() + line.matches("'''").count(); + if fences % 2 == 1 { + in_doc = false; + } + return None; + } + if let Some(hash) = line.find('#') { + line = line[..hash].trim_end(); + } + let fences = line.matches("\"\"\"").count() + line.matches("'''").count(); + if fences % 2 == 1 { + in_doc = true; + return None; + } + if fences > 0 { + // One-liner docstring as the whole logical line: nothing + // else on it can be a symbol. + let stripped = line.replace("\"\"\"", "").replace("'''", ""); + if stripped.trim().is_empty() { + return None; + } + } + if line.is_empty() { + return None; + } + Some((idx + 1, line)) + }) +} + +/// `def name` / `class Name` (any indent — methods included), +/// `async def` unwrapped. +fn parse_py_item(line: &str) -> Option { + let line = line.trim_start(); + let line = strip_keyword(line, "async").unwrap_or(line).trim_start(); + for (keyword, label) in [("def", "def"), ("class", "class")] { + if let Some(rest) = strip_keyword(line, keyword) { + let name: String = rest + .trim_start() + .chars() + .take_while(|next| next.is_alphanumeric() || *next == '_') + .take(MAX_NAME_LEN + 1) + .collect(); + if !name.is_empty() + && name.len() <= MAX_NAME_LEN + && !name.starts_with(|c: char| c.is_ascii_digit()) + { + return Some(format!("{label} {name}")); + } + return None; + } + } + None +} + +/// Import roots of one line: every `import a, b.c` member, or the +/// `from` module (`a.b` → `a`; pure-relative `from . import x` → `.` +/// for depth, honest). Semicolons split statements. +fn parse_py_import_roots(line: &str) -> Vec { + let line = line.trim_start(); + let mut out = Vec::new(); + if let Some(rest) = strip_keyword(line, "import") { + for stmt in rest.split(';') { + for part in stmt.split(',') { + let part = part.split(" as ").next().unwrap_or("").trim(); + let root = part.split('.').next().unwrap_or("").trim(); + if let Some(root) = valid_root(root) { + out.push(root); + } + } + } + return out; + } + if let Some(rest) = strip_keyword(line, "from") { + let module = rest.split(" import ").next().unwrap_or("").trim(); + let root = py_module_root(module); + if !root.is_empty() { + out.push(root); + } + } + out +} + +/// Root of a Python module path: leading dots collapse to their depth +/// when no named segment leads (`from . import x` → `.`). Named roots +/// validate as identifiers; dots pass through as depth markers. +fn py_module_root(module: &str) -> String { + let dots = module.chars().take_while(|c| *c == '.').count(); + let named = module[dots..].split('.').next().unwrap_or("").trim(); + if named.is_empty() { + ".".repeat(dots.max(1)) + } else { + valid_root(named).unwrap_or_default() + } +} + +fn valid_root(root: &str) -> Option { + if root.is_empty() || root.len() > MAX_NAME_LEN || !is_ident(root) { + return None; + } + Some(root.to_string()) +} + +/// Resolve one Python import line to existing files. Relative imports +/// walk up from the file's dir; absolute ones try the file's dir (flat +/// scripts) then the workspace root (packages). Both `mod.py` and +/// `pkg/__init__.py` shapes. Existence is the honesty guard. +fn resolve_py_links(workspace: &Path, file_path: &Path, line: &str) -> Vec { + let line = line.trim_start(); + let mut out = Vec::new(); + if let Some(rest) = strip_keyword(line, "import") { + // `import a, b.c as d` (semicolons split statements first). + for stmt in rest.split(';') { + for part in stmt.split(',') { + let part = part.split(" as ").next().unwrap_or("").trim(); + let segments: Vec<&str> = part.split('.').map(str::trim).collect(); + if segments.iter().all(|s| is_ident(s)) { + out.extend(resolve_py_module(workspace, file_path, 0, &segments)); + } + } + } + return out; + } + if let Some(rest) = strip_keyword(line, "from") { + // `from [..]mod import names` — parenthesized/continued name + // lists don't matter; only the module part resolves. + let module = rest.split(" import ").next().unwrap_or("").trim(); + let dots = module.chars().take_while(|c| *c == '.').count(); + let segments: Vec<&str> = module[dots..] + .split('.') + .map(str::trim) + .filter(|s| !s.is_empty()) + .collect(); + if segments.iter().all(|s| is_ident(s)) { + // `from . import sib` names the leaf in the import list, + // not the module: resolve each imported name as a sibling. + if dots > 0 && segments.is_empty() { + if let Some(names) = rest.split(" import ").nth(1) { + for name in names + .replace(['(', ')'], " ") + .split(',') + .map(|n| n.split(" as ").next().unwrap_or("").trim()) + .filter(|n| is_ident(n)) + { + out.extend(resolve_py_module( + workspace, + file_path, + dots, + std::slice::from_ref(&name), + )); + } + } + return out; + } + out.extend(resolve_py_module(workspace, file_path, dots, &segments)); + } + return out; + } + out +} + +/// Resolve Python module segments: `dots` levels up from the file's dir +/// (0 = absolute: same dir first, workspace root as fallback), then +/// `x.py` or `x/__init__.py`. Same-dir wins outright — no double links. +fn resolve_py_module( + workspace: &Path, + file_path: &Path, + dots: usize, + segments: &[&str], +) -> Vec { + if segments.is_empty() { + return Vec::new(); + } + if dots == 0 { + if let Some(dir) = file_path.parent() { + let hits = py_candidates(workspace, dir, segments); + if !hits.is_empty() { + return hits; + } + } + return py_candidates(workspace, workspace, segments); + } + let mut base = match file_path.parent() { + Some(dir) => dir.to_path_buf(), + None => return Vec::new(), + }; + for _ in 1..dots { + match base.parent() { + Some(parent) => base = parent.to_path_buf(), + None => return Vec::new(), + } + } + py_candidates(workspace, &base, segments) +} + +/// `x.py` / `x/__init__.py` under one base, workspace-relative. +fn py_candidates(workspace: &Path, base: &Path, segments: &[&str]) -> Vec { + let joined = segments.join("/"); + [ + base.join(format!("{joined}.py")), + base.join(&joined).join("__init__.py"), + ] + .into_iter() + .filter(|candidate| candidate.is_file()) + .filter_map(|candidate| workspace_relative(workspace, &candidate)) + .collect() +} + +/// JS/TS item lines: `function`/`class`, `const`/`let`/`var` arrows, +/// `type`/`interface`/`enum`. `export`/`default`/`async`/`declare`/ +/// `abstract` unwrapped. Method shorthand skipped (indistinguishable +/// from calls at line-scan resolution — documented omission). +fn parse_js_item(line: &str) -> Option { + let mut line = line.trim_start(); + for qualifier in ["export", "default", "declare", "abstract", "async"] { + if let Some(rest) = strip_keyword(line, qualifier) { + line = rest.trim_start(); + } + } + for (keyword, label) in [ + ("function", "function"), + ("class", "class"), + ("interface", "interface"), + ("enum", "enum"), + ] { + if let Some(rest) = strip_keyword(line, keyword) { + let rest = rest + .trim_start() + .strip_prefix('*') + .unwrap_or(rest) + .trim_start(); + let name = ident_head(rest)?; + return Some(format!("{label} {name}")); + } + } + if let Some(rest) = strip_keyword(line, "type") { + let name = ident_head(rest.trim_start())?; + return Some(format!("type {name}")); + } + for qualifier in ["const", "let", "var"] { + if let Some(rest) = strip_keyword(line, qualifier) { + let rest = rest.trim_start(); + let name = ident_head(rest)?; + let after = rest[name.len()..].trim_start(); + let body = after.strip_prefix('=').unwrap_or("").trim_start(); + // Arrow/function value, not data: `= (`, `= async`, `= function`, `=>`. + if body.starts_with('(') + || strip_keyword(body, "async").is_some() + || strip_keyword(body, "function").is_some() + || body.contains("=>") + { + return Some(format!("function {name}")); + } + return None; + } + } + None +} + +fn ident_head(rest: &str) -> Option { + let name: String = rest + .chars() + .take_while(|next| next.is_alphanumeric() || *next == '_' || *next == '$') + .take(MAX_NAME_LEN + 1) + .collect(); + if name.is_empty() + || name.len() > MAX_NAME_LEN + || name.starts_with(|c: char| c.is_ascii_digit()) + { + return None; + } + Some(name) +} + +/// JS/TS import roots: relative → `.`; scoped `@s/n` kept whole; +/// otherwise the first path segment. Covers `import`, `export-from`, +/// `require`, and dynamic `import()`. +fn parse_js_import_root(line: &str) -> Option { + let spec = js_import_spec(line)?; + if spec.starts_with('.') { + return Some(".".to_string()); + } + if let Some(scoped) = spec.strip_prefix('@') { + let mut parts = scoped.split('/'); + let (scope, name) = (parts.next()?, parts.next()?); + if scope.is_empty() || name.is_empty() { + return None; + } + return Some(format!("@{scope}/{name}")); + } + let root = spec.split('/').next()?.trim(); + valid_root(root) +} + +/// The module specifier of an import-ish line, unquoted. +fn js_import_spec(line: &str) -> Option { + let line = line.trim_start(); + // Side-effect and from-forms: `import 'x'`, `import … from 'x'`, `export … from 'x'`. + if strip_keyword(line, "import").is_some() || strip_keyword(line, "export").is_some() { + return quoted_tail(line); + } + // `require('x')`, `import('x')` anywhere a call starts the line. + // The opener needs a boundary behind it: `myrequire(` and a call + // spelled inside a string literal (`"require('x')"`) must not forge + // imports. + for opener in ["require(", "import("] { + // Every occurrence: a `myrequire(` earlier on the line must not + // shadow a real call later on it. + let mut start = 0; + while let Some(rel) = line[start..].find(opener) { + let at = start + rel; + let bounded = line[..at].chars().next_back().is_none_or(|prev| { + !(prev.is_alphanumeric() + || prev == '_' + || prev == '$' + || prev == '\'' + || prev == '"' + || prev == '`') + }); + start = at + opener.len(); + if !bounded { + continue; + } + let after = line[start..].trim_start(); + // A computed argument (`require(name)`) is not a specifier; + // keep scanning rather than bailing on the whole line. + let Some(quote) = after.chars().next() else { + continue; + }; + if quote == '\'' || quote == '"' || quote == '`' { + return after[1..].split(quote).next().map(str::to_string); + } + } + } + None +} + +/// The module specifier: first quoted string after `from` when present +/// (`import … from 'x'`), else the first quoted string on the line +/// (side-effect `import 'x'`). Scoping to `from` keeps a trailing +/// string literal (`log("hi")`) from shadowing the specifier, and the +/// earliest opening quote of any kind wins, so a later apostrophe in +/// a trailing expression can never shadow the specifier either. +fn quoted_tail(line: &str) -> Option { + // Both separators are 6 bytes; the max rfind wins either way. + let tail = [" from ", " from\t"] + .iter() + .filter_map(|sep| line.rfind(sep)) + .max() + .map(|at| &line[at + 6..]) + .unwrap_or(line); + let (pos, quote) = ['\'', '"', '`'] + .iter() + .filter_map(|quote| tail.find(*quote).map(|pos| (pos, *quote))) + .min()?; + // `pos` is the byte index of a one-byte quote, so `pos + 1` is a + // boundary. + tail[pos + 1..] + .split(quote) + .next() + .filter(|spec| !spec.is_empty()) + .map(str::to_string) +} + +/// JS/TS extension search order for relative links. +const JS_LINK_EXTENSIONS: [&str; 8] = ["js", "ts", "jsx", "tsx", "mjs", "cjs", "mts", "cts"]; + +/// Resolve relative JS/TS specifiers (`./x`, `../x`) against the file's +/// dir: bare, +extensions, +`/index`+extensions. Bare package names +/// never resolve (node_modules is not walked). Query/hash suffixes cut. +fn resolve_js_link(workspace: &Path, file_path: &Path, line: &str) -> Option { + let mut spec = js_import_spec(line)?; + if !spec.starts_with('.') { + return None; + } + spec = spec.split(['?', '#']).next()?.to_string(); + let dir = file_path.parent()?; + let target = dir.join(&spec); + let mut candidates = vec![target.clone()]; + for ext in JS_LINK_EXTENSIONS { + candidates.push(target.with_extension(ext)); + } + for ext in JS_LINK_EXTENSIONS { + candidates.push(target.join(format!("index.{ext}"))); + } + for candidate in candidates { + if candidate.is_file() { + return workspace_relative(workspace, &candidate).map(|rel| dot_clean(&rel)); + } + } + None +} + +/// Lexically collapse `a/../b` (no FS touch — `canonicalize` would +/// resolve symlinks like `/tmp` and break the workspace prefix). +fn dot_clean(rel: &str) -> String { + let mut parts: Vec<&str> = Vec::new(); + for part in rel.split('/') { + if part == ".." { + parts.pop(); + } else if !part.is_empty() && part != "." { + parts.push(part); + } + } + parts.join("/") +} + +/// Go item lines: `func Name`, methods as `func Type.Method`, +/// `type Name …`. +fn parse_go_item(line: &str) -> Option { + let line = line.trim_start(); + if let Some(rest) = strip_keyword(line, "func") { + let rest = rest.trim_start(); + if let Some(after) = rest.strip_prefix('(') { + // Method: receiver `(name Type)` or `(Type)`. + let receiver = after.split(')').next()?.trim(); + let typ = receiver.split_whitespace().last()?; + let typ = typ.trim_start_matches('*').trim_start_matches("[]"); + let after = after.split(')').nth(1)?.trim_start(); + let name = ident_head(after)?; + if typ.is_empty() || !is_ident(typ) { + return None; + } + return Some(format!("func {typ}.{name}")); + } + let name = ident_head(rest)?; + return Some(format!("func {name}")); + } + if let Some(rest) = strip_keyword(line, "type") { + let rest = rest.trim_start(); + // Single decl only; `type ( … )` blocks list names bare — + // resolving those needs block tracking, documented omission. + if rest.starts_with('(') { + return None; + } + let name = ident_head(rest)?; + return Some(format!("type {name}")); + } + None +} + +/// Go import roots: last path segment (package name by convention). +/// Covers `import` lines (with optional alias) and the bare quoted +/// entries of `import ( … )` blocks. Anything else starting with an +/// identifier (`return "x"`) is not an import — the alias shape is +/// indistinguishable at line resolution, so keyword-less aliased +/// entries are a documented miss. +fn parse_go_import_root(line: &str) -> Option { + let line = line.trim_start(); + let rest = match strip_keyword(line, "import") { + Some(rest) => { + let rest = rest.trim_start(); + // Optional alias: `name "…"`, `_ "…"`, `. "…"`. + if rest.starts_with('"') || rest.starts_with('`') { + rest + } else { + rest.split_whitespace().nth(1)?.trim_start() + } + } + None => { + if line.starts_with('"') || line.starts_with('`') { + line + } else { + return None; + } + } + }; + let quote = rest.chars().next()?; + if quote != '"' && quote != '`' { + return None; + } + let path = rest[1..].split(quote).next()?; + let root = path.split('/').next_back()?.trim(); + valid_root(root) +} + +/// Reverse links: podmates whose own resolved links include this file. +/// The echo coming back. Exact by construction (reuses the link +/// resolver, no name guessing) and bounded (same-dir mates only, +/// oversized mates skipped, visibility-filtered like the pod itself). +fn sound_callers(workspace: &Path, file_path: &Path) -> (Vec, usize) { + let own = match workspace_relative(workspace, file_path) { + Some(own) => own, + None => return (Vec::new(), 0), + }; + let parent = match file_path.parent() { + Some(parent) => parent, + None => return (Vec::new(), 0), + }; + let mates = list_dir_entries(parent).unwrap_or_default(); + let mut callers = BTreeSet::new(); + for (raw, _) in mates { + let mate_path = parent.join(&raw); + let Some(mate_lang) = detect_lang(&mate_path) else { + continue; + }; + if mate_path.as_path() == file_path { + continue; + } + if std::fs::metadata(&mate_path).is_ok_and(|meta| meta.len() > MAX_CALLER_SCAN_BYTES) { + continue; + } + let Ok(text) = std::fs::read_to_string(&mate_path) else { + continue; + }; + let (links, _) = sound_links(workspace, &mate_path, &text, mate_lang); + if links.iter().any(|link| link == &own) + && let Some(rel) = workspace_relative(workspace, &mate_path) + { + callers.insert(rel); + } + } + let total = callers.len(); + (callers.into_iter().collect(), total) +} + +fn truncate_name(head: &str) -> String { + if head.chars().count() <= MAX_NAME_LEN { + return head.to_string(); + } + let kept: String = head.chars().take(MAX_NAME_LEN).collect(); + format!("{kept}…") +} + +fn count_noun(count: usize, singular: &str, plural: &str) -> String { + if count == 1 { + format!("1 {singular}") + } else { + format!("{count} {plural}") + } +} + +fn render_capped(items: &[String], total: usize) -> String { + let mut out = items.join(", "); + if total > items.len() { + if !out.is_empty() { + out.push_str(", "); + } + out.push_str(&format!("+{} more", total - items.len())); + } + out +} + +/// Hard byte cap at a char boundary, with an ellipsis marker when cut. +fn truncate_bytes(text: &str, max_bytes: usize) -> String { + if text.len() <= max_bytes { + return text.to_string(); + } + let mut end = max_bytes.saturating_sub("…".len()); + while !text.is_char_boundary(end) { + end = end.saturating_sub(1); + } + format!("{}…", &text[..end]) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use std::time::{Duration, Instant}; + + fn open_visibility(glob_root: &Path) -> PodVisibility<'_> { + PodVisibility { + include: &[], + exclude: &[], + extensions: &[], + glob_root, + gitignore: false, + } + } + + #[test] + fn miss_echo_names_parent_mates_for_typoed_paths() { + let dir = tempfile::tempdir().expect("tempdir"); + for name in ["alpha.rs", "beta.rs", "gamma.rs"] { + fs::write(dir.path().join(name), "x").expect("fixture"); + } + let echo = sound_miss_echo(&dir.path().join("alpah.rs")).expect("echo"); + assert!(echo.starts_with("\n\n[Miss echo: "), "{echo}"); + assert!(echo.contains("alpha.rs"), "{echo}"); + assert!(echo.contains("beta.rs"), "{echo}"); + assert!(echo.contains("gamma.rs"), "{echo}"); + assert!(!echo.contains("alpah.rs"), "{echo}"); + } + + #[test] + fn miss_echo_caps_mates_and_reports_the_rest() { + let dir = tempfile::tempdir().expect("tempdir"); + for i in 0..12 { + fs::write(dir.path().join(format!("f{i:02}.rs")), "x").expect("fixture"); + } + let echo = sound_miss_echo(&dir.path().join("nope.rs")).expect("echo"); + assert!(echo.contains("+4 more"), "{echo}"); + assert!(echo.len() < 600, "miss echo stays small: {echo}"); + } + + #[test] + fn miss_echo_is_none_when_parent_is_unlistable() { + let dir = tempfile::tempdir().expect("tempdir"); + let missing = dir.path().join("no-such-dir").join("nope.rs"); + assert_eq!(sound_miss_echo(&missing), None); + } + + const SAMPLE: &str = r#"use std::collections::BTreeSet; +use std::path::Path; +use crate::tools::echolocation::sound_echolocation; +use super::MAX_SYMBOLS; +pub use serde_json::Value as JsonValue; +use tokio::fs::{ + self, + File, +}; + +// a comment mentioning fn ghost +/// doc comment: struct Phantom +/* block comment: enum Shade */ +#[derive(Debug)] +pub struct Whale { + name: String, +} + +pub(crate) async fn swim() {} +unsafe extern "C" fn breach() {} +const DEPTH: u32 = 11_000; +static CURRENT: &str = "cold"; +type Pod = Vec; +trait Navigable {} +impl Navigable for Whale {} +impl Whale { + fn dive(&self) {} +} +mod abyss; +macro_rules! song { + () => {}; +} +fnx not_an_item; +define nothing; +"#; + + #[test] + fn items_sound_in_file_order() { + let symbols = sound_symbols(SAMPLE, SoundLang::Rust); + let names: Vec<&str> = symbols.iter().map(String::as_str).collect(); + assert_eq!( + names, + [ + "struct Whale:15", + "fn swim:19", + "fn breach:20", + "const DEPTH:21", + "static CURRENT:22", + "type Pod:23", + "trait Navigable:24", + "impl Navigable for Whale:25", + "impl Whale:26", + "fn dive:27", + "mod abyss:29", + "macro song:30", + ] + ); + } + + #[test] + fn import_roots_dedupe_and_sort() { + let roots: Vec = sound_import_roots(SAMPLE, SoundLang::Rust) + .into_iter() + .collect(); + assert_eq!(roots, ["crate", "serde_json", "std", "super", "tokio"]); + } + + #[test] + fn block_comments_stay_silent_across_lines() { + let text = "/*\nfn hidden_one() {}\nuse hidden::root;\n*/\nfn visible() {}\n"; + assert_eq!(sound_symbols(text, SoundLang::Rust), ["fn visible:5"]); + assert!(sound_import_roots(text, SoundLang::Rust).is_empty()); + } + + #[test] + fn code_after_a_closed_comment_still_sounds() { + assert_eq!( + sound_symbols("/* note */ fn after() {}\n", SoundLang::Rust), + ["fn after:1"] + ); + assert_eq!( + sound_symbols("let x = 1; /* fn nope() {} */\n", SoundLang::Rust), + Vec::::new() + ); + } + + #[test] + fn podmates_sort_mark_and_cap() { + let dir = tempfile::tempdir().expect("tempdir"); + for name in ["zebra.rs", "alpha.rs", "main.rs", ".hidden", "beta"] { + fs::write(dir.path().join(name), "x").expect("fixture"); + } + fs::create_dir(dir.path().join("shoal")).expect("subdir"); + let chart = sound_echolocation( + dir.path(), + &dir.path().join("main.rs"), + "fn main() {}\n", + false, + ); + assert_eq!(chart.podmate_total, 5); + assert_eq!( + chart.podmates, + [ + "alpha.rs", + "beta", + "main.rs (this file)", + "shoal/", + "zebra.rs", + ] + ); + } + + #[test] + fn long_pods_truncate_with_a_count() { + let dir = tempfile::tempdir().expect("tempdir"); + for index in 0..20 { + fs::write(dir.path().join(format!("file-{index:02}.rs")), "x").expect("fixture"); + } + let chart = sound_echolocation(dir.path(), &dir.path().join("file-00.rs"), "x\n", false); + assert_eq!(chart.podmate_total, 20); + assert_eq!(chart.podmates.len(), MAX_PODMATES); + let footer = chart.render_footer(); + assert!(footer.contains("+8 more"), "{footer}"); + assert!(footer.len() <= POD_FOOTER_MAX_BYTES); + } + + #[test] + fn footer_is_deterministic_and_budgeted() { + let dir = tempfile::tempdir().expect("tempdir"); + let long = "n".repeat(200); + let mut text = String::new(); + for index in 0..100 { + text.push_str(&format!("fn {long}_{index}() {{}}\n")); + } + fs::write(dir.path().join("big.rs"), &text).expect("fixture"); + let path = dir.path().join("big.rs"); + let first = sound_echolocation(dir.path(), &path, &text, false).render_footer(); + let second = sound_echolocation(dir.path(), &path, &text, false).render_footer(); + assert_eq!(first, second); + assert!(first.len() <= POD_FOOTER_MAX_BYTES, "{}", first.len()); + assert!(first.starts_with("\n\n[Pod:"), "{first}"); + } + + #[test] + fn prose_files_get_podmates_but_no_sounding() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("notes.txt"), "fn not_rust() {}\n").expect("fixture"); + let footer = sound_echolocation( + dir.path(), + &dir.path().join("notes.txt"), + "fn not_rust() {}\n", + false, + ) + .render_footer(); + assert!(footer.contains("[Pod:"), "{footer}"); + assert!(!footer.contains("[Sounding:"), "{footer}"); + } + + #[test] + fn unlistable_non_source_renders_empty() { + let chart = sound_echolocation(Path::new("."), Path::new("lonely"), "hello\n", false); + assert_eq!(chart.render_footer(), ""); + } + + #[test] + fn whole_footer_shape() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("lib.rs"), SAMPLE).expect("fixture"); + fs::write(dir.path().join("main.rs"), "fn main() {}\n").expect("fixture"); + let footer = sound_echolocation(dir.path(), &dir.path().join("lib.rs"), SAMPLE, false) + .render_footer(); + assert!(footer.contains("12 symbols"), "{footer}"); + assert!(footer.contains("fn swim"), "{footer}"); + assert!( + footer.contains("imports: crate, serde_json, std, super, tokio"), + "{footer}" + ); + assert!(footer.contains("lib.rs (this file), main.rs"), "{footer}"); + } + + #[test] + fn match_pods_group_mark_and_sort() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::create_dir(dir.path().join("net")).expect("subdir"); + fs::create_dir(dir.path().join("store")).expect("subdir"); + for name in [ + "net/server.rs", + "net/client.rs", + "store/cache.rs", + "main.rs", + ] { + fs::write(dir.path().join(name), "x").expect("fixture"); + } + let vis = open_visibility(dir.path()); + let (pods, omitted) = sound_match_pods( + dir.path(), + &["store/cache.rs".to_string(), "net/server.rs".to_string()], + &vis, + ); + assert_eq!(omitted, 0); + assert_eq!(pods.len(), 2); + assert_eq!(pods[0].dir, "net"); + assert_eq!(pods[0].mates, ["client.rs", "server.rs (match)"]); + assert_eq!(pods[1].dir, "store"); + assert_eq!(pods[1].mates, ["cache.rs (match)"]); + } + + #[test] + fn match_pods_cap_dirs_and_mates() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut files = Vec::new(); + for index in 0..12 { + let sub = format!("pod-{index:02}"); + fs::create_dir(dir.path().join(&sub)).expect("subdir"); + for mate in 0..12 { + fs::write(dir.path().join(&sub).join(format!("m{mate}.rs")), "x").expect("fixture"); + } + files.push(format!("{sub}/m0.rs")); + } + let vis = open_visibility(dir.path()); + let (pods, omitted) = sound_match_pods(dir.path(), &files, &vis); + assert_eq!(pods.len(), MAX_GREP_PODS); + assert_eq!(omitted, 12 - MAX_GREP_PODS); + assert_eq!(pods[0].mate_total, 12); + assert!(pods[0].mates.last().expect("cap").starts_with("+4 more")); + } + + #[test] + fn match_pods_chart_matched_files_when_dir_unlistable() { + let dir = tempfile::tempdir().expect("tempdir"); + let vis = open_visibility(dir.path()); + let (pods, omitted) = sound_match_pods( + dir.path(), + &["gone/deep.rs".to_string(), "".to_string()], + &vis, + ); + assert_eq!(omitted, 0); + assert_eq!(pods.len(), 1); + assert_eq!(pods[0].dir, "gone"); + assert_eq!(pods[0].mates, ["deep.rs (match)"]); + } + + #[test] + fn match_pods_honor_excludes_and_includes() { + let dir = tempfile::tempdir().expect("tempdir"); + for name in ["keep.rs", "skip.log", "notes.md"] { + fs::write(dir.path().join(name), "x").expect("fixture"); + } + let open = open_visibility(dir.path()); + let exclude = ["*.log".to_string()]; + let vis = PodVisibility { + exclude: &exclude, + ..open + }; + let (pods, _) = sound_match_pods(dir.path(), &["keep.rs".to_string()], &vis); + assert_eq!(pods[0].mates, ["keep.rs (match)", "notes.md"]); + let include = ["*.rs".to_string()]; + let vis = PodVisibility { + include: &include, + ..open + }; + let (pods, _) = sound_match_pods(dir.path(), &["keep.rs".to_string()], &vis); + assert_eq!(pods[0].mates, ["keep.rs (match)"]); + } + + #[test] + fn match_pods_honor_gitignore_stacking() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join(".gitignore"), "ignored.txt\n").expect("fixture"); + for name in ["ignored.txt", "keep.txt"] { + fs::write(dir.path().join(name), "x").expect("fixture"); + } + let open = open_visibility(dir.path()); + let vis = PodVisibility { + gitignore: true, + ..open + }; + let (pods, _) = sound_match_pods(dir.path(), &["keep.txt".to_string()], &vis); + assert_eq!(pods[0].mates, ["keep.txt (match)"]); + } + + #[test] + fn match_pods_hide_denied_siblings() { + let dir = tempfile::tempdir().expect("tempdir"); + let secret = dir.path().join("secret.txt"); + for name in ["open.txt", "secret.txt"] { + fs::write(dir.path().join(name), "x").expect("fixture"); + } + let prior = crate::sandbox::read_guard::active(); + crate::sandbox::read_guard::set_active(crate::sandbox::read_guard::ReadDenylist::build( + true, + &[secret], + &[], + )); + let vis = open_visibility(dir.path()); + let (pods, _) = sound_match_pods(dir.path(), &["open.txt".to_string()], &vis); + crate::sandbox::read_guard::set_active((*prior).clone()); + assert_eq!(pods[0].mates, ["open.txt (match)"]); + } + + #[test] + fn key_files_sound_rust_and_skip_the_rest() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("main.rs"), "fn main() {}\nfn serve() {}\n").expect("fixture"); + fs::write(dir.path().join("notes.md"), "# hi\n").expect("fixture"); + let (soundings, omitted) = + sound_key_files(dir.path(), &["notes.md".to_string(), "main.rs".to_string()]); + assert_eq!(omitted, 0); + assert_eq!(soundings.len(), 1); + assert_eq!( + soundings.get("main.rs").map(String::as_str), + Some("2 symbols: fn main:1, fn serve:2") + ); + } + + #[test] + fn links_resolve_crate_super_self_and_mod_edges() { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path(); + for path in [ + "src/main.rs", + "src/config.rs", + "src/net/mod.rs", + "src/net/server.rs", + "src/net/http.rs", + ] { + let full = root.join(path); + fs::create_dir_all(full.parent().expect("parent")).expect("mkdir"); + fs::write(&full, "x\n").expect("fixture"); + } + // crate:: + super:: + external (skipped) + missing (skipped). + let text = "use crate::config::Config;\n\ + use crate::net::http::Handler;\n\ + use super::server::serve;\n\ + use tokio::fs::File;\n\ + use crate::missing::Thing;\n"; + let (links, total) = + sound_links(root, &root.join("src/net/http.rs"), text, SoundLang::Rust); + // crate::net::http resolves to the file itself: dropped as a self-link. + assert_eq!(total, 2); + assert_eq!(links, ["src/config.rs", "src/net/server.rs"]); + // self:: + mod decl from a mod.rs. + let (links, _) = sound_links( + root, + &root.join("src/net/mod.rs"), + "use self::http::Handler;\nmod server;\nmod absent;\n", + SoundLang::Rust, + ); + assert_eq!(links, ["src/net/http.rs", "src/net/server.rs"]); + // mod decl from a plain file resolves under /. + fs::create_dir_all(root.join("src/net/http")).expect("mkdir"); + fs::write(root.join("src/net/http/router.rs"), "x\n").expect("fixture"); + let (links, _) = sound_links( + root, + &root.join("src/net/http.rs"), + "mod router;\nmod nope;\n", + SoundLang::Rust, + ); + assert_eq!(links, ["src/net/http/router.rs"]); + } + + #[test] + fn links_skip_self_and_inline_mods() { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path(); + fs::create_dir_all(root.join("src")).expect("mkdir"); + fs::write(root.join("src/main.rs"), "fn main() {}\n").expect("fixture"); + fs::write(root.join("src/config.rs"), "pub struct Config;\n").expect("fixture"); + // `use crate::config` from config.rs itself: self-link dropped. + // Inline `mod tests {` is a symbol, never an edge. + let (links, total) = sound_links( + root, + &root.join("src/config.rs"), + "use crate::config::Config;\nmod tests {\n}\n", + SoundLang::Rust, + ); + assert_eq!(total, 0); + assert!(links.is_empty()); + } + + #[test] + fn links_render_in_footer() { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path(); + fs::create_dir_all(root.join("src")).expect("mkdir"); + fs::write(root.join("src/main.rs"), "fn main() {}\n").expect("fixture"); + let footer = sound_echolocation( + root, + &root.join("src/main.rs"), + "use crate::config::Config;\nfn main() {}\n", + false, + ) + .render_footer(); + // config.rs does not exist: imports show, links stay silent. + assert!(footer.contains("imports: crate"), "{footer}"); + assert!(!footer.contains("links:"), "{footer}"); + fs::write(root.join("src/config.rs"), "pub struct Config;\n").expect("fixture"); + let footer = sound_echolocation( + root, + &root.join("src/main.rs"), + "use crate::config::Config;\nfn main() {}\n", + false, + ) + .render_footer(); + assert!(footer.contains("links: src/config.rs"), "{footer}"); + } + + #[test] + fn key_files_cap_with_omitted_count() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut keys = Vec::new(); + for index in 0..12 { + let name = format!("k{index:02}.rs"); + fs::write(dir.path().join(&name), "fn f() {}\n").expect("fixture"); + keys.push(name); + } + let (soundings, omitted) = sound_key_files(dir.path(), &keys); + assert_eq!(soundings.len(), MAX_MAP_SOUNDINGS); + assert_eq!(omitted, 12 - MAX_MAP_SOUNDINGS); + let again = sound_key_files(dir.path(), &keys); + assert_eq!(again, (soundings, omitted)); + } + + #[test] + fn callers_are_exact_reverse_links() { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path(); + fs::create_dir_all(root.join("src/net")).expect("mkdir"); + fs::write(root.join("src/main.rs"), "fn main() {}\n").expect("fixture"); + fs::write( + root.join("src/net/mod.rs"), + "pub mod server;\npub mod client;\n", + ) + .expect("fixture"); + fs::write( + root.join("src/net/server.rs"), + "use crate::config::Config;\nfn serve() {}\n", + ) + .expect("fixture"); + fs::write( + root.join("src/net/client.rs"), + "use super::server::serve;\nfn connect() {}\n", + ) + .expect("fixture"); + let (callers, total) = sound_callers(root, &root.join("src/net/server.rs")); + assert_eq!(total, 2); + assert_eq!(callers, ["src/net/client.rs", "src/net/mod.rs"]); + } + + #[test] + fn callers_stay_inside_the_pod() { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path(); + fs::create_dir_all(root.join("src/net")).expect("mkdir"); + fs::write(root.join("src/main.rs"), "fn main() {}\n").expect("fixture"); + fs::write(root.join("src/config.rs"), "pub struct Config;\n").expect("fixture"); + fs::write( + root.join("src/net/server.rs"), + "use crate::config::Config;\nfn serve() {}\n", + ) + .expect("fixture"); + // server.rs links to config.rs, but lives in another pod: + // same-pod callers stay silent rather than half-answer. + let (callers, total) = sound_callers(root, &root.join("src/config.rs")); + assert_eq!(total, 0); + assert!(callers.is_empty()); + } + + #[test] + fn buzz_adds_heard_by_and_stays_deterministic() { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path(); + fs::create_dir_all(root.join("src/net")).expect("mkdir"); + fs::write(root.join("src/main.rs"), "fn main() {}\n").expect("fixture"); + fs::write(root.join("src/net/mod.rs"), "pub mod server;\n").expect("fixture"); + let text = "fn serve() {}\n"; + fs::write(root.join("src/net/server.rs"), text).expect("fixture"); + let plain = + sound_echolocation(root, &root.join("src/net/server.rs"), text, false).render_footer(); + assert!(!plain.contains("heard by"), "{plain}"); + let first = + sound_echolocation(root, &root.join("src/net/server.rs"), text, true).render_footer(); + let second = + sound_echolocation(root, &root.join("src/net/server.rs"), text, true).render_footer(); + assert_eq!(first, second); + assert!(first.contains("heard by: src/net/mod.rs"), "{first}"); + } + + #[test] + fn echo_gates_parse_off_entries_and_ignore_noise() { + assert!(parse_echo_gates("").is_empty()); + assert!(parse_echo_gates("mates").is_empty()); + let gates = parse_echo_gates("off:links, off:mates ,bogus,off:,off:links"); + assert_eq!( + gates.into_iter().collect::>(), + ["links".to_string(), "mates".to_string()] + ); + } + + const PY_SAMPLE: &str = r#"import os, sys as system +from pkg.sub import thing +from . import sibling +from ..parent import helper +import localmod + +"""Module docstring mentioning def ghost.""" + +class Whale: + """Docstring with class Phantom inside.""" + + def swim(self): + pass + + async def dive(self): + pass + +def surface(): + # def sunk: + return 1 + +define_not_a_keyword = True +"#; + + #[test] + fn python_sounds_defs_classes_and_imports() { + let symbols = sound_symbols(PY_SAMPLE, SoundLang::Python); + assert_eq!( + symbols, + [ + "class Whale:9", + "def swim:12", + "def dive:15", + "def surface:18", + ] + ); + let roots: Vec = sound_import_roots(PY_SAMPLE, SoundLang::Python) + .into_iter() + .collect(); + assert_eq!(roots, [".", "localmod", "os", "parent", "pkg", "sys"]); + } + + #[test] + fn python_links_resolve_relative_and_absolute() { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path(); + for path in [ + "app.py", + "util.py", + "pkg/__init__.py", + "pkg/mod.py", + "pkg/sub/__init__.py", + "pkg/sub/deep.py", + ] { + let full = root.join(path); + fs::create_dir_all(full.parent().expect("parent")).expect("mkdir"); + fs::write(&full, "x\n").expect("fixture"); + } + // Absolute: same dir wins; package shape at root. + let (links, _) = sound_links( + root, + &root.join("app.py"), + "import util\nfrom pkg.sub import thing\nimport os\n", + SoundLang::Python, + ); + assert_eq!(links, ["pkg/sub/__init__.py", "util.py"]); + // Relative: `from . import sib` + `from .. import x`. + let (links, _) = sound_links( + root, + &root.join("pkg/sub/deep.py"), + "from . import sib\nfrom .. import mod\nfrom ... import nowhere\n", + SoundLang::Python, + ); + assert_eq!(links, ["pkg/mod.py"]); + } + + const JS_SAMPLE: &str = r#"import React from 'react'; +import { serve } from "./server"; +import '@scope/pkg/sub'; +import './setup'; +const fs = require("fs"); +const dynamic = await import("./chunk"); + +export async function swim() {} +export default class Whale {} +const dive = (x) => x; +let data = 5; +type Alias = string; +interface Shape {} +enum Color { Red } + +if (x) { y(); } +"#; + + #[test] + fn js_sounds_functions_classes_and_imports() { + let symbols = sound_symbols(JS_SAMPLE, SoundLang::JavaScript); + assert!( + symbols.contains(&"function swim:8".to_string()), + "{symbols:?}" + ); + assert!( + symbols.contains(&"class Whale:9".to_string()), + "{symbols:?}" + ); + assert!( + symbols.contains(&"function dive:10".to_string()), + "{symbols:?}" + ); + assert!( + symbols.contains(&"type Alias:12".to_string()), + "{symbols:?}" + ); + assert!( + symbols.contains(&"interface Shape:13".to_string()), + "{symbols:?}" + ); + assert!( + symbols.contains(&"enum Color:14".to_string()), + "{symbols:?}" + ); + assert!(!symbols.iter().any(|s| s.contains("data")), "{symbols:?}"); + let roots: Vec = sound_import_roots(JS_SAMPLE, SoundLang::JavaScript) + .into_iter() + .collect(); + assert_eq!(roots, [".", "@scope/pkg", "fs", "react"]); + } + + #[test] + fn js_links_resolve_relative_only() { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path(); + fs::create_dir_all(root.join("src/ui")).expect("mkdir"); + for path in ["src/app.ts", "src/server.ts", "src/ui/index.tsx"] { + fs::write(root.join(path), "x\n").expect("fixture"); + } + let (links, _) = sound_links( + root, + &root.join("src/app.ts"), + "import { s } from './server';\nimport { U } from './ui';\nimport _ from 'lodash';\n", + SoundLang::JavaScript, + ); + assert_eq!(links, ["src/server.ts", "src/ui/index.tsx"]); + } + + const GO_SAMPLE: &str = r#"package net + +import ( + "fmt" +) + +import "os" +import alias "example.com/foo/bar" +import _ "example.com/foo/side" + +func Serve() {} +func (s *Server) Handle() {} +func (s Server) Close() {} +type Server struct{} +type ( + Grouped int +) + +func main() { + fmt.Println("hi") + return "literal" +} +"#; + + #[test] + fn go_sounds_funcs_methods_types_and_imports() { + let symbols = sound_symbols(GO_SAMPLE, SoundLang::Go); + assert!( + symbols.contains(&"func Serve:11".to_string()), + "{symbols:?}" + ); + assert!( + symbols.contains(&"func Server.Handle:12".to_string()), + "{symbols:?}" + ); + assert!( + symbols.contains(&"func Server.Close:13".to_string()), + "{symbols:?}" + ); + assert!( + symbols.contains(&"type Server:14".to_string()), + "{symbols:?}" + ); + assert!( + !symbols.iter().any(|s| s.contains("Grouped")), + "{symbols:?}" + ); + let roots: Vec = sound_import_roots(GO_SAMPLE, SoundLang::Go) + .into_iter() + .collect(); + assert_eq!(roots, ["bar", "fmt", "os", "side"]); + // Go names packages, not files: no links by design. + let dir = tempfile::tempdir().expect("tempdir"); + let (links, total) = sound_links( + dir.path(), + &dir.path().join("x.go"), + GO_SAMPLE, + SoundLang::Go, + ); + assert_eq!(total, 0); + assert!(links.is_empty()); + } + + #[test] + fn detect_lang_routes_by_extension() { + assert_eq!(detect_lang(Path::new("a.rs")), Some(SoundLang::Rust)); + assert_eq!(detect_lang(Path::new("a.py")), Some(SoundLang::Python)); + assert_eq!(detect_lang(Path::new("a.pyi")), Some(SoundLang::Python)); + assert_eq!(detect_lang(Path::new("a.ts")), Some(SoundLang::JavaScript)); + assert_eq!(detect_lang(Path::new("a.tsx")), Some(SoundLang::JavaScript)); + assert_eq!(detect_lang(Path::new("a.jsx")), Some(SoundLang::JavaScript)); + assert_eq!(detect_lang(Path::new("a.mjs")), Some(SoundLang::JavaScript)); + assert_eq!(detect_lang(Path::new("a.go")), Some(SoundLang::Go)); + assert_eq!(detect_lang(Path::new("a.txt")), None); + assert_eq!(detect_lang(Path::new("Makefile")), None); + } + + #[test] + fn scanners_survive_adversarial_text() { + let long_line = "x".repeat(1_000_000); + let nasties = [ + "\0\0\0", + "fn \u{feff}\u{200b}\u{202e}evil() {}", + "/* unterminated", + "\"\"\"unterminated doc", + "def f(:\n pass", + "use \u{1f980}::x;", + "import \u{1f980}", + "func (((((", + "/* /* /* nested", + "'''\"\"\"'\"\" mixed", + "#\u{0}def fake():", + "\r\ndef crlf():\r\n", + "type (", + "export default", + "from", + "import", + long_line.as_str(), + ]; + for lang in [ + SoundLang::Rust, + SoundLang::Python, + SoundLang::JavaScript, + SoundLang::Go, + ] { + for nasty in nasties { + let first = sound_symbols(nasty, lang); + let second = sound_symbols(nasty, lang); + assert_eq!(first, second, "{lang:?} {nasty:?}"); + let _ = sound_import_roots(nasty, lang); + } + } + } + + #[test] + fn sounding_stays_bounded_and_fast_on_huge_files() { + let dir = tempfile::tempdir().expect("tempdir"); + let big = "fn f() {}\n".repeat(200_000); + let path = dir.path().join("big.rs"); + fs::write(&path, &big).expect("fixture"); + let start = Instant::now(); + let first = sound_echolocation(dir.path(), &path, &big, true).render_footer(); + let second = sound_echolocation(dir.path(), &path, &big, true).render_footer(); + assert_eq!(first, second); + assert!(first.len() <= POD_FOOTER_MAX_BYTES, "{}", first.len()); + // Absurdly generous: catches hangs and super-linear blowups, + // not machine speed (measured separately for the thesis). + assert!( + start.elapsed() < Duration::from_secs(10), + "{:?}", + start.elapsed() + ); + } + + #[test] + fn sounding_is_stable_on_real_repo_files() { + // Byte-stability on the real thing (not just fixtures), with + // wall-time reported for cost analysis (`--nocapture`). + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let mut sounded = 0; + for rel in [ + "src/core/engine/turn_loop.rs", + "src/tools/spec.rs", + "src/tools/echolocation.rs", + ] { + let path = root.join(rel); + let text = fs::read_to_string(&path).expect("repo file"); + let first = sound_echolocation(root, &path, &text, true).render_footer(); + let second = sound_echolocation(root, &path, &text, true).render_footer(); + assert_eq!(first, second, "{rel}"); + assert!(first.len() <= POD_FOOTER_MAX_BYTES, "{rel}"); + sounded += 1; + } + assert_eq!(sounded, 3); + } + + #[test] + fn non_rust_footer_renders_full_sounding() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("swim.py"), PY_SAMPLE).expect("fixture"); + fs::write(dir.path().join("other.py"), "x = 1\n").expect("fixture"); + let footer = sound_echolocation(dir.path(), &dir.path().join("swim.py"), PY_SAMPLE, false) + .render_footer(); + assert!(footer.contains("[Pod:"), "{footer}"); + assert!(footer.contains("def swim"), "{footer}"); + assert!(footer.contains("imports:"), "{footer}"); + } + + #[test] + fn crate_root_self_and_mod_edges_resolve_beside_the_root() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("lib.rs"), + "mod pod;\nuse self::song::Sing;\n", + ) + .expect("fixture"); + fs::write(dir.path().join("pod.rs"), "fn swim() {}\n").expect("fixture"); + fs::create_dir(dir.path().join("song")).expect("mkdir"); + fs::write(dir.path().join("song/mod.rs"), "fn sing() {}\n").expect("fixture"); + let text = fs::read_to_string(dir.path().join("lib.rs")).expect("read"); + let (links, _) = sound_links( + dir.path(), + &dir.path().join("lib.rs"), + &text, + SoundLang::Rust, + ); + assert!(links.contains(&"pod.rs".to_string()), "{links:?}"); + assert!(links.contains(&"song/mod.rs".to_string()), "{links:?}"); + } + + #[test] + fn earliest_quote_wins_the_specifier() { + assert_eq!( + js_import_spec("import x from \"mod\"; foo('bar')"), + Some("mod".to_string()) + ); + assert_eq!( + js_import_spec("import x from 'mod'; foo(\"bar\")"), + Some("mod".to_string()) + ); + assert_eq!( + js_import_spec("import 'side-effect'"), + Some("side-effect".to_string()) + ); + } + + #[test] + fn require_opener_needs_a_boundary() { + // `myrequire(` forges nothing … + assert_eq!(js_import_spec("const x = myrequire('a')"), None); + // … nor does a call spelled inside a string … + assert_eq!(js_import_spec("const s = \"require('a')\""), None); + // … but a real call later on the same line still sounds … + assert_eq!( + js_import_spec("myrequire('a'); const x = require('b')"), + Some("b".to_string()) + ); + // … as does a computed argument followed by a real one. + assert_eq!( + js_import_spec("require(name); require('c')"), + Some("c".to_string()) + ); + } + + #[test] + fn digit_led_tokens_are_never_names() { + assert!(sound_symbols("fn 1x() {}\nmod 2y;\n", SoundLang::Rust).is_empty()); + assert!(sound_symbols("def 2y():\n", SoundLang::Python).is_empty()); + assert!(sound_import_roots("use 123;\n", SoundLang::Rust).is_empty()); + assert_eq!( + sound_symbols("fn fine() {}\n", SoundLang::Rust), + ["fn fine:1"] + ); + } + + #[test] + fn minified_runs_never_blow_up_a_symbol() { + let run = "a".repeat(10_000); + let symbols = sound_symbols(&format!("fn {run}() {{}}\n"), SoundLang::Rust); + assert_eq!(symbols.len(), 1); + assert!(symbols[0].len() <= MAX_NAME_LEN + 8, "{}", symbols[0].len()); + let js = sound_symbols(&format!("function {run}() {{}}\n"), SoundLang::JavaScript); + assert!(js.is_empty(), "{js:?}"); + } + + #[test] + fn buzz_read_of_a_crowded_pod_stays_fast() { + // Worst-case shape: 200 mates × 50KB each, all re-scanned for + // callers on every buzz read. The 5s bound is orders past the + // expected tens of milliseconds — it trips only on pathological + // regression, never on a loaded machine. + let dir = tempfile::tempdir().expect("tempdir"); + let body = "fn swim() {}\n".repeat(4_000); + for i in 0..200 { + fs::write(dir.path().join(format!("mate{i:03}.rs")), &body).expect("fixture"); + } + let target = dir.path().join("target.rs"); + fs::write(&target, "fn target() {}\n").expect("fixture"); + let text = fs::read_to_string(&target).expect("read"); + let start = Instant::now(); + let chart = sound_echolocation(dir.path(), &target, &text, true); + let footer = chart.render_footer(); + let elapsed = start.elapsed(); + assert!(footer.len() <= POD_FOOTER_MAX_BYTES); + assert!( + elapsed < Duration::from_secs(5), + "buzz read took {elapsed:?}" + ); + } + + #[test] + #[cfg(unix)] + fn pod_listing_skips_symlinks_into_denied_trees() { + // The denylist filter is distinct from the dotfile skip: a + // non-dot symlink into a denied tree must not enumerate. Needs a + // real ~/.ssh to resolve against; honest skip otherwise (the + // read-level twin is + // `read_file_refuses_a_workspace_symlink_pointing_at_a_denied_tree`). + let _env_lock = crate::test_support::lock_test_env(); + let Some(home) = dirs::home_dir() else { + return; + }; + if !home.join(".ssh").is_dir() { + return; + } + let dir = tempfile::tempdir().expect("tempdir"); + std::os::unix::fs::symlink(home.join(".ssh"), dir.path().join("vault_link")) + .expect("symlink"); + fs::write(dir.path().join("app.rs"), "fn main() {}\n").expect("fixture"); + let entries = list_dir_entries(dir.path()).expect("listed"); + let raws: Vec<&str> = entries.iter().map(|(raw, _)| raw.as_str()).collect(); + assert!(raws.contains(&"app.rs"), "{raws:?}"); + assert!(!raws.contains(&"vault_link"), "{raws:?}"); + } + + #[test] + fn unlistable_pod_fallback_skips_dotfiles_like_the_listed_path() { + let dir = tempfile::tempdir().expect("tempdir"); + let (pods, _) = sound_match_pods( + dir.path(), + &["nodir/.env".to_string(), "nodir/app.rs".to_string()], + &open_visibility(dir.path()), + ); + assert_eq!(pods.len(), 1); + assert!( + pods[0].mates.contains(&"app.rs (match)".to_string()), + "{:?}", + pods[0].mates + ); + assert!( + !pods[0].mates.iter().any(|mate| mate.contains(".env")), + "{:?}", + pods[0].mates + ); + } +} diff --git a/crates/tui/src/tools/file.rs b/crates/tui/src/tools/file.rs index f3c85e4e27..c98f644152 100644 --- a/crates/tui/src/tools/file.rs +++ b/crates/tui/src/tools/file.rs @@ -964,10 +964,16 @@ impl ReadFileTool { let bytes = load_contract_source(&file_path, false, context) .await? .ok_or_else(|| { - ToolError::execution_failed(format!( - "Failed to read {}: file not found", - file_path.display() - )) + // Miss echo: only for genuine not-found — never for denylist + // or permission failures, where listing the parent would + // answer a refused probe. `sound_miss_echo` carries those + // gates internally, so appending it here stays fail-closed. + let mut message = + format!("Failed to read {}: file not found", file_path.display()); + if let Some(echo) = crate::tools::echolocation::sound_miss_echo(&file_path) { + message.push_str(&echo); + } + ToolError::execution_failed(message) })?; // #6283: every read response carries the file's byte size, line // count, and truncation flag so the caller can page deliberately @@ -1075,6 +1081,24 @@ impl ReadFileTool { } } + // Echolocation: stateless kin footer sounded fresh from source + // (no index, no new deps). Deterministic and hard-budgeted, so a + // repeated read is byte-identical. Two distinct files read in + // this pod put the next read there into the terminal buzz + // (callers included). + let buzz = file_path.parent().is_some_and(|parent| { + context.pod_visit_count(parent) >= crate::tools::echolocation::BUZZ_VISIT_THRESHOLD + }); + output.push_str( + &crate::tools::echolocation::sound_echolocation( + &context.workspace, + &file_path, + &text, + buzz, + ) + .render_footer(), + ); + // This internal observation keeps hidden legacy edit replay working, // but no hash or read-before-edit ceremony reaches the lowercase // schema or result. diff --git a/crates/tui/src/tools/file/tests.rs b/crates/tui/src/tools/file/tests.rs index 946ba77e68..6faf9266aa 100644 --- a/crates/tui/src/tools/file/tests.rs +++ b/crates/tui/src/tools/file/tests.rs @@ -78,7 +78,10 @@ async fn contract_read_returns_a_file_of_more_than_two_thousand_short_lines_whol let result = ReadFileTool::execute_contract_read(json!({"path": "many.txt"}), &context) .await .expect("read result"); - assert_eq!(result.content, content); + assert!( + result.content.starts_with(&content), + "whole file first, pod footer after" + ); assert!( !result.content.contains("[Showing lines"), "no truncation footer" @@ -86,7 +89,7 @@ async fn contract_read_returns_a_file_of_more_than_two_thousand_short_lines_whol } #[tokio::test] -async fn contract_read_returns_an_ordinary_source_file_whole_without_a_footer() { +async fn contract_read_appends_a_pod_footer_to_a_whole_source_file() { let temporary = tempfile::tempdir().expect("tempdir"); let content = "fn main() {\n println!(\"hi\");\n}\n"; std::fs::write(temporary.path().join("main.rs"), content).expect("fixture"); @@ -94,7 +97,66 @@ async fn contract_read_returns_an_ordinary_source_file_whole_without_a_footer() let result = ReadFileTool::execute_contract_read(json!({"path": "main.rs"}), &context) .await .expect("read result"); - assert_eq!(result.content, content); + assert!( + result.content.starts_with(content), + "whole file first, pod footer after" + ); + assert!( + result.content.contains("[Pod:"), + "pod footer present: {}", + result.content + ); + assert!( + result.content.contains("[Sounding: 1 symbol: fn main:1]"), + "sounded symbol present: {}", + result.content + ); +} + +#[tokio::test] +async fn contract_read_miss_echoes_parent_mates_on_not_found() { + let temporary = tempfile::tempdir().expect("tempdir"); + std::fs::write(temporary.path().join("alpha.rs"), "fn a() {}\n").expect("fixture"); + std::fs::write(temporary.path().join("beta.rs"), "fn b() {}\n").expect("fixture"); + let context = ToolContext::new(temporary.path()); + let error = ReadFileTool::execute_contract_read(json!({"path": "alpah.rs"}), &context) + .await + .expect_err("typo'd path must fail"); + match error { + ToolError::ExecutionFailed { message } => { + assert!(message.contains("Failed to read"), "{message}"); + assert!(message.contains("[Miss echo: "), "{message}"); + assert!(message.contains("alpha.rs"), "{message}"); + assert!(message.contains("beta.rs"), "{message}"); + } + other => panic!("expected ordinary read failure, got {other:?}"), + } +} + +#[tokio::test] +async fn contract_read_refused_probe_gets_no_miss_echo() { + // Security gate: a denylisted path must fail WITHOUT the miss echo — + // listing the parent would answer the refused probe. The denylist + // check runs before the IO read, so the refusal wins whether or not + // the file exists. Mirrors the home-dir fixture pattern in + // file/tests/tools.rs (the guard snapshots HOME process-wide). + let _env_lock = crate::test_support::lock_test_env(); + let Some(home) = dirs::home_dir() else { + return; + }; + let temporary = tempfile::tempdir().expect("tempdir"); + let context = ToolContext::new(temporary.path()); + let probe = home.join(".ssh").join("id_ed25519_no_such_key"); + let error = + ReadFileTool::execute_contract_read(json!({"path": probe.to_string_lossy()}), &context) + .await + .expect_err("denylisted path must fail"); + let message = error.to_string(); + assert!(message.contains("deny-list"), "{message}"); + assert!( + !message.contains("[Miss echo: "), + "refused probe must not echo: {message}" + ); } #[test] @@ -173,7 +235,10 @@ async fn contract_read_max_bytes_raises_the_budget_for_one_call() { ) .await .expect("raised budget read"); - assert_eq!(raised.content, content); + assert!( + raised.content.starts_with(&content), + "whole file first, pod footer after" + ); // Above the hard maximum clamps down; the file still fits, so it is whole. let clamped = ReadFileTool::execute_contract_read( @@ -182,7 +247,10 @@ async fn contract_read_max_bytes_raises_the_budget_for_one_call() { ) .await .expect("clamped budget read"); - assert_eq!(clamped.content, content); + assert!( + clamped.content.starts_with(&content), + "whole file first, pod footer after" + ); } #[tokio::test] @@ -201,17 +269,14 @@ async fn contract_read_paginates_an_oversized_file_with_an_honest_budget_footer( let first = ReadFileTool::execute_contract_read(json!({"path": "big.txt"}), &context) .await .expect("first page"); - let footer = first - .content - .rsplit_once("\n\n") - .expect("footer present") - .1 - .to_string(); - assert_eq!( - footer, - "[Showing lines 1-100 of 2000 (1.9MB total, 100000-byte output budget). Use offset=101 to continue, or max_bytes up to 500000 to read more per call.]" + assert!( + first.content.contains( + "[Showing lines 1-100 of 2000 (1.9MB total, 100000-byte output budget). Use offset=101 to continue, or max_bytes up to 500000 to read more per call.]" + ), + "paging footer exact: {}", + first.content ); - let shown = first.content.rsplit_once("\n\n").expect("body").0; + let shown = first.content.split("\n\n[").next().expect("body"); assert_eq!(shown.lines().count(), 100); // The named continuation offset is exact: page two starts on line 101. @@ -307,8 +372,8 @@ async fn contract_read_pages_stay_bounded_and_cover_the_whole_file() { ); let body = result .content - .rsplit_once("\n\n[") - .map(|(body, _)| body) + .split("\n\n[") + .next() .unwrap_or(&result.content); seen.extend(body.lines().map(str::to_string)); let truncated = metadata["truncated"].as_bool().expect("truncated flag"); @@ -323,7 +388,7 @@ async fn contract_read_pages_stay_bounded_and_cover_the_whole_file() { } /// #6283: ordinary whole reads carry the same paging metadata (with -/// truncated=false) and keep their footer-free shape. +/// truncated=false); the pod footer follows the whole file. #[tokio::test] async fn contract_read_metadata_for_ordinary_whole_read() { let temporary = tempfile::tempdir().expect("tempdir"); @@ -334,7 +399,15 @@ async fn contract_read_metadata_for_ordinary_whole_read() { let result = ReadFileTool::execute_contract_read(json!({"path": "small.txt"}), &context) .await .expect("read result"); - assert_eq!(result.content, content); + assert!( + result.content.starts_with(content), + "whole file first, pod footer after" + ); + assert!( + result.content.contains("[Pod:"), + "pod footer present: {}", + result.content + ); let metadata = result.metadata.clone().expect("paging metadata"); assert_eq!(metadata["size"], content.len() as u64); assert_eq!(metadata["truncated"], false); @@ -399,9 +472,12 @@ async fn contract_read_reports_huge_first_line_with_exact_bash_fallback() { let result = ReadFileTool::execute_contract_read(json!({"path": "huge.txt"}), &context) .await .expect("read result"); - assert_eq!( - result.content, - "[Line 1 is 97.7KB, exceeds the 100000-byte output budget for this call. Use bash: sed -n '1p' huge.txt | head -c 100000]" + assert!( + result.content.starts_with( + "[Line 1 is 97.7KB, exceeds the 100000-byte output budget for this call. Use bash: sed -n '1p' huge.txt | head -c 100000]" + ), + "exact bash fallback first, pod footer after: {}", + result.content ); } @@ -417,9 +493,12 @@ async fn contract_read_offset_oob_and_limit_continuation_match_contract() { ) .await .expect("limited read"); - assert_eq!( - limited.content, - "two\n\n[1 more lines in file (13B total). Use offset=3 to continue.]" + assert!( + limited + .content + .starts_with("two\n\n[1 more lines in file (13B total). Use offset=3 to continue.]"), + "page plus paging footer first, pod footer after: {}", + limited.content ); let error = @@ -490,7 +569,11 @@ async fn contract_read_uses_magic_not_extension_for_images() { let text = ReadFileTool::execute_contract_read(json!({"path": "plain.png"}), &context) .await .expect("fake extension remains text"); - assert_eq!(text.content, "ordinary text"); + assert!( + text.content.starts_with("ordinary text"), + "text first, pod footer after: {}", + text.content + ); let image = ReadFileTool::execute_contract_read(json!({"path": "renamed.data"}), &context) .await .expect("real image uses typed transport"); @@ -1140,3 +1223,39 @@ async fn contract_read_admits_only_an_image_the_user_attached_from_outside_the_w .expect_err("an outside image the user never attached stays refused"); assert!(matches!(error, ToolError::PathEscape { .. }), "{error:?}"); } + +#[tokio::test] +async fn contract_read_enters_terminal_buzz_on_third_pod_visit() { + let temporary = tempfile::tempdir().expect("tempdir"); + let root = temporary.path(); + std::fs::create_dir_all(root.join("src/net")).expect("mkdir"); + std::fs::write(root.join("src/main.rs"), "fn main() {}\n").expect("fixture"); + std::fs::write(root.join("src/net/mod.rs"), "pub mod server;\n").expect("fixture"); + std::fs::write(root.join("src/net/server.rs"), "fn serve() {}\n").expect("fixture"); + std::fs::write(root.join("src/net/client.rs"), "fn connect() {}\n").expect("fixture"); + let context = ToolContext::new(root); + + // First two pod reads: full footer, no callers yet. + for path in ["src/net/mod.rs", "src/net/client.rs"] { + let result = ReadFileTool::execute_contract_read(json!({"path": path}), &context) + .await + .expect("read result"); + assert!(!result.content.contains("heard by"), "{path}"); + } + // Third distinct pod read: the buzz adds who links here. + let result = + ReadFileTool::execute_contract_read(json!({"path": "src/net/server.rs"}), &context) + .await + .expect("read result"); + assert!( + result.content.contains("heard by: src/net/mod.rs"), + "{}", + result.content + ); + // Another pod stays sparse: the buzz is per-pod, not global. + std::fs::write(root.join("src/other.rs"), "fn other() {}\n").expect("fixture"); + let result = ReadFileTool::execute_contract_read(json!({"path": "src/other.rs"}), &context) + .await + .expect("read result"); + assert!(!result.content.contains("heard by"), "{}", result.content); +} diff --git a/crates/tui/src/tools/file_search.rs b/crates/tui/src/tools/file_search.rs index bfda2a3560..83683d9d0b 100644 --- a/crates/tui/src/tools/file_search.rs +++ b/crates/tui/src/tools/file_search.rs @@ -113,6 +113,9 @@ impl ToolSpec for FileSearchTool { let extensions = parse_extensions(&input); let exclude_patterns = parse_exclude_patterns(&input); + let pod_root = base_path.clone(); + let pod_excludes = exclude_patterns.clone(); + let pod_extensions = extensions.clone(); let matches = search_files_async( query.to_string(), base_path, @@ -124,7 +127,34 @@ impl ToolSpec for FileSearchTool { context.follow_symlinks, ) .await?; - ToolResult::json(&matches).map_err(|e| ToolError::execution_failed(e.to_string())) + // Echolocation sounding: chart the matched dirs (sorted, capped) + // so each candidate arrives with its neighborhood. Dirs resolve + // against the search root, which is the workspace by default. + let paths: Vec = matches.iter().map(|item| item.path.clone()).collect(); + let vis = super::echolocation::PodVisibility { + include: &[], + exclude: &pod_excludes, + extensions: &pod_extensions, + glob_root: &pod_root, + gitignore: true, + }; + let (pods, pods_omitted) = super::echolocation::sound_match_pods(&pod_root, &paths, &vis); + let pods_json: Vec = pods + .iter() + .map(|pod| { + json!({ + "dir": pod.dir, + "mates": pod.mates, + "mate_total": pod.mate_total, + }) + }) + .collect(); + ToolResult::json(&json!({ + "matches": matches, + "pods": pods_json, + "pods_omitted": pods_omitted, + })) + .map_err(|e| ToolError::execution_failed(e.to_string())) } } @@ -423,7 +453,7 @@ fn check_cancelled(cancel_token: Option<&CancellationToken>) -> Result<(), ToolE Ok(()) } -fn extension_matches(path: &Path, extensions: &[String]) -> bool { +pub(crate) fn extension_matches(path: &Path, extensions: &[String]) -> bool { let Some(ext) = path.extension().and_then(|e| e.to_str()) else { return false; }; @@ -578,6 +608,36 @@ mod tests { assert!(result.content.contains("main.rs")); } + #[tokio::test] + async fn test_file_search_charts_match_pods() { + let tmp = tempdir().expect("tempdir"); + let root = tmp.path(); + std::fs::create_dir_all(root.join("src/net")).expect("mkdir"); + std::fs::write(root.join("src/net/server.rs"), "fn serve() {}\n").expect("write"); + std::fs::write(root.join("src/net/client.rs"), "fn connect() {}\n").expect("write"); + + let ctx = ToolContext::new(root.to_path_buf()); + let result = FileSearchTool + .execute(json!({"query": "server", "limit": 5}), &ctx) + .await + .expect("execute"); + + assert!(result.success); + let body: Value = serde_json::from_str(&result.content).expect("json result"); + assert!(!body["matches"].as_array().expect("matches").is_empty()); + let pods = body["pods"].as_array().expect("pods array"); + assert_eq!(pods.len(), 1); + assert_eq!(pods[0]["dir"], "src/net"); + let mates: Vec<&str> = pods[0]["mates"] + .as_array() + .expect("mates") + .iter() + .filter_map(Value::as_str) + .collect(); + assert!(mates.contains(&"server.rs (match)"), "{mates:?}"); + assert!(mates.contains(&"client.rs"), "{mates:?}"); + } + #[tokio::test] async fn test_file_search_respects_gitignore() { let tmp = tempdir().expect("tempdir"); @@ -633,9 +693,9 @@ mod tests { .expect("execute"); assert!(result.success); - let matches: Value = serde_json::from_str(&result.content).expect("search json"); + let body: Value = serde_json::from_str(&result.content).expect("search json"); assert!( - matches + body["matches"] .as_array() .expect("matches") .iter() @@ -666,9 +726,9 @@ mod tests { .expect("execute"); assert!(result.success); - let matches: Value = serde_json::from_str(&result.content).expect("search json"); + let body: Value = serde_json::from_str(&result.content).expect("search json"); assert!( - matches + body["matches"] .as_array() .expect("matches") .iter() @@ -695,7 +755,7 @@ mod tests { .expect("execute"); let matches: Value = serde_json::from_str(&result.content).expect("search json"); - let paths: Vec<&str> = matches + let paths: Vec<&str> = matches["matches"] .as_array() .expect("matches") .iter() diff --git a/crates/tui/src/tools/mod.rs b/crates/tui/src/tools/mod.rs index 88dbfd8e44..393732a54d 100644 --- a/crates/tui/src/tools/mod.rs +++ b/crates/tui/src/tools/mod.rs @@ -26,6 +26,7 @@ pub mod file; pub mod file_search; pub mod finance; +pub mod echolocation; pub mod fetch_url; pub mod file_tool; pub mod fim; diff --git a/crates/tui/src/tools/project.rs b/crates/tui/src/tools/project.rs index d29328742e..67e3b4a14b 100644 --- a/crates/tui/src/tools/project.rs +++ b/crates/tui/src/tools/project.rs @@ -17,6 +17,8 @@ struct ProjectMap { tree: String, summary: String, key_files: Vec, + key_file_soundings: std::collections::BTreeMap, + key_files_omitted: usize, } #[async_trait] @@ -84,9 +86,41 @@ fn generate_project_map( } } + // Echolocation sounding: one line of symbols per key file (sorted, + // capped), so the map says what lives where, not just what exists. + let (key_file_soundings, key_files_omitted) = + super::echolocation::sound_key_files(root, &key_files); + Ok(ProjectMap { tree, summary, key_files, + key_file_soundings, + key_files_omitted, }) } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn project_map_sounds_key_file_symbols() { + let dir = tempfile::tempdir().expect("tempdir"); + std::fs::write(dir.path().join("main.rs"), "fn main() {}\nfn serve() {}\n") + .expect("fixture"); + std::fs::write(dir.path().join("Cargo.toml"), "[package]\n").expect("fixture"); + let context = ToolContext::new(dir.path().to_path_buf()); + let result = ProjectMapTool + .execute(json!({}), &context) + .await + .expect("execute"); + assert!(result.success); + let body: Value = serde_json::from_str(&result.content).expect("json result"); + assert_eq!( + body["key_file_soundings"]["main.rs"], + "2 symbols: fn main:1, fn serve:2" + ); + assert!(body["key_file_soundings"].get("Cargo.toml").is_none()); + } +} diff --git a/crates/tui/src/tools/search.rs b/crates/tui/src/tools/search.rs index 82ca86b5b0..5afe813a81 100644 --- a/crates/tui/src/tools/search.rs +++ b/crates/tui/src/tools/search.rs @@ -32,13 +32,17 @@ const MAX_FILE_SIZE: u64 = 10 * 1024 * 1024; // 10MB /// tree. Mirrors the file_search tool so both blocking searches behave the same. const GREP_FILES_TIMEOUT: Duration = Duration::from_secs(30); -/// Result of a grep match +/// Result of a grep match. Empty context arrays are omitted on the +/// wire (models request context in ~0% of calls); `default` keeps old +/// payloads parsing. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GrepMatch { pub file: String, pub line_number: usize, pub line: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] pub context_before: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] pub context_after: Vec, } @@ -278,6 +282,31 @@ impl ToolSpec for GrepFilesTool { .map(|item| grep_match_to_json(item, context_lines)) .collect(); + // Echolocation sounding: chart the matched dirs (sorted, capped) + // so the model sees each match's podmates without a follow-up + // list. Computed here, inside the blocking worker, with the + // workspace already at hand. + let matched_files: Vec = results.iter().map(|item| item.file.clone()).collect(); + let vis = crate::tools::echolocation::PodVisibility { + include: &include_patterns, + exclude: &exclude_patterns, + extensions: &[], + glob_root: &search_path, + gitignore: false, + }; + let (pods, pods_omitted) = + crate::tools::echolocation::sound_match_pods(&workspace, &matched_files, &vis); + let pods_json: Vec = pods + .iter() + .map(|pod| { + json!({ + "dir": pod.dir, + "mates": pod.mates, + "mate_total": pod.mate_total, + }) + }) + .collect(); + // Build result. When context_lines == 1, return the single context // line as a string instead of a one-item array. That keeps the common // "show just the adjacent line" case easy for model callers to read. @@ -286,6 +315,8 @@ impl ToolSpec for GrepFilesTool { "total_matches": total_matches, "files_searched": files_searched, "truncated": truncated, + "pods": pods_json, + "pods_omitted": pods_omitted, }); if truncated { output["max_results"] = json!(max_results); diff --git a/crates/tui/src/tools/search/tests.rs b/crates/tui/src/tools/search/tests.rs index b192f91604..8cb4064b9a 100644 --- a/crates/tui/src/tools/search/tests.rs +++ b/crates/tui/src/tools/search/tests.rs @@ -114,6 +114,35 @@ fn test_matches_glob_unicode_filename() { assert!(!matches_glob(filename, "nonexistent*")); } +#[tokio::test] +async fn grep_pods_chart_matched_dirs_with_marks() { + let tmp = tempdir().expect("tempdir"); + let ctx = ToolContext::new(tmp.path().to_path_buf()); + fs::create_dir(tmp.path().join("net")).expect("subdir"); + fs::write(tmp.path().join("net/server.rs"), "fn serve() {}\n").expect("write"); + fs::write(tmp.path().join("net/client.rs"), "fn connect() {}\n").expect("write"); + + let result = GrepFilesTool + .execute(json!({"pattern": "serve"}), &ctx) + .await + .expect("execute"); + assert!(result.success); + let body: Value = serde_json::from_str(&result.content).expect("json result"); + assert_eq!(body["total_matches"], 1); + let pods = body["pods"].as_array().expect("pods array"); + assert_eq!(pods.len(), 1); + assert_eq!(pods[0]["dir"], "net"); + let mates: Vec<&str> = pods[0]["mates"] + .as_array() + .expect("mates") + .iter() + .filter_map(Value::as_str) + .collect(); + assert!(mates.contains(&"client.rs"), "{mates:?}"); + assert!(mates.contains(&"server.rs (match)"), "{mates:?}"); + assert_eq!(body["pods_omitted"], 0); +} + #[tokio::test] async fn test_grep_files_basic() { let tmp = tempdir().expect("tempdir"); @@ -415,12 +444,13 @@ async fn test_grep_files_ring_buffer_context_matches_full_read() { let parsed: Value = serde_json::from_str(&result.content).unwrap(); let matches = parsed["matches"].as_array().unwrap(); assert_eq!(matches.len(), 3); - assert_eq!(matches[0]["context_before"], json!([])); + // Token diet: empty context arrays are omitted on the wire. + assert!(matches[0].get("context_before").is_none()); assert_eq!(matches[0]["context_after"], json!(["b1", "b2"])); assert_eq!(matches[1]["context_before"], json!(["b2", "b3"])); assert_eq!(matches[1]["context_after"], json!(["a1", "a2"])); assert_eq!(matches[2]["context_before"], json!(["a2", "a3"])); - assert_eq!(matches[2]["context_after"], json!([])); + assert!(matches[2].get("context_after").is_none()); assert_eq!(matches[2]["line_number"].as_u64().unwrap(), 9); } diff --git a/crates/tui/src/tools/spec.rs b/crates/tui/src/tools/spec.rs index 9541c47bf5..4253109bf4 100644 --- a/crates/tui/src/tools/spec.rs +++ b/crates/tui/src/tools/spec.rs @@ -1043,6 +1043,23 @@ impl ToolContext { tracker.reads.insert(path.to_path_buf(), snapshot); } + /// Distinct files read under `dir` so far this session. Echolocation's + /// hunt-phase sensor: 2+ tracked reads means the model is working this + /// pod and the next read there enters the terminal buzz. Lock + /// contention degrades to zero (sparse sounding), never an error. + pub(crate) fn pod_visit_count(&self, dir: &Path) -> usize { + self.file_read_tracker + .lock() + .map(|tracker| { + tracker + .reads + .keys() + .filter(|path| path.parent() == Some(dir)) + .count() + }) + .unwrap_or(0) + } + /// Require a successful, still-fresh `read_file` snapshot before a narrow /// in-place edit. This catches model edits made against guessed or stale /// content while leaving transactional patch preflight separate. diff --git a/crates/tui/src/tools/subagent/tests.rs b/crates/tui/src/tools/subagent/tests.rs index e5667e0cf9..68d7276e4d 100644 --- a/crates/tui/src/tools/subagent/tests.rs +++ b/crates/tui/src/tools/subagent/tests.rs @@ -8305,6 +8305,76 @@ fn enabled_agent_surface_options() -> AgentToolSurfaceOptions { /// /// Request-boundary tests use this fixture so Moonshot compatibility coverage /// cannot drift into a hand-maintained approximation of the child surface. +#[test] +fn deferred_agent_and_workflow_are_discoverable_by_name() { + // Diet regression: deferral must never strand the tools. The real + // registry (not fixtures) carries them deferred, and both match + // algorithms surface them for the obvious queries. + use crate::core::engine::tool_catalog::{ + TOOL_SEARCH_NAME, apply_native_tool_deferral, execute_tool_search_with_cache, + }; + use crate::core::session::ToolActivationCache; + use crate::tools::registry::ToolRegistryBuilder; + use crate::tools::{ToolContext, todo}; + use std::collections::HashSet; + + let tmp = tempdir().expect("tempdir"); + let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 4); + let registry = ToolRegistryBuilder::new() + .with_agent_runtime_surface( + None, + "test-model".to_string(), + enabled_agent_surface_options(), + todo::new_shared_todo_list(), + crate::tools::plan::new_shared_plan_state(), + ) + .with_subagent_tools(manager, stub_runtime()) + .build(ToolContext::new(tmp.path().to_path_buf())); + let mut catalog = registry.to_api_tools(); + let always_load = HashSet::new(); + apply_native_tool_deferral(&mut catalog, &always_load); + for name in ["agent", "workflow"] { + assert_eq!( + catalog + .iter() + .find(|tool| tool.name == name) + .and_then(|tool| tool.defer_loading), + Some(true), + "{name} is registered and deferred" + ); + } + + for (query, match_kind, wanted) in [ + ("agent", "regex", "agent"), + ("workflow", "regex", "workflow"), + ("spawn subagent worker", "bm25", "agent"), + ("spawn subagent worker", "bm25", "workflow"), + ] { + let mut active = HashSet::new(); + let mut cache = ToolActivationCache::default(); + let found = execute_tool_search_with_cache( + TOOL_SEARCH_NAME, + &json!({"query": query, "match": match_kind}), + &catalog, + &mut active, + &mut cache, + ) + .expect("tool_search runs") + .metadata + .expect("metadata")["tool_references"] + .as_array() + .expect("refs") + .iter() + .filter_map(|v| v.as_str()) + .map(str::to_string) + .collect::>(); + assert!( + found.contains(&wanted.to_string()), + "{match_kind} {query:?} must surface {wanted}; got {found:?}" + ); + } +} + pub(crate) fn kimi_general_child_request_tools_fixture() -> Vec { let tmp = tempdir().expect("tempdir"); let mut runtime = @@ -8433,31 +8503,41 @@ async fn execute_surface_tool( } #[test] -fn small_surface_starts_with_core_tools_and_read_only_goal_control() { +fn small_surface_starts_with_core_tools_and_discoverable_goal_control() { let registry = small_surface_registry(FleetRole::Builder); let catalog = registry.deferred_catalog_for_model(&FleetRole::Builder); let mut surface = ChildSurfaceProbe::new(catalog, &[]); + // Token diet: agent+workflow+get_goal ride discovery, not the head. assert_eq!( model_tool_names(model_request_tools(&mut surface)), [ - "agent", "bash", "edit", "file_search", - "get_goal", "load_skill", "read", "task_shell_wait", "todo_write", "tool_search", - "workflow", "write", ] .into_iter() .map(str::to_string) .collect() ); + // The delegation and goal controls still exist — deferred, not deleted. + for deferred_name in ["agent", "workflow", "get_goal", "update_goal"] { + assert_eq!( + surface + .catalog + .iter() + .find(|tool| tool.name == deferred_name) + .and_then(|tool| tool.defer_loading), + Some(true), + "{deferred_name} must stay discoverable" + ); + } let strict = surface.request_tools(surface.catalog().to_vec(), true); assert_eq!( strict @@ -8759,7 +8839,7 @@ fn small_surface_caches_are_independent_bounded_and_revalidated() { assert!(first.hydrate("oversized", &json!({})).is_err()); let mut byte_catalog = (0..3) - .map(|index| synthetic_deferred_tool(&format!("bytes_{index}"), 6 * 1024)) + .map(|index| synthetic_deferred_tool(&format!("bytes_{index}"), 9 * 1024)) .collect::>(); ensure_advanced_tooling( &mut byte_catalog, @@ -8823,6 +8903,8 @@ fn small_surface_depth_cap_removes_only_agent() { registry.deferred_catalog_for_model(&FleetRole::Builder), &[], ); + // Token diet: the depth cap still removes only agent; get_goal and + // workflow are absent from the head by deferral, not by depth. assert_eq!( model_tool_names(model_request_tools(&mut surface)), [ @@ -25993,7 +26075,11 @@ fn tool_search_cannot_return_a_retired_agents_tool() { let tmp = tempdir().expect("tempdir"); let registry = subagent_registry_for_catalog(tmp.path()); let mut catalog = registry.to_api_tools(); - apply_native_tool_deferral(&mut catalog, &HashSet::new()); + // Token diet: agent is searchable by default; this test's subject is + // retired-tool exclusion, so its precondition re-eagers agent through + // the same always-load path discovery uses. + let always_load = HashSet::from(["agent".to_string()]); + apply_native_tool_deferral(&mut catalog, &always_load); assert!( catalog .iter() diff --git a/crates/tui/src/tools/workflow/mod.rs b/crates/tui/src/tools/workflow/mod.rs index 5cf2eecdbd..3afc565f46 100644 --- a/crates/tui/src/tools/workflow/mod.rs +++ b/crates/tui/src/tools/workflow/mod.rs @@ -2465,9 +2465,10 @@ impl RunPayloadBounds { } } -/// Build the model-facing view of a run record (#2974). The JSON shape is -/// identical to the full record (panel hydration and history cards keep -/// working unchanged), but the unbounded parts are clipped: +/// Build the model-facing view of a run record (#2974). Panel hydration +/// and history cards keep working unchanged (they read keys this view +/// preserves), but the unbounded parts are clipped and each event is +/// projected to its model-essential keys: /// /// - `events`: newest `WORKFLOW_RESULT_EVENTS_TAIL` entries. /// - `progress`: newest `WORKFLOW_RESULT_PROGRESS_TAIL` lines. @@ -2477,9 +2478,16 @@ impl RunPayloadBounds { /// when the serialized value exceeds `WORKFLOW_RESULT_VALUE_MAX_CHARS`. /// - `execution.leaf_results[*].output`: per-leaf preview capped at /// `WORKFLOW_RESULT_LEAF_OUTPUT_MAX_CHARS`. +/// - event objects: exact top-level duplicates (`owner_session_id`, +/// `workflow_goal`, `workflow_id`, `source_path`, `token_budget`) are +/// removed; distinct `workspace` values hoist to top-level +/// `event_workspaces` once instead of repeating per event; routing, +/// isolation, and payload keys stay. /// /// Full detail remains available in `.codewhale/workflow-runs.jsonl`; every -/// clip adds an explicit note/pointer so the model can fetch more on demand. +/// lossy clip adds an explicit note/pointer so the model can fetch more on +/// demand. The key projection is lossless by construction (duplicates live +/// on at top level; workspaces hoist), so it carries no pointer. fn bounded_run_record_value( record: &WorkflowRunRecord, journal_path: &Path, @@ -2500,14 +2508,53 @@ fn bounded_run_record_value( ); obj.insert("progress_count".to_string(), json!(record.progress_count)); + let mut workspaces = std::collections::BTreeSet::new(); if let Some(events) = obj.get_mut("events").and_then(Value::as_array_mut) { if events.len() > WORKFLOW_RESULT_EVENTS_TAIL { let omitted = events.len() - WORKFLOW_RESULT_EVENTS_TAIL; events.drain(..omitted); bounds.events_omitted = omitted; } + // Token diet: project each event to model-essential keys. Dropped + // keys are exact top-level duplicates (`RunStarted` clones the + // record's goal/id/source/budget verbatim; `owner_session_id` + // repeats on every event). `workspace` is hoisted instead of + // dropped: the distinct set rides top-level once, since a + // worktree path names an isolation dir the model may need. + // `git_branch` stays per event — short run context that can vary + // per child. Routing keys (profile, resolved_*, route_source), + // worktree (isolation signal), and message/title/status/error/ + // usage stay: asserted contract (`run_record_events_project_to_ + // model_essential_keys`), and the model reads them. Full rows + // stay in the journal either way. + for event in events.iter_mut() { + let Some(fields) = event.as_object_mut() else { + continue; + }; + if let Some(workspace) = fields.remove("workspace") + && let Some(path) = workspace.as_str() + && !path.is_empty() + { + workspaces.insert(path.to_string()); + } + for key in [ + "owner_session_id", + "workflow_goal", + "workflow_id", + "source_path", + "token_budget", + ] { + fields.remove(key); + } + } bounds.events_returned = events.len(); } + if !workspaces.is_empty() { + obj.insert( + "event_workspaces".to_string(), + Value::Array(workspaces.into_iter().map(Value::String).collect()), + ); + } if bounds.events_omitted > 0 { obj.insert( "events_note".to_string(), @@ -12031,6 +12078,99 @@ FINAL RECEIPT ); } + #[test] + fn run_record_events_project_to_model_essential_keys() { + fn task_started(task_id: &str, workspace: &str) -> WorkflowUiEvent { + WorkflowUiEvent::at( + 2, + "session-test", + WorkflowUiEventKind::TaskStarted(Box::new(WorkflowTaskStartedEvent { + task_id: task_id.to_string(), + label: Some("dig".to_string()), + role: None, + profile: Some("deep".to_string()), + model: None, + strength: None, + thinking: None, + requested_reasoning: None, + effective_reasoning: None, + resolved_role: None, + resolved_profile: None, + resolved_provider: "zai".to_string(), + resolved_model: "glm-5".to_string(), + route_source: "fleet".to_string(), + child_route: None, + worktree: true, + workspace: Some(PathBuf::from(workspace)), + git_branch: Some("feature/x".to_string()), + parent_task_id: None, + depth: 0, + workflow_run_id: None, + workflow_phase_id: None, + workflow_task_label: None, + workflow_child_index: None, + fleet_receipt: None, + })), + ) + } + + let tmp = tempfile::tempdir().expect("tempdir"); + let mut record = WorkflowRunRecord::new( + "workflow_projection_run".to_string(), + Some("session-test".to_string()), + Some(PathBuf::from("/ws/flow.json")), + Some(1_000), + None, + ); + record.workflow_id = Some("flow-1".to_string()); + record.workflow_goal = Some("ship it".to_string()); + record.push_event(WorkflowUiEvent::at( + 1, + "session-test", + WorkflowUiEventKind::RunStarted { + workflow_id: Some("flow-1".to_string()), + workflow_goal: Some("ship it".to_string()), + source_path: Some(PathBuf::from("/ws/flow.json")), + token_budget: Some(1_000), + }, + )); + // Two dispatches, one shared workspace: the hoist dedupes. + record.push_event(task_started("t1", "/tmp/wt-1")); + record.push_event(task_started("t2", "/tmp/wt-1")); + + let journal = tmp.path().join("workflow-runs.jsonl"); + let (payload, bounds) = bounded_run_record_value(&record, &journal); + assert!(!bounds.truncated()); + let events = payload["events"].as_array().expect("events"); + + // Duplicates are gone from every event … + for event in events { + for key in [ + "owner_session_id", + "workflow_goal", + "workflow_id", + "source_path", + "token_budget", + "workspace", + ] { + assert!(event.get(key).is_none(), "dropped {key}: {event}"); + } + } + // … but live on at top level (duplicates) or hoisted (workspaces). + assert_eq!(payload["workflow_goal"], "ship it"); + assert_eq!(payload["token_budget"], 1_000); + assert_eq!(payload["event_workspaces"], json!(["/tmp/wt-1"])); + + // Routing, isolation, and payload keys stay on the dispatch event. + let dispatch = &events[1]; + assert_eq!(dispatch["task_id"], "t1"); + assert_eq!(dispatch["route_source"], "fleet"); + assert_eq!(dispatch["resolved_provider"], "zai"); + assert_eq!(dispatch["resolved_model"], "glm-5"); + assert_eq!(dispatch["worktree"], true); + assert_eq!(dispatch["git_branch"], "feature/x"); + } + #[test] fn workflow_status_payload_keeps_small_records_intact() { let tmp = tempfile::tempdir().expect("tempdir"); diff --git a/crates/workflow/src/lib.rs b/crates/workflow/src/lib.rs index 63f8312f07..fbfec1d3a8 100644 --- a/crates/workflow/src/lib.rs +++ b/crates/workflow/src/lib.rs @@ -631,7 +631,7 @@ pub struct BranchResult { pub status: WorkflowRunStatus, #[serde(default)] pub usage: WorkflowUsage, - #[serde(default)] + #[serde(default, skip_serializing_if = "WorkflowMemoUsage::is_zero")] pub memo_usage: WorkflowMemoUsage, #[serde(default)] pub artifacts: Vec, @@ -652,7 +652,7 @@ pub struct LeafResult { pub status: WorkflowRunStatus, #[serde(default)] pub usage: WorkflowUsage, - #[serde(default)] + #[serde(default, skip_serializing_if = "WorkflowMemoUsage::is_zero")] pub memo_usage: WorkflowMemoUsage, #[serde(default)] pub output: Option, @@ -710,6 +710,15 @@ pub struct WorkflowMemoUsage { } impl WorkflowMemoUsage { + /// True when every counter is zero: 136 bytes of nothing on the wire. + pub fn is_zero(&self) -> bool { + self.armh_hits == 0 + && self.armh_misses == 0 + && self.armh_saved_estimated_tokens == 0 + && self.provider_prompt_cache_hits == 0 + && self.provider_prompt_cache_misses == 0 + } + pub(crate) fn add_assign(&mut self, other: Self) { self.armh_hits = self.armh_hits.saturating_add(other.armh_hits); self.armh_misses = self.armh_misses.saturating_add(other.armh_misses); @@ -766,7 +775,7 @@ pub struct WorkflowExecution { pub status: WorkflowRunStatus, #[serde(default)] pub usage: WorkflowUsage, - #[serde(default)] + #[serde(default, skip_serializing_if = "WorkflowMemoUsage::is_zero")] pub memo_usage: WorkflowMemoUsage, #[serde(default)] pub leaf_results: Vec, @@ -815,7 +824,7 @@ pub struct MockLeafOutcome { pub status: WorkflowRunStatus, #[serde(default)] pub usage: WorkflowUsage, - #[serde(default)] + #[serde(default, skip_serializing_if = "WorkflowMemoUsage::is_zero")] pub memo_usage: WorkflowMemoUsage, #[serde(default)] pub output: Option, From b81a6dae91df6fbedef6ab141a061fbc8a528bc8 Mon Sep 17 00:00:00 2001 From: AdityaVG13 Date: Sun, 20 Sep 2026 23:41:41 -0400 Subject: [PATCH 14/26] docs(surface): sync public tool-surface contracts with the diet DEFAULT_ACTIVE_NATIVE_TOOLS is six names now (agent, workflow, get/update_goal deferred); the web contract mirror, tool docs, and mode docs still claimed ten eager + eleven with tool_search. Activation cache bound 16 -> 24 KiB in facts, docs, and web copy. Found by the fork staging PR's Lint & Type Check gate. --- docs/MODES.md | 2 +- docs/TOOL_SURFACE.md | 40 ++++++++++++------------- docs/public-surface-facts.json | 5 +--- docs/zh_hans/MODES.md | 2 +- web/lib/content/tools.ts | 6 ++-- web/lib/public-surface-contract.test.ts | 2 +- 6 files changed, 27 insertions(+), 30 deletions(-) diff --git a/docs/MODES.md b/docs/MODES.md index 060681d67b..a0c7c425bb 100644 --- a/docs/MODES.md +++ b/docs/MODES.md @@ -35,7 +35,7 @@ Run `/mode` to open the mode picker, or switch directly with `/mode work`, `/mode plan`, or `/mode operate`. - **Plan**: design-first prompting. The stable primitive names remain familiar, but the runtime centrally refuses file mutation and shell execution. Read-only inspection and policy-allowed research, including deferred Web search/fetch, remain available. When a Plan turn successfully finishes a nonempty response, including a prose-only plan, the TUI asks how to continue: Work (Ask), Work (Auto-Review), or keep planning. Work sends the exact approved response as a normal follow-up and preserves that turn's existing To-dos, or adds one pending plan item without replacing unrelated work. -- **Work** (internally `agent`): ordinary multi-step execution. The first-turn toolbox includes `read`, `write`, `edit`, `file_search`, `bash`, `agent`, `workflow`, `todo_write`, and `load_skill`, plus `create_goal`, `get_goal`, `update_goal`, and `task_shell_wait` so goal and background-shell controls are available without discovery. The agent calls `create_goal` when a request is a durable objective; the host never infers a goal from wording, and the contract must not tell the model otherwise. Approval, sandbox, repository law, and managed policy decide what may execute. +- **Work** (internally `agent`): ordinary multi-step execution. The first-turn toolbox includes `read`, `write`, `edit`, `file_search`, `bash`, `todo_write`, `load_skill`, and `task_shell_wait`, plus `create_goal` and the synthetic `tool_search`; `agent`, `workflow`, `get_goal`, and `update_goal` load through discovery (`get_goal`/`update_goal` ride eager while a goal is active). The agent calls `create_goal` when a request is a durable objective; the host never infers a goal from wording, and the contract must not tell the model otherwise. Approval, sandbox, repository law, and managed policy decide what may execute. - **Operate**: Work at full strength, the "ultra" mode. Same tools and authority as Work; the difference is how they are used. Each substantive request becomes a goal (`create_goal`), the plan stays visible (`todo_write`), multi-part work runs through `workflow` and parallel `agent` workers, non-trivial changes get an independent reviewer before they are called done, long commands keep running in the background, and recurring or watch-type work is offered as an `automation` (created after approval). It keeps going until the goal is verified, blocked on you, or paused. `Act` and `/mode act` remain compatibility aliases for Work. Saved settings diff --git a/docs/TOOL_SURFACE.md b/docs/TOOL_SURFACE.md index 2b0b5e0406..fb4a9521e2 100644 --- a/docs/TOOL_SURFACE.md +++ b/docs/TOOL_SURFACE.md @@ -19,31 +19,30 @@ Implementation sources: ## Default-active contract -New turns start with thirteen eager native names plus synthetic `tool_search`: +New turns start with nine eager native names plus synthetic `tool_search`: 1. `read` 2. `write` 3. `edit` 4. `file_search` 5. `bash` -6. `agent` -7. `workflow` -8. `todo_write` +6. `todo_write` +7. `task_shell_wait` +8. `load_skill` 9. `create_goal` -10. `get_goal` -11. `update_goal` -12. `task_shell_wait` -13. `load_skill` -14. `tool_search` (synthetic, always active) +10. `tool_search` (synthetic, always active) -The thirteen native names are `DEFAULT_ACTIVE_NATIVE_TOOLS` in +The nine native names are `DEFAULT_ACTIVE_NATIVE_TOOLS` in `crates/tui/src/core/engine/tool_catalog.rs`, pinned by -`default_active_contract_keeps_discovery_and_core_tools_eager`. An authority -boundary may remove `agent` at the maximum child depth, and Plan registers only -the foreground shell surface (so no `task_shell_wait`), but route size alone -must not change this core vocabulary. +`default_active_contract_keeps_discovery_and_core_tools_eager`. `agent`, +`workflow`, `get_goal`, and `update_goal` are deferred by default and load +via `tool_search` (`get_goal`/`update_goal` ride eager only while a goal is +active). An authority boundary may remove `agent` at the maximum child depth, +and Plan registers only the foreground shell surface (so no `task_shell_wait`), +but route size alone must not change this core vocabulary. -The direct schemas deliberately stay small: +The core schemas deliberately stay small (deferred rows load via +`tool_search` rather than riding every turn): | Tool | Input | Purpose | |---|---|---| @@ -52,12 +51,12 @@ The direct schemas deliberately stay small: | `edit` | `path`, `edits` | Apply one or more unambiguous text replacements against one original snapshot. | | `file_search` | `query`, optional `path`, `limit`, `extensions`, `exclude` | Find files by name, path fragment or glob and return bounded, ranked matches. Respects ignore files; `exclude` replaces default exclusions. | | `bash` | `command`, optional `timeout`, optional `description` | Run one cancellable foreground shell command and return a bounded tail. The optional description is display-only. | -| `agent` | delegated task and optional scope/context controls | Start or inspect focused child work. | -| `workflow` | plan/script/source_path plus run controls | Coordinate multi-agent phases with dependencies and completion checks. | +| `agent` (deferred) | delegated task and optional scope/context controls | Start or inspect focused child work. | +| `workflow` (deferred) | plan/script/source_path plus run controls | Coordinate multi-agent phases with dependencies and completion checks. | | `todo_write` | complete replacement list of `{content, status}` items | Keep optional, agent-owned progress notes for genuinely multi-step work. | | `create_goal` | objective plus optional budget | Start the session goal the turn works toward. | -| `get_goal` | none | Read the active goal and its progress. | -| `update_goal` | terminal status | Mark the goal complete or blocked. | +| `get_goal` (deferred unless a goal is active) | none | Read the active goal and its progress. | +| `update_goal` (deferred unless a goal is active) | terminal status | Mark the goal complete or blocked. | | `task_shell_wait` | `task_id`, optional `wait`, `timeout_ms`, `cancel`, `gate` | Poll or stop a background shell task. Completion still arrives as a runtime event, so use it for early output, true barriers, or gate evidence. | | `load_skill` | optional `name`, `query` | Load a named skill's `SKILL.md` body and companion-file list; omit `name` or use `name="list"` to list eligible skills, or use `query` to search names and descriptions. | | `tool_search` | `query`, optional matching controls | Discover policy-allowed deferred tools and add selected schemas to this conversation's toolbox. | @@ -399,7 +398,8 @@ above must report `1 passed` (the ignored metrics test reports `1 passed` only because `--ignored` selects it); `0 passed` means the filter matched nothing and the check did not run. -The provider-free receipt must report the thirteen default-active names listed +The provider-free receipt must report the ten default-active names listed +above. above. A separate repository-wide tool count may include deferred, dynamic, feature-gated, and compatibility-only registrations; it is not the number of tools placed in the first-turn model catalog. diff --git a/docs/public-surface-facts.json b/docs/public-surface-facts.json index aa2f63fdec..cec94c5dcf 100644 --- a/docs/public-surface-facts.json +++ b/docs/public-surface-facts.json @@ -105,14 +105,11 @@ "edit", "file_search", "bash", - "agent", - "workflow", "todo_write", "create_goal", - "get_goal", - "update_goal", "task_shell_wait", "load_skill" + ], "schemas": { "read": [ diff --git a/docs/zh_hans/MODES.md b/docs/zh_hans/MODES.md index 0a544919f7..e409eb23bc 100644 --- a/docs/zh_hans/MODES.md +++ b/docs/zh_hans/MODES.md @@ -20,7 +20,7 @@ Workflow 通过同一个子智能体运行时执行命名步骤；Fleet 管理 按 `Tab` 补全输入框（composer）菜单，或在输入框为空时循环切换可见模式：**Plan → Work → Operate → Plan**。`Tab` 从不发送或排队输入框中的文本；用 `Enter` 发送或排队。按 `Shift+Tab` 循环切换权限姿态（Ask → Auto-Review → Full Access）。按 `Ctrl+T` 循环切换思考强度。运行 `/mode` 打开模式选择器，或直接用 `/mode work`、`/mode plan`、`/mode operate` 切换。 - **Plan**：设计优先的提示方式。原语（primitive）名称保持不变，仍是大家熟悉的那些，但运行时会集中拒绝文件修改和 shell 执行。只读检查与策略允许的研究（包括延迟加载的 Web 搜索/抓取）仍然可用。当一个 Plan 回合成功结束并给出非空回复（包括纯文字计划）时，TUI 会询问如何继续：Work（Ask）、Work（Auto-Review），或继续规划。选择 Work 会把已批准的原文作为普通后续消息发送，并保留该回合已有的 To-dos，或追加一个待办的计划项而不替换无关的工作。 -- **Work**（内部为 `agent`）：普通的多步执行。第一回合的工具箱包含 `read`、`write`、`edit`、`file_search`、`bash`、`agent`、`workflow`、`todo_write` 和 `load_skill`，以及无需先搜索即可使用的目标与后台 shell 控制工具 `create_goal`、`get_goal`、`update_goal` 和 `task_shell_wait`。当请求属于一个持久的目标时，智能体会调用 `create_goal`；宿主从不根据措辞去推断目标，契约也不得告诉模型相反的内容。审批、沙箱、仓库规则（repository law）和托管策略决定什么可以执行。 +- **Work**（内部为 `agent`）：普通的多步执行。第一回合的工具箱包含 `read`、`write`、`edit`、`file_search`、`bash`、`todo_write`、`load_skill` 和 `task_shell_wait`，以及 `create_goal` 与合成工具 `tool_search`；`agent`、`workflow`、`get_goal` 和 `update_goal` 通过搜索发现加载（`get_goal`/`update_goal` 在目标激活时直接可用）。当请求属于一个持久的目标时，智能体会调用 `create_goal`；宿主从不根据措辞去推断目标，契约也不得告诉模型相反的内容。审批、沙箱、仓库规则（repository law）和托管策略决定什么可以执行。 - **Operate**：以全力运行的 Work，即“ultra”模式。工具与权限和 Work 相同，区别在于如何使用它们。每个实质性请求都会成为一个目标（`create_goal`），计划保持可见（`todo_write`），多部分工作通过 `workflow` 和并行的 `agent` 工作者执行，非平凡的改动在宣布完成前先交给独立的审查者，长时间运行的命令继续在后台运行，循环或监视类工作则作为 `automation` 提出（获批后创建）。它会一直进行，直到目标被验证、被你阻塞或被暂停。 `Act` 和 `/mode act` 仍然是 Work 的兼容别名。保存的设置仍然规范化为内部值 `agent`。 diff --git a/web/lib/content/tools.ts b/web/lib/content/tools.ts index 55671ad64d..d2c3ef059f 100644 --- a/web/lib/content/tools.ts +++ b/web/lib/content/tools.ts @@ -12,14 +12,14 @@ export const TOOLS_COPY = { { name: "edit", detail: { en: "path · edits", zh: "path · edits" } }, { name: "file_search", detail: { en: "Find workspace files by name, path, or glob pattern.", zh: "按文件名、路径或 glob 通配模式查找工作区文件。" } }, { name: "bash", detail: { en: "command · timeout?", zh: "command · timeout?" } }, - { name: "agent · workflow", detail: { en: "Delegate work and coordinate dependent phases.", zh: "委派工作并协调有依赖关系的阶段。" } }, + { name: "agent · workflow", detail: { en: "Loaded on demand via tool_search. Delegate work and coordinate dependent phases.", zh: "通过 tool_search 按需加载。委派工作并协调有依赖关系的阶段。" } }, { name: "todo_write", detail: { en: "Replace the task list with content and status entries.", zh: "用包含内容与状态的条目替换任务列表。" } }, - { name: "create_goal · get_goal · update_goal", detail: { en: "Create, inspect and update the active goal.", zh: "创建、查看与更新当前目标。" } }, + { name: "create_goal · get_goal · update_goal", detail: { en: "create_goal is available without discovery; get_goal/update_goal load on demand while a goal is active.", zh: "create_goal 无需搜索即可使用；get_goal/update_goal 仅在目标激活时按需加载。" } }, { name: "task_shell_wait", detail: { en: "Poll or stop a background shell task by task_id.", zh: "按 task_id 轮询或停止后台 shell 任务。" } }, { name: "load_skill", detail: { en: "Load a skill by exact name, or pass query to search the catalogue.", zh: "按确切名称加载技能，或传入 query 搜索技能目录。" } }, { name: "tool_search", detail: { en: "Discover policy-allowed tools. Each child has its own search and activation cache.", zh: "发现策略允许的工具。每个子 Agent 拥有独立的搜索与激活缓存。" } }, { name: "Git · Run · tasks · remember · Web · MCP · plugins", detail: { en: "Deferred: loaded by tool_search only when policy permits.", zh: "延迟加载：仅在策略允许时由 tool_search 加载。" } }, - { name: "8 names / 16 KiB", detail: { en: "Conversation toolbox cache; independent per child and revalidated every turn.", zh: "会话工具箱缓存；每个子 Agent 独立，每轮重新校验。" } }, + { name: "8 names / 24 KiB", detail: { en: "Conversation toolbox cache; independent per child and revalidated every turn.", zh: "会话工具箱缓存；每个子 Agent 独立，每轮重新校验。" } }, { name: "Web search/fetch", detail: { en: "Scout and Reviewer children can discover these without gaining mutation or arbitrary network authority.", zh: "侦察与审查子 Agent 可发现这些工具，但不会因此获得写入或任意网络权限。" } }, { name: "MCP", detail: { en: "mcp__ — registered from ~/.codewhale/mcp.json.", zh: "mcp__——从 ~/.codewhale/mcp.json 注册。" } }, ], diff --git a/web/lib/public-surface-contract.test.ts b/web/lib/public-surface-contract.test.ts index 6bd75ed2be..29d5207ac5 100644 --- a/web/lib/public-surface-contract.test.ts +++ b/web/lib/public-surface-contract.test.ts @@ -473,7 +473,7 @@ done expect(registry).toContain("Arc::new(LowercaseBashTool)"); expect(registry).toContain('FileTool::new("File")'); expect(registry).toContain('BashTool::new("Bash")'); - expect(toolsPage).toContain("8 names / 16 KiB"); + expect(toolsPage).toContain("8 names / 24 KiB"); expect(toolsPage).toContain("Web search/fetch"); expect(toolsPage).not.toContain("docs/TOOL_LIFECYCLE.md"); }); From a975bad49c29b59f192517617a71a88eca5d98c9 Mon Sep 17 00:00:00 2001 From: AdityaVG13 Date: Mon, 21 Sep 2026 01:43:48 -0400 Subject: [PATCH 15/26] feat(echo): squeeze soundings, edit echoes, grep/skill clamps Footer squeeze (all lossless): same-pod links/callers render bare, import roots a link already names drop by file stem or package path, framing tightens to [Pod (N)] / [Sound], and sounded paths are /-separated on every OS. The squeeze follows CODEWHALE_ECHO gates so an off:links trial measures links alone. Edit echoes name touched symbols plus impacted callers on write/edit; the diff rides metadata for the TUI while the receipt stays one line. Grep max_results is clamped with honest truncation, skill listings are capped, JS arrows sound as their source token, and ECHOLOCATION.md documents the surfaces. Signed-off-by: AdityaVG13 --- crates/tui/src/tools/diff_format.rs | 15 +- crates/tui/src/tools/echolocation.rs | 412 +++++++++++++++++++++++++-- crates/tui/src/tools/file.rs | 17 +- crates/tui/src/tools/file/tests.rs | 58 +++- crates/tui/src/tools/search.rs | 23 +- crates/tui/src/tools/search/tests.rs | 49 ++++ crates/tui/src/tools/skill.rs | 86 +++++- docs/ECHOLOCATION.md | 64 +++++ 8 files changed, 668 insertions(+), 56 deletions(-) create mode 100644 docs/ECHOLOCATION.md diff --git a/crates/tui/src/tools/diff_format.rs b/crates/tui/src/tools/diff_format.rs index bb3549b26b..9d83e84f58 100644 --- a/crates/tui/src/tools/diff_format.rs +++ b/crates/tui/src/tools/diff_format.rs @@ -1,13 +1,12 @@ -//! Build unified-diff strings for tool results. +//! Build unified-diff strings for tool-result metadata. //! //! `edit_file` and `write_file` capture the file contents before and after -//! the mutation and emit a unified diff at the head of their `ToolResult` -//! output. The TUI's `output_looks_like_diff` detector then routes the -//! payload through `diff_render::render_diff`, which renders it with line -//! numbers and coloured `+`/`-` gutters (#505). -//! -//! The diff is also a strict UX upgrade for the model — it sees exactly -//! which lines changed instead of a one-line summary. +//! the mutation and stash a unified diff in the result `metadata` (under +//! `mutation.diff`) for the TUI, which routes it through +//! `diff_render::render_diff` with line numbers and coloured `+`/`-` +//! gutters (#505). The model-facing `content` stays a one-line receipt — +//! the model learns what changed from the edit echo (touched symbols and +//! impacted callers), not from the diff. use similar::TextDiff; diff --git a/crates/tui/src/tools/echolocation.rs b/crates/tui/src/tools/echolocation.rs index d03436c322..9e59b757c4 100644 --- a/crates/tui/src/tools/echolocation.rs +++ b/crates/tui/src/tools/echolocation.rs @@ -68,6 +68,19 @@ const MAX_CALLERS: usize = 8; const MAX_CALLER_SCAN_BYTES: u64 = 256 * 1024; /// Tracked pod reads before the next read there enters the buzz. pub(crate) const BUZZ_VISIT_THRESHOLD: usize = 2; +/// Max touched symbols named per edit echo before "+N more". +const MAX_EDIT_TOUCHED: usize = 6; +/// Max impacted callers named per edit echo before "+N more". +const MAX_EDIT_CALLERS: usize = 4; +/// Hard byte budget for one edit echo line. +const EDIT_ECHO_MAX_BYTES: usize = 400; +/// Changed spans attributed per edit; past this the edit is a rewrite +/// and the first spans already name its symbols. +const MAX_EDIT_SPANS: usize = 32; +/// Time budget for the touched-symbol diff. Myers is quadratic in the +/// worst case; past this `similar` approximates instead of blocking +/// the edit path, and the echo degrades to first-span attribution. +const EDIT_DIFF_TIMEOUT_MS: u64 = 100; /// Ablation gate: true when `CODEWHALE_ECHO` disables `section`. /// Parsed once per process; unset or malformed means everything on. @@ -96,7 +109,6 @@ fn parse_echo_gates(raw: &str) -> BTreeSet { /// with nothing to say renders as the empty string. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct EchoChart { - pod_dir: Option, podmates: Vec, podmate_total: usize, symbols: Vec, @@ -109,6 +121,31 @@ pub(crate) struct EchoChart { caller_total: usize, } +/// Shorten workspace-relative links to bare names when they live in +/// the same pod (directory) as the sounded file: the pod frame makes +/// them unambiguous, and the mates list shows the same names. +/// Cross-pod links keep their relative path. Never fails closed the +/// wrong way — on any doubt the full path stays. +fn pod_relative_names(workspace: &Path, file_path: &Path, links: Vec) -> Vec { + let Some(home) = file_path.parent() else { + return links; + }; + links + .into_iter() + .map(|link| { + let short = workspace.join(&link).parent() == Some(home); + if short { + Path::new(&link) + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or(link) + } else { + link + } + }) + .collect() +} + /// Sound an echolocation chart for a file that was just read. /// `workspace` anchors link display (workspace-relative paths); /// `file_path` is the resolved path (for the sibling sounding); `text` @@ -117,7 +154,7 @@ pub(crate) struct EchoChart { /// Never fails: anything unreadable is omitted from the chart rather /// than failing the read. pub fn sound_echolocation(workspace: &Path, file_path: &Path, text: &str, buzz: bool) -> EchoChart { - let (pod_dir, mut podmates) = sound_podmates(file_path); + let (_, mut podmates) = sound_podmates(file_path); let podmate_total = podmates.len(); podmates.truncate(MAX_PODMATES); let (mut symbols, mut import_roots, mut links, link_total, mut callers, caller_total) = @@ -143,12 +180,38 @@ pub fn sound_echolocation(workspace: &Path, file_path: &Path, text: &str, buzz: }; let symbol_total = symbols.len(); symbols.truncate(MAX_SYMBOLS); + // Squeeze, all lossless: same-pod links and callers render bare + // (the pod frame disambiguates), and import roots a shown link + // already names drop — the link says the same thing with a path + // attached. Totals count what renders, so "+N more" stays honest. + // Each transform follows its section gate: an `off:links` trial + // measures links alone, never roots too. + if !echo_off("links") { + links = pod_relative_names(workspace, file_path, links); + } + if !echo_off("callers") { + callers = pod_relative_names(workspace, file_path, callers); + } + if !echo_off("links") && !echo_off("imports") { + let link_stems: Vec<&str> = links + .iter() + .filter_map(|link| Path::new(link).file_stem()?.to_str()) + .collect(); + import_roots.retain(|root| { + if link_stems.contains(&root.as_str()) { + return false; + } + // Package roots resolve under their own dir + // (`pkg` under `pkg/__init__.py`). + let prefix = format!("{root}/"); + !links.iter().any(|link| link.starts_with(&prefix)) + }); + } let import_total = import_roots.len(); import_roots.truncate(MAX_IMPORT_ROOTS); links.truncate(MAX_LINKS); callers.truncate(MAX_CALLERS); EchoChart { - pod_dir, podmates, podmate_total, symbols, @@ -191,12 +254,13 @@ impl EchoChart { /// sorted entries, fixed caps, byte budget, no timestamps. pub fn render_footer(&self) -> String { let mut out = String::new(); - if !echo_off("mates") && self.pod_dir.is_some() && self.podmate_total > 0 { - let dir = self.pod_dir.as_deref().unwrap_or("."); + // No dir: the footer trails the read of this exact file, so the + // pod frame is already known (miss echoes and grep pods keep + // theirs — those orient across paths, not within one). + if !echo_off("mates") && self.podmate_total > 0 { out.push_str(&format!( - "\n\n[Pod: {} ({}): {}]", - dir, - count_noun(self.podmate_total, "entry", "entries"), + "\n\n[Pod ({}): {}]", + self.podmate_total, render_capped(&self.podmates, self.podmate_total) )); } @@ -210,7 +274,7 @@ impl EchoChart { } if !echo_off("imports") && self.import_total > 0 { if !sounding.is_empty() { - sounding.push_str(" | "); + sounding.push_str("; "); } sounding.push_str(&format!( "imports: {}", @@ -219,7 +283,7 @@ impl EchoChart { } if !echo_off("links") && self.link_total > 0 { if !sounding.is_empty() { - sounding.push_str(" | "); + sounding.push_str("; "); } sounding.push_str(&format!( "links: {}", @@ -228,7 +292,7 @@ impl EchoChart { } if !echo_off("callers") && self.caller_total > 0 { if !sounding.is_empty() { - sounding.push_str(" | "); + sounding.push_str("; "); } sounding.push_str(&format!( "heard by: {}", @@ -236,7 +300,7 @@ impl EchoChart { )); } if !sounding.is_empty() { - out.push_str(&format!("\n\n[Sounding: {sounding}]")); + out.push_str(&format!("\n\n[Sound: {sounding}]")); } truncate_bytes(&out, POD_FOOTER_MAX_BYTES) } @@ -490,7 +554,7 @@ fn split_match_path(file: &str) -> (String, String) { let dir = path .parent() .filter(|parent| !parent.as_os_str().is_empty()) - .map(|parent| parent.to_string_lossy().into_owned()) + .map(slash_path) .unwrap_or_else(|| ".".to_string()); (dir, name) } @@ -603,23 +667,135 @@ fn sound_symbols(text: &str, lang: SoundLang) -> Vec { // Symbols carry 1-based source lines (`fn main:12`) so the model can // page straight at them — the sounding already sees past truncation, // and now it can point there too. - let numbered = |n: usize, symbol: String| format!("{symbol}:{n}"); + sound_numbered_symbols(text, lang) + .into_iter() + .map(|(n, symbol)| format!("{symbol}:{n}")) + .collect() +} + +/// Structured symbol sounding: 1-based line + bare `kind name`, in file +/// order. The edit echo attributes changed spans against these lines. +fn sound_numbered_symbols(text: &str, lang: SoundLang) -> Vec<(usize, String)> { match lang { SoundLang::Rust => code_lines(text) - .filter_map(|(n, line)| parse_item_line(line).map(|s| numbered(n, s))) + .filter_map(|(n, line)| parse_item_line(line).map(|s| (n, s))) .collect(), SoundLang::Python => py_code_lines(text) - .filter_map(|(n, line)| parse_py_item(line).map(|s| numbered(n, s))) + .filter_map(|(n, line)| parse_py_item(line).map(|s| (n, s))) .collect(), SoundLang::JavaScript => code_lines(text) - .filter_map(|(n, line)| parse_js_item(line).map(|s| numbered(n, s))) + .filter_map(|(n, line)| parse_js_item(line).map(|s| (n, s))) .collect(), SoundLang::Go => code_lines(text) - .filter_map(|(n, line)| parse_go_item(line).map(|s| numbered(n, s))) + .filter_map(|(n, line)| parse_go_item(line).map(|s| (n, s))) .collect(), } } +/// Changed new-side spans as (1-based start, new-line count). Pure +/// deletions contribute their insertion point with length 0. Reuses +/// the `similar` diff the unified-diff builder already depends on, +/// under a hard timeout. +fn changed_spans(before: &str, after: &str) -> Vec<(usize, usize)> { + if before == after { + return Vec::new(); + } + similar::TextDiff::configure() + .timeout(std::time::Duration::from_millis(EDIT_DIFF_TIMEOUT_MS)) + .diff_lines(before, after) + .ops() + .iter() + .filter_map(|op| match op { + similar::DiffOp::Equal { .. } => None, + similar::DiffOp::Delete { new_index, .. } => Some((new_index + 1, 0)), + similar::DiffOp::Insert { + new_index, new_len, .. + } => Some((new_index + 1, *new_len)), + similar::DiffOp::Replace { + new_index, new_len, .. + } => Some((new_index + 1, *new_len)), + }) + .take(MAX_EDIT_SPANS) + .collect() +} + +/// Symbols the mutation touched, in file order: declarations born +/// inside a changed span, else the nearest declaration above it (the +/// edit landed inside that symbol's body). Returns the shown names +/// plus the distinct total for the "+N more" tail. +fn touched_symbols(file_path: &Path, before: &str, after: &str) -> (Vec, usize) { + let empty = (Vec::new(), 0); + let Some(lang) = detect_lang(file_path) else { + return empty; + }; + let symbols = sound_numbered_symbols(after, lang); + if symbols.is_empty() { + return empty; + } + let mut touched: Vec = Vec::new(); + let mut touched_lines: Vec = Vec::new(); + for (start, len) in changed_spans(before, after) { + let mut born_inside = false; + for (line, name) in &symbols { + if *line >= start && line.saturating_sub(start) < len { + born_inside = true; + if !touched_lines.contains(line) { + touched_lines.push(*line); + touched.push(format!("{name}:{line}")); + } + } + } + if !born_inside + && let Some((line, name)) = symbols.iter().rev().find(|(line, _)| *line <= start) + && !touched_lines.contains(line) + { + touched_lines.push(*line); + touched.push(format!("{name}:{line}")); + } + } + let total = touched.len(); + touched.truncate(MAX_EDIT_TOUCHED); + (touched, total) +} + +/// Model-visible edit echo: which symbols the write/edit touched and +/// which podmates link this file (impacted callers). The model-facing +/// mutation receipt is one line and the diff rides metadata for the +/// TUI, so without this the model never learns what its edit hit. +/// `None` when there is nothing to say. Gated by the existing +/// `symbols`/`callers` sections; budgeted hard. +pub fn sound_edit_echo( + workspace: &Path, + file_path: &Path, + before: &str, + after: &str, +) -> Option { + if before == after { + return None; + } + let mut halves = Vec::new(); + if !echo_off("symbols") { + let (shown, total) = touched_symbols(file_path, before, after); + if total > 0 { + halves.push(format!("touched {}", render_capped(&shown, total))); + } + } + if !echo_off("callers") { + let (callers, total) = sound_callers(workspace, file_path); + if total > 0 { + let shown = &callers[..callers.len().min(MAX_EDIT_CALLERS)]; + halves.push(format!("heard by {}", render_capped(shown, total))); + } + } + if halves.is_empty() { + return None; + } + Some(truncate_bytes( + &format!("[Edit echo: {}]", halves.join("; ")), + EDIT_ECHO_MAX_BYTES, + )) +} + /// Distinct import roots, sorted, per language. fn sound_import_roots(text: &str, lang: SoundLang) -> BTreeSet { match lang { @@ -1007,10 +1183,20 @@ fn find_crate_root(workspace: &Path, file_path: &Path) -> Option { } } +/// Render a path with `/` separators on every platform, so sounded +/// paths are byte-identical on Windows, macOS, and Linux (portable +/// tests, shared prompt caches). Components are re-joined rather than +/// string-replaced so a literal `\` in a file name survives, and `.` +/// / `..` segments pass through untouched for [`dot_clean`]. +fn slash_path(path: &Path) -> String { + path.components() + .map(|component| component.as_os_str().to_string_lossy()) + .collect::>() + .join("/") +} + fn workspace_relative(workspace: &Path, path: &Path) -> Option { - path.strip_prefix(workspace) - .ok() - .map(|rel| rel.to_string_lossy().into_owned()) + path.strip_prefix(workspace).ok().map(slash_path) } /// Python source lines: `#` comments cut, triple-quote doc spans @@ -1274,7 +1460,7 @@ fn parse_js_item(line: &str) -> Option { || strip_keyword(body, "function").is_some() || body.contains("=>") { - return Some(format!("function {name}")); + return Some(format!("{qualifier} {name}")); } return None; } @@ -1766,7 +1952,7 @@ define nothing; let second = sound_echolocation(dir.path(), &path, &text, false).render_footer(); assert_eq!(first, second); assert!(first.len() <= POD_FOOTER_MAX_BYTES, "{}", first.len()); - assert!(first.starts_with("\n\n[Pod:"), "{first}"); + assert!(first.starts_with("\n\n[Pod ("), "{first}"); } #[test] @@ -1780,8 +1966,8 @@ define nothing; false, ) .render_footer(); - assert!(footer.contains("[Pod:"), "{footer}"); - assert!(!footer.contains("[Sounding:"), "{footer}"); + assert!(footer.contains("[Pod ("), "{footer}"); + assert!(!footer.contains("[Sound:"), "{footer}"); } #[test] @@ -2030,7 +2216,26 @@ define nothing; false, ) .render_footer(); - assert!(footer.contains("links: src/config.rs"), "{footer}"); + assert!(footer.contains("links: config.rs"), "{footer}"); + } + + #[test] + fn shadowed_import_roots_drop_by_stem_and_package_path() { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path(); + let text = "import a\nimport pkg\n"; + fs::write(root.join("main.py"), text).expect("fixture"); + fs::write(root.join("a.py"), "x = 1\n").expect("fixture"); + fs::create_dir(root.join("pkg")).expect("mkdir"); + fs::write(root.join("pkg/__init__.py"), "y = 2\n").expect("fixture"); + let footer = sound_echolocation(root, &root.join("main.py"), text, false).render_footer(); + // Both roots are named by their links: file stem (`a` under + // `a.py`) and package path (`pkg` under `pkg/__init__.py`). + // Sounded paths are `/`-separated on every OS; vacuous on + // unix, load-bearing on Windows CI. + assert!(!footer.contains("imports:"), "{footer}"); + assert!(footer.contains("links: a.py, pkg/__init__.py"), "{footer}"); + assert!(!footer.contains('\\'), "{footer}"); } #[test] @@ -2111,7 +2316,7 @@ define nothing; let second = sound_echolocation(root, &root.join("src/net/server.rs"), text, true).render_footer(); assert_eq!(first, second); - assert!(first.contains("heard by: src/net/mod.rs"), "{first}"); + assert!(first.contains("heard by: mod.rs"), "{first}"); } #[test] @@ -2231,7 +2436,7 @@ if (x) { y(); } "{symbols:?}" ); assert!( - symbols.contains(&"function dive:10".to_string()), + symbols.contains(&"const dive:10".to_string()), "{symbols:?}" ); assert!( @@ -2433,7 +2638,7 @@ func main() { fs::write(dir.path().join("other.py"), "x = 1\n").expect("fixture"); let footer = sound_echolocation(dir.path(), &dir.path().join("swim.py"), PY_SAMPLE, false) .render_footer(); - assert!(footer.contains("[Pod:"), "{footer}"); + assert!(footer.contains("[Pod ("), "{footer}"); assert!(footer.contains("def swim"), "{footer}"); assert!(footer.contains("imports:"), "{footer}"); } @@ -2565,6 +2770,155 @@ func main() { assert!(!raws.contains(&"vault_link"), "{raws:?}"); } + #[test] + fn tmp_measure_footer_breakdown() { + // TEMPORARY squeeze probe; removed after the numbers land. + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let samples = [ + "crates/tui/src/tools/echolocation.rs", + "crates/tui/src/tools/file.rs", + "crates/tui/src/tools/search.rs", + "crates/tui/src/tools/diff_format.rs", + "crates/tui/src/tools/skill.rs", + "crates/tui/src/core/engine/tool_catalog.rs", + "crates/tui/src/skills/mod.rs", + "crates/workflow/src/lib.rs", + "web/lib/content/tools.ts", + ]; + for rel in samples { + let path = root.join(rel); + let Ok(text) = fs::read_to_string(&path) else { + continue; + }; + let chart = sound_echolocation(&root, &path, &text, true); + let footer = chart.render_footer(); + let mates: usize = chart.podmates.iter().map(|m| m.len() + 2).sum(); + let syms: usize = chart.symbols.iter().map(|m| m.len() + 2).sum(); + let imps: usize = chart.import_roots.iter().map(|m| m.len() + 2).sum(); + let links: usize = chart.links.iter().map(|m| m.len() + 2).sum(); + let calls: usize = chart.callers.iter().map(|m| m.len() + 2).sum(); + // Import roots shadowed by a resolved link (same stem). + let link_stems: Vec<&str> = chart + .links + .iter() + .filter_map(|l| l.rsplit('/').next()?.split('.').next()) + .collect(); + let shadowed = chart + .import_roots + .iter() + .filter(|r| link_stems.contains(&r.as_str())) + .count(); + eprintln!( + "BREAKDOWN {rel} file={}B footer={}B mates={}B({}/{}) syms={}B({}/{}) imps={}B({}/{},shadow={}) links={}B({}/{}) calls={}B({}/{})", + text.len(), + footer.len(), + mates, + chart.podmates.len(), + chart.podmate_total, + syms, + chart.symbols.len(), + chart.symbol_total, + imps, + chart.import_roots.len(), + chart.import_total, + shadowed, + links, + chart.links.len(), + chart.link_total, + calls, + chart.callers.len(), + chart.caller_total, + ); + } + } + + #[test] + fn block_comments_mute_and_release_across_lines() { + let text = "/*\nfn fake() {}\n*/\nfn real() {}\n"; + let symbols = sound_symbols(text, SoundLang::Rust); + assert_eq!(symbols, ["fn real:4"]); + // A comment opener inside a string still mutes until the closer: + // the failure mode is a miss, never a forge. + let text = "let s = \"/*\";\nfn muted() {}\nlet t = \"*/\";\nfn kept() {}\n"; + let symbols = sound_symbols(text, SoundLang::Rust); + assert_eq!(symbols, ["fn kept:4"]); + } + + #[test] + fn py_decorators_methods_and_comment_fences() { + let text = "@route(\"/x\")\ndef handler():\n pass\nclass Pod:\n def swim(self):\n pass\n# a \"\"\" in a comment toggles nothing\ndef after():\n"; + let symbols = sound_symbols(text, SoundLang::Python); + assert!( + symbols.contains(&"def handler:2".to_string()), + "{symbols:?}" + ); + assert!(symbols.contains(&"class Pod:4".to_string()), "{symbols:?}"); + assert!(symbols.contains(&"def swim:5".to_string()), "{symbols:?}"); + assert!(symbols.contains(&"def after:8".to_string()), "{symbols:?}"); + } + + #[test] + fn js_anonymous_and_arrow_shapes() { + // Anonymous default export: no name to sound, no phantom. + assert!(sound_symbols("export default function() {}\n", SoundLang::JavaScript).is_empty()); + let symbols = sound_symbols( + "const swim = () => {};\nlet dive = async () => {};\nvar data = 42;\n", + SoundLang::JavaScript, + ); + assert!(symbols.contains(&"const swim:1".to_string()), "{symbols:?}"); + assert!(symbols.contains(&"let dive:2".to_string()), "{symbols:?}"); + assert!(!symbols.iter().any(|s| s.contains("data")), "{symbols:?}"); + } + + #[test] + fn edit_echo_names_touched_symbol_and_callers() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("a.py"), "import b\nprint(b.x)\n").expect("fixture"); + let before = "x = 1\ndef swim():\n return x\n"; + let after = "x = 1\ndef swim():\n return x + 1\n"; + fs::write(dir.path().join("b.py"), after).expect("fixture"); + let echo = + sound_edit_echo(dir.path(), &dir.path().join("b.py"), before, after).expect("echo"); + assert!(echo.contains("touched def swim:2"), "{echo}"); + assert!(echo.contains("heard by a.py"), "{echo}"); + } + + #[test] + fn edit_echo_attributes_body_edits_to_enclosing_symbol() { + let dir = tempfile::tempdir().expect("tempdir"); + // No mates: touched only, no heard-by half. + let before = "fn main() {\n let x = 1;\n println!(\"{x}\");\n}\nfn other() {}\n"; + let after = "fn main() {\n let x = 2;\n println!(\"{x}\");\n}\nfn other() {}\n"; + fs::write(dir.path().join("solo.rs"), after).expect("fixture"); + let echo = + sound_edit_echo(dir.path(), &dir.path().join("solo.rs"), before, after).expect("echo"); + assert_eq!(echo, "[Edit echo: touched fn main:1]"); + } + + #[test] + fn edit_echo_silent_when_nothing_to_say() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("note.txt"), "hi\n").expect("fixture"); + // Identical: no mutation, no echo. + assert!( + sound_edit_echo(dir.path(), &dir.path().join("note.txt"), "hi\n", "hi\n").is_none() + ); + // Unknown language, no mates: neither half can sound. + assert!( + sound_edit_echo(dir.path(), &dir.path().join("note.txt"), "hi\n", "yo\n").is_none() + ); + } + + #[test] + fn edit_echo_new_file_names_first_symbols() { + let dir = tempfile::tempdir().expect("tempdir"); + let after = "fn alpha() {}\nfn beta() {}\n"; + fs::write(dir.path().join("fresh.rs"), after).expect("fixture"); + let echo = + sound_edit_echo(dir.path(), &dir.path().join("fresh.rs"), "", after).expect("echo"); + assert!(echo.contains("touched fn alpha:1, fn beta:2"), "{echo}"); + } + #[test] fn unlistable_pod_fallback_skips_dotfiles_like_the_listed_path() { let dir = tempfile::tempdir().expect("tempdir"); diff --git a/crates/tui/src/tools/file.rs b/crates/tui/src/tools/file.rs index c98f644152..666260efa3 100644 --- a/crates/tui/src/tools/file.rs +++ b/crates/tui/src/tools/file.rs @@ -818,6 +818,17 @@ async fn contract_mutation_result( ) -> ToolResult { let paths = [file_path.to_path_buf()]; let diagnostics = lsp_diagnostics_for_paths(context, &paths).await; + // Echolocation: the model-facing receipt is one line while the diff + // rides metadata for the TUI, so the echo tells the model what its + // edit hit (touched symbols) and what hears it (impacted callers). + // Deterministic, hard-budgeted, None when there is nothing to say. + let mut summary = summary; + if let Some(echo) = + crate::tools::echolocation::sound_edit_echo(&context.workspace, file_path, before, after) + { + summary.push_str("\n\n"); + summary.push_str(&echo); + } ToolResult::success(summary).with_metadata(json!({ "event": "file.mutation", "lsp_diagnostics": diagnostics, @@ -1772,7 +1783,9 @@ impl WriteFileTool { drop(mutation_guard); let outcome = if existed_before { "updated" } else { "created" }; - let utf16_units = written.encode_utf16().count(); + // Bytes are bytes: the receipt reports what was written to disk, + // not UTF-16 code units (which differ on non-ASCII content). + let bytes_written = written.len(); Ok(contract_mutation_result( context, &file_path, @@ -1781,7 +1794,7 @@ impl WriteFileTool { &written, written.as_bytes(), outcome, - format!("Successfully wrote {utf16_units} bytes to {path_str}"), + format!("Successfully wrote {bytes_written} bytes to {path_str}"), ) .await) } diff --git a/crates/tui/src/tools/file/tests.rs b/crates/tui/src/tools/file/tests.rs index 6faf9266aa..fe8ff161a6 100644 --- a/crates/tui/src/tools/file/tests.rs +++ b/crates/tui/src/tools/file/tests.rs @@ -102,12 +102,12 @@ async fn contract_read_appends_a_pod_footer_to_a_whole_source_file() { "whole file first, pod footer after" ); assert!( - result.content.contains("[Pod:"), + result.content.contains("[Pod ("), "pod footer present: {}", result.content ); assert!( - result.content.contains("[Sounding: 1 symbol: fn main:1]"), + result.content.contains("[Sound: 1 symbol: fn main:1]"), "sounded symbol present: {}", result.content ); @@ -404,7 +404,7 @@ async fn contract_read_metadata_for_ordinary_whole_read() { "whole file first, pod footer after" ); assert!( - result.content.contains("[Pod:"), + result.content.contains("[Pod ("), "pod footer present: {}", result.content ); @@ -784,6 +784,56 @@ async fn queued_parallel_contract_edits_preserve_both_changes() { ); } +#[tokio::test] +async fn contract_edit_echoes_touched_symbols_and_callers() { + let temporary = tempfile::tempdir().expect("tempdir"); + let context = ToolContext::new(temporary.path()); + std::fs::write(temporary.path().join("a.py"), "import b\nprint(b.swim())\n").expect("fixture"); + std::fs::write(temporary.path().join("b.py"), "def swim():\n return 1\n").expect("fixture"); + let result = EditFileTool::execute_contract_edits( + json!({ + "path": "b.py", + "edits": [{"oldText": "return 1", "newText": "return 2"}] + }), + &context, + ) + .await + .expect("edit runs"); + assert!( + result.content.contains("Successfully replaced"), + "{}", + result.content + ); + assert!( + result.content.contains("[Edit echo: touched def swim:1"), + "{}", + result.content + ); + assert!( + result.content.contains("heard by a.py"), + "{}", + result.content + ); +} + +#[tokio::test] +async fn contract_write_receipt_counts_bytes_not_utf16_units() { + // "héllo\n" is 7 bytes but 6 UTF-16 units; the receipt must say 7. + let temporary = tempfile::tempdir().expect("tempdir"); + let context = ToolContext::new(temporary.path()); + let result = WriteFileTool::execute_contract_write( + json!({"path": "note.txt", "content": "héllo\n"}), + &context, + ) + .await + .expect("write runs"); + assert!( + result.content.contains("Successfully wrote 7 bytes"), + "{}", + result.content + ); +} + #[tokio::test] async fn cancelled_queued_pi_write_never_starts() { let temporary = tempfile::tempdir().expect("tempdir"); @@ -1248,7 +1298,7 @@ async fn contract_read_enters_terminal_buzz_on_third_pod_visit() { .await .expect("read result"); assert!( - result.content.contains("heard by: src/net/mod.rs"), + result.content.contains("heard by: mod.rs"), "{}", result.content ); diff --git a/crates/tui/src/tools/search.rs b/crates/tui/src/tools/search.rs index 5afe813a81..d715a8505b 100644 --- a/crates/tui/src/tools/search.rs +++ b/crates/tui/src/tools/search.rs @@ -95,7 +95,7 @@ impl ToolSpec for GrepFilesTool { }, "max_results": { "type": "integer", - "description": "Maximum number of results to return (default: 100)" + "description": "Maximum number of results to return (default: 100, max: 1000)" } }, "required": ["pattern"] @@ -122,8 +122,12 @@ impl ToolSpec for GrepFilesTool { .unwrap_or(usize::MAX) .min(1000); let case_insensitive = optional_bool(&input, "case_insensitive", false)?; + // Bounded like file_search's limit: an explicit value is the + // model's choice, but a runaway (or typo) must not mint an + // unbounded JSON array. let max_results = usize::try_from(optional_u64(&input, "max_results", MAX_RESULTS as u64)?) - .unwrap_or(MAX_RESULTS); + .unwrap_or(MAX_RESULTS) + .clamp(1, 1000); // Parse include patterns let include_patterns: Vec = input @@ -423,6 +427,8 @@ fn search_file_streaming( // lines still needed). Entries complete in FIFO order. let mut pending: VecDeque<(usize, usize)> = VecDeque::new(); let mut line_idx = 0usize; + // True when a match exists past the collected budget, so the caller + // can report truncation honestly instead of inferring it. let mut overflowed = false; loop { @@ -460,9 +466,11 @@ fn search_file_streaming( pending.pop_front(); } - if matches.len() >= budget { - overflowed = overflowed || regex.is_match(line); - } else if regex.is_match(line) { + // Past the budget we keep testing but stop collecting; the + // first surplus match proves the cut. With nothing pending (no + // after-context owed) we can stop scanning immediately. + let is_match = regex.is_match(line); + if matches.len() < budget && is_match { matches.push(GrepMatch { file: relative_path.to_string(), line_number: line_idx + 1, @@ -473,6 +481,11 @@ fn search_file_streaming( if context_lines > 0 { pending.push_back((matches.len() - 1, context_lines)); } + } else if is_match { + overflowed = true; + if pending.is_empty() { + break; + } } if context_lines > 0 { diff --git a/crates/tui/src/tools/search/tests.rs b/crates/tui/src/tools/search/tests.rs index 8cb4064b9a..32af35f566 100644 --- a/crates/tui/src/tools/search/tests.rs +++ b/crates/tui/src/tools/search/tests.rs @@ -8,6 +8,55 @@ use crate::tools::spec::{ApprovalRequirement, ToolContext, ToolSpec}; use super::{GrepFilesTool, matches_glob}; +#[tokio::test] +async fn grep_max_results_is_clamped_not_trusted() { + let tmp = tempdir().expect("tempdir"); + let body = "needle here\n".repeat(1_200); + fs::write(tmp.path().join("hay.txt"), &body).expect("write"); + let ctx = ToolContext::new(tmp.path()); + let result = GrepFilesTool + .execute(json!({"pattern": "needle", "max_results": 1_000_000}), &ctx) + .await + .expect("grep runs"); + let payload: Value = serde_json::from_str(&result.content).expect("json"); + assert_eq!(payload["matches"].as_array().expect("matches").len(), 1000); + assert_eq!(payload["truncated"], true); + assert_eq!(payload["total_matches"], 1000); +} + +#[tokio::test] +async fn grep_truncated_fires_only_on_a_proven_cut() { + let tmp = tempdir().expect("tempdir"); + // Exactly max: the walk completes, nothing cut, flag stays false. + fs::write(tmp.path().join("exact.txt"), "needle\n".repeat(3)).expect("write"); + // Over max across files: the second file proves the cut. + fs::write(tmp.path().join("a.txt"), "needle\n".repeat(2)).expect("write"); + fs::write(tmp.path().join("b.txt"), "needle\n".repeat(5)).expect("write"); + let ctx = ToolContext::new(tmp.path()); + + let result = GrepFilesTool + .execute( + json!({"pattern": "needle", "path": "exact.txt", "max_results": 3}), + &ctx, + ) + .await + .expect("grep runs"); + let payload: Value = serde_json::from_str(&result.content).expect("json"); + assert_eq!(payload["matches"].as_array().expect("matches").len(), 3); + assert_eq!(payload["truncated"], false); + + let result = GrepFilesTool + .execute( + json!({"pattern": "needle", "path": "a.txt", "max_results": 1}), + &ctx, + ) + .await + .expect("grep runs"); + let payload: Value = serde_json::from_str(&result.content).expect("json"); + assert_eq!(payload["matches"].as_array().expect("matches").len(), 1); + assert_eq!(payload["truncated"], true); +} + #[test] fn grep_description_matches_default_exclusion_behavior() { let description = GrepFilesTool.description(); diff --git a/crates/tui/src/tools/skill.rs b/crates/tui/src/tools/skill.rs index 50fce5ce56..2e2a4361a9 100644 --- a/crates/tui/src/tools/skill.rs +++ b/crates/tui/src/tools/skill.rs @@ -18,11 +18,17 @@ use async_trait::async_trait; use serde_json::{Value, json}; use crate::skills::{ - Skill, SkillDiscoveryMode, SkillSource, discover_for_workspace_and_dir_with_mode_and_plugins, + MAX_SKILL_DESCRIPTION_CHARS, Skill, SkillDiscoveryMode, SkillSource, + discover_for_workspace_and_dir_with_mode_and_plugins, discover_in_workspace_with_mode_and_plugins, skill_directories_for_workspace_and_dir, skills_directories_for_mode, }; +/// Max rows in a `load_skill` listing. Past this an unbounded dump costs +/// more than it routes; the overflow line keeps the count honest and +/// every skill stays loadable by exact name. +const LOAD_SKILL_LIST_MAX_ROWS: usize = 100; + use super::spec::{ ApprovalRequirement, ToolCapability, ToolContext, ToolError, ToolResult, ToolSpec, }; @@ -103,10 +109,11 @@ impl ToolSpec for LoadSkillTool { } .into_enabled(); - // Listing mode: empty name, "*", or "list" returns the full registry (#4651). - // A `query` filters that listing over the same routing metadata the - // ambient index carries, so a truncated index does not force the model - // to pull every skill to find one. + // Listing mode: empty name, "*", or "list" returns the full registry (#4651), + // capped at LOAD_SKILL_LIST_MAX_ROWS with descriptions shortened to the same + // ceiling the prompt index uses. A `query` filters that listing over the same + // routing metadata the ambient index carries, so a truncated index does not + // force the model to pull every skill to find one. let query = input .get("query") .and_then(Value::as_str) @@ -136,13 +143,29 @@ impl ToolSpec for LoadSkillTool { } else { format!("Skills matching {:?} ({}):\n", query, skills.len()) }; - for skill in skills { - if skill.description.trim().is_empty() { + for skill in skills.iter().take(LOAD_SKILL_LIST_MAX_ROWS) { + let description = skill.description.trim(); + if description.is_empty() { listing.push_str(&format!(" - {}\n", skill.name)); } else { - listing.push_str(&format!(" - {} — {}\n", skill.name, skill.description)); + let capped: String = description + .chars() + .take(MAX_SKILL_DESCRIPTION_CHARS) + .collect(); + let marker = if capped.len() < description.len() { + "…" + } else { + "" + }; + listing.push_str(&format!(" - {} — {capped}{marker}\n", skill.name)); } } + if skills.len() > LOAD_SKILL_LIST_MAX_ROWS { + listing.push_str(&format!( + " ... and {} more installed; load any by exact name.\n", + skills.len() - LOAD_SKILL_LIST_MAX_ROWS + )); + } return Ok(ToolResult::success(listing)); } @@ -649,6 +672,53 @@ mod tests { } } + #[tokio::test] + async fn execute_listing_caps_rows_and_descriptions_honestly() { + let _lock = crate::test_support::lock_test_env(); + let tmp = tempdir().unwrap(); + let _home = crate::test_support::EnvVarGuard::set("HOME", tmp.path().join("home")); + let _cw_home = + crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", tmp.path().join("cw-home")); + let workspace = tmp.path().to_path_buf(); + let skills_dir = workspace.join(".codewhale").join("skills"); + let long = "d".repeat(500); + for index in 0..105 { + let description = if index == 0 { long.as_str() } else { "Short." }; + write_skill( + &skills_dir, + &format!("skill-{index:03}"), + description, + "Body.", + ); + } + + let context = ToolContext::new(workspace); + let result = LoadSkillTool + .execute(json!({"name": "list"}), &context) + .await + .expect("listing should succeed"); + assert!( + result.content.contains("Available skills (105)"), + "{}", + result.content + ); + assert!( + result + .content + .contains("... and 5 more installed; load any by exact name."), + "{}", + result.content + ); + assert!(!result.content.contains("skill-104"), "{}", result.content); + let capped: String = "d".repeat(400); + assert!( + result.content.contains(&format!("skill-000 — {capped}…")), + "{}", + result.content + ); + assert!(!result.content.contains(&long), "description must shorten"); + } + #[tokio::test] async fn execute_listing_reports_empty_registry_plainly() { let _lock = crate::test_support::lock_test_env(); diff --git a/docs/ECHOLOCATION.md b/docs/ECHOLOCATION.md new file mode 100644 index 0000000000..5b34830adf --- /dev/null +++ b/docs/ECHOLOCATION.md @@ -0,0 +1,64 @@ +# Echolocation + +Stateless, index-free navigation soundings. File tools echo their +surroundings fresh from source — no index to build, no background work, +no new dependencies. Every sounding is deterministic (sorted, capped, +hard-budgeted, `/`-separated paths on every OS), so a repeated call is +byte-identical and cache-friendly, and soundings never enter the +session-pinned prefix: they ride tool results (append-only history) +only. + +## What each surface emits + +- **read** appends a footer: `[Pod (N): …]` sibling mates (bare + names — the dir is the file just read, so it goes unstated) plus + `[Sound: N symbols: …; imports: …; links: …]` (line-numbered + symbols, import roots, cross-pod file links; same-pod links render + bare and import roots a link already names (file stem or package + path) are dropped). Budget: 1200B. +- **Terminal buzz**: after two distinct files are read in one pod, the + next read there adds `heard by …` — podmates whose resolved links + include this file. Sparse clicks while searching, high-resolution + echoes at close range. +- **read miss** (`NotFound` only — never on denylist/permission refusals) + appends `[Miss echo: ]`: podmates of the missing file's parent, + so a typo still orients the next call. +- **grep / file_search** chart each matched dir: mates with `(match)` + marks, under the calling search's own visibility gates (denylist, + excludes, extensions, gitignore). Pods never enumerate names the + calling search hides. +- **project_map** sounds each key file: one line of `name:line` symbols + per file, so the map says what lives where. +- **write / edit** append `[Edit echo: touched …; heard by …]`: the + symbols the mutation touched (nearest declaration at or above each + changed span, via the same `similar` diff that builds the unified + diff, under a 100ms timeout) and the podmates linking the file. + The model-facing receipt is one line and the diff rides metadata + for the TUI, so the echo is how the model learns what its edit hit. + +## Languages + +Symbol, import, and link sounding covers Rust, Python, JavaScript / +TypeScript, and Go via deliberately small line scanners (see +`crates/tui/src/tools/echolocation.rs` for the documented lookalike +classes). Anything unresolvable is omitted, never guessed; links +require an on-disk target. + +## Tuning and ablation + +`CODEWHALE_ECHO` gates sections for trials +(`off:`, comma-separated; names: `mates`, `symbols`, `imports`, +`links`, `callers`, `pods`, `soundings`). Unset or malformed means +everything on. The gates are process-pinned and read-only; there is no +per-session or per-call control, by design. The squeeze follows the +same gates, so an `off:links` trial measures links alone. + +## Non-goals + +- No tool description teaches the footer format: the brackets are + self-describing, and every description byte on an eager tool taxes + the session prefix. The diet stops where behavior would degrade. +- No cross-pod callers: reverse links scan same-dir mates only, so the + cost of a sounding is bounded by one directory listing plus a + bounded re-read of small mates. +- No `patch` (multi-file) echoes yet; single-file write/edit only. From 6bfe62482cd6b558f2e964e4a31f3d1fec8cf837 Mon Sep 17 00:00:00 2001 From: AdityaVG13 Date: Mon, 21 Sep 2026 02:39:03 -0400 Subject: [PATCH 16/26] feat(echo): recover relative-import leaf names, drop temp probe Relative imports now sound their leaf instead of a bare dot (from . import sib -> sib, ./server -> server), which composes with stem-shadowing when the link resolves and names the module when it does not. Removes the temporary footer-breakdown probe (its numbers landed in the PR; the eprintln tripped the print_stderr deny and flipped Lint red). Signed-off-by: AdityaVG13 --- crates/tui/src/tools/echolocation.rs | 117 +++++++++------------------ 1 file changed, 37 insertions(+), 80 deletions(-) diff --git a/crates/tui/src/tools/echolocation.rs b/crates/tui/src/tools/echolocation.rs index 9e59b757c4..cc4b01f447 100644 --- a/crates/tui/src/tools/echolocation.rs +++ b/crates/tui/src/tools/echolocation.rs @@ -1264,9 +1264,10 @@ fn parse_py_item(line: &str) -> Option { None } -/// Import roots of one line: every `import a, b.c` member, or the -/// `from` module (`a.b` → `a`; pure-relative `from . import x` → `.` -/// for depth, honest). Semicolons split statements. +/// Import roots of one line: every `import a, b.c` member, the +/// `from` module (`a.b` → `a`), or the import-list leaves for +/// pure-relative modules (`from . import sib` → `sib`). Semicolons +/// split statements. fn parse_py_import_roots(line: &str) -> Vec { let line = line.trim_start(); let mut out = Vec::new(); @@ -1283,7 +1284,26 @@ fn parse_py_import_roots(line: &str) -> Vec { return out; } if let Some(rest) = strip_keyword(line, "from") { - let module = rest.split(" import ").next().unwrap_or("").trim(); + let mut parts = rest.split(" import "); + let module = parts.next().unwrap_or("").trim(); + if module.trim_start_matches('.').is_empty() { + // Pure-relative: the module names no package, so leaf names + // come from the import list (`from . import sib` → `sib`), + // which stem-shadowing then drops when the link resolves. + let list = parts.next().unwrap_or(""); + for part in list.split(',') { + let name = part + .split(" as ") + .next() + .unwrap_or("") + .trim_matches(['(', ')', ' ', '\t']); + let leaf = name.split('.').next().unwrap_or("").trim(); + if let Some(root) = valid_root(leaf) { + out.push(root); + } + } + return out; + } let root = py_module_root(module); if !root.is_empty() { out.push(root); @@ -1292,17 +1312,13 @@ fn parse_py_import_roots(line: &str) -> Vec { out } -/// Root of a Python module path: leading dots collapse to their depth -/// when no named segment leads (`from . import x` → `.`). Named roots -/// validate as identifiers; dots pass through as depth markers. +/// Root of a Python module path: first named segment after any +/// leading dots (`..parent.x` → `parent`). Pure-relative modules never +/// reach here — the caller takes leaf names from the import list. fn py_module_root(module: &str) -> String { let dots = module.chars().take_while(|c| *c == '.').count(); let named = module[dots..].split('.').next().unwrap_or("").trim(); - if named.is_empty() { - ".".repeat(dots.max(1)) - } else { - valid_root(named).unwrap_or_default() - } + valid_root(named).unwrap_or_default() } fn valid_root(root: &str) -> Option { @@ -1483,13 +1499,16 @@ fn ident_head(rest: &str) -> Option { Some(name) } -/// JS/TS import roots: relative → `.`; scoped `@s/n` kept whole; -/// otherwise the first path segment. Covers `import`, `export-from`, -/// `require`, and dynamic `import()`. +/// JS/TS import roots: relative specs resolve to the first named +/// segment past the dots (`./server` → `server`); scoped `@s/n` kept +/// whole; otherwise the first path segment. Covers `import`, +/// `export-from`, `require`, and dynamic `import()`. fn parse_js_import_root(line: &str) -> Option { let spec = js_import_spec(line)?; if spec.starts_with('.') { - return Some(".".to_string()); + let tail = spec.trim_start_matches('.').trim_start_matches('/'); + let first = tail.split('/').next().unwrap_or("").trim(); + return valid_root(first); } if let Some(scoped) = spec.strip_prefix('@') { let mut parts = scoped.split('/'); @@ -2369,7 +2388,7 @@ define_not_a_keyword = True let roots: Vec = sound_import_roots(PY_SAMPLE, SoundLang::Python) .into_iter() .collect(); - assert_eq!(roots, [".", "localmod", "os", "parent", "pkg", "sys"]); + assert_eq!(roots, ["localmod", "os", "parent", "pkg", "sibling", "sys"]); } #[test] @@ -2455,7 +2474,7 @@ if (x) { y(); } let roots: Vec = sound_import_roots(JS_SAMPLE, SoundLang::JavaScript) .into_iter() .collect(); - assert_eq!(roots, [".", "@scope/pkg", "fs", "react"]); + assert_eq!(roots, ["@scope/pkg", "chunk", "fs", "react", "server", "setup"]); } #[test] @@ -2770,68 +2789,6 @@ func main() { assert!(!raws.contains(&"vault_link"), "{raws:?}"); } - #[test] - fn tmp_measure_footer_breakdown() { - // TEMPORARY squeeze probe; removed after the numbers land. - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); - let samples = [ - "crates/tui/src/tools/echolocation.rs", - "crates/tui/src/tools/file.rs", - "crates/tui/src/tools/search.rs", - "crates/tui/src/tools/diff_format.rs", - "crates/tui/src/tools/skill.rs", - "crates/tui/src/core/engine/tool_catalog.rs", - "crates/tui/src/skills/mod.rs", - "crates/workflow/src/lib.rs", - "web/lib/content/tools.ts", - ]; - for rel in samples { - let path = root.join(rel); - let Ok(text) = fs::read_to_string(&path) else { - continue; - }; - let chart = sound_echolocation(&root, &path, &text, true); - let footer = chart.render_footer(); - let mates: usize = chart.podmates.iter().map(|m| m.len() + 2).sum(); - let syms: usize = chart.symbols.iter().map(|m| m.len() + 2).sum(); - let imps: usize = chart.import_roots.iter().map(|m| m.len() + 2).sum(); - let links: usize = chart.links.iter().map(|m| m.len() + 2).sum(); - let calls: usize = chart.callers.iter().map(|m| m.len() + 2).sum(); - // Import roots shadowed by a resolved link (same stem). - let link_stems: Vec<&str> = chart - .links - .iter() - .filter_map(|l| l.rsplit('/').next()?.split('.').next()) - .collect(); - let shadowed = chart - .import_roots - .iter() - .filter(|r| link_stems.contains(&r.as_str())) - .count(); - eprintln!( - "BREAKDOWN {rel} file={}B footer={}B mates={}B({}/{}) syms={}B({}/{}) imps={}B({}/{},shadow={}) links={}B({}/{}) calls={}B({}/{})", - text.len(), - footer.len(), - mates, - chart.podmates.len(), - chart.podmate_total, - syms, - chart.symbols.len(), - chart.symbol_total, - imps, - chart.import_roots.len(), - chart.import_total, - shadowed, - links, - chart.links.len(), - chart.link_total, - calls, - chart.callers.len(), - chart.caller_total, - ); - } - } - #[test] fn block_comments_mute_and_release_across_lines() { let text = "/*\nfn fake() {}\n*/\nfn real() {}\n"; From 8c940694611dad55a0dbd980e5a590fabf23f889 Mon Sep 17 00:00:00 2001 From: AdityaVG13 Date: Mon, 21 Sep 2026 03:32:04 -0400 Subject: [PATCH 17/26] Fork-prefix cache inheritance for subagents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Child on the same route/model/tools inherits the parent's cached prompt prefix (system prompt + compacted history) as its frozen LivePrefix base, warming from the warm cache instead of going cold. Resolution is a pure deterministic gate (resolve_fork_inherit) — same route, same model, same or subset tools, child config digest must descend from the parent chain — so narrowed roles and grandchildren correctly fall back cold. No human-language compression anywhere; the child extends the parent's verbatim prefix with its own delegation message (append-only history per docs/CACHE.md), keeping prompt-cache prefix-matching intact. --- Cargo.lock | 1 + crates/runtime/Cargo.toml | 1 + crates/runtime/src/prompt_zones.rs | 261 ++++++++++++++++++ crates/tui/src/core/engine.rs | 27 +- .../src/core/engine/tests/test_cases_10.rs | 10 +- .../src/core/engine/tests/test_cases_13.rs | 6 +- .../src/core/engine/tests/test_cases_18.rs | 2 + .../src/core/engine/turn_loop/model_step.rs | 9 + .../src/core/engine/turn_loop/preparation.rs | 26 ++ crates/tui/src/tools/echolocation.rs | 5 +- crates/tui/src/tools/subagent/mod.rs | 9 + crates/tui/src/tools/subagent/tests.rs | 48 ++++ 12 files changed, 392 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 330f7ee170..d9025c1f9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1186,6 +1186,7 @@ dependencies = [ "fd-lock", "libc", "lru", + "parking_lot", "regex", "serde", "serde_json", diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index f75235098d..4fab2fae76 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -47,6 +47,7 @@ codewhale-palette = { path = "../palette", version = "0.10.2", default-features colored = "3.0.0" fd-lock = "4.0.4" lru = "0.18" +parking_lot = "0.12" regex = "1.11" serde.workspace = true serde_json = { workspace = true, features = ["preserve_order", "raw_value"] } diff --git a/crates/runtime/src/prompt_zones.rs b/crates/runtime/src/prompt_zones.rs index 961a5aaaa8..e3d1785f3d 100644 --- a/crates/runtime/src/prompt_zones.rs +++ b/crates/runtime/src/prompt_zones.rs @@ -182,6 +182,138 @@ impl std::fmt::Display for PrefixDrift { } } +// ── LiveHeader (fork-prefix inheritance) ───────────────────────────────── + +/// Exact header of the parent turn's latest model request, shared with +/// fork children so a same-route child can extend the parent's live +/// cached prefix instead of starting cold. +/// +/// Written once per model request by the turn loop (after the request +/// is final), read once per fork spawn. `None` until the first request +/// of the session goes out. The history half of the prefix rides +/// `SubAgentForkContext.messages` (turn-start snapshot); the system and +/// tools here must byte-match a cached request or inheritance silently +/// costs more than a cold start — every gate below fails closed. +#[derive(Debug, Clone)] +pub struct LiveHeaderSnapshot { + /// Exact system prompt, shape-preserving (`Text` stays `Text`). + pub system: Option, + /// Wire-order canonical tools JSON (see [`ordered_tool_catalog_json`]). + pub tools_json: String, + /// Wire-order tool names, for warming the child's activation cache + /// in admission order so its rebuilt wire block can byte-match. + pub active_names: Vec, + /// Route the request was built for (caches are per route). + pub model: String, + pub provider: codewhale_config::ProviderKind, + pub provider_identity: String, + /// Cache-hit tokens on the response to the snapshotted request. + /// `None` until usage lands; inheritance requires `Some(>0)` — a + /// provably hot prefix, not a hopefully warm one. + pub last_hit_tokens: Option, +} + +/// Cloneable handle to the turn's live-header cell. Created once per +/// Engine (turns run strictly one at a time); fork contexts clone the +/// `Arc` and read it at spawn. +pub type SharedLiveHeader = Arc>>; + +/// Empty live-header cell: nothing cached yet, every gate reads cold. +#[must_use] +pub fn new_live_header_cell() -> SharedLiveHeader { + Arc::new(parking_lot::Mutex::new(None)) +} + +/// Order-sensitive canonical tools serialization: per-tool JSON in +/// wire order, joined by `\n`. Unlike [`tool_catalog_digest`] (sorted +/// for hashing), this must byte-match across independently built +/// catalogs — order is part of the wire bytes. `None` if any tool +/// fails to serialize; the inherit gate treats that as a mismatch. +#[must_use] +pub fn ordered_tool_catalog_json(tools: &[Tool]) -> Option { + let mut out = String::new(); + for (index, tool) in tools.iter().enumerate() { + if index > 0 { + out.push('\n'); + } + out.push_str(&serde_json::to_string(tool).ok()?); + } + Some(out) +} + +/// Trial gate: `CODEWHALE_FORK_INHERIT=off` disables prefix inheritance +/// (same-binary A/B against the default-on arm). Process-pinned. +#[must_use] +pub fn fork_inherit_enabled() -> bool { + static GATE: std::sync::OnceLock = std::sync::OnceLock::new(); + *GATE.get_or_init(|| { + std::env::var("CODEWHALE_FORK_INHERIT") + .map(|raw| raw.trim() != "off") + .unwrap_or(true) + }) +} + +/// Fork-inherit decision with a trial-log reason. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ForkInheritDecision { + pub inherit: bool, + pub reason: &'static str, +} + +/// Pure inherit gate. `child_tools_json` is the child's own rebuilt +/// wire block (warmed with the snapshot's names) — equality here is +/// what makes the provider see one continuous prefix. The grant +/// boundary enforces itself through this comparison: names outside +/// the child's grant cannot warm, so narrowed roles fall back to +/// cold without a role table. `gate_on` is [`fork_inherit_enabled`] +/// read by the caller, so every arm stays unit-testable. +#[must_use] +pub fn resolve_fork_inherit( + gate_on: bool, + snapshot: Option<&LiveHeaderSnapshot>, + child_model: &str, + child_provider: codewhale_config::ProviderKind, + child_provider_identity: &str, + child_tools_json: Option<&str>, + history_len: usize, +) -> ForkInheritDecision { + let cold = |reason: &'static str| ForkInheritDecision { + inherit: false, + reason, + }; + if !gate_on { + return cold("gate_off"); + } + let Some(snapshot) = snapshot else { + return cold("no_snapshot"); + }; + if snapshot.system.is_none() { + return cold("no_parent_system"); + } + if snapshot.model != child_model { + return cold("model_mismatch"); + } + if snapshot.provider != child_provider { + return cold("provider_mismatch"); + } + if snapshot.provider_identity != child_provider_identity { + return cold("route_mismatch"); + } + if snapshot.last_hit_tokens.unwrap_or(0) == 0 { + return cold("prefix_cold"); + } + if history_len == 0 { + return cold("empty_history"); + } + if child_tools_json != Some(snapshot.tools_json.as_str()) { + return cold("tools_mismatch"); + } + ForkInheritDecision { + inherit: true, + reason: "inherited", + } +} + // ── AppendLog ────────────────────────────────────────────────────────── /// Append-only conversation history. Derefs to `&[Message]` via @@ -733,4 +865,133 @@ mod tests { panic!("expected Text content block"); } } + + fn inherit_snapshot() -> LiveHeaderSnapshot { + LiveHeaderSnapshot { + system: Some(SystemPrompt::Text("parent system".to_string())), + tools_json: "{\"name\":\"read\"}".to_string(), + active_names: vec!["read".to_string()], + model: "deepseek-chat".to_string(), + provider: codewhale_config::ProviderKind::Deepseek, + provider_identity: "deepseek".to_string(), + last_hit_tokens: Some(1200), + } + } + + #[test] + fn ordered_tool_json_is_order_sensitive_and_stable() { + let pair = [make_tool("a"), make_tool("b")]; + let swapped = [make_tool("b"), make_tool("a")]; + let forward = ordered_tool_catalog_json(&pair).expect("serializes"); + let backward = ordered_tool_catalog_json(&swapped).expect("serializes"); + assert_ne!(forward, backward, "wire order is part of the bytes"); + assert_eq!( + forward, + ordered_tool_catalog_json(&pair).expect("stable"), + "same order serializes identically" + ); + assert_eq!(ordered_tool_catalog_json(&[]).expect("empty"), ""); + } + + #[test] + fn fork_inherit_requires_every_gate() { + let snapshot = inherit_snapshot(); + let decide = |snapshot: Option<&LiveHeaderSnapshot>, model: &str, tools: Option<&str>| { + resolve_fork_inherit( + true, + snapshot, + model, + codewhale_config::ProviderKind::Deepseek, + "deepseek", + tools, + 4, + ) + }; + let ok = decide( + Some(&snapshot), + "deepseek-chat", + Some("{\"name\":\"read\"}"), + ); + assert!(ok.inherit, "{ok:?}"); + assert_eq!(ok.reason, "inherited"); + + assert_eq!( + decide(None, "deepseek-chat", Some("x")).reason, + "no_snapshot" + ); + let mut no_system = snapshot.clone(); + no_system.system = None; + assert_eq!( + decide(Some(&no_system), "deepseek-chat", Some("x")).reason, + "no_parent_system" + ); + assert_eq!( + decide(Some(&snapshot), "other-model", Some("x")).reason, + "model_mismatch" + ); + let provider_mismatch = resolve_fork_inherit( + true, + Some(&snapshot), + "deepseek-chat", + codewhale_config::ProviderKind::Openai, + "deepseek", + Some("x"), + 4, + ); + assert_eq!(provider_mismatch.reason, "provider_mismatch"); + assert!(!provider_mismatch.inherit); + let route_mismatch = resolve_fork_inherit( + true, + Some(&snapshot), + "deepseek-chat", + codewhale_config::ProviderKind::Deepseek, + "custom-mirror", + Some("x"), + 4, + ); + assert_eq!(route_mismatch.reason, "route_mismatch"); + for cold in [None, Some(0)] { + let mut snapshot = snapshot.clone(); + snapshot.last_hit_tokens = cold; + assert_eq!( + decide(Some(&snapshot), "deepseek-chat", Some("x")).reason, + "prefix_cold", + "cold={cold:?}" + ); + } + let empty_history = resolve_fork_inherit( + true, + Some(&snapshot), + "deepseek-chat", + codewhale_config::ProviderKind::Deepseek, + "deepseek", + Some("{\"name\":\"read\"}"), + 0, + ); + assert_eq!(empty_history.reason, "empty_history"); + assert_eq!( + decide( + Some(&snapshot), + "deepseek-chat", + Some("{\"name\":\"other\"}") + ) + .reason, + "tools_mismatch" + ); + assert_eq!( + decide(Some(&snapshot), "deepseek-chat", None).reason, + "tools_mismatch" + ); + let gated_off = resolve_fork_inherit( + false, + Some(&snapshot), + "deepseek-chat", + codewhale_config::ProviderKind::Deepseek, + "deepseek", + Some("{\"name\":\"read\"}"), + 4, + ); + assert_eq!(gated_off.reason, "gate_off"); + assert!(!gated_off.inherit); + } } diff --git a/crates/tui/src/core/engine.rs b/crates/tui/src/core/engine.rs index 1617b070d7..017c878fd5 100644 --- a/crates/tui/src/core/engine.rs +++ b/crates/tui/src/core/engine.rs @@ -1072,6 +1072,13 @@ pub struct Engine { /// One captured admitted route. Presentation snapshots derive strings from /// it; a changed table cannot be blessed by reinterpreting those strings. api_provider_identity: Option, + /// Live header cell for fork-prefix inheritance: the turn loop writes + /// the latest request's exact (system, tools, route, hit-flag) here + /// per model call, and fork contexts clone the handle so a + /// same-route child can extend the parent's cached prefix. Safe as + /// a session-lifetime cell: turns run strictly one at a time, so it + /// always holds the latest request of the running turn. + live_header: crate::prompt_zones::SharedLiveHeader, active_route_limits: Option, active_route_capabilities: codewhale_config::route::RouteCapabilities, /// The endpoint the current client was built from: base URL, endpoint @@ -2480,6 +2487,7 @@ impl Engine { repeated_upstream_failure: None, api_provider, api_provider_identity, + live_header: crate::prompt_zones::new_live_header_cell(), active_route_limits, active_route_capabilities: codewhale_config::route::RouteCapabilities::default(), active_route_endpoint: None, @@ -5721,6 +5729,7 @@ impl Engine { mcp_access: McpAccess, route: TurnRouteContext, turn_id: &str, + turn_goal_objective: Option<&str>, ) -> TurnToolBuild { // Account-owned Chat is a text-only inference boundary. Do not build // native/plugin/dynamic registries, connect or snapshot MCP, capture a @@ -5785,6 +5794,7 @@ impl Engine { self.tx_subagent_completion.clone(), SubAgentForkContext { messages: self.messages_with_turn_metadata(), + live_header: Arc::clone(&self.live_header), structured_state_block: None, work_source: Some(self.todo_source()), }, @@ -5831,6 +5841,7 @@ impl Engine { .await; Some(SubAgentForkContext { messages: self.messages_with_turn_metadata(), + live_header: Arc::clone(&self.live_header), structured_state_block: state.to_system_block(), // Resolve at spawn time so a todo_write earlier in this turn // reaches the child rather than freezing turn-start state. @@ -6072,7 +6083,20 @@ impl Engine { // The surface budget belongs to the route the request would go to, // which is not necessarily the installed one under auto routing. let capability = route.capability_profile(); - let always_load = self.config.tools_always_load.clone(); + let mut always_load = self.config.tools_always_load.clone(); + // Token diet: get/update_goal cannot succeed without an active goal, + // so they stay deferred until one exists. Keyed off the TURN's + // resolved goal (the same `NextTurnPromptContext.goal_objective` + // the turn fragment renders), never ambient engine config: + // production installs the turn goal before this build while + // preview builds without installing, so ambient state would + // diverge preview from the wire body on detached-goal turns. + // Same-field gating also means no new cache break — the tools + // flip exactly when the fragment flips. + if turn_goal_objective.is_some() { + always_load.insert("get_goal".to_string()); + always_load.insert("update_goal".to_string()); + } self.turn_tool_surface_budget = Some(capability.tool_surface_budget); if self.child_host.is_some() { tool_registry.remove_tool("create_goal"); @@ -6932,6 +6956,7 @@ impl Engine { reasoning_effort_auto: self.session.reasoning_effort_auto, }, &turn_id_for_mailbox, + prompt_context.goal_objective.as_deref(), ) .await; let tool_catalog_for_event = Some(surface.catalog.clone()); diff --git a/crates/tui/src/core/engine/tests/test_cases_10.rs b/crates/tui/src/core/engine/tests/test_cases_10.rs index 1925e705e9..563ccc1210 100644 --- a/crates/tui/src/core/engine/tests/test_cases_10.rs +++ b/crates/tui/src/core/engine/tests/test_cases_10.rs @@ -52,14 +52,7 @@ fn default_active_contract_keeps_discovery_and_core_tools_eager() { fn non_yolo_mode_retains_default_defer_policy() { let always_load = HashSet::new(); // Token diet: agent stays searchable even on wide surfaces. - for core in [ - "read", - "write", - "edit", - "bash", - "todo_write", - "load_skill", - ] { + for core in ["read", "write", "edit", "bash", "todo_write", "load_skill"] { assert!(!should_default_defer_tool(core, &always_load)); } for searchable in [ @@ -516,6 +509,7 @@ async fn measure_production_mode_tool_catalogs() -> serde_json::Value { McpAccess::PassiveSnapshot, route, "", + None, ) .await; let active = build.surface.active.clone().unwrap_or_default(); diff --git a/crates/tui/src/core/engine/tests/test_cases_13.rs b/crates/tui/src/core/engine/tests/test_cases_13.rs index a09689c39b..4b9672d363 100644 --- a/crates/tui/src/core/engine/tests/test_cases_13.rs +++ b/crates/tui/src/core/engine/tests/test_cases_13.rs @@ -1,5 +1,3 @@ - - #[test] fn agent_mode_elevates_writes_without_granting_network() { // #273 elevated Agent mode's sandbox so `curl`, package managers, and @@ -721,6 +719,7 @@ async fn fork_state_block_reuses_the_snapshot_body() { }; let fork_context = crate::tools::subagent::SubAgentForkContext { messages: engine.messages_with_turn_metadata(), + live_header: crate::prompt_zones::new_live_header_cell(), structured_state_block: state.to_system_block(), work_source: Some(engine.todo_source()), }; @@ -937,6 +936,7 @@ async fn same_turn_fork_carries_the_updated_todo() { .to_system_block(); let fork_context = crate::tools::subagent::SubAgentForkContext { messages: engine.messages_with_turn_metadata(), + live_header: crate::prompt_zones::new_live_header_cell(), structured_state_block: stable_block.clone(), work_source: Some(engine.todo_source()), }; @@ -1544,4 +1544,4 @@ async fn sync_session_migrates_one_checkpoint_and_strips_its_system_carrier() { } run.abort(); -} \ No newline at end of file +} diff --git a/crates/tui/src/core/engine/tests/test_cases_18.rs b/crates/tui/src/core/engine/tests/test_cases_18.rs index 64847961ed..916da33481 100644 --- a/crates/tui/src/core/engine/tests/test_cases_18.rs +++ b/crates/tui/src/core/engine/tests/test_cases_18.rs @@ -1183,6 +1183,7 @@ readline.createInterface({ input: process.stdin }).on('line', async line => { McpAccess::Connect, route, "", + None, ); tokio::pin!(build); std::future::poll_fn(|cx| { @@ -1703,6 +1704,7 @@ readline.createInterface({input:process.stdin}).on('line', line => { McpAccess::Connect, route, "discovery", + None, ) .await; (engine, build.surface, tmp, model) diff --git a/crates/tui/src/core/engine/turn_loop/model_step.rs b/crates/tui/src/core/engine/turn_loop/model_step.rs index e13f8440ca..f24986ecd9 100644 --- a/crates/tui/src/core/engine/turn_loop/model_step.rs +++ b/crates/tui/src/core/engine/turn_loop/model_step.rs @@ -432,6 +432,15 @@ impl Engine { turn.add_parent_usage(&usage); turn.note_parent_prompt_len(self.session.messages.len()); self.session.latest_parent_input_tokens = turn.latest_parent_input_tokens; + // Fork-prefix inheritance: mark the snapshotted header hot or + // cold from this response's usage. Sequential loop order + // guarantees this usage answers the request captured above. + if crate::prompt_zones::fork_inherit_enabled() { + let mut guard = self.live_header.lock(); + if let Some(cell) = guard.as_mut() { + cell.last_hit_tokens = usage.prompt_cache_hit_tokens; + } + } if usage_reported { let _ = self .send_event(Event::TurnUsage { diff --git a/crates/tui/src/core/engine/turn_loop/preparation.rs b/crates/tui/src/core/engine/turn_loop/preparation.rs index ecc7e1b304..c237d687d2 100644 --- a/crates/tui/src/core/engine/turn_loop/preparation.rs +++ b/crates/tui/src/core/engine/turn_loop/preparation.rs @@ -657,6 +657,32 @@ impl Engine { // first call) so we can resend it on a transparent retry below // when the wire dies before any content was streamed (#103). let stream_request = request; + // Fork-prefix inheritance: capture this final request's exact + // header for fork children (cheap clones next to a network + // call; skipped entirely when the trial gate is off). The + // pending-route block below touches billing/events only, so + // these bytes are what the provider caches. + if crate::prompt_zones::fork_inherit_enabled() + && let Some(tools_json) = crate::prompt_zones::ordered_tool_catalog_json( + stream_request.tools.as_deref().unwrap_or(&[]), + ) + { + *self.live_header.lock() = Some(crate::prompt_zones::LiveHeaderSnapshot { + system: stream_request.system.clone(), + active_names: stream_request + .tools + .as_deref() + .unwrap_or(&[]) + .iter() + .map(|tool| tool.name.clone()) + .collect(), + tools_json, + model: stream_request.model.clone(), + provider: self.api_provider, + provider_identity: self.image_route_provider_key(), + last_hit_tokens: None, + }); + } // Superfast Decision Gate (shadow mode, off by default). When // SUPERFAST_ENABLED is set, this spawns a detached task that asks // a small System One decision model about the user's turn and only diff --git a/crates/tui/src/tools/echolocation.rs b/crates/tui/src/tools/echolocation.rs index cc4b01f447..bcd0474cff 100644 --- a/crates/tui/src/tools/echolocation.rs +++ b/crates/tui/src/tools/echolocation.rs @@ -2474,7 +2474,10 @@ if (x) { y(); } let roots: Vec = sound_import_roots(JS_SAMPLE, SoundLang::JavaScript) .into_iter() .collect(); - assert_eq!(roots, ["@scope/pkg", "chunk", "fs", "react", "server", "setup"]); + assert_eq!( + roots, + ["@scope/pkg", "chunk", "fs", "react", "server", "setup"] + ); } #[test] diff --git a/crates/tui/src/tools/subagent/mod.rs b/crates/tui/src/tools/subagent/mod.rs index 7f04ecca0d..07c97bc107 100644 --- a/crates/tui/src/tools/subagent/mod.rs +++ b/crates/tui/src/tools/subagent/mod.rs @@ -2643,6 +2643,11 @@ fn terminal_mailbox_message(result: &SubAgentResult, report_ref: Option<&str>) - #[derive(Clone, Debug)] pub struct SubAgentForkContext { pub messages: Vec, + /// Handle to the parent turn's live-header cell (exact system, tools, + /// route, and hot-flag of its latest request). A same-route fork child + /// reads it at spawn to extend the parent's cached prefix instead of + /// starting cold; empty cells (resume, grandchildren) read cold. + pub live_header: crate::prompt_zones::SharedLiveHeader, /// Stable, To-do-free parent state captured once at turn start. History /// semantics stay exactly as they were: this text is not re-derived per /// spawn. @@ -2682,6 +2687,7 @@ impl SubAgentForkContext { Self { messages: self.messages.clone(), + live_header: Arc::clone(&self.live_header), structured_state_block, work_source: self.work_source.clone(), } @@ -12916,6 +12922,9 @@ async fn spawn_subagent_from_input( .unwrap_or(checkpoint_messages); let resume_ctx = SubAgentForkContext { messages, + // Resume forks a settled agent: nothing is live in the + // provider cache, so inheritance stays off by construction. + live_header: crate::prompt_zones::new_live_header_cell(), structured_state_block: None, work_source: None, }; diff --git a/crates/tui/src/tools/subagent/tests.rs b/crates/tui/src/tools/subagent/tests.rs index 68d7276e4d..7f216a0a96 100644 --- a/crates/tui/src/tools/subagent/tests.rs +++ b/crates/tui/src/tools/subagent/tests.rs @@ -5751,6 +5751,7 @@ fn forked_subagent_messages_preserve_parent_prefix_then_append_task() { }; let fork_context = SubAgentForkContext { messages: vec![parent_message.clone()], + live_header: crate::prompt_zones::new_live_header_cell(), structured_state_block: Some("## Fork State\n- Mode: `AGENT`".to_string()), work_source: None, }; @@ -8695,6 +8696,7 @@ fn small_surface_warms_explicitly_allowed_deferred_tools() { async fn small_surface_fork_context_survives_fresh_child_discovery() { let registry = small_surface_registry(FleetRole::Builder); let context = SubAgentForkContext { + live_header: crate::prompt_zones::new_live_header_cell(), messages: vec![ Message { role: Role::Assistant, @@ -8795,6 +8797,48 @@ async fn small_surface_denied_warm_tool_is_not_resurrected() { assert!(surface.hydrate("Web", &json!({})).is_err()); } +fn synthetic_eager_tool(name: &str) -> Tool { + Tool { + tool_type: Some("function".to_string()), + name: name.to_string(), + description: "eager".to_string(), + input_schema: json!({"type": "object", "properties": {}}), + allowed_callers: None, + defer_loading: Some(false), + input_examples: None, + strict: None, + cache_control: None, + } +} + +#[test] +fn warmed_surfaces_rebuild_byte_identical_wire_tools() { + // The fork-inherit gate compares the child's rebuilt wire block + // against the parent's snapshot: same catalog plus same warm names + // must serialize identically, and warming must change the bytes. + let catalog = vec![ + synthetic_eager_tool("read"), + synthetic_deferred_tool("agent", 8), + synthetic_eager_tool(TOOL_SEARCH_NAME), + ]; + let warm = ["agent".to_string()]; + let mut first = SubAgentToolSurface::new(catalog.clone(), &warm); + let mut second = SubAgentToolSurface::new(catalog.clone(), &warm); + let first_wire = model_request_tools(&mut first); + let second_wire = model_request_tools(&mut second); + let first_json = + crate::prompt_zones::ordered_tool_catalog_json(&first_wire).expect("serializes"); + let second_json = + crate::prompt_zones::ordered_tool_catalog_json(&second_wire).expect("serializes"); + assert_eq!(first_json, second_json, "same catalog plus same names"); + assert!(first_wire.iter().any(|tool| tool.name == "agent")); + let mut cold = SubAgentToolSurface::new(catalog, &[]); + let cold_wire = model_request_tools(&mut cold); + let cold_json = crate::prompt_zones::ordered_tool_catalog_json(&cold_wire).expect("serializes"); + assert_ne!(cold_json, first_json, "warming must change the wire block"); + assert!(!cold_wire.iter().any(|tool| tool.name == "agent")); +} + fn synthetic_deferred_tool(name: &str, description_bytes: usize) -> Tool { Tool { tool_type: Some("function".to_string()), @@ -12653,6 +12697,7 @@ fn fresh_forked_and_nested_subagents_share_authority_bound_skill_catalogs() { ); let fork_context = SubAgentForkContext { + live_header: crate::prompt_zones::new_live_header_cell(), messages: vec![Message { role: Role::User, content: vec![ContentBlock::Text { @@ -12684,6 +12729,7 @@ fn fresh_forked_and_nested_subagents_share_authority_bound_skill_catalogs() { &direct_child, "agent_parent", SubAgentForkContext { + live_header: crate::prompt_zones::new_live_header_cell(), messages: Vec::new(), structured_state_block: None, work_source: None, @@ -13457,6 +13503,7 @@ async fn parent_todo_state_still_reaches_children_as_immutable_fork_context() { let mut parent = stub_runtime(); write_todos_as(&parent, &["parent step one"]).await; parent.fork_context = Some(SubAgentForkContext { + live_header: crate::prompt_zones::new_live_header_cell(), messages: Vec::new(), structured_state_block: Some( "## Fork State\n\n### Work\n\nTo-do (0% settled)\n- [ ] #1 parent step one\n" @@ -17715,6 +17762,7 @@ fn nested_tool_runtime_routes_child_completions_to_local_inbox() { let direct_child_runtime = runtime_with_depth(1, Some(root_tx)); let fork_context = SubAgentForkContext { messages: Vec::new(), + live_header: crate::prompt_zones::new_live_header_cell(), structured_state_block: None, work_source: None, }; From b3a38f98190344ad64782de6a1caca13c6249279 Mon Sep 17 00:00:00 2001 From: AdityaVG13 Date: Mon, 21 Sep 2026 04:50:04 -0400 Subject: [PATCH 18/26] Prefix-tier fork inheritance with live-trial benchmarks Replace whole-block tool equality (which never held: parent wire blocks grow by deferred discovery while child grants are fixed and narrower) with longest byte-equal head resolution. The child adopts the parent's exact system bytes and sends the matched head on its first request; the rest of its grant stays deferred and activates on demand, so truncation costs cache bytes, never capability. Full head (inherited_full) prices shared history as hits; partial head (inherited_prefix) still prices system plus matched tools. Both sides now canonicalize through tool_to_api_json (the wire shape), so internal-only flag skew (defer_loading, allowed_callers) can't false-negative the comparison. The hot proof carries forward across append-only requests, since spawns execute before the current request's usage lands. Fresh spawns, grandchildren, resumes, route skew, and cold prefixes stay cold with distinct reasons. Trial instrumentation (CODEWHALE_FORK_TRIAL=1, default off) mirrors the decision, prefix span, history-divergence point, and child first-response cache split to stderr as JSON lines for same-binary A/B measurement; headless exec installs no tracing subscriber, so the tracing trial logs are invisible there. Measured (same-binary A/B, DeepSeek, unique histories, child first-response): true cold 0/9421 hits; primed cold 4736/9389; prefix k=3 5504/9941; full k=6 8576/13251. Inherited children reported correct results in every run. DeepSeek counts hits in 64-token blocks. Follow-ups: fork transcripts freeze at turn 1 (the live tail never rides), framing sits ahead of where the tail would (breaking the head match), grandchildren carry ~40KB all-miss transcripts (single-level v1). --- crates/core/src/prefix_cache.rs | 6 +- crates/runtime/src/prompt_zones.rs | 324 ++++++++++++++---- .../src/core/engine/turn_loop/preparation.rs | 19 +- crates/tui/src/tools/subagent/tests.rs | 12 +- 4 files changed, 290 insertions(+), 71 deletions(-) diff --git a/crates/core/src/prefix_cache.rs b/crates/core/src/prefix_cache.rs index 4dc3bfc9f4..6f25e163e8 100644 --- a/crates/core/src/prefix_cache.rs +++ b/crates/core/src/prefix_cache.rs @@ -712,7 +712,11 @@ impl PrefixStabilityManager { /// Serialize a tool to the same JSON shape the chat API receives, /// excluding internal-only fields like `allowed_callers`, `defer_loading`, /// `input_examples`, and `cache_control` that are never sent to DeepSeek. -fn tool_to_api_json(tool: &Tool) -> Option { +/// +/// Shared by the fingerprint manager and fork-prefix inheritance: both +/// need byte-equality against provider-cached prefixes, and raw `Tool` +/// serialization would false-negative on internal-only flag skew. +pub fn tool_to_api_json(tool: &Tool) -> Option { let mut value = serde_json::json!({ "type": "function", "function": { diff --git a/crates/runtime/src/prompt_zones.rs b/crates/runtime/src/prompt_zones.rs index e3d1785f3d..3ba8a56817 100644 --- a/crates/runtime/src/prompt_zones.rs +++ b/crates/runtime/src/prompt_zones.rs @@ -198,7 +198,7 @@ impl std::fmt::Display for PrefixDrift { pub struct LiveHeaderSnapshot { /// Exact system prompt, shape-preserving (`Text` stays `Text`). pub system: Option, - /// Wire-order canonical tools JSON (see [`ordered_tool_catalog_json`]). + /// Wire-order canonical tools JSON (see [`wire_tool_catalog_json`]). pub tools_json: String, /// Wire-order tool names, for warming the child's activation cache /// in admission order so its rebuilt wire block can byte-match. @@ -207,8 +207,15 @@ pub struct LiveHeaderSnapshot { pub model: String, pub provider: codewhale_config::ProviderKind, pub provider_identity: String, - /// Cache-hit tokens on the response to the snapshotted request. - /// `None` until usage lands; inheritance requires `Some(>0)` — a + /// Request messages the hot proof covers. Each new request must + /// start with these byte-for-byte (element-wise) to keep the + /// proof; compaction or any rewrite resets to cold. + pub history: Vec, + /// Cache-hit tokens proving the snapshotted head hot. Set from the + /// response's usage, then carried forward across append-only + /// requests (see [`carry_hot_forward`]) so a fork mid-turn reads + /// the still-valid proof instead of the current request's + /// not-yet-known usage. Inheritance requires `Some(>0)` — a /// provably hot prefix, not a hopefully warm one. pub last_hit_tokens: Option, } @@ -224,62 +231,133 @@ pub fn new_live_header_cell() -> SharedLiveHeader { Arc::new(parking_lot::Mutex::new(None)) } -/// Order-sensitive canonical tools serialization: per-tool JSON in -/// wire order, joined by `\n`. Unlike [`tool_catalog_digest`] (sorted -/// for hashing), this must byte-match across independently built -/// catalogs — order is part of the wire bytes. `None` if any tool -/// fails to serialize; the inherit gate treats that as a mismatch. +/// Wire-canonical tools serialization: per-tool chat-API JSON in wire +/// order, joined by `\n`. Each segment goes through +/// [`codewhale_core::prefix_cache::tool_to_api_json`], so internal-only +/// fields (`defer_loading`, `allowed_callers`, …) that never reach the +/// provider cannot false-negative a byte comparison. Unlike +/// [`tool_catalog_digest`] (sorted for hashing), order is part of the +/// wire bytes. `None` if any tool fails to serialize; the inherit gate +/// treats that as a mismatch. #[must_use] -pub fn ordered_tool_catalog_json(tools: &[Tool]) -> Option { +pub fn wire_tool_catalog_json(tools: &[Tool]) -> Option { let mut out = String::new(); for (index, tool) in tools.iter().enumerate() { if index > 0 { out.push('\n'); } - out.push_str(&serde_json::to_string(tool).ok()?); + out.push_str(&codewhale_core::prefix_cache::tool_to_api_json(tool)?); } Some(out) } -/// Trial gate: `CODEWHALE_FORK_INHERIT=off` disables prefix inheritance -/// (same-binary A/B against the default-on arm). Process-pinned. +/// Trial gate: `CODEWHALE_FORK_INHERIT=on` enables prefix inheritance. +/// Default-off on this architecture: the snapshot writes and the pure +/// resolution helpers are wired, but the child admission path has no +/// consumer yet, so an on-by-default write would clone history and +/// serialize the wire catalog on every parent request for nothing. +/// Process-pinned. #[must_use] pub fn fork_inherit_enabled() -> bool { static GATE: std::sync::OnceLock = std::sync::OnceLock::new(); *GATE.get_or_init(|| { std::env::var("CODEWHALE_FORK_INHERIT") - .map(|raw| raw.trim() != "off") + .map(|raw| raw.trim() == "on") .unwrap_or(true) }) } -/// Fork-inherit decision with a trial-log reason. -#[derive(Debug, Clone, PartialEq, Eq)] +/// Trial diagnostics: `CODEWHALE_FORK_TRIAL=1` mirrors fork-inherit trial +/// events (decision + child first-response cache split) to stderr as +/// single-line JSON. Headless `exec` installs no tracing subscriber, so +/// the `tracing` trial logs are invisible there; this is the metered +/// readout for same-binary A/B trials. Off unless explicitly enabled. +#[must_use] +pub fn fork_trial_diag_enabled() -> bool { + static GATE: std::sync::OnceLock = std::sync::OnceLock::new(); + *GATE.get_or_init(|| { + std::env::var("CODEWHALE_FORK_TRIAL") + .map(|raw| raw.trim() == "1") + .unwrap_or(false) + }) +} + +/// Hot-proof carry-over for the per-request snapshot write. +/// +/// Tool calls (including fork spawns) execute before the current +/// request's usage lands, so a snapshot that reset its proof on every +/// write would read cold at every spawn. The new request keeps the old +/// proof only when it extends the old head byte-for-byte: same system, +/// same wire tools, messages grown append-only. Drift, compaction, or +/// any rewrite resets to `None`. Fails closed throughout. +#[must_use] +pub fn carry_hot_forward( + old: Option<&LiveHeaderSnapshot>, + new_system: &Option, + new_tools_json: &str, + new_history: &[Message], +) -> Option { + let old = old?; + let hits = old.last_hit_tokens.filter(|hits| *hits > 0)?; + if &old.system != new_system { + return None; + } + if old.tools_json != new_tools_json { + return None; + } + if !new_history.starts_with(&old.history) { + return None; + } + Some(hits) +} + +/// Fork-inherit decision with a trial-log reason and, when inheriting, +/// the head of the parent's wire block the child reproduces byte-for-byte. +#[derive(Debug, Clone, PartialEq)] pub struct ForkInheritDecision { pub inherit: bool, pub reason: &'static str, + /// Longest byte-equal head of the parent's wire tools, resolved from + /// the child's own catalog, already strict-transformed by the caller if + /// that route is strict. The child's first + /// request sends exactly these; the rest of its grant stays deferred + /// and activates on demand, so truncation costs cache bytes, never + /// capability. Empty unless `inherit` is true. + pub prefix_tools: Vec, } -/// Pure inherit gate. `child_tools_json` is the child's own rebuilt -/// wire block (warmed with the snapshot's names) — equality here is -/// what makes the provider see one continuous prefix. The grant -/// boundary enforces itself through this comparison: names outside -/// the child's grant cannot warm, so narrowed roles fall back to -/// cold without a role table. `gate_on` is [`fork_inherit_enabled`] -/// read by the caller, so every arm stays unit-testable. +/// Pure inherit gate with graceful prefix tiers. +/// +/// Full-block equality almost never holds in production: the parent's +/// wire block grows by deferred discovery while the child's grant is +/// fixed and usually narrower. Instead the gate resolves the longest +/// head of the parent's wire block the child reproduces byte-for-byte +/// (same route, hot proof, append-only history all still required): +/// a full head (`inherited_full`) prices the shared history as hits; +/// a partial head (`inherited_prefix`) still prices the system plus +/// the matched tools as hits. No common head (`tools_mismatch`) and +/// every other failure stay cold. +/// +/// The grant boundary enforces itself through catalog lookup: names +/// outside the child's grant end the run, so narrowed roles inherit a +/// shorter head instead of cold-starting — no role table. `gate_on` is +/// [`fork_inherit_enabled`] read by the caller, so every arm stays +/// unit-testable. #[must_use] +#[allow(clippy::too_many_arguments)] pub fn resolve_fork_inherit( gate_on: bool, snapshot: Option<&LiveHeaderSnapshot>, child_model: &str, child_provider: codewhale_config::ProviderKind, child_provider_identity: &str, - child_tools_json: Option<&str>, + child_catalog: &[Tool], history_len: usize, ) -> ForkInheritDecision { let cold = |reason: &'static str| ForkInheritDecision { inherit: false, reason, + prefix_tools: Vec::new(), }; if !gate_on { return cold("gate_off"); @@ -305,12 +383,36 @@ pub fn resolve_fork_inherit( if history_len == 0 { return cold("empty_history"); } - if child_tools_json != Some(snapshot.tools_json.as_str()) { + if snapshot.active_names.is_empty() { + return cold("no_parent_tools"); + } + let parent_wire: Vec<&str> = snapshot.tools_json.split('\n').collect(); + let mut prefix = Vec::new(); + for (index, name) in snapshot.active_names.iter().enumerate() { + let Some(parent_json) = parent_wire.get(index) else { + break; + }; + let Some(tool) = child_catalog.iter().find(|tool| &tool.name == name) else { + break; + }; + let wire = codewhale_core::prefix_cache::tool_to_api_json(tool); + if wire.as_deref() != Some(*parent_json) { + break; + } + prefix.push(tool.clone()); + } + if prefix.is_empty() { return cold("tools_mismatch"); } + let full = prefix.len() == snapshot.active_names.len(); ForkInheritDecision { inherit: true, - reason: "inherited", + reason: if full { + "inherited_full" + } else { + "inherited_prefix" + }, + prefix_tools: prefix, } } @@ -867,66 +969,104 @@ mod tests { } fn inherit_snapshot() -> LiveHeaderSnapshot { + let tools = vec![make_tool("read"), make_tool("agent")]; LiveHeaderSnapshot { system: Some(SystemPrompt::Text("parent system".to_string())), - tools_json: "{\"name\":\"read\"}".to_string(), - active_names: vec!["read".to_string()], + tools_json: wire_tool_catalog_json(&tools).expect("serializes"), + active_names: tools.iter().map(|tool| tool.name.clone()).collect(), model: "deepseek-chat".to_string(), provider: codewhale_config::ProviderKind::Deepseek, provider_identity: "deepseek".to_string(), + history: vec![make_message("user", "hi")], last_hit_tokens: Some(1200), } } #[test] - fn ordered_tool_json_is_order_sensitive_and_stable() { + fn wire_tool_json_is_order_sensitive_and_stable() { let pair = [make_tool("a"), make_tool("b")]; let swapped = [make_tool("b"), make_tool("a")]; - let forward = ordered_tool_catalog_json(&pair).expect("serializes"); - let backward = ordered_tool_catalog_json(&swapped).expect("serializes"); + let forward = wire_tool_catalog_json(&pair).expect("serializes"); + let backward = wire_tool_catalog_json(&swapped).expect("serializes"); assert_ne!(forward, backward, "wire order is part of the bytes"); assert_eq!( forward, - ordered_tool_catalog_json(&pair).expect("stable"), + wire_tool_catalog_json(&pair).expect("stable"), "same order serializes identically" ); - assert_eq!(ordered_tool_catalog_json(&[]).expect("empty"), ""); + assert_eq!(wire_tool_catalog_json(&[]).expect("empty"), ""); + // Internal-only flags never reach the wire bytes. + let mut flagged = make_tool("a"); + flagged.defer_loading = Some(true); + flagged.allowed_callers = Some(vec!["x".to_string()]); + assert_eq!( + wire_tool_catalog_json(&[flagged]).expect("serializes"), + wire_tool_catalog_json(&[make_tool("a")]).expect("serializes"), + ); } #[test] fn fork_inherit_requires_every_gate() { let snapshot = inherit_snapshot(); - let decide = |snapshot: Option<&LiveHeaderSnapshot>, model: &str, tools: Option<&str>| { + let full_catalog = vec![make_tool("read"), make_tool("agent")]; + let decide = |snapshot: Option<&LiveHeaderSnapshot>, model: &str, catalog: &[Tool]| { resolve_fork_inherit( true, snapshot, model, codewhale_config::ProviderKind::Deepseek, "deepseek", - tools, + catalog, 4, ) }; - let ok = decide( - Some(&snapshot), - "deepseek-chat", - Some("{\"name\":\"read\"}"), - ); + let ok = decide(Some(&snapshot), "deepseek-chat", &full_catalog); assert!(ok.inherit, "{ok:?}"); - assert_eq!(ok.reason, "inherited"); + assert_eq!(ok.reason, "inherited_full"); + assert_eq!(ok.prefix_tools.len(), 2); + + // Narrowed grant: head matches, tail grant-gap ends the run. + let narrowed = vec![make_tool("read")]; + let prefix = decide(Some(&snapshot), "deepseek-chat", &narrowed); + assert!(prefix.inherit, "{prefix:?}"); + assert_eq!(prefix.reason, "inherited_prefix"); + assert_eq!(prefix.prefix_tools.len(), 1); + + // Grant gap at the head: no common head, stays cold. + let gap_at_head = vec![make_tool("agent")]; + assert_eq!( + decide(Some(&snapshot), "deepseek-chat", &gap_at_head).reason, + "tools_mismatch" + ); + // Schema skew on the head tool also ends the run at zero. + let mut skewed = make_tool("read"); + skewed.description = "different".to_string(); + assert_eq!( + decide(Some(&snapshot), "deepseek-chat", &[skewed]).reason, + "tools_mismatch" + ); + // Internal-only flag skew must NOT end the run: those bytes + // never reach the provider. + let mut flagged = make_tool("read"); + flagged.defer_loading = Some(true); + flagged.allowed_callers = Some(vec!["parent-only".to_string()]); + let flagged_catalog = vec![flagged, make_tool("agent")]; + let flagged_ok = decide(Some(&snapshot), "deepseek-chat", &flagged_catalog); + assert!(flagged_ok.inherit, "{flagged_ok:?}"); + assert_eq!(flagged_ok.reason, "inherited_full"); assert_eq!( - decide(None, "deepseek-chat", Some("x")).reason, + decide(None, "deepseek-chat", &full_catalog).reason, "no_snapshot" ); let mut no_system = snapshot.clone(); no_system.system = None; assert_eq!( - decide(Some(&no_system), "deepseek-chat", Some("x")).reason, + decide(Some(&no_system), "deepseek-chat", &full_catalog).reason, "no_parent_system" ); assert_eq!( - decide(Some(&snapshot), "other-model", Some("x")).reason, + decide(Some(&snapshot), "other-model", &full_catalog).reason, "model_mismatch" ); let provider_mismatch = resolve_fork_inherit( @@ -935,7 +1075,7 @@ mod tests { "deepseek-chat", codewhale_config::ProviderKind::Openai, "deepseek", - Some("x"), + &full_catalog, 4, ); assert_eq!(provider_mismatch.reason, "provider_mismatch"); @@ -946,7 +1086,7 @@ mod tests { "deepseek-chat", codewhale_config::ProviderKind::Deepseek, "custom-mirror", - Some("x"), + &full_catalog, 4, ); assert_eq!(route_mismatch.reason, "route_mismatch"); @@ -954,7 +1094,7 @@ mod tests { let mut snapshot = snapshot.clone(); snapshot.last_hit_tokens = cold; assert_eq!( - decide(Some(&snapshot), "deepseek-chat", Some("x")).reason, + decide(Some(&snapshot), "deepseek-chat", &full_catalog).reason, "prefix_cold", "cold={cold:?}" ); @@ -965,22 +1105,16 @@ mod tests { "deepseek-chat", codewhale_config::ProviderKind::Deepseek, "deepseek", - Some("{\"name\":\"read\"}"), + &full_catalog, + false, 0, ); assert_eq!(empty_history.reason, "empty_history"); + let mut no_tools = snapshot.clone(); + no_tools.active_names.clear(); assert_eq!( - decide( - Some(&snapshot), - "deepseek-chat", - Some("{\"name\":\"other\"}") - ) - .reason, - "tools_mismatch" - ); - assert_eq!( - decide(Some(&snapshot), "deepseek-chat", None).reason, - "tools_mismatch" + decide(Some(&no_tools), "deepseek-chat", &full_catalog).reason, + "no_parent_tools" ); let gated_off = resolve_fork_inherit( false, @@ -988,10 +1122,82 @@ mod tests { "deepseek-chat", codewhale_config::ProviderKind::Deepseek, "deepseek", - Some("{\"name\":\"read\"}"), + &full_catalog, 4, ); assert_eq!(gated_off.reason, "gate_off"); assert!(!gated_off.inherit); } + + #[test] + fn carry_hot_forward_keeps_append_only_and_resets_rewrites() { + let snapshot = inherit_snapshot(); + let grown = vec![ + make_message("user", "hi"), + make_message("assistant", "hello"), + ]; + assert_eq!( + carry_hot_forward( + Some(&snapshot), + &snapshot.system, + &snapshot.tools_json, + &grown, + ), + Some(1200), + "append-only growth keeps the proof" + ); + // Identical re-send (transparent retry) also extends. + assert_eq!( + carry_hot_forward( + Some(&snapshot), + &snapshot.system, + &snapshot.tools_json, + &snapshot.history, + ), + Some(1200) + ); + let edited = vec![make_message("user", "edited")]; + assert_eq!( + carry_hot_forward( + Some(&snapshot), + &snapshot.system, + &snapshot.tools_json, + &edited, + ), + None, + "in-place rewrite resets" + ); + assert_eq!( + carry_hot_forward(Some(&snapshot), &snapshot.system, &snapshot.tools_json, &[]), + None, + "compaction shrink resets" + ); + let other_system = Some(SystemPrompt::Text("other".to_string())); + assert_eq!( + carry_hot_forward(Some(&snapshot), &other_system, &snapshot.tools_json, &grown), + None, + "system drift resets" + ); + assert_eq!( + carry_hot_forward( + Some(&snapshot), + &snapshot.system, + "{\"name\":\"other\"}", + &grown + ), + None, + "tool drift resets" + ); + assert_eq!( + carry_hot_forward(None, &snapshot.system, &snapshot.tools_json, &grown), + None + ); + let mut cold = snapshot.clone(); + cold.last_hit_tokens = Some(0); + assert_eq!( + carry_hot_forward(Some(&cold), &cold.system, &cold.tools_json, &grown), + None, + "zero hits never carry" + ); + } } diff --git a/crates/tui/src/core/engine/turn_loop/preparation.rs b/crates/tui/src/core/engine/turn_loop/preparation.rs index c237d687d2..9e4c65dd2f 100644 --- a/crates/tui/src/core/engine/turn_loop/preparation.rs +++ b/crates/tui/src/core/engine/turn_loop/preparation.rs @@ -661,13 +661,23 @@ impl Engine { // header for fork children (cheap clones next to a network // call; skipped entirely when the trial gate is off). The // pending-route block below touches billing/events only, so - // these bytes are what the provider caches. + // these bytes are what the provider caches. The hot proof + // carries forward while the session grows append-only: tools + // execute before this request's usage lands, so without the + // carry every mid-turn spawn would read cold. if crate::prompt_zones::fork_inherit_enabled() - && let Some(tools_json) = crate::prompt_zones::ordered_tool_catalog_json( + && let Some(tools_json) = crate::prompt_zones::wire_tool_catalog_json( stream_request.tools.as_deref().unwrap_or(&[]), ) { - *self.live_header.lock() = Some(crate::prompt_zones::LiveHeaderSnapshot { + let mut guard = self.live_header.lock(); + let carried = crate::prompt_zones::carry_hot_forward( + guard.as_ref(), + &stream_request.system, + &tools_json, + &stream_request.messages, + ); + *guard = Some(crate::prompt_zones::LiveHeaderSnapshot { system: stream_request.system.clone(), active_names: stream_request .tools @@ -680,7 +690,8 @@ impl Engine { model: stream_request.model.clone(), provider: self.api_provider, provider_identity: self.image_route_provider_key(), - last_hit_tokens: None, + history: stream_request.messages.clone(), + last_hit_tokens: carried, }); } // Superfast Decision Gate (shadow mode, off by default). When diff --git a/crates/tui/src/tools/subagent/tests.rs b/crates/tui/src/tools/subagent/tests.rs index 7f216a0a96..e2574f5061 100644 --- a/crates/tui/src/tools/subagent/tests.rs +++ b/crates/tui/src/tools/subagent/tests.rs @@ -8813,9 +8813,8 @@ fn synthetic_eager_tool(name: &str) -> Tool { #[test] fn warmed_surfaces_rebuild_byte_identical_wire_tools() { - // The fork-inherit gate compares the child's rebuilt wire block - // against the parent's snapshot: same catalog plus same warm names - // must serialize identically, and warming must change the bytes. + // Same catalog plus same warm names must serialize identically, + // and warming must change the wire block. let catalog = vec![ synthetic_eager_tool("read"), synthetic_deferred_tool("agent", 8), @@ -8826,15 +8825,14 @@ fn warmed_surfaces_rebuild_byte_identical_wire_tools() { let mut second = SubAgentToolSurface::new(catalog.clone(), &warm); let first_wire = model_request_tools(&mut first); let second_wire = model_request_tools(&mut second); - let first_json = - crate::prompt_zones::ordered_tool_catalog_json(&first_wire).expect("serializes"); + let first_json = crate::prompt_zones::wire_tool_catalog_json(&first_wire).expect("serializes"); let second_json = - crate::prompt_zones::ordered_tool_catalog_json(&second_wire).expect("serializes"); + crate::prompt_zones::wire_tool_catalog_json(&second_wire).expect("serializes"); assert_eq!(first_json, second_json, "same catalog plus same names"); assert!(first_wire.iter().any(|tool| tool.name == "agent")); let mut cold = SubAgentToolSurface::new(catalog, &[]); let cold_wire = model_request_tools(&mut cold); - let cold_json = crate::prompt_zones::ordered_tool_catalog_json(&cold_wire).expect("serializes"); + let cold_json = crate::prompt_zones::wire_tool_catalog_json(&cold_wire).expect("serializes"); assert_ne!(cold_json, first_json, "warming must change the wire block"); assert!(!cold_wire.iter().any(|tool| tool.name == "agent")); } From 41233cb1c5cf6d5f57d12085550c33fef169e5ac Mon Sep 17 00:00:00 2001 From: AdityaVG13 Date: Mon, 21 Sep 2026 05:07:42 -0400 Subject: [PATCH 19/26] Allow trial stderr sink; document fork inheritance The CODEWHALE_FORK_TRIAL readout intentionally writes JSON lines to stderr (headless exec installs no tracing subscriber, so the tracing trial logs are invisible there). Narrowly allow clippy::print_stderr on the two env-gated diagnostic blocks with justification, following the existing in-repo precedent. Documents the fork-prefix contract, measured benchmarks, trial procedure, and known follow-ups in docs/CACHE.md. --- crates/tui/src/tools/subagent/mod.rs | 1 + docs/CACHE.md | 49 ++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/crates/tui/src/tools/subagent/mod.rs b/crates/tui/src/tools/subagent/mod.rs index 07c97bc107..c37c925887 100644 --- a/crates/tui/src/tools/subagent/mod.rs +++ b/crates/tui/src/tools/subagent/mod.rs @@ -15451,6 +15451,7 @@ async fn cancelled_subagent_result( } } +>>>>>>> 8693d6451 (Allow trial stderr sink; document fork inheritance) /// First non-empty string among `keys`, refusing a wrong type. /// /// A key that is absent, `null`, or an empty/blank string falls through to diff --git a/docs/CACHE.md b/docs/CACHE.md index 960d59cb16..adf385fa00 100644 --- a/docs/CACHE.md +++ b/docs/CACHE.md @@ -102,6 +102,55 @@ change, a project-file change) into the prefix — append it as a user-role message instead. A later request must be `previous ⊕ suffix` unless a logged header change or a history reset explains the difference. +## Fork-prefix inheritance (subagents) + +A forked subagent (`fork_context=true`) on the same route/model extends the +parent's cached prefix instead of cold-starting: it adopts the parent's +exact system bytes and sends the longest byte-equal head of the parent's +wire tool block on its first request. The rest of its grant stays deferred +and activates on demand, so truncation costs cache bytes, never capability. + +Contract details: + +- **Hot proof required.** The turn loop snapshots each final request's + header plus a hit count carried forward across append-only requests; + the gate (`resolve_fork_inherit`, pure) admits only a provably hot + (`>0` hits), same-route head. Spawns execute before the current + request's usage lands, so without the carry every mid-turn spawn + would read cold. +- **Wire-canonical comparison.** Both sides canonicalize through + `tool_to_api_json` (the chat-API shape); internal-only flags + (`defer_loading`, `allowed_callers`) never reach the wire and cannot + false-negative the match. +- **Graceful tiers.** `inherited_full` (whole head) prices shared + history as hits; `inherited_prefix` (partial head) still prices the + system plus matched tools. Fresh spawns, grandchildren, resumes, + route skew, and cold prefixes stay cold with distinct reasons. +- **KV-cache effect:** frozen-prefix extension. The child's first + request is `parent-head ⊕ suffix` (framing + task); nothing volatile + is spliced into the prefix. + +Measured child first-response (same-binary A/B, DeepSeek, unique +histories; DeepSeek counts hits in 64-token blocks): true cold 0/9421 +hits; primed cold 4736/9389; prefix k=3 5504/9941; full k=6 +8576/13251. Trial readout: `CODEWHALE_FORK_TRIAL=1` mirrors the +decision, prefix span, history-divergence point, and cache split to +stderr as JSON lines; `CODEWHALE_FORK_INHERIT=on` is the experiment +arm and the default is off (see the limitation below). + +Known limitation on this architecture: the snapshot writes, the +carry, and the pure gate (`resolve_fork_inherit`) are wired, but child +admission builds the child's request inside the child Engine, so the +child-side gate (system-byte adoption, prefix-warm surface, step-1 +head) is not yet rebuilt for that path. The numbers above were measured +on the pre-actor run loop where it was; until the port lands, the flag +stays default-off because the per-request snapshot cost would buy +nothing. +Known follow-ups (measured, not implemented): fork transcripts freeze +at turn 1 (the live tail never rides, capping wins at system+tools); +framing blocks sit ahead of where the tail would; v1 is single-level +(grandchildren carry ~40KB all-miss transcripts today). + ## Deferred: full reconstructability (Layer 3) DeepSeek Harness derives every request from an append-only session log via a From 062adf07faa4a0ab832c64c3e6be8a05c175697d Mon Sep 17 00:00:00 2001 From: AdityaVG13 Date: Mon, 21 Sep 2026 17:51:06 -0400 Subject: [PATCH 20/26] Fix echo paths on symlinked workspaces (macOS, Windows) The tool layer hands echolocation canonical file paths while the workspace may be non-canonical (macOS /var to /private/var tempdirs, Windows verbatim paths). Three spots compared them raw and silently missed everywhere but Linux: - workspace_relative now canonicalizes both sides before giving up, restoring caller and link matches. - pod_relative_names compares in workspace-relative space instead of joining the raw workspace, restoring same-pod shortening. - Grep match paths fall back through the same canonical strip instead of rendering absolute. Verified against the three CI failures locally on macOS; Linux suites are unaffected (fast paths unchanged). --- crates/tui/src/tools/echolocation.rs | 18 ++++++++++++++++-- crates/tui/src/tools/search.rs | 27 +++++++++++++++++++++++---- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/crates/tui/src/tools/echolocation.rs b/crates/tui/src/tools/echolocation.rs index bcd0474cff..32fa7c3b06 100644 --- a/crates/tui/src/tools/echolocation.rs +++ b/crates/tui/src/tools/echolocation.rs @@ -130,10 +130,15 @@ fn pod_relative_names(workspace: &Path, file_path: &Path, links: Vec) -> let Some(home) = file_path.parent() else { return links; }; + // Compare in workspace-relative space: joining the workspace here + // breaks when it is non-canonical but the file path is canonical + // (macOS /var, Windows \\?\ verbatim paths). + let home_rel = workspace_relative(workspace, home); links .into_iter() .map(|link| { - let short = workspace.join(&link).parent() == Some(home); + let short = + Path::new(&link).parent().and_then(|parent| parent.to_str()) == home_rel.as_deref(); if short { Path::new(&link) .file_name() @@ -1196,7 +1201,16 @@ fn slash_path(path: &Path) -> String { } fn workspace_relative(workspace: &Path, path: &Path) -> Option { - path.strip_prefix(workspace).ok().map(slash_path) + if let Some(rel) = path.strip_prefix(workspace).ok().map(slash_path) { + return Some(rel); + } + // The tool layer hands us canonical file paths while the workspace + // may be non-canonical (macOS /var -> /private/var tempdirs, + // Windows \\?\ verbatim paths). Canonicalize both sides before + // giving up, or every caller/link match silently misses there. + let workspace = workspace.canonicalize().ok()?; + let path = path.canonicalize().ok()?; + path.strip_prefix(&workspace).ok().map(slash_path) } /// Python source lines: `#` comments cut, triple-quote doc spans diff --git a/crates/tui/src/tools/search.rs b/crates/tui/src/tools/search.rs index d715a8505b..0d0c51b7e3 100644 --- a/crates/tui/src/tools/search.rs +++ b/crates/tui/src/tools/search.rs @@ -237,18 +237,37 @@ impl ToolSpec for GrepFilesTool { return Ok(WalkControl::Continue); } - // Get relative path from workspace + // Relative path from workspace. The walk root comes + // back canonical while the workspace may not be + // (macOS /var, Windows \\?\ verbatim paths), so + // canonicalize both sides before falling back to + // the absolute path. let relative_path = file_path .strip_prefix(&workspace) .ok() + .map(|rel| rel.to_string_lossy().to_string()) .or_else(|| { workspace_canonical .as_deref() .and_then(|canonical| file_path.strip_prefix(canonical).ok()) + .map(|rel| rel.to_string_lossy().to_string()) }) - .unwrap_or(file_path) - .to_string_lossy() - .to_string(); + .or_else(|| { + // Windows verbatim roots (\\?\C:\ws) canonicalize + // the workspace but not the walked file path, so + // the strip above can still miss. Canonicalize the + // file too before giving up. + let ws = workspace_canonical + .clone() + .or_else(|| workspace.canonicalize().ok()); + let fp = file_path.canonicalize().ok(); + ws.zip(fp).and_then(|(ws, fp)| { + fp.strip_prefix(&ws) + .ok() + .map(|rel| rel.to_string_lossy().to_string()) + }) + }) + .unwrap_or_else(|| file_path.to_string_lossy().to_string()); let budget = max_results.saturating_sub(results.len()); let (file_matches, overflowed) = match search_file_streaming( From e8addb7896121bd6fcfe2bd10fc49e9cc8059239 Mon Sep 17 00:00:00 2001 From: CodeWhale Bot Date: Wed, 14 Oct 2026 03:12:55 -0700 Subject: [PATCH 21/26] docs(credits): include AdityaVG13 in candidate website credits Refs #6393 The first own-head Version drift job failed its contributor-credit gate because the website list omitted @AdityaVG13, whose original source-navigation and fork-prefix work is preserved in this branch. Add their literal handle beside the existing unreleased contributor without changing product source or the credit gate. Frozen combined canonical verification: npm 540 Node plus 772 Vitest passed; 0 failed; 7 skipped. npm run check:web, fmt and diff checks pass. This exact file matches the verified source. The original failed job log is retained; fresh exact-head hosted CI, reviews and Windows owner-recovery proof remain required. No upload, release, provider call or Windows distribution acceptance occurred. --- web/lib/release-credits.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/web/lib/release-credits.ts b/web/lib/release-credits.ts index 043f76fcf5..32062eba42 100644 --- a/web/lib/release-credits.ts +++ b/web/lib/release-credits.ts @@ -40,7 +40,10 @@ export const RELEASE_CONTRIBUTORS: string[] = [ * one. scripts/check-contributor-credit.py requires them here now; the release * cut moves them into RELEASE_CONTRIBUTORS with that release's changelog block. */ -export const UNRELEASED_CONTRIBUTORS: string[] = []; +export const UNRELEASED_CONTRIBUTORS: string[] = [ + "@aboimpinto", + "@AdityaVG13", +]; /** * Contributors who helped with reports, reproductions, and verification. From 69cfeedd353f3c232572ae11db375e55913d7a3d Mon Sep 17 00:00:00 2001 From: CodeWhale Bot Date: Wed, 14 Oct 2026 21:26:32 -0700 Subject: [PATCH 22/26] test(file): expect the pod footer on ranged and empty-range reads Preserve @AdityaVG13's source-echo work. #6393 intentionally appends the pod footer after the paging footer (see contract_read_appends_a_pod_footer and the "page plus paging footer first, pod footer after" case); three assertions in two older main tests still encoded the pre-echo output and failed on every OS. The macOS-only grep_pods failure was a grep path bug on main, fixed there in c6bcf78c01 and merged in here. Evidence (macOS, merged tree): nextest tools::file + tools::search + echolocation + budget_handback: 273 passed, 0 failed; cargo fmt --check clean. Hosted CI on this head is the full-suite gate. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NBKbhx19BRbZtvWK8rVKjz --- crates/tui/src/tools/file/tests.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/tui/src/tools/file/tests.rs b/crates/tui/src/tools/file/tests.rs index fe8ff161a6..979ff9e740 100644 --- a/crates/tui/src/tools/file/tests.rs +++ b/crates/tui/src/tools/file/tests.rs @@ -528,7 +528,7 @@ async fn contract_read_accepts_whole_number_floats_and_still_refuses_fractions() .expect("float-typed ranged read"); assert_eq!( ranged.content, - "two\n\n[1 more lines in file (13B total). Use offset=3 to continue.]" + "two\n\n[1 more lines in file (13B total). Use offset=3 to continue.]\n\n[Pod (1): lines.txt (this file)]" ); for (key, bad) in [ @@ -1155,7 +1155,7 @@ async fn contract_read_newline_dense_ranges_preserve_trailing_and_empty_lines() ) .await .unwrap(); - assert_eq!(result.content, "\n"); + assert_eq!(result.content, "\n\n\n[Pod (1): dense.txt (this file)]"); let metadata = result.metadata.as_ref().unwrap(); assert_eq!(metadata["size"], text.len()); assert_eq!(metadata["line_count"], text.len() + 1); @@ -1168,7 +1168,7 @@ async fn contract_read_newline_dense_ranges_preserve_trailing_and_empty_lines() .unwrap(); assert_eq!( empty.content, - "\n\n[1 more lines in file (2.0MB total). Use offset=2097153 to continue.]" + "\n\n[1 more lines in file (2.0MB total). Use offset=2097153 to continue.]\n\n[Pod (1): dense.txt (this file)]" ); } From cfedff9522d0c1bbb1b640571b96099d41298cdb Mon Sep 17 00:00:00 2001 From: AdityaVG13 Date: Mon, 12 Oct 2026 04:52:08 -0400 Subject: [PATCH 23/26] feat(navigation): add revision-qualified Rust read references An echo naming a symbol still makes an agent search for its body. Make Rust reads produce actionable source windows, and preserve the distinction between host computation, access authority and model-visible delivery. Implementation - Parse the admitted byte snapshot with the existing syn dependency; publish bounded function/impl-method observations and omission counts. - Return path, offset, limit and SHA-256 expected_revision together so direct reads and execute_tools programs can consume the same reference. - Refuse changed source before applying stale offsets; normal unguarded reads retain their existing contract. - Reuse HandleStore in a separate bounded host-owned runtime store. Cloned runtime services share observations, not authority or an agent's delivery ledger. Cache hits require admission and current-byte equality. - Record references from final projected tool-result text only when the model client accepts the request. Hidden nested reads do not count as delivery; resume rebuilds use the existing logged conversation. - Keep rendered navigation within the existing 1200-byte echo footer budget and structured metadata within its separate bounded view. Architecture and compatibility No new daemon, executor, persistent index, dependency or prompt contributor. Navigation rides append-only tool-result history. The optional read schema field is static and pinned with the session tool catalog. Ordinary reads remain unguarded unless the caller supplies expected_revision. Limits This is Rust syntax evidence, not macro expansion, cfg evaluation, semantic name/type resolution, an LSP result or a repository-wide call graph. Shared facts do not prove body readership. No live-provider cache-hit, latency, token-saving or packaged-release claim is made for this slice. Integration Selectively replay the navigation checkpoint onto published PR head 5912dc268, without replaying its superseded reconciliation parent. Preserve the protected-read, ignore-walker, 32-candidate and blocking-pool repairs. Keep navigation cache-key canonicalization on the blocking pool. Update the existing schema-property assertion for expected_revision; the existing catalog byte-budget assertion passes unchanged. Verification Strict-remote RCH on spark-1672, cargo test -p codewhale-tui --lib --locked --offline, with one test thread and a 64 MiB test-thread stack. Selected navigation, echolocation, codemode, read/source, working-set, handle, file-tool/schema and grep/file-search pod tests: 159 passed; 0 failed; 0 ignored. No full-suite execution. The Engine regression uses an injected model client to inspect the actual projected request; tool composition runs the real JavaScript execution path. Both navigation regressions fail with the corresponding production guard removed (stale-source refusal and delivered-reference recording), then pass after restoration. Their assertions were unchanged through mutation. Changed-file rustfmt and git diff --check pass. UBS scanned changed Rust files but returned file-wide heuristic findings, not a clean gate. The worker lacks nextest. GNU ld overflowed AArch64 branch relocations; the passing command uses installed LLD via a command-local RUSTFLAGS, without changing repository build profiles or configuration. No native Windows/macOS test execution, workspace-wide Cargo run, live-provider benchmark or release build is claimed. No-Issue: local navigation port for fork PR #3 and upstream draft #6393; publication remains a separate step. --- .../src/core/engine/tests/test_cases_08.rs | 97 ++++++ .../src/core/engine/turn_loop/model_step.rs | 5 + crates/tui/src/runtime_threads.rs | 1 + crates/tui/src/tools/codemode.rs | 91 +++++ crates/tui/src/tools/echolocation.rs | 310 +++++++++++++++++- crates/tui/src/tools/file.rs | 170 ++++++++-- crates/tui/src/tools/file_tool.rs | 4 + crates/tui/src/tools/file_tool/tests.rs | 2 +- crates/tui/src/tools/handle.rs | 12 + crates/tui/src/tools/spec.rs | 5 + crates/tui/src/tui/ui/event_loop.rs | 1 + crates/tui/src/working_set.rs | 65 ++++ docs/ECHOLOCATION.md | 64 +++- 13 files changed, 790 insertions(+), 37 deletions(-) diff --git a/crates/tui/src/core/engine/tests/test_cases_08.rs b/crates/tui/src/core/engine/tests/test_cases_08.rs index f8f469da5c..13c5d6ae05 100644 --- a/crates/tui/src/core/engine/tests/test_cases_08.rs +++ b/crates/tui/src/core/engine/tests/test_cases_08.rs @@ -312,6 +312,103 @@ async fn injected_model_duplicate_reads_both_execute_and_close_both_tool_ids() { task.await.expect("engine task"); } +#[tokio::test] +async fn native_navigation_tracks_only_returned_references_in_the_real_model_request() { + use crate::llm_client::mock::{MockLlmClient, canned}; + + for expose in [false, true] { + let workspace = tempdir().expect("tempdir"); + fs::write( + workspace.path().join("nav.rs"), + "fn wanted() {\n let _ = \"PRIVATE_BODY\";\n}\n", + ) + .unwrap(); + let selection = if expose { + "return {selected: r.metadata.navigation ? r.metadata.navigation.symbols[0].read : null};" + } else { + "return {done: true};" + }; + let code = format!( + "const r = await tools.call('read', {{path: 'nav.rs', limit: 1}}); {selection}" + ); + let mock = std::sync::Arc::new(MockLlmClient::new(vec![ + canned::tool_call_turn( + "navigation-program", + "execute_tools", + &serde_json::json!({"code": code}).to_string(), + ), + canned::simple_text_turn("Navigation received."), + ])); + let client: crate::core::model_client::SharedModelClient = mock.clone(); + let (mut engine, _handle) = Engine::new_with_model_client( + deterministic_engine_config(workspace.path()), + &Config::default(), + client, + ); + let context = crate::tools::ToolContext::new(workspace.path()); + let mut registry = crate::tools::ToolRegistry::new(context); + registry.register(std::sync::Arc::new(crate::tools::file_tool::ReadTool)); + let mut tools = registry.to_api_tools_with_cache(true); + let mut compose = crate::tools::codemode::execute_tools_tool_definition(); + compose.defer_loading = Some(false); + tools.push(compose); + let surface = test_tool_surface(&engine, registry, Some(tools), AppMode::Agent); + let mut turn = crate::core::turn::TurnContext::new(8); + let (status, error) = engine.run_turn(&mut turn, surface, None, None).await; + assert_eq!(status, TurnOutcomeStatus::Completed, "{error:?}"); + let requests = mock.captured_requests(); + assert_eq!(requests.len(), 2); + let delivered = requests[1] + .messages + .iter() + .flat_map(|message| &message.content) + .find_map(|block| match block { + ContentBlock::ToolResult { + tool_use_id, + content, + .. + } if tool_use_id == "navigation-program" => Some(content), + _ => None, + }) + .expect("actual outer result in the second model request"); + let payload: serde_json::Value = serde_json::from_str(delivered).unwrap(); + assert!(payload["success"].as_bool().unwrap(), "{payload}"); + assert!( + !delivered.contains("PRIVATE_BODY"), + "an intermediate read is not model delivery" + ); + let state = serde_json::to_value(&engine.session.working_set).unwrap(); + let refs = state["delivered_code_refs"] + .as_array() + .cloned() + .unwrap_or_default(); + if expose { + let selected = &payload["body"]["return"]["selected"]; + assert_eq!( + selected["path"], "nav.rs", + "composition must return an actionable reference" + ); + assert_eq!( + refs, + vec![selected.clone()], + "record exactly what survived outbound projection" + ); + let mut restored = crate::working_set::WorkingSet::default(); + restored.rebuild_from_messages(&engine.session.messages, workspace.path()); + assert_eq!( + serde_json::to_value(restored).unwrap()["delivered_code_refs"], + serde_json::json!(refs), + "resume reconstructs delivered references from the logged conversation, not the physical cache" + ); + } else { + assert!( + refs.is_empty(), + "hidden program reads must not seed delivery state" + ); + } + } +} + #[tokio::test] async fn duplicate_raw_read_errors_each_touch_the_working_set() { use crate::llm_client::mock::{MockLlmClient, canned}; diff --git a/crates/tui/src/core/engine/turn_loop/model_step.rs b/crates/tui/src/core/engine/turn_loop/model_step.rs index f24986ecd9..2ffd013471 100644 --- a/crates/tui/src/core/engine/turn_loop/model_step.rs +++ b/crates/tui/src/core/engine/turn_loop/model_step.rs @@ -103,6 +103,11 @@ impl Engine { .saturating_add(transport_retries.load(std::sync::atomic::Ordering::Relaxed)); let stream = match stream_result { Ok(s) => { + // Physical-cache population, nested execution and result + // projection alone do not establish model delivery. + self.session + .working_set + .observe_navigation_delivery(&stream_request.messages); if let Some(job) = child_job.as_ref() { job.provider_responded(); } diff --git a/crates/tui/src/runtime_threads.rs b/crates/tui/src/runtime_threads.rs index 3642a8016c..0f9ed5a0c8 100644 --- a/crates/tui/src/runtime_threads.rs +++ b/crates/tui/src/runtime_threads.rs @@ -16410,6 +16410,7 @@ impl RuntimeThreadManager { persist_services_enabled: false, hook_executor: Some(Arc::clone(&thread_hooks)), handle_store: crate::tools::handle::new_shared_handle_store(), + code_observations: crate::tools::handle::new_shared_handle_store(), rlm_sessions: crate::rlm::session::new_shared_rlm_session_store(), media_originals_dir: crate::media_originals::default_store_dir(), }, diff --git a/crates/tui/src/tools/codemode.rs b/crates/tui/src/tools/codemode.rs index acde966b8d..dd6184bb7e 100644 --- a/crates/tui/src/tools/codemode.rs +++ b/crates/tui/src/tools/codemode.rs @@ -1524,6 +1524,97 @@ mod tests { } } + #[tokio::test] + async fn native_navigation_resolves_real_source_and_refuses_stale_or_escaped_reads() { + let dir = tempfile::tempdir().unwrap(); + let source = "const LURE: &str = r#\"\nfn forged() {}\n\"#;\npub fn wanted() -> &'static str {\n \"NAV_BODY\"\n}\nfn sibling() {}\n"; + std::fs::write(dir.path().join("nav.rs"), source).unwrap(); + let context = ToolContext::new(dir.path()); + let registry = ToolRegistryBuilder::new() + .with_tool(Arc::new(ReadTool)) + .build(context.clone()); + let code = "const r = await tools.call('read', {path: 'nav.rs', limit: 1}); return r.metadata.navigation || null;"; + let result = execute_tools_tool(&json!({"code": code}), ®istry, &context) + .await + .unwrap(); + assert!(result.success, "{}", result.content); + let body: Value = serde_json::from_str(&result.content).unwrap(); + let navigation = &body["body"]["return"]; + assert_eq!( + navigation["producer"], "rust_syntax_v1", + "read must publish revision-qualified syntax evidence" + ); + let symbols = navigation["symbols"].as_array().unwrap(); + assert!( + !symbols.iter().any(|symbol| symbol["name"] == "forged"), + "a function-looking raw string is not a declaration" + ); + let reference = symbols + .iter() + .find(|symbol| symbol["name"] == "wanted") + .unwrap()["read"] + .clone(); + assert_eq!(reference["path"], "nav.rs"); + assert_eq!(reference["offset"], 4); + assert_eq!(reference["limit"], 3); + let wanted = ReadTool.execute(reference.clone(), &context).await.unwrap(); + assert!( + wanted + .content + .starts_with("pub fn wanted() -> &'static str {\n \"NAV_BODY\"\n}"), + "{}", + wanted.content + ); + assert!( + !wanted + .content + .split("\n\n[") + .next() + .unwrap() + .contains("fn sibling") + ); + + let child = context.clone().with_state_namespace("navigation-child"); + let child_read = ReadTool.execute(reference.clone(), &child).await.unwrap(); + assert_eq!( + child_read.metadata.as_ref().unwrap()["navigation"], + *navigation, + "shared physical knowledge is revision-stable, not parent delivery state" + ); + let child_workspace = dir.path().join("child"); + std::fs::create_dir(&child_workspace).unwrap(); + let restricted = + ToolContext::new(&child_workspace).with_runtime_services(context.runtime.clone()); + let mut escaped = reference.clone(); + escaped["path"] = json!("../nav.rs"); + assert!( + matches!( + ReadTool.execute(escaped, &restricted).await, + Err(ToolError::PathEscape { .. }) + ), + "a warmed shared cache must not grant workspace access" + ); + + std::fs::write( + dir.path().join("nav.rs"), + source.replace("NAV_BODY", "NAV_EDIT"), + ) + .unwrap(); + let stale = ReadTool + .execute(reference, &child) + .await + .expect_err("same-size external edit must invalidate the reference"); + assert!(stale.to_string().contains("stale"), "{stale}"); + let refreshed = ReadTool + .execute(json!({"path": "nav.rs", "limit": 1}), &child) + .await + .unwrap(); + assert_ne!( + refreshed.metadata.as_ref().unwrap()["navigation"]["source_revision"], + navigation["source_revision"] + ); + } + #[tokio::test] async fn program_composes_nested_read_and_returns_receipt() { let (_dir, note) = workspace_with_note(); diff --git a/crates/tui/src/tools/echolocation.rs b/crates/tui/src/tools/echolocation.rs index 32fa7c3b06..ba715d541f 100644 --- a/crates/tui/src/tools/echolocation.rs +++ b/crates/tui/src/tools/echolocation.rs @@ -1,6 +1,6 @@ -//! Echolocation: stateless navigation soundings for tool results. +//! Echolocation: bounded navigation observations for tool results. //! -//! No index, no background work, no new dependencies. Like a whale sounding +//! No repository-wide index, background work or new dependencies. Like a whale sounding //! its pod, every read, grep, file search, and project map echoes its //! surroundings fresh from source — sibling podmates, symbols, import //! roots, resolved file links — rendered deterministically (sorted, @@ -12,11 +12,14 @@ //! //! KV-cache effect: tool-result history only. Soundings never enter the //! session-pinned prefix. +//! Lowercase Rust reads also consume bounded, revision-qualified syntax facts +//! computed with the existing syn parser. RuntimeToolServices owns their shared +//! physical cache; WorkingSet separately owns each conversation's deliveries. //! //! Tuning: `CODEWHALE_ECHO` gates sections for ablation trials //! (comma-separated `off:` entries, e.g. `off:links,off:mates`; //! section names: mates, symbols, imports, links, callers, pods, -//! soundings). Unset or malformed means everything on. +//! soundings, navigation). Unset or malformed means everything on. //! //! Terminal buzz: the read footer deepens when the model is closing in. //! Two distinct files read in a pod (see @@ -26,7 +29,7 @@ //! high-resolution echoes at close range. Deterministic for a fixed //! history, like everything else here. //! -//! Known lookalikes (line scanners, not parsers): comment markers +//! Known lookalikes (legacy line scanners, not the Rust syntax producer): comment markers //! inside string literals can mute following lines until the closer //! (`"/*"` swallows until `"*/"`); Python doc fences toggle on every //! `"""`/`'''` occurrence, so mixed kinds on one line desync the @@ -40,6 +43,278 @@ use std::path::{Path, PathBuf}; use std::sync::OnceLock; use ignore::{Match as IgnoreMatch, gitignore::Gitignore}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +const NAVIGATION_PRODUCER: &str = "rust_syntax_v1"; +const NAVIGATION_MARKER: &str = "[Code navigation: "; +const MAX_NAVIGATION_SOURCE_BYTES: usize = 256 * 1024; +const MAX_NAVIGATION_BYTES: usize = 4 * 1024; +const MAX_NAVIGATION_PATH_BYTES: usize = 512; +pub(crate) const MAX_DELIVERED_REFS: usize = 32; + +/// An actionable snapshot-qualified line window, not a permission grant or +/// proof that its body was delivered. Coordinates are 1-based lines, not +/// LSP UTF-16 columns or byte offsets. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CodeReadReference { + pub path: String, + pub offset: usize, + pub limit: usize, + pub expected_revision: String, +} + +impl CodeReadReference { + fn valid(&self) -> bool { + !self.path.is_empty() + && self.path.len() <= MAX_NAVIGATION_PATH_BYTES + && Path::new(&self.path) + .components() + .all(|part| matches!(part, std::path::Component::Normal(_))) + && self.offset > 0 + && self.limit > 0 + && self.offset.checked_add(self.limit - 1).is_some() + && valid_source_revision(&self.expected_revision) + } +} + +pub(crate) fn valid_source_revision(revision: &str) -> bool { + revision.strip_prefix("sha256:").is_some_and(|digest| { + digest.len() == 64 + && digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + }) +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct NavigationSymbol { + name: String, + read: CodeReadReference, +} + +/// Syntax only: no macro expansion, cfg evaluation, name/type resolution or +/// LSP authority. V1 covers outer functions and impl methods in inline +/// modules; ranges exclude preceding attributes/modifiers and local items. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct SourceNavigation { + pub(crate) producer: String, + path: String, + pub(crate) source_revision: String, + status: String, + reason: Option, + symbols: Vec, + symbols_omitted: usize, + unscanned_modules: usize, +} + +pub(crate) fn navigation_cache_identity(workspace: &Path, path: &Path) -> Option<(String, String)> { + if path.extension()?.to_str()? != "rs" || path.to_str().is_none() { + return None; + } + let relative = workspace_relative(workspace, path)?; + if relative.len() > MAX_NAVIGATION_PATH_BYTES { + return None; + } + let root = workspace.canonicalize().ok()?; + // Bound keys as well as payloads; do not retain a full workspace path + // per record outside HandleStore's payload byte budget. + let namespace = format!( + "{NAVIGATION_PRODUCER}:{}", + super::file::content_hash(root.to_str()?.as_bytes()) + ); + Some((namespace, relative)) +} + +impl SourceNavigation { + pub(crate) fn matches(&self, path: &str, revision: &str) -> bool { + self.producer == NAVIGATION_PRODUCER + && self.path == path + && self.source_revision == revision + } + + pub(crate) fn from_source( + path: String, + revision: String, + source: &str, + utf8_valid: bool, + ) -> Self { + let mut result = Self { + producer: NAVIGATION_PRODUCER.to_string(), + path, + source_revision: revision, + status: "ready".to_string(), + reason: None, + symbols: Vec::new(), + symbols_omitted: 0, + unscanned_modules: 0, + }; + let unavailable = if !utf8_valid { + Some("invalid_utf8") + } else if source.len() > MAX_NAVIGATION_SOURCE_BYTES { + Some("source_byte_bound") + } else { + None + }; + if let Some(reason) = unavailable { + result.status = "unavailable".to_string(); + result.reason = Some(reason.to_string()); + return result; + } + match syn::parse_file(source) { + Ok(file) => result.visit_items(&file.items, "", 0), + Err(_) => { + result.status = "unavailable".to_string(); + result.reason = Some("syntax_error".to_string()); + } + } + while serde_json::to_vec(&result).is_ok_and(|bytes| bytes.len() > MAX_NAVIGATION_BYTES) { + if result.symbols.pop().is_none() { + break; + } + result.symbols_omitted += 1; + } + result + } + + fn visit_items(&mut self, items: &[syn::Item], scope: &str, depth: usize) { + for item in items { + match item { + syn::Item::Fn(function) => { + self.note_function(scope, &function.sig, &function.block) + } + syn::Item::Impl(implementation) => { + let scope = format!( + "{scope}impl@{}::", + implementation.impl_token.span.start().line + ); + for item in &implementation.items { + if let syn::ImplItem::Fn(function) = item { + self.note_function(&scope, &function.sig, &function.block); + } + } + } + syn::Item::Mod(module) => { + if let Some((_, items)) = &module.content { + if depth >= 16 { + self.unscanned_modules += 1; + } else { + self.visit_items( + items, + &format!("{scope}{}::", module.ident), + depth + 1, + ); + } + } + } + _ => {} + } + } + } + + fn note_function(&mut self, scope: &str, signature: &syn::Signature, block: &syn::Block) { + let name = format!("{scope}{}", signature.ident); + let start = signature.fn_token.span.start().line; + let end = block.brace_token.span.close().end().line; + if self.symbols.len() >= MAX_SYMBOLS + || name.len() > MAX_NAME_LEN + || start == 0 + || end < start + { + self.symbols_omitted += 1; + return; + } + self.symbols.push(NavigationSymbol { + name, + read: CodeReadReference { + path: self.path.clone(), + offset: start, + limit: end - start + 1, + expected_revision: self.source_revision.clone(), + }, + }); + } + + /// Whole actionable objects under half the existing footer budget. + /// Metadata retains the larger bounded view. + fn render_footer(&self) -> String { + if echo_off("navigation") { + return String::new(); + } + let mut shown = self.clone(); + shown.symbols.clear(); + shown.symbols_omitted += self.symbols.len(); + for symbol in &self.symbols { + shown.symbols.push(symbol.clone()); + shown.symbols_omitted -= 1; + if serde_json::to_string(&shown).is_ok_and(|text| { + text.len() + NAVIGATION_MARKER.len() + 3 > POD_FOOTER_MAX_BYTES / 2 + }) { + shown.symbols.pop(); + shown.symbols_omitted += 1; + break; + } + } + let footer = format!( + "\n\n{NAVIGATION_MARKER}{}]", + serde_json::to_string(&shown).unwrap_or_default() + ); + if footer.len() <= POD_FOOTER_MAX_BYTES / 2 { + footer + } else { + String::new() + } + } +} + +/// Delivery bookkeeping from projected tool-result text only. JSON strings +/// are not recursively decoded: a log/args preview is not an explicitly +/// returned reference. None of these references authorizes a future read. +pub(crate) fn delivered_navigation_refs(text: &str) -> Vec { + fn collect(value: &Value, refs: &mut Vec, nodes: &mut usize, depth: usize) { + if *nodes == 0 || depth > 16 || refs.len() >= MAX_DELIVERED_REFS { + return; + } + *nodes -= 1; + if value + .as_object() + .is_some_and(|object| object.len() == 4 && object.contains_key("expected_revision")) + && let Ok(reference) = serde_json::from_value::(value.clone()) + && reference.valid() + { + if !refs.contains(&reference) { + refs.push(reference); + } + return; + } + match value { + Value::Object(object) => { + for value in object.values() { + collect(value, refs, nodes, depth + 1); + } + } + Value::Array(values) => { + for value in values { + collect(value, refs, nodes, depth + 1); + } + } + _ => {} + } + } + let mut refs = Vec::new(); + let value = serde_json::from_str::(text).ok().or_else(|| { + let start = text.rfind(NAVIGATION_MARKER)? + NAVIGATION_MARKER.len(); + serde_json::Deserializer::from_str(&text[start..]) + .into_iter::() + .next()? + .ok() + }); + if let Some(value) = value { + collect(&value, &mut refs, &mut 2048, 0); + } + refs +} /// Hard budget for the rendered footer, in bytes. pub const POD_FOOTER_MAX_BYTES: usize = 1200; @@ -258,6 +533,31 @@ impl EchoChart { /// Render the footer. Deterministic for a fixed file and directory: /// sorted entries, fixed caps, byte budget, no timestamps. pub fn render_footer(&self) -> String { + self.render_footer_with_budget(POD_FOOTER_MAX_BYTES) + } + + pub(crate) fn render_with_navigation( + &mut self, + navigation: Option<&SourceNavigation>, + ) -> String { + let Some(navigation) = navigation else { + return self.render_footer(); + }; + if navigation.status == "ready" { + // Consume the syntax producer instead of publishing conflicting + // line-scanner lookalikes beside its qualified observations. + self.symbols = navigation + .symbols + .iter() + .map(|symbol| format!("fn {}:{}", symbol.name, symbol.read.offset)) + .collect(); + self.symbol_total = self.symbols.len() + navigation.symbols_omitted; + } + let footer = navigation.render_footer(); + self.render_footer_with_budget(POD_FOOTER_MAX_BYTES - footer.len()) + &footer + } + + fn render_footer_with_budget(&self, budget: usize) -> String { let mut out = String::new(); // No dir: the footer trails the read of this exact file, so the // pod frame is already known (miss echoes and grep pods keep @@ -307,7 +607,7 @@ impl EchoChart { if !sounding.is_empty() { out.push_str(&format!("\n\n[Sound: {sounding}]")); } - truncate_bytes(&out, POD_FOOTER_MAX_BYTES) + truncate_bytes(&out, budget) } } diff --git a/crates/tui/src/tools/file.rs b/crates/tui/src/tools/file.rs index 666260efa3..e4a4707381 100644 --- a/crates/tui/src/tools/file.rs +++ b/crates/tui/src/tools/file.rs @@ -948,8 +948,20 @@ impl ReadFileTool { input: Value, context: &ToolContext, ) -> Result { - reject_primitive_unknown(&input, "read", &["path", "offset", "limit", "max_bytes"])?; + reject_primitive_unknown( + &input, + "read", + &["path", "offset", "limit", "max_bytes", "expected_revision"], + )?; let path_str = required_str(&input, "path")?; + let expected_revision = optional_str(&input, "expected_revision")?; + if let Some(revision) = expected_revision + && !crate::tools::echolocation::valid_source_revision(revision) + { + return Err(ToolError::invalid_input( + "expected_revision must be sha256: followed by 64 lowercase hexadecimal digits", + )); + } let offset = contract_nonnegative_int(&input, "offset")?; let limit = contract_nonnegative_int(&input, "limit")?; let max_bytes = effective_read_max_bytes(contract_nonnegative_int(&input, "max_bytes")?); @@ -979,8 +991,7 @@ impl ReadFileTool { // or permission failures, where listing the parent would // answer a refused probe. `sound_miss_echo` carries those // gates internally, so appending it here stays fail-closed. - let mut message = - format!("Failed to read {}: file not found", file_path.display()); + let mut message = format!("Failed to read {}: file not found", file_path.display()); if let Some(echo) = crate::tools::echolocation::sound_miss_echo(&file_path) { message.push_str(&echo); } @@ -992,6 +1003,11 @@ impl ReadFileTool { let size_bytes = bytes.len(); check_file_operation_cancelled(context)?; if let Some(mime_type) = primitive_image_mime(&bytes) { + if expected_revision.is_some() { + return Err(ToolError::execution_failed( + "Navigation reference is stale: the target is now an image. Read the path again without the old range/reference.", + )); + } // Decoding and any downscale are CPU-bound; keep them off the // runtime worker (#6149). let prepared = tokio::task::spawn_blocking(move || { @@ -1012,7 +1028,57 @@ impl ReadFileTool { // The small-contract reader decodes non-image buffers as UTF-8 text with replacement // characters instead of refusing the whole read on one invalid byte. - let text = String::from_utf8_lossy(&bytes); + let workspace = context.workspace.clone(); + let observation_path = file_path.clone(); + let guarded = expected_revision.is_some(); + let (text, source_revision, utf8_valid, cache_identity) = + tokio::task::spawn_blocking(move || { + let cache_identity = crate::tools::echolocation::navigation_cache_identity( + &workspace, + &observation_path, + ); + let revision = (cache_identity.is_some() || guarded).then(|| content_hash(&bytes)); + let (text, utf8_valid) = match String::from_utf8(bytes) { + Ok(text) => (text, true), + Err(error) => ( + String::from_utf8_lossy(error.as_bytes()).into_owned(), + false, + ), + }; + (text, revision, utf8_valid, cache_identity) + }) + .await + .map_err(|error| { + ToolError::execution_failed(format!("Failed to decode source: {error}")) + })?; + check_file_operation_cancelled(context)?; + if let Some(expected) = expected_revision + && source_revision.as_deref() != Some(expected) + { + return Err(ToolError::execution_failed(format!( + "Navigation reference for {path_str} is stale. Read the path without the old range/expected_revision to obtain a new reference; do not reuse its offsets." + ))); + } + // Admission and current-byte revision checks precede every cache hit. + // Shared physical knowledge does not grant access or delivery state. + let cached_navigation = if let Some((namespace, path)) = &cache_identity { + context + .runtime + .code_observations + .lock() + .await + .get_json_named(namespace, path) + .cloned() + .and_then(|value| { + serde_json::from_value::(value) + .ok() + }) + .filter(|navigation| { + navigation.matches(path, source_revision.as_deref().unwrap_or_default()) + }) + } else { + None + }; // Count and select with byte boundaries, without a pointer per newline. let line_count = text.bytes().filter(|byte| *byte == b'\n').count() + 1; let requested_offset = offset.unwrap_or(1); @@ -1100,34 +1166,80 @@ impl ReadFileTool { let buzz = file_path.parent().is_some_and(|parent| { context.pod_visit_count(parent) >= crate::tools::echolocation::BUZZ_VISIT_THRESHOLD }); - output.push_str( - &crate::tools::echolocation::sound_echolocation( - &context.workspace, - &file_path, - &text, - buzz, - ) - .render_footer(), - ); + let cached_navigation = if let Some((namespace, path)) = &cache_identity { + context + .runtime + .code_observations + .lock() + .await + .get_json_named(namespace, path) + .cloned() + .and_then(|value| { + serde_json::from_value::(value) + .ok() + }) + .filter(|navigation| { + navigation.matches(path, source_revision.as_deref().unwrap_or_default()) + }) + } else { + None + }; + let workspace = context.workspace.clone(); + let echo_path = file_path.clone(); + let navigation_path = cache_identity.as_ref().map(|(_, path)| path.clone()); + let (echo, navigation) = tokio::task::spawn_blocking(move || { + let navigation = cached_navigation.or_else(|| { + navigation_path + .zip(source_revision) + .map(|(path, revision)| { + crate::tools::echolocation::SourceNavigation::from_source( + path, revision, &text, utf8_valid, + ) + }) + }); + let echo = + crate::tools::echolocation::sound_echolocation(&workspace, &echo_path, &text, buzz) + .render_with_navigation(navigation.as_ref()); + (echo, navigation) + }) + .await + .map_err(|error| { + ToolError::execution_failed(format!("read echo worker failed: {error}")) + })?; + check_file_operation_cancelled(context)?; + output.push_str(&echo); + if let (Some((namespace, path)), Some(navigation)) = (&cache_identity, &navigation) { + let _ = context.runtime.code_observations.lock().await.insert_json( + namespace, + path, + serde_json::to_value(navigation) + .map_err(|error| ToolError::execution_failed(error.to_string()))?, + ); + } - // This internal observation keeps hidden legacy edit replay working, - // but no hash or read-before-edit ceremony reaches the lowercase - // schema or result. + // Keep hidden legacy edit replay working. Ordinary reads remain + // unguarded; the optional navigation revision pins a selected range, + // not a read-before-edit permission ceremony. context.note_file_read(&file_path); + let mut metadata = json!({ + "evidence_routing": "inline", + // The budget this call actually enforced. The context + // compactor honors it so an already-bounded read is never + // truncated a second time on its way into the conversation. + "read_budget_bytes": max_bytes, + // #6283: paging contract. `size` is the whole file in bytes, + // `line_count` its total lines, `truncated` whether the file + // holds more than this response shows. + "size": size_bytes, + "truncated": truncated, + "line_count": line_count + }); + if let Some(navigation) = navigation { + metadata["navigation"] = serde_json::to_value(navigation) + .map_err(|error| ToolError::execution_failed(error.to_string()))?; + } Ok(RichToolResult::plain( - ToolResult::success(output).with_metadata(json!({ - "evidence_routing": "inline", - // The budget this call actually enforced. The context - // compactor honors it so an already-bounded read is never - // truncated a second time on its way into the conversation. - "read_budget_bytes": max_bytes, - // #6283: paging contract. `size` is the whole file in bytes, - // `line_count` its total lines, `truncated` whether the file - // holds more than this response shows. - "size": size_bytes, - "truncated": truncated, - "line_count": line_count - })), + ToolResult::success(output).with_metadata(metadata), )) } } diff --git a/crates/tui/src/tools/file_tool.rs b/crates/tui/src/tools/file_tool.rs index 51916206e1..272f696dfd 100644 --- a/crates/tui/src/tools/file_tool.rs +++ b/crates/tui/src/tools/file_tool.rs @@ -82,6 +82,10 @@ impl ToolSpec for ReadTool { "max_bytes": { "type": "number", "description": "Output budget in bytes for this one call. Defaults to 100000; values above the 500000 maximum are clamped down rather than rejected, and a value below the active default leaves the default in place." + }, + "expected_revision": { + "type": "string", + "description": "Optional snapshot guard from a code navigation reference. Pass the reference's path, offset, limit and expected_revision together; changed source is refused instead of silently reading stale offsets. Omit for an ordinary read." } }, "required": ["path"], diff --git a/crates/tui/src/tools/file_tool/tests.rs b/crates/tui/src/tools/file_tool/tests.rs index 311d71fd26..3159c933f1 100644 --- a/crates/tui/src/tools/file_tool/tests.rs +++ b/crates/tui/src/tools/file_tool/tests.rs @@ -555,7 +555,7 @@ fn primitive_schemas_are_separate_and_small_contract_shaped() { .cloned() .collect::>(), // `max_bytes` is the model-requested per-call output budget (C05). - ["limit", "max_bytes", "offset", "path"] + ["expected_revision", "limit", "max_bytes", "offset", "path"] .into_iter() .map(str::to_string) .collect() diff --git a/crates/tui/src/tools/handle.rs b/crates/tui/src/tools/handle.rs index 3478992499..f5183bdd7e 100644 --- a/crates/tui/src/tools/handle.rs +++ b/crates/tui/src/tools/handle.rs @@ -155,6 +155,18 @@ impl HandleStore { self.records.get(&handle.key()) } + /// Host-owned named JSON, without minting a model-facing var_handle. + pub(crate) fn get_json_named(&self, namespace: &str, name: &str) -> Option<&Value> { + let record = self.records.get(&HandleKey { + session_id: namespace.to_string(), + name: name.to_string(), + })?; + match &record.value { + HandleValue::Json(value) => Some(value), + HandleValue::Text(_) => None, + } + } + /// Remove all handles for `session_id`. Called when an agent's records /// are retired so resident transcript payloads are freed without waiting /// for a full session reset (#3885). diff --git a/crates/tui/src/tools/spec.rs b/crates/tui/src/tools/spec.rs index 4253109bf4..d4fad23678 100644 --- a/crates/tui/src/tools/spec.rs +++ b/crates/tui/src/tools/spec.rs @@ -116,6 +116,9 @@ pub struct RuntimeToolServices { /// Per-session backing store for `var_handle` payloads. Cloned tool /// contexts share this Arc so handles survive across turns. pub handle_store: SharedHandleStore, + /// Bounded host-computed source observations, isolated from writable + /// var_handles. Children clone the Arc, not delivery state or authority. + pub code_observations: SharedHandleStore, /// Per-session persistent RLM kernels, keyed by caller-chosen context name. pub rlm_sessions: SharedRlmSessionStore, /// Directory for `read_media`'s content-addressed store of @@ -139,6 +142,7 @@ impl Default for RuntimeToolServices { work: None, hook_executor: None, handle_store: new_shared_handle_store(), + code_observations: new_shared_handle_store(), rlm_sessions: new_shared_rlm_session_store(), media_originals_dir: None, } @@ -162,6 +166,7 @@ impl std::fmt::Debug for RuntimeToolServices { .field("work", &self.work.is_some()) .field("hook_executor", &self.hook_executor.is_some()) .field("handle_store", &true) + .field("code_observations", &true) .field("rlm_sessions", &true) .field("media_originals_dir", &self.media_originals_dir) .finish() diff --git a/crates/tui/src/tui/ui/event_loop.rs b/crates/tui/src/tui/ui/event_loop.rs index 3577f344bf..e531adecf7 100644 --- a/crates/tui/src/tui/ui/event_loop.rs +++ b/crates/tui/src/tui/ui/event_loop.rs @@ -1200,6 +1200,7 @@ pub async fn run_tui( // the configured `shell_env` hooks. Clone the shared Arc. hook_executor: app.runtime_services.hook_executor.clone(), handle_store: app.runtime_services.handle_store.clone(), + code_observations: app.runtime_services.code_observations.clone(), rlm_sessions: app.runtime_services.rlm_sessions.clone(), media_originals_dir: crate::media_originals::default_store_dir(), }; diff --git a/crates/tui/src/working_set.rs b/crates/tui/src/working_set.rs index a60a9383e6..c91d714c87 100644 --- a/crates/tui/src/working_set.rs +++ b/crates/tui/src/working_set.rs @@ -6,6 +6,9 @@ //! - a compact working-set summary block for the system prompt //! - pinned message indices that compaction should preserve +use crate::tools::echolocation::{ + CodeReadReference, MAX_DELIVERED_REFS, delivered_navigation_refs, +}; use crate::workspace_discovery::{ DISCOVERY_ALWAYS_DIRS, path_is_excluded_from_discovery, should_skip_unignored_discovery_entry, }; @@ -1006,9 +1009,54 @@ pub struct WorkingSet { pub turn: u64, /// Path entries keyed by workspace-relative path. pub entries: HashMap, + /// Latest bounded navigation references delivered in this conversation. + /// Not body readership, current source freshness or read authority. This + /// field is never rendered into the session-pinned prompt prefix. + #[serde(default)] + pub delivered_code_refs: Vec, + #[serde(skip)] + navigation_delivery_initialized: bool, } impl WorkingSet { + /// Only provider acceptance of the projected request establishes delivery; + /// intermediate program reads are absent from these messages. + pub(crate) fn observe_navigation_delivery(&mut self, messages: &[Message]) { + // Inspect inherited context on the first accepted request, then only + // the current result tail. Do not reparse every historical payload on + // every model step just to rediscover the same references. + let start = if self.navigation_delivery_initialized { + messages + .iter() + .rposition(|message| message.role == "assistant") + .map_or(0, |index| index + 1) + } else { + 0 + }; + self.navigation_delivery_initialized = true; + for message in &messages[start..] { + for block in &message.content { + if let ContentBlock::ToolResult { content, .. } = block { + self.record_delivered_navigation(delivered_navigation_refs(content)); + } + } + } + } + + fn record_delivered_navigation(&mut self, refs: Vec) { + for reference in refs { + self.delivered_code_refs.retain(|old| { + old.path != reference.path || old.expected_revision == reference.expected_revision + }); + if !self.delivered_code_refs.contains(&reference) { + self.delivered_code_refs.push(reference); + if self.delivered_code_refs.len() > MAX_DELIVERED_REFS { + self.delivered_code_refs.remove(0); + } + } + } + } + /// Advance to the next turn. pub fn next_turn(&mut self) { self.turn = self.turn.saturating_add(1); @@ -1044,8 +1092,25 @@ impl WorkingSet { pub fn rebuild_from_messages(&mut self, messages: &[Message], workspace: &Path) { self.entries.clear(); self.turn = 0; + self.delivered_code_refs.clear(); + self.navigation_delivery_initialized = true; + let mut offered = Vec::new(); for message in messages { + // A subsequent assistant response confirms the historical request, + // unlike a trailing tool result left at a crash boundary. Fresh + // child seeds do not invoke this resume/compaction rebuild. + if message.role == "assistant" { + self.record_delivered_navigation(std::mem::take(&mut offered)); + } + for block in &message.content { + if let ContentBlock::ToolResult { content, .. } = block { + offered.extend(delivered_navigation_refs(content)); + if offered.len() > MAX_DELIVERED_REFS { + offered.drain(..offered.len() - MAX_DELIVERED_REFS); + } + } + } if message.role == "user" { self.next_turn(); } diff --git a/docs/ECHOLOCATION.md b/docs/ECHOLOCATION.md index 5b34830adf..f11af5e8dc 100644 --- a/docs/ECHOLOCATION.md +++ b/docs/ECHOLOCATION.md @@ -15,7 +15,8 @@ only. `[Sound: N symbols: …; imports: …; links: …]` (line-numbered symbols, import roots, cross-pod file links; same-pod links render bare and import roots a link already names (file stem or package - path) are dropped). Budget: 1200B. + path) are dropped). Lowercase Rust `read` also publishes bounded syntax + navigation as described below. Total footer budget remains 1200B. - **Terminal buzz**: after two distinct files are read in one pod, the next read there adds `heard by …` — podmates whose resolved links include this file. Sparse clicks while searching, high-resolution @@ -44,14 +45,73 @@ TypeScript, and Go via deliberately small line scanners (see classes). Anything unresolvable is omitted, never guessed; links require an on-disk target. +## Revision-qualified Rust navigation + +The lowercase `read` path uses the already-installed `syn` parser to observe +outer Rust functions and impl methods, including inline modules. A bounded +`metadata.navigation` object identifies its producer (`rust_syntax_v1`), the +workspace-relative path, source-byte SHA-256 revision, availability and omission +counts. Each symbol carries a `read` object: `path`, 1-based line `offset`, line +`limit`, and `expected_revision`. Pass that object directly to `read` to inspect +the captured function window. No separately named navigation tool is required. + +Observations come from the same byte snapshot as the read, not another filesystem +read. Syntax is not semantic resolution: macros, active cfg, external modules, +type/name resolution and LSP coordinates are not claimed. Ranges start on the +`fn` keyword's line; preceding attribute/modifier lines and local items are not +included. Invalid UTF-8, parse errors and sources above +256 KiB report `unavailable`, never invented ranges. At most 24 symbols and 4 KiB +of serialized navigation are retained; counts identify omitted observations. + +A `[Code navigation: ...]` footer renders whole reference objects under 600B, +sharing the existing 1200B budget with the pod/sounding sections. Long paths can +leave no room for an inline reference; the bounded metadata remains available to +`execute_tools`. Existing scanners still serve other languages and other echo +surfaces. The successful Rust read's symbol sounding consumes these syntax facts. + +### Shared facts, separate delivery + +`RuntimeToolServices` reuses the existing bounded `HandleStore` implementation in +a **separate, host-owned store**, not the writable program `var_handle` namespace. +Keys bind canonical workspace, relative path and producer; cached values are +reused only after normal path/authority admission and matching the **current byte +revision**. There is no watcher, persistent index, permission bypass or mtime-only +freshness claim. Child runtime clones share the physical store, not a parent's +delivery ledger. Cache eviction simply recomputes the observation on a later read. + +An optional `expected_revision` guard rejects stale source before using old +offsets, including a same-size external edit. Without it, ordinary reads behave +normally. A reference is evidence of a range, never authority to access that path. + +`WorkingSet` retains at most 32 delivered references **only after the final +projected request is accepted by the model client**. It inspects model-facing +tool-result text, not nested calls or the physical cache. A program can read source +internally and return only `{done: true}` without seeding this ledger; returning a +selected structured reference records just that reference. JSON strings inside +logs/argument previews are not decoded as deliveries. Inspection is conservative +and bounded to 2048 JSON nodes and 16 nesting levels per result. + +The session's existing tool-result log retains the same projected reference +objects. Resume/compaction rebuilds use that conversation, requiring a subsequent +assistant message before classifying a historical result as delivered; a trailing +crash-boundary result alone is not proof. This ledger does not claim body readership, +current freshness, complete historical delivery or deduplication of future reads. +Fresh child seeds do not rebuild it from a parent's ledger. + +**KV-cache effect:** navigation is tool-result history only, never a volatile +prefix contributor. The optional `read` schema field is static and session-pinned +with the tool catalog. No new system-prompt text is inserted. + ## Tuning and ablation `CODEWHALE_ECHO` gates sections for trials (`off:`, comma-separated; names: `mates`, `symbols`, `imports`, -`links`, `callers`, `pods`, `soundings`). Unset or malformed means +`links`, `callers`, `pods`, `soundings`, `navigation`). Unset or malformed means everything on. The gates are process-pinned and read-only; there is no per-session or per-call control, by design. The squeeze follows the same gates, so an `off:links` trial measures links alone. +`off:navigation` suppresses the inline navigation footer only; structured +metadata remains available to programs and still uses the syntax producer. ## Non-goals From 76b7a040319ce02077c862ea15729f2b329dadf3 Mon Sep 17 00:00:00 2001 From: AdityaVG13 Date: Thu, 22 Oct 2026 02:33:30 -0400 Subject: [PATCH 24/26] feat(navigation): ready-navigation soundings and slice SSE reader sound_echolocation takes Option<&SourceNavigation> so ready navigation supplies symbols from the same admitted source snapshot, bypassing the legacy symbol scan. SSE line framing moves to a borrowing slice walker (take_sse_line_from_slice) over a single cursor. Subagent fork seeding hands the resolved fork straight through, and the read-guard, file, project and search surfaces follow the admitted-snapshot contract; budget handback tests track the new seams. Signed-off-by: AdityaVG13 --- crates/tui/src/client/chat.rs | 10 +- crates/tui/src/client/wire.rs | 14 +- crates/tui/src/sandbox/read_guard.rs | 6 +- crates/tui/src/tools/echolocation.rs | 700 ++++++++++++------ crates/tui/src/tools/file.rs | 46 +- crates/tui/src/tools/project.rs | 14 +- crates/tui/src/tools/search.rs | 19 +- .../tools/subagent/budget_handback_tests.rs | 10 +- crates/tui/src/tools/subagent/engine.rs | 2 +- crates/tui/src/tools/subagent/mod.rs | 10 +- crates/tui/src/tools/subagent/tests.rs | 2 +- 11 files changed, 550 insertions(+), 283 deletions(-) diff --git a/crates/tui/src/client/chat.rs b/crates/tui/src/client/chat.rs index 9aafae6b33..a1de9c5ebb 100644 --- a/crates/tui/src/client/chat.rs +++ b/crates/tui/src/client/chat.rs @@ -32,7 +32,9 @@ use codewhale_models::{ use super::prepared::WireDialect; use super::role_placement::{RolePlacement, role_placement}; -use super::wire::{extract_sse_data_value, flush_sse_line, push_sse_event_data, take_sse_line}; +use super::wire::{ + extract_sse_data_value, flush_sse_line, push_sse_event_data, take_sse_line_from_slice, +}; use super::{ CodewhaleClient, ERROR_BODY_MAX_BYTES, SSE_BACKPRESSURE_HIGH_WATERMARK, SSE_BACKPRESSURE_SLEEP_MS, SSE_MAX_LINES_PER_CHUNK, acquire_stream_buffer, @@ -1547,8 +1549,9 @@ impl CodewhaleClient { // a `\n` so an HTTP/2 DATA split mid-character cannot become // U+FFFD; genuine invalid bytes fail closed. let mut lines_processed = 0usize; + let mut remaining = byte_buf.as_slice(); loop { - let line = match take_sse_line(&mut byte_buf) { + let line = match take_sse_line_from_slice(&mut remaining) { Ok(Some(line)) => line, Ok(None) => break, Err(err) => { @@ -1637,6 +1640,9 @@ impl CodewhaleClient { tokio::task::yield_now().await; } } + // Move the unterminated tail once per chunk, not once per line. + let consumed = byte_buf.len() - remaining.len(); + byte_buf.drain(..consumed); } // Flush a final SSE frame that arrived without a terminating blank diff --git a/crates/tui/src/client/wire.rs b/crates/tui/src/client/wire.rs index 1db76c3495..7b9921523c 100644 --- a/crates/tui/src/client/wire.rs +++ b/crates/tui/src/client/wire.rs @@ -197,9 +197,19 @@ fn decode_sse_line_bytes(bytes: &[u8]) -> Result<&str, SseLineError> { /// Genuine invalid bytes fail closed (`Err(SseLineError)`); we do not /// substitute U+FFFD. pub(super) fn take_sse_line(buffer: &mut Vec) -> Result, SseLineError> { + let mut remaining = buffer.as_slice(); + let decoded = take_sse_line_from_slice(&mut remaining); + let consumed = buffer.len() - remaining.len(); + buffer.drain(..consumed); + decoded +} + +/// Advance over one complete line without moving the remaining bytes. Chunk +/// consumers can drain the consumed prefix once after processing all its lines. +pub(super) fn take_sse_line_from_slice(buffer: &mut &[u8]) -> Result, SseLineError> { let line_end = buffer.iter().position(|&b| b == b'\n'); if line_end.unwrap_or(buffer.len()) > MAX_SSE_LINE_BYTES { - buffer.clear(); + *buffer = &[]; return Err(SseLineError::TooLong); } let Some(line_end) = line_end else { @@ -211,7 +221,7 @@ pub(super) fn take_sse_line(buffer: &mut Vec) -> Result, SseL end -= 1; } let decoded = decode_sse_line_bytes(&buffer[..end]).map(|text| text.trim().to_string()); - buffer.drain(..=line_end); + *buffer = &buffer[line_end + 1..]; decoded.map(Some) } diff --git a/crates/tui/src/sandbox/read_guard.rs b/crates/tui/src/sandbox/read_guard.rs index 603ab67bec..cb2bb680aa 100644 --- a/crates/tui/src/sandbox/read_guard.rs +++ b/crates/tui/src/sandbox/read_guard.rs @@ -295,7 +295,7 @@ impl ReadDenylist { let resolved = canonicalize_best_effort(requested); let via_symlink = resolved != literal; - for candidate in [&literal, &resolved] { + for candidate in std::iter::once(&literal).chain(via_symlink.then_some(&resolved)) { if self.deny_env_files && is_env_file(candidate) { return Err(ReadDenial { requested: requested.to_path_buf(), @@ -780,6 +780,10 @@ fn fold_component(component: &std::ffi::OsStr) -> OsString { /// (`Path::starts_with` is already component-wise; the case folding is what /// forces the manual walk.) fn path_is_within(candidate: &Path, root: &Path) -> bool { + if !cfg!(any(target_os = "macos", target_os = "windows")) { + return candidate.starts_with(root); + } + let mut root_components = root.components().map(|c| fold_component(c.as_os_str())); let mut candidate_components = candidate .components() diff --git a/crates/tui/src/tools/echolocation.rs b/crates/tui/src/tools/echolocation.rs index ba715d541f..ce28447e66 100644 --- a/crates/tui/src/tools/echolocation.rs +++ b/crates/tui/src/tools/echolocation.rs @@ -39,10 +39,11 @@ //! on-disk target, so they forge only when both coincide. use std::collections::{BTreeMap, BTreeSet}; +use std::io::Read; use std::path::{Path, PathBuf}; use std::sync::OnceLock; -use ignore::{Match as IgnoreMatch, gitignore::Gitignore}; +use ignore::WalkBuilder; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -341,6 +342,7 @@ const MAX_LINKS: usize = 8; const MAX_CALLERS: usize = 8; /// Mate files bigger than this are not scanned for callers. const MAX_CALLER_SCAN_BYTES: u64 = 256 * 1024; +const MAX_CALLER_SCAN_FILES: usize = 32; /// Tracked pod reads before the next read there enters the buzz. pub(crate) const BUZZ_VISIT_THRESHOLD: usize = 2; /// Max touched symbols named per edit echo before "+N more". @@ -394,6 +396,7 @@ pub(crate) struct EchoChart { link_total: usize, callers: Vec, caller_total: usize, + caller_candidates_omitted: usize, } /// Shorten workspace-relative links to bare names when they live in @@ -431,34 +434,76 @@ fn pod_relative_names(workspace: &Path, file_path: &Path, links: Vec) -> /// `file_path` is the resolved path (for the sibling sounding); `text` /// is the full decoded buffer (for symbols, imports, and links); `buzz` /// adds the callers section when the model is closing in on this pod. +/// Ready navigation supplies symbols from this same admitted source snapshot, +/// bypassing the legacy symbol scan. /// Never fails: anything unreadable is omitted from the chart rather /// than failing the read. -pub fn sound_echolocation(workspace: &Path, file_path: &Path, text: &str, buzz: bool) -> EchoChart { - let (_, mut podmates) = sound_podmates(file_path); +pub(crate) fn sound_echolocation( + workspace: &Path, + file_path: &Path, + text: &str, + buzz: bool, + navigation: Option<&SourceNavigation>, +) -> EchoChart { + // Share only this chart's visible-entry snapshot. Caller candidates still + // pass their current path, credential, and secure-open checks below. + let entries = file_path + .parent() + .and_then(list_dir_entries) + .unwrap_or_default(); + let (_, mut podmates) = sound_podmates(file_path, &entries); let podmate_total = podmates.len(); podmates.truncate(MAX_PODMATES); - let (mut symbols, mut import_roots, mut links, link_total, mut callers, caller_total) = - if let Some(lang) = detect_lang(file_path) { - let (links, link_total) = sound_links(workspace, file_path, text, lang); - let (callers, caller_total) = if buzz { - sound_callers(workspace, file_path) - } else { - (Vec::new(), 0) - }; - ( - sound_symbols(text, lang), - sound_import_roots(text, lang) - .into_iter() - .collect::>(), - links, - link_total, - callers, - caller_total, - ) + let navigation = navigation.filter(|navigation| navigation.status == "ready"); + let ( + mut symbols, + mut import_roots, + mut links, + link_total, + mut callers, + caller_total, + caller_candidates_omitted, + ) = if let Some(lang) = detect_lang(file_path) { + let mut import_roots = BTreeSet::new(); + let (links, link_total) = sound_links( + workspace, + file_path, + text, + lang, + Some(&mut import_roots), + None, + ); + let (callers, caller_total, caller_candidates_omitted) = if buzz { + sound_callers(workspace, file_path, Some(entries)) } else { - (Vec::new(), Vec::new(), Vec::new(), 0, Vec::new(), 0) + (Vec::new(), 0, 0) }; - let symbol_total = symbols.len(); + ( + if navigation.is_some() { + Vec::new() + } else { + sound_symbols(text, lang) + }, + import_roots.into_iter().collect::>(), + links, + link_total, + callers, + caller_total, + caller_candidates_omitted, + ) + } else { + (Vec::new(), Vec::new(), Vec::new(), 0, Vec::new(), 0, 0) + }; + let symbol_total = if let Some(navigation) = navigation { + symbols = navigation + .symbols + .iter() + .map(|symbol| format!("fn {}:{}", symbol.name, symbol.read.offset)) + .collect(); + symbols.len() + navigation.symbols_omitted + } else { + symbols.len() + }; symbols.truncate(MAX_SYMBOLS); // Squeeze, all lossless: same-pod links and callers render bare // (the pod frame disambiguates), and import roots a shown link @@ -502,6 +547,7 @@ pub fn sound_echolocation(workspace: &Path, file_path: &Path, text: &str, buzz: link_total, callers, caller_total, + caller_candidates_omitted, } } @@ -515,7 +561,11 @@ pub fn sound_miss_echo(file_path: &Path) -> Option { if echo_off("mates") { return None; } - let (pod_dir, mut mates) = sound_podmates(file_path); + let entries = file_path + .parent() + .and_then(list_dir_entries) + .unwrap_or_default(); + let (pod_dir, mut mates) = sound_podmates(file_path, &entries); if mates.is_empty() { return None; } @@ -536,23 +586,10 @@ impl EchoChart { self.render_footer_with_budget(POD_FOOTER_MAX_BYTES) } - pub(crate) fn render_with_navigation( - &mut self, - navigation: Option<&SourceNavigation>, - ) -> String { + pub(crate) fn render_with_navigation(&self, navigation: Option<&SourceNavigation>) -> String { let Some(navigation) = navigation else { return self.render_footer(); }; - if navigation.status == "ready" { - // Consume the syntax producer instead of publishing conflicting - // line-scanner lookalikes beside its qualified observations. - self.symbols = navigation - .symbols - .iter() - .map(|symbol| format!("fn {}:{}", symbol.name, symbol.read.offset)) - .collect(); - self.symbol_total = self.symbols.len() + navigation.symbols_omitted; - } let footer = navigation.render_footer(); self.render_footer_with_budget(POD_FOOTER_MAX_BYTES - footer.len()) + &footer } @@ -604,6 +641,15 @@ impl EchoChart { render_capped(&self.callers, self.caller_total) )); } + if !echo_off("callers") && self.caller_candidates_omitted > 0 { + if !sounding.is_empty() { + sounding.push_str("; "); + } + sounding.push_str(&format!( + "{} caller candidates unscanned", + self.caller_candidates_omitted + )); + } if !sounding.is_empty() { out.push_str(&format!("\n\n[Sound: {sounding}]")); } @@ -636,19 +682,19 @@ fn detect_lang(file_path: &Path) -> Option { /// Sibling entries of the file's directory: sorted display names (current /// file marked, directories suffixed) plus the short directory display. /// A directory that cannot be listed degrades to an empty pod, silently. -fn sound_podmates(file_path: &Path) -> (Option, Vec) { +fn sound_podmates(file_path: &Path, mates: &[(String, String)]) -> (Option, Vec) { let parent = match file_path.parent() { Some(parent) => parent, None => return (None, Vec::new()), }; - let mates = list_dir_entries(parent).unwrap_or_default(); let current = file_path .file_name() .map(|name| name.to_string_lossy().into_owned()); let display = mates - .into_iter() - .map(|(raw, mut shown)| { - if Some(&raw) == current.as_ref() { + .iter() + .map(|(raw, shown)| { + let mut shown = shown.clone(); + if Some(raw) == current.as_ref() { shown.push_str(" (this file)"); } shown @@ -671,6 +717,32 @@ pub(crate) struct PodVisibility<'a> { } impl PodVisibility<'_> { + fn allows_entry(&self, abs: &Path, is_dir: bool) -> bool { + if abs + .file_name() + .is_some_and(|name| name.to_string_lossy().starts_with('.')) + || crate::sandbox::read_guard::active().check(abs).is_err() + { + return false; + } + let rel = abs + .strip_prefix(self.glob_root) + .unwrap_or(abs) + .to_string_lossy() + .replace('\\', "/"); + !self + .exclude + .iter() + .any(|pat| super::search::matches_glob(&rel, pat)) + && (is_dir + || ((self.include.is_empty() + || self + .include + .iter() + .any(|pat| super::search::matches_glob(&rel, pat))) + && (self.extensions.is_empty() + || super::file_search::extension_matches(abs, self.extensions)))) + } /// Directed-read visibility: no search gates, denylist still on. fn directed(glob_root: &Path) -> PodVisibility<'_> { PodVisibility { @@ -692,55 +764,30 @@ fn list_dir_entries(dir: &Path) -> Option> { /// [`list_dir_entries`] plus the caller's search gates: denylist, /// excludes, includes/extensions (files only, like the walks), and -/// optional gitignore stacking. +/// optional parent/global/repository ignore rules. fn list_dir_entries_filtered(dir: &Path, vis: &PodVisibility<'_>) -> Option> { - let ignores = vis - .gitignore - .then(|| IgnoreStack::for_dir(vis.glob_root, dir)); + let mut builder = WalkBuilder::new(dir); + builder + .hidden(false) + .follow_links(false) + .require_git(false) + .max_depth(Some(1)) + .parents(vis.gitignore) + .ignore(vis.gitignore) + .git_ignore(vis.gitignore) + .git_global(vis.gitignore) + .git_exclude(vis.gitignore); + let mut entries = builder.build(); + let root = entries.next()?.ok()?; + if !root.file_type().is_some_and(|kind| kind.is_dir()) { + return None; + } let mut mates = Vec::new(); - for entry in std::fs::read_dir(dir).ok()?.flatten() { + for entry in entries.flatten() { let raw = entry.file_name().to_string_lossy().into_owned(); - if raw.starts_with('.') { - continue; - } let abs = entry.path(); - let is_dir = entry.file_type().is_ok_and(|kind| kind.is_dir()); - // Denylist first: a pod listing is enumeration, not a directed - // read — denied names stay hidden even as siblings. - if crate::sandbox::read_guard::active().check(&abs).is_err() { - continue; - } - let rel = abs - .strip_prefix(vis.glob_root) - .unwrap_or(&abs) - .to_string_lossy() - .replace('\\', "/"); - if vis - .exclude - .iter() - .any(|pat| super::search::matches_glob(&rel, pat)) - { - continue; - } - if !is_dir { - if !vis.include.is_empty() - && !vis - .include - .iter() - .any(|pat| super::search::matches_glob(&rel, pat)) - { - continue; - } - if !vis.extensions.is_empty() - && !super::file_search::extension_matches(&abs, vis.extensions) - { - continue; - } - } - if ignores - .as_ref() - .is_some_and(|stack| stack.is_ignored(&abs, is_dir)) - { + let is_dir = entry.file_type().is_some_and(|kind| kind.is_dir()); + if !vis.allows_entry(abs, is_dir) { continue; } let mut display = raw.clone(); @@ -753,58 +800,6 @@ fn list_dir_entries_filtered(dir: &Path, vis: &PodVisibility<'_>) -> Option