From 949f1a95b40c1f1ff229a4a654c3cb773c5a0d6a Mon Sep 17 00:00:00 2001 From: ivanmilevtues Date: Mon, 26 Oct 2026 11:23:10 +0100 Subject: [PATCH] feat(sessions): add structured local history search --- .../docs/user-guide/17-sessions.md | 45 +++ .../xai-grok-pager/src/sessions_cmd.rs | 50 ++- .../src/views/session_picker.rs | 48 +++ .../xai-grok-session-search/src/fts.rs | 73 +---- .../xai-grok-session-search/src/lib.rs | 5 +- .../xai-grok-session-search/src/query.rs | 198 ++++++++++++ .../src/query_tests.rs | 131 ++++++++ .../tests/structured_search.rs | 293 ++++++++++++++++++ .../tests/structured_search_manager.rs | 81 +++++ .../src/extensions/session_search.rs | 3 +- .../src/session/storage/search.rs | 3 +- 11 files changed, 855 insertions(+), 75 deletions(-) create mode 100644 crates/codegen/xai-grok-session-search/src/query.rs create mode 100644 crates/codegen/xai-grok-session-search/src/query_tests.rs create mode 100644 crates/codegen/xai-grok-session-search/tests/structured_search.rs create mode 100644 crates/codegen/xai-grok-session-search/tests/structured_search_manager.rs diff --git a/crates/codegen/xai-grok-pager/docs/user-guide/17-sessions.md b/crates/codegen/xai-grok-pager/docs/user-guide/17-sessions.md index 93b190805b0..e08ac4984a9 100644 --- a/crates/codegen/xai-grok-pager/docs/user-guide/17-sessions.md +++ b/crates/codegen/xai-grok-pager/docs/user-guide/17-sessions.md @@ -271,6 +271,51 @@ grok sessions search "rate limit" `grok sessions list` shows sessions for the current working directory, grouped by worktree label. Each row lists the session ID, the creation and update dates, the source status, and the summary. `grok sessions search` combines a local SQLite index with remote results. +### Structured local search + +Use quotes, field scopes, and exclusions to narrow the local session history: + +```bash +# An exact sequence of words in either the title or prompts +grok sessions search '"rate limit"' + +# Combine a title prefix, an exact prompt phrase, and a title exclusion +grok sessions search 'title:retry content:"rate limit" -title:archived' + +# Exclude a word or phrase from both fields +grok sessions search 'rust -tokio -"out of date"' + +# Ordinary keyword search without remote results +grok sessions search --local 'borrow checker' +``` + +| Syntax | Meaning | +|---|---| +| `rate limit` | Match both word prefixes; if there are no matches, offer sessions matching either word | +| `"rate limit"` | Match adjacent words in order, without prefix expansion or plural stemming | +| `title:retry` | Match a word prefix in the session title | +| `content:"rate limit"` | Match a phrase in the indexed prompts | +| `-archived` | Exclude sessions matching this prefix in either field | +| `-title:archived` | Exclude sessions matching this prefix in the title | + +Explicit syntax is strict: all positive clauses must match and every exclusion +must be absent. These queries never fall back to partial matches. Quotes match +SQLite's indexed words, so case and punctuation are normalized. An unfinished +quote extends to the end of the query while you type. Empty clauses are ignored; +a query containing only exclusions returns no results. Field names are lowercase +and must touch their value. Unknown prefixes and words such as `OR` are searched +as ordinary text, not interpreted as operators. + +Structured CLI queries automatically search the local index only and print a +notice. The remote keyword service does not implement this syntax. `--local` +also forces local-only behavior for ordinary keywords. If local session search +is disabled, both forms report an error instead of returning unconstrained +remote results. Local results remain scoped to the current working directory. + +The same syntax is available in the native session picker's deep content search +and the `x.ai/session/search` ACP extension's `query` field. Foreign-agent +histories and remote chat search retain their existing search behavior. + --- ## The grok usage Subcommand diff --git a/crates/codegen/xai-grok-pager/src/sessions_cmd.rs b/crates/codegen/xai-grok-pager/src/sessions_cmd.rs index 7c1c8821d93..a7803b878f0 100644 --- a/crates/codegen/xai-grok-pager/src/sessions_cmd.rs +++ b/crates/codegen/xai-grok-pager/src/sessions_cmd.rs @@ -3,6 +3,7 @@ use clap::Subcommand; use xai_grok_login::{AuthManager, try_ensure_fresh_auth}; use xai_grok_shell::agent::config::Config as AgentConfig; use xai_grok_shell::session::merge::MergedSession; +use xai_grok_shell::session::storage::search::SessionQuery; use xai_grok_shell::util::grok_home::grok_home; #[derive(Debug, clap::Args, Clone)] pub struct SessionsArgs { @@ -18,10 +19,14 @@ enum SessionsCommand { #[arg(short = 'n', long, default_value = "20")] limit: usize, }, - /// Search sessions by keyword + /// Search session titles and prompts with keywords, phrases, or field filters Search { - /// Search query (searches summaries and first prompts). + /// Query: words, "exact phrases", title:term, content:term, or -term. + /// Structured queries search the local index only. query: String, + /// Search only the local index, without remote keyword results + #[arg(long)] + local: bool, /// Maximum number of sessions to show #[arg(short = 'n', long, default_value = "20")] limit: usize, @@ -34,14 +39,24 @@ enum SessionsCommand { } pub async fn run(args: SessionsArgs, agent_config: &AgentConfig) -> Result<()> { + let local_only = matches!( + &args.command, + SessionsCommand::Search { query, local, .. } + if *local || SessionQuery::parse(query).is_structured() + ); // Best-effort only: never force an interactive public login here. Enterprise deployments may configure only a // deployment_key and a custom xai_api_base_url. Otherwise we still proceed so the SessionRegistryClient can use // the deployment_key when talking to the custom proxy. - let auth = try_ensure_fresh_auth( - &agent_config.grok_com_config, - agent_config.endpoints.proxy_url(), - ) - .await; + // A local search must not wait for an authentication refresh either. + let auth = if local_only { + None + } else { + try_ensure_fresh_auth( + &agent_config.grok_com_config, + agent_config.endpoints.proxy_url(), + ) + .await + }; let auth_manager = std::sync::Arc::new(AuthManager::new_with_proxy_base_url( &grok_home(), @@ -73,7 +88,11 @@ pub async fn run(args: SessionsArgs, agent_config: &AgentConfig) -> Result<()> { .await; print_sessions_grouped(&sessions); } - SessionsCommand::Search { query, limit } => { + SessionsCommand::Search { + query, + limit, + local, + } => { use std::collections::HashSet; use xai_grok_shell::session::merge::REMOTE_TIMEOUT; use xai_grok_shell::session::storage::search::{ @@ -83,6 +102,16 @@ pub async fn run(args: SessionsArgs, agent_config: &AgentConfig) -> Result<()> { // Search is the only subcommand that reads the index, so it is the only one to start one let search = xai_grok_shell::session::storage::search::start_if_enabled(agent_config); + let structured = SessionQuery::parse(&query).is_structured(); + if local_only && let Some(by) = search.off_reason() { + anyhow::bail!( + "local session search is off ({by}); enable it to use local or structured search" + ); + } + if structured && !local { + eprintln!("Structured query: searching local sessions only."); + } + let req = SessionSearchRequest { query, cwd: Some(cwd.to_string_lossy().to_string()), @@ -92,10 +121,13 @@ pub async fn run(args: SessionsArgs, agent_config: &AgentConfig) -> Result<()> { }; let root = grok_home(); - let remote_limit = (limit * 3).max(100) as i64; + let remote_limit = i64::try_from(limit.saturating_mul(3).max(100)).unwrap_or(i64::MAX); let (local_resp, remote_results) = tokio::join!( execute_search(IndexDecision::settled(&search), &root, &req), async { + if local_only { + return Vec::new(); + } tokio::time::timeout( REMOTE_TIMEOUT, client.search(Some(&req.query), remote_limit), diff --git a/crates/codegen/xai-grok-pager/src/views/session_picker.rs b/crates/codegen/xai-grok-pager/src/views/session_picker.rs index cad4e4e297d..72502b4ddce 100644 --- a/crates/codegen/xai-grok-pager/src/views/session_picker.rs +++ b/crates/codegen/xai-grok-pager/src/views/session_picker.rs @@ -413,6 +413,14 @@ pub(crate) fn filter_session_entries( let Some(entries) = entries else { return vec![]; }; + // Literal title/ID matching cannot enforce phrase and content constraints. + // Native structured queries are served exclusively by the deep-search hits. + // Foreign histories keep their existing literal search behavior. + if !source_filter.is_content_search_disabled() + && xai_grok_shell::session::storage::search::SessionQuery::parse(query).is_structured() + { + return vec![]; + } let q = query.to_lowercase(); entries .iter() @@ -1036,6 +1044,46 @@ mod tests { matched_fields: vec![], } } + + #[test] + fn structured_search_only_displays_verified_content_hits() { + // A literal title containing the query is not evidence that the + // content: clause actually matches this session's prompts. + let mut unverified = make_entry("unverified", "repo"); + unverified.summary = "content:rust -title:archived".into(); + let entries = vec![unverified]; + let hits = vec![make_content_hit("verified")]; + let items = build_virtual_list( + Some(&entries), + Some(&hits), + "content:rust -title:archived", + SourceFilter::Grok, + ); + assert!(matches!( + items.as_slice(), + [PickerItem::Content { hit_index: 0 }] + )); + assert!( + build_virtual_list( + Some(&entries), + None, + "content:rust -title:archived", + SourceFilter::Grok, + ) + .is_empty() + ); + } + + #[test] + fn external_history_keeps_literal_search_for_structured_looking_titles() { + let mut external = make_entry("external", "repo"); + external.source = "codex".into(); + external.summary = "title:rust notes".into(); + assert_eq!( + filter_session_entries(Some(&[external]), "title:rust", SourceFilter::External,), + [0] + ); + } /// Grouped entry map places repo headers and resolves mouse-click indices to the correct original session index. #[test] fn grouped_entry_map_resolves_correct_session() { diff --git a/crates/codegen/xai-grok-session-search/src/fts.rs b/crates/codegen/xai-grok-session-search/src/fts.rs index b16f0c2b37f..322ec84569b 100644 --- a/crates/codegen/xai-grok-session-search/src/fts.rs +++ b/crates/codegen/xai-grok-session-search/src/fts.rs @@ -19,6 +19,7 @@ use std::time::Duration; use rusqlite::{Connection, OptionalExtension, params}; use xai_sqlite_journal::JournalMode; +use crate::query::SessionQuery; use crate::recovery; /// Bump when making breaking schema changes that require dropping and recreating tables, or to force a rebuild of stale index content. @@ -439,7 +440,8 @@ impl SessionSearchIndex { Ok(ids) } - /// When that intersection matches nothing the query reruns as an OR, so partial matches still show up. A query shaped + /// Plain keywords use AND first, then OR when there are no hits. Explicit + /// phrases, field scopes, and exclusions use a strict plan with no fallback. A query shaped /// like a session id (a full UUID or a hyphenated hex prefix) matches `session_docs.session_id` directly. FTS only /// indexes title and content, and a hyphenated UUID `MATCH` looks for tokens that were never indexed. pub fn query( @@ -457,7 +459,8 @@ impl SessionSearchIndex { } } - let Some((and_query, or_query)) = Self::build_match_queries(query) else { + let plan = SessionQuery::parse(query); + let Some(primary) = plan.primary_match() else { return Ok(QueryResult { results: Vec::new(), next_offset: None, @@ -465,10 +468,12 @@ impl SessionSearchIndex { }); }; - let result = self.run_match_query(&and_query, cwd, limit, offset, include_content)?; + let result = self.run_match_query(primary, cwd, limit, offset, include_content)?; // Gate the fallback on the total (not the page) so every offset of one logical query is served by the same match string - if result.total_estimate == Some(0) && and_query != or_query { - return self.run_match_query(&or_query, cwd, limit, offset, include_content); + if result.total_estimate == Some(0) + && let Some(fallback) = plan.fallback_match() + { + return self.run_match_query(fallback, cwd, limit, offset, include_content); } Ok(result) } @@ -623,64 +628,6 @@ impl SessionSearchIndex { total_estimate: Some(total_usize), }) } - - /// Build the AND-joined and OR-joined FTS5 MATCH strings for a query. - /// - /// The strings are identical for single-token queries, which lets the caller skip the fallback rerun. - fn build_match_queries(query: &str) -> Option<(String, String)> { - let prefixes: Vec = query - .split_whitespace() - .flat_map(Self::sanitize_token) - .map(Self::token_prefix) - .collect(); - - if prefixes.is_empty() { - let fallback = query.trim(); - if fallback.is_empty() { - return None; - } - let cleaned = fallback.replace('"', ""); - let phrase = format!("\"{cleaned}\" *"); - return Some((phrase.clone(), phrase)); - } - - Some((prefixes.join(" AND "), prefixes.join(" OR "))) - } - - /// Split a query word on every stripped character instead of gluing: `session_picker.rs` searches as `session_picker` and `rs`. - /// Gluing produced the never-indexed `session_pickerrs`. - /// Fragments with no alphanumeric (`-`, `->`, `_`) are dropped: they tokenize to empty phrases; an empty phrase inside an AND matches nothing. - fn sanitize_token(token: &str) -> impl Iterator { - token - .split(|c: char| !(c.is_ascii_alphanumeric() || c == '_' || c == '-')) - .filter(|part| part.chars().any(|c| c.is_ascii_alphanumeric())) - } - - /// One quoted FTS5 prefix per token, stemmed on the query side only. Plural queries reach singular docs by searching the - /// shorter stem (`sessions` becomes `session*`, `caches` becomes `cach*`). Words shorter than four letters, identifiers - /// with digits/`_`/`-`, and words ending in `ss` (`pass`, `class`) stay exact. - fn token_prefix(token: &str) -> String { - let stem = if token.len() < 4 || !token.chars().all(|c| c.is_ascii_alphabetic()) { - token - } else { - let lower = token.to_ascii_lowercase(); - if lower.ends_with("es") { - // Stemming `caches` to `cach` still finds `cache`: the trailing `*` covers the dropped `e` - match token.len().checked_sub(2).and_then(|n| token.get(..n)) { - Some(s) => s, - None => token, - } - } else if lower.ends_with('s') && !lower.ends_with("ss") { - match token.len().checked_sub(1).and_then(|n| token.get(..n)) { - Some(s) => s, - None => token, - } - } else { - token - } - }; - format!("\"{stem}\" *") - } } /// True when `query` is a UUID or a hyphenated hex prefix long enough to be an intentional session-id search (not a normal keyword). diff --git a/crates/codegen/xai-grok-session-search/src/lib.rs b/crates/codegen/xai-grok-session-search/src/lib.rs index 18f875c0672..c1449717934 100644 --- a/crates/codegen/xai-grok-session-search/src/lib.rs +++ b/crates/codegen/xai-grok-session-search/src/lib.rs @@ -2,7 +2,8 @@ //! //! Layering, bottom up: //! -//! - [`fts`] owns the schema, the BM25 query, and the fenced `meta` writes. +//! - [`query`] parses user syntax into strict or relaxed FTS5 search plans. +//! - [`fts`] owns the schema, executes those plans with BM25 ranking, and fences `meta` writes. //! - `recovery` classifies an unusable database file, quarantines it, and recreates an empty one; `db` adds the open-and-retry helpers on top. //! - `doc` turns a session plus its extracted text into an index document. //! - `bootstrap` runs the full reindex, held to one process at a time by a lease. @@ -18,6 +19,7 @@ mod db; mod doc; pub mod fts; mod manager; +pub mod query; mod recovery; mod source; @@ -25,4 +27,5 @@ pub use manager::{ SearchIndexManager, SearchIndexStatus, SessionSearchRequest, SessionSearchResponse, evict_session, execute_search, }; +pub use query::SessionQuery; pub use source::{ContentExtractor, IndexableSession, SessionSource, SessionSourceFactory}; diff --git a/crates/codegen/xai-grok-session-search/src/query.rs b/crates/codegen/xai-grok-session-search/src/query.rs new file mode 100644 index 00000000000..54bf890970f --- /dev/null +++ b/crates/codegen/xai-grok-session-search/src/query.rs @@ -0,0 +1,198 @@ +//! User-facing session search syntax, independent of storage and transport. +//! +//! Plain words retain prefix matching and the AND-then-OR fallback. Quoted +//! phrases, `title:`/`content:` scopes, and `-` exclusions opt into strict +//! matching: relaxing those queries would violate the user's constraints. + +/// A parsed query and its safe FTS5 execution plan. +/// +/// Parsing is forgiving for live search: an unfinished quote extends to the +/// end of the input and empty clauses are ignored. Queries without a positive +/// searchable clause return no matches, rather than searching all history. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SessionQuery { + primary: Option, + fallback: Option, + structured: bool, +} + +#[derive(Debug, Clone, Copy)] +enum Field { + Title, + Content, +} + +impl Field { + fn name(self) -> &'static str { + match self { + Self::Title => "title", + Self::Content => "content", + } + } +} + +impl SessionQuery { + pub fn parse(input: &str) -> Self { + let mut rest = input.trim_start(); + let mut positive = Vec::new(); + let mut negative = Vec::new(); + let mut structured = false; + + while !rest.is_empty() { + let excluded = rest.strip_prefix('-').is_some_and(|tail| { + tail.chars() + .next() + .is_some_and(|c| c.is_alphanumeric() || c == '"' || c == '_') + }); + if excluded { + rest = rest.strip_prefix('-').unwrap_or(rest); + structured = true; + } + + let field = if let Some(tail) = rest.strip_prefix("title:") { + rest = tail; + Some(Field::Title) + } else if let Some(tail) = rest.strip_prefix("content:") { + rest = tail; + Some(Field::Content) + } else { + None + }; + structured |= field.is_some(); + + let clauses = if let Some(tail) = rest.strip_prefix('"') { + structured = true; + let (phrase, tail) = take_phrase(tail); + rest = tail; + // FTS5 tokenizes punctuation itself. Do not stem or prefix a + // phrase: "rate limit" must not match "rate limiting". + if phrase.chars().any(char::is_alphanumeric) { + // FTS5 treats NUL as the end of the expression even when + // SQLite receives a length-delimited parameter. + let escaped = phrase.replace('\0', " ").replace('"', "\"\""); + vec![format!("\"{escaped}\"")] + } else { + Vec::new() + } + } else { + let end = rest.find(char::is_whitespace).unwrap_or(rest.len()); + let (word, tail) = rest.split_at(end); + rest = tail; + sanitize_token(word).map(token_prefix).collect() + }; + + // A scoped filename can split into several tokens; scope all of + // them. Likewise, -foo.rs excludes the combination, not either word. + if !clauses.is_empty() { + let clause = if field.is_some() || excluded { + let joined = clauses.join(" AND "); + match field { + Some(field) => format!("{} : ({joined})", field.name()), + None => format!("({joined})"), + } + } else { + // Keep ordinary fragments separate for the OR fallback. + positive.extend(clauses); + rest = rest.trim_start(); + continue; + }; + if excluded { + negative.push(clause); + } else { + positive.push(clause); + } + } + rest = rest.trim_start(); + } + + let fallback = (!structured && positive.len() > 1).then(|| positive.join(" OR ")); + let primary = if positive.is_empty() { + None + } else { + let mut expression = positive.join(" AND "); + if !negative.is_empty() { + expression = format!("({expression}) NOT ({})", negative.join(" OR ")); + } + Some(expression) + }; + Self { + primary, + fallback, + structured, + } + } + + /// True when the query uses syntax the remote keyword search cannot honor. + /// Clients can use this to route to the local index and suppress literal + /// title matches that have not been checked against the full query. + pub fn is_structured(&self) -> bool { + self.structured + } + + /// Primary MATCH expression, or `None` for an empty/negative-only query. + pub fn primary_match(&self) -> Option<&str> { + self.primary.as_deref() + } + + /// Optional relaxed expression. Execute only if the primary has zero hits + /// across *all* pages, never merely because the requested page is empty. + pub fn fallback_match(&self) -> Option<&str> { + self.fallback.as_deref() + } +} + +/// Consume a quote-delimited phrase, accepting escaped quotes and backslashes. +fn take_phrase(input: &str) -> (String, &str) { + let mut phrase = String::new(); + let mut chars = input.char_indices().peekable(); + while let Some((index, ch)) = chars.next() { + if ch == '"' { + return ( + phrase, + input.get(index + ch.len_utf8()..).unwrap_or_default(), + ); + } + if ch == '\\' + && chars + .peek() + .is_some_and(|(_, next)| matches!(next, '"' | '\\')) + { + if let Some((_, escaped)) = chars.next() { + phrase.push(escaped); + } + } else { + phrase.push(ch); + } + } + (phrase, "") +} + +/// Split punctuation rather than gluing filename fragments into nonexistent +/// tokens. Keep Unicode words; SQLite's unicode61 tokenizer indexes them too. +fn sanitize_token(token: &str) -> impl Iterator { + token + .split(|c: char| !(c.is_alphanumeric() || c == '_' || c == '-')) + .filter(|part| part.chars().any(char::is_alphanumeric)) +} + +/// Preserve the existing query-side plural stemming for plain ASCII words. +fn token_prefix(token: &str) -> String { + let stem = if token.len() < 4 || !token.chars().all(|c| c.is_ascii_alphabetic()) { + token + } else { + let lower = token.to_ascii_lowercase(); + let suffix_len = if lower.ends_with("es") { + 2 + } else if lower.ends_with('s') && !lower.ends_with("ss") { + 1 + } else { + 0 + }; + token.get(..token.len() - suffix_len).unwrap_or(token) + }; + format!("\"{stem}\" *") +} + +#[cfg(test)] +#[path = "query_tests.rs"] +mod tests; diff --git a/crates/codegen/xai-grok-session-search/src/query_tests.rs b/crates/codegen/xai-grok-session-search/src/query_tests.rs new file mode 100644 index 00000000000..5e5f94dbcba --- /dev/null +++ b/crates/codegen/xai-grok-session-search/src/query_tests.rs @@ -0,0 +1,131 @@ +use super::SessionQuery; + +#[test] +fn plain_words_keep_relaxed_plan_and_plural_stemming() { + let query = SessionQuery::parse("sessions caches class pass api_v2 foo-bar"); + assert!(!query.is_structured()); + assert_eq!( + query.primary_match(), + Some( + "\"session\" * AND \"cach\" * AND \"class\" * AND \"pass\" * AND \"api_v2\" * AND \"foo-bar\" *" + ) + ); + assert!(query.fallback_match().unwrap().contains(" OR ")); +} + +#[test] +fn punctuation_splits_filename_tokens() { + let query = SessionQuery::parse("fix -> session_picker.rs!!!"); + assert_eq!( + query.primary_match(), + Some("\"fix\" * AND \"session_picker\" * AND \"rs\" *") + ); + assert_eq!( + query.fallback_match(), + Some("\"fix\" * OR \"session_picker\" * OR \"rs\" *") + ); +} + +#[test] +fn phrases_are_exact_and_never_relaxed() { + let query = SessionQuery::parse("\"rate limits\" retry"); + assert!(query.is_structured()); + assert_eq!( + query.primary_match(), + Some("\"rate limits\" AND \"retry\" *") + ); + assert_eq!(query.fallback_match(), None); +} + +#[test] +fn scopes_apply_to_every_fragment() { + let query = SessionQuery::parse("title:session_picker.rs content:\"borrow checker\""); + assert_eq!( + query.primary_match(), + Some("title : (\"session_picker\" * AND \"rs\" *) AND content : (\"borrow checker\")") + ); + assert_eq!(query.fallback_match(), None); +} + +#[test] +fn exclusions_bind_to_the_entire_positive_expression() { + let query = SessionQuery::parse("rust tokio -title:archive -\"out of date\""); + assert_eq!( + query.primary_match(), + Some("(\"rust\" * AND \"tokio\" *) NOT (title : (\"archive\" *) OR (\"out of date\"))") + ); + assert_eq!(query.fallback_match(), None); +} + +#[test] +fn empty_or_negative_only_queries_cannot_search_all_history() { + for input in [ + "", + " ", + "-> _ …", + "\"\"", + "title:", + "content:", + "-cache", + "-title:cache", + "-\"rate limit\"", + ] { + let query = SessionQuery::parse(input); + assert_eq!(query.primary_match(), None, "{input}"); + assert_eq!(query.fallback_match(), None, "{input}"); + } +} + +#[test] +fn unfinished_input_is_safe_for_live_search() { + let query = SessionQuery::parse("title:\"rate limit"); + assert_eq!(query.primary_match(), Some("title : (\"rate limit\")")); + assert_eq!(query, SessionQuery::parse("title:\"rate limit\"")); + assert_eq!( + SessionQuery::parse("title: retry").primary_match(), + Some("\"retry\" *") + ); +} + +#[test] +fn escaped_quotes_are_quoted_again_for_fts() { + let query = SessionQuery::parse(r#""say \"hello\"""#); + assert_eq!(query.primary_match(), Some(r#""say ""hello""""#)); + assert_eq!(query.fallback_match(), None); +} + +#[test] +fn unknown_fields_and_fts_operators_are_literal_words() { + let query = SessionQuery::parse("owner:alice OR NEAR(foo) NOT bar"); + assert!(!query.is_structured()); + assert_eq!( + query.primary_match(), + Some( + "\"owner\" * AND \"alice\" * AND \"OR\" * AND \"NEAR\" * AND \"foo\" * AND \"NOT\" * AND \"bar\" *" + ) + ); +} + +#[test] +fn unicode_terms_are_preserved() { + let query = SessionQuery::parse("café 日本語"); + assert_eq!(query.primary_match(), Some("\"café\" * AND \"日本語\" *")); + assert_eq!(query.fallback_match(), Some("\"café\" * OR \"日本語\" *")); +} + +#[test] +fn hyphens_inside_words_are_not_exclusions() { + let query = SessionQuery::parse("rate-limit"); + assert!(!query.is_structured()); + assert_eq!(query.primary_match(), Some("\"rate-limit\" *")); + assert_eq!(query.fallback_match(), None); +} + +#[test] +fn quoting_can_search_literal_syntax() { + let query = SessionQuery::parse("\"title:cache\" \"-retry\""); + assert_eq!( + query.primary_match(), + Some("\"title:cache\" AND \"-retry\"") + ); +} diff --git a/crates/codegen/xai-grok-session-search/tests/structured_search.rs b/crates/codegen/xai-grok-session-search/tests/structured_search.rs new file mode 100644 index 00000000000..d215ffca4c1 --- /dev/null +++ b/crates/codegen/xai-grok-session-search/tests/structured_search.rs @@ -0,0 +1,293 @@ +//! Exercise the public API against a real SQLite FTS5 index, including ranking, +//! snippets, pagination, and compatibility with existing keyword searches. +use tempfile::TempDir; +use xai_grok_session_search::fts::{QueryResult, SessionDoc, SessionSearchIndex}; + +fn fixture() -> (TempDir, SessionSearchIndex) { + let temp = TempDir::new().unwrap(); + let index = SessionSearchIndex::open_or_create(&temp.path().join("search.sqlite")).unwrap(); + for (id, cwd, title, content) in [ + ( + "exact", + "/project", + "Retry handling", + "fix the rate limit with exponential backoff", + ), + ( + "separated", + "/project", + "Retry handling", + "rate control with a configurable limit", + ), + ( + "prefix", + "/project", + "Retry handling", + "rate limiting with exponential backoff", + ), + ( + "archived", + "/project", + "Archived retry handling", + "rate limit old implementation", + ), + ("title-only", "/project", "Rate limit", "unrelated body"), + ( + "other-workspace", + "/other", + "Retry handling", + "rate limit exponential backoff", + ), + ] { + index + .upsert_doc(&SessionDoc { + session_id: id.into(), + cwd: cwd.into(), + title: title.into(), + content: content.into(), + updated_at_unix: 1_700_000_000, + content_hash: String::new(), + }) + .unwrap(); + } + (temp, index) +} + +fn ids(result: &QueryResult) -> Vec<&str> { + let mut ids: Vec<_> = result + .results + .iter() + .map(|hit| hit.session_id.as_str()) + .collect(); + ids.sort_unstable(); + ids +} + +#[test] +fn phrases_require_adjacent_exact_tokens() { + let (_temp, index) = fixture(); + let result = index + .query("content:\"rate limit\"", Some("/project"), 20, 0, true) + .unwrap(); + assert_eq!(ids(&result), ["archived", "exact"]); + for hit in result.results { + assert_eq!(hit.matched_fields, ["content"]); + assert!(hit.snippet.unwrap().contains("[rate limit]")); + } +} + +#[test] +fn field_scopes_and_exclusions_compose() { + let (_temp, index) = fixture(); + let result = index + .query( + "title:retry content:\"rate limit\" -title:archived", + Some("/project"), + 20, + 0, + true, + ) + .unwrap(); + assert_eq!(ids(&result), ["exact"]); + assert_eq!( + result.results.first().unwrap().matched_fields, + ["title", "content"] + ); +} + +#[test] +fn exclusions_search_both_fields_unless_scoped() { + let (_temp, index) = fixture(); + let result = index + .query( + "\"rate limit\" -archived -title:rate", + Some("/project"), + 20, + 0, + false, + ) + .unwrap(); + assert_eq!(ids(&result), ["exact"]); + let result = index + .query( + "\"rate limit\" -content:archived", + Some("/project"), + 20, + 0, + false, + ) + .unwrap(); + assert_eq!(ids(&result), ["archived", "exact", "title-only"]); +} + +#[test] +fn strict_queries_never_fall_back_to_partial_hits() { + let (_temp, index) = fixture(); + for query in [ + "title:missing retry", + "\"rate limit\" missing", + "retry -retry", + "title:retry content:unrelated", + ] { + let result = index.query(query, None, 20, 0, false).unwrap(); + assert!(result.results.is_empty(), "{query}"); + assert_eq!(result.total_estimate, Some(0)); + } + // Existing plain keyword queries still offer partial matches. + assert!( + !index + .query("missing retry", None, 20, 0, false) + .unwrap() + .results + .is_empty() + ); +} + +#[test] +fn structured_pagination_uses_the_same_constraints_and_stable_order() { + let (_temp, index) = fixture(); + let query = "title:retry -title:archived"; + let all = index.query(query, Some("/project"), 20, 0, false).unwrap(); + let mut paged = Vec::new(); + for offset in 0..3 { + let page = index + .query(query, Some("/project"), 1, offset, false) + .unwrap(); + assert_eq!(page.total_estimate, Some(3)); + assert_eq!(page.next_offset, (offset < 2).then_some(offset + 1)); + assert!(page.results.first().unwrap().snippet.is_none()); + paged.extend(page.results.into_iter().map(|hit| hit.session_id)); + } + assert_eq!( + paged, + all.results + .into_iter() + .map(|hit| hit.session_id) + .collect::>() + ); + let beyond = index.query(query, Some("/project"), 1, 99, false).unwrap(); + assert!(beyond.results.is_empty()); + assert_eq!(beyond.total_estimate, Some(3)); + assert_eq!(beyond.next_offset, None); +} + +#[test] +fn empty_negative_and_incomplete_queries_do_not_error() { + let (_temp, index) = fixture(); + for query in ["-retry", "-title:retry", "\"\"", "title:", "content:", "…"] { + let result = index.query(query, None, 20, 0, true).unwrap(); + assert!(result.results.is_empty(), "{query}"); + assert_eq!(result.total_estimate, Some(0)); + } + let result = index + .query("content:\"rate limit", Some("/project"), 20, 0, false) + .unwrap(); + assert_eq!(ids(&result), ["archived", "exact"]); +} + +#[test] +fn user_text_cannot_inject_fts_or_sql_operators() { + let (_temp, index) = fixture(); + for query in [ + "title:missing OR retry", + "title:missing) OR (retry", + "\"missing\" OR retry", + "title:missing'; DROP TABLE session_docs; --", + r#""say \"hello\"""#, + "content:{title}:NEAR(retry)", + ] { + let result = index.query(query, None, 20, 0, false).unwrap(); + assert!(result.results.is_empty(), "{query}"); + } + assert_eq!( + index + .query("retry", None, 20, 0, false) + .unwrap() + .total_estimate, + Some(5) + ); +} + +#[test] +fn punctuation_and_partial_input_produce_valid_match_expressions() { + let (_temp, index) = fixture(); + for prefix in ["", "title:", "content:", "retry -", "retry -title:"] { + for value in [ + "\"", + "\"\"", + "\"retry", + "\"retry\"tail", + "\"rate\0limit\"", + "\"rate\\\"limit\"", + "{retry}", + "(retry)", + "_", + "->", + "*", + "日本語", + ] { + let query = format!("{prefix}{value}"); + assert!(index.query(&query, None, 20, 0, false).is_ok(), "{query:?}"); + } + } +} + +#[test] +fn unicode_queries_match_indexed_text() { + let (_temp, index) = fixture(); + index + .upsert_doc(&SessionDoc { + session_id: "unicode".into(), + cwd: "/project".into(), + updated_at_unix: 0, + title: "café 日本語".into(), + content: "résumé café".into(), + content_hash: String::new(), + }) + .unwrap(); + for query in ["日本語", "title:café", "content:\"résumé café\""] { + assert_eq!( + ids(&index.query(query, None, 20, 0, false).unwrap()), + ["unicode"] + ); + } +} + +#[test] +fn plain_query_empty_page_does_not_activate_or_fallback() { + let (_temp, index) = fixture(); + let result = index + .query("retry old", Some("/project"), 1, 1, false) + .unwrap(); + assert!(result.results.is_empty()); + assert_eq!(result.total_estimate, Some(1)); + assert_eq!(result.next_offset, None); +} + +#[test] +fn scoped_filenames_keep_all_fragments_in_the_same_field() { + let (_temp, index) = fixture(); + index + .upsert_doc(&SessionDoc { + session_id: "filename".into(), + cwd: "/project".into(), + updated_at_unix: 0, + title: "session_picker.rs".into(), + content: "session_picker".into(), + content_hash: String::new(), + }) + .unwrap(); + assert_eq!( + ids(&index + .query("title:session_picker.rs", None, 20, 0, false) + .unwrap()), + ["filename"] + ); + assert!( + index + .query("content:session_picker.rs", None, 20, 0, false) + .unwrap() + .results + .is_empty() + ); +} diff --git a/crates/codegen/xai-grok-session-search/tests/structured_search_manager.rs b/crates/codegen/xai-grok-session-search/tests/structured_search_manager.rs new file mode 100644 index 00000000000..ec235a60eb4 --- /dev/null +++ b/crates/codegen/xai-grok-session-search/tests/structured_search_manager.rs @@ -0,0 +1,81 @@ +//! Cover the async entry point used by the CLI and ACP, including a cold index. +use std::io; +use std::path::{Path, PathBuf}; + +use xai_grok_session_search::{ + IndexableSession, SearchIndexManager, SessionSearchRequest, SessionSource, execute_search, +}; + +struct Source(PathBuf); + +#[async_trait::async_trait] +impl SessionSource for Source { + async fn list_sessions(&self) -> io::Result> { + Ok([ + ("current", "Retry handling"), + ("old", "Archived retry handling"), + ] + .into_iter() + .map(|(id, title)| IndexableSession { + session_id: id.into(), + cwd: "/workspace".into(), + updated_at_unix: 1_700_000_000, + title: title.into(), + updates_path: Some(self.0.join("prompt.txt")), + }) + .collect()) + } + + async fn load_session(&self, id: &str, cwd: &str) -> io::Result> { + Ok(self + .list_sessions() + .await? + .into_iter() + .find(|session| session.session_id == id && session.cwd == cwd)) + } +} + +fn extract(path: &Path) -> io::Result<(String, u64)> { + let content = std::fs::read_to_string(path)?; + let bytes = content.len() as u64; + Ok((content, bytes)) +} + +#[tokio::test] +async fn cold_bootstrap_and_async_query_honor_structured_constraints() { + let temp = tempfile::TempDir::new().unwrap(); + std::fs::write(temp.path().join("prompt.txt"), "fix the rate limit").unwrap(); + let manager = SearchIndexManager::start(|root| Box::new(Source(root)), extract); + let mut req = SessionSearchRequest { + query: "title:retry content:\"rate limit\" -title:archived".into(), + cwd: Some("/workspace".into()), + limit: 1, + offset: 0, + include_content: true, + }; + let response = execute_search(Some(&manager), temp.path(), &req) + .await + .unwrap(); + assert!(!response.bootstrapping); + assert_eq!(response.total_estimate, Some(1)); + assert_eq!(response.next_offset, None); + let hit = response.results.first().unwrap(); + assert_eq!(hit.session_id, "current"); + assert_eq!(hit.matched_fields, ["title", "content"]); + assert!(hit.snippet.as_deref().unwrap().contains("[rate limit]")); + + req.offset = 1; + let response = execute_search(Some(&manager), temp.path(), &req) + .await + .unwrap(); + assert!(response.results.is_empty()); + assert_eq!(response.total_estimate, Some(1)); + + req.offset = 0; + req.cwd = Some("/another-workspace".into()); + let response = execute_search(Some(&manager), temp.path(), &req) + .await + .unwrap(); + assert!(response.results.is_empty()); + assert_eq!(response.total_estimate, Some(0)); +} diff --git a/crates/codegen/xai-grok-shell/src/extensions/session_search.rs b/crates/codegen/xai-grok-shell/src/extensions/session_search.rs index 393b99fe3c7..0a2883edf33 100644 --- a/crates/codegen/xai-grok-shell/src/extensions/session_search.rs +++ b/crates/codegen/xai-grok-shell/src/extensions/session_search.rs @@ -25,7 +25,8 @@ use super::ExtResult; #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct SearchSessionsRequest { - /// The search query string. + /// Local search query: words, "exact phrases", title:/content: scopes, + /// and -exclusions. Explicit syntax is strict; plain words allow partial matches. pub query: String, /// Optional workspace directory to scope results to. #[serde(default)] diff --git a/crates/codegen/xai-grok-shell/src/session/storage/search.rs b/crates/codegen/xai-grok-shell/src/session/storage/search.rs index 02391659891..e2ea970088a 100644 --- a/crates/codegen/xai-grok-shell/src/session/storage/search.rs +++ b/crates/codegen/xai-grok-shell/src/session/storage/search.rs @@ -14,7 +14,8 @@ use crate::session::persistence::Summary; use xai_grok_session_search::{IndexableSession, SessionSource}; pub use xai_grok_session_search::{ - SearchIndexManager, SearchIndexStatus, SessionSearchRequest, SessionSearchResponse, + SearchIndexManager, SearchIndexStatus, SessionQuery, SessionSearchRequest, + SessionSearchResponse, }; /// Private on purpose: [`start_if_enabled`] is the only way to a manager, so the feature cannot be bypassed.