From 4026e4617b14ea22d3ec9eacb1f7562064598de3 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Sat, 31 Oct 2026 14:44:42 -0700 Subject: [PATCH 1/3] perf(query): tokenize a query once per completion, hover and check Completion tokenized the whole text twice per keystroke (to rule out a comment or string, then for the outline), and hover twice. The check's underlines tokenized the whole text again for every diagnostic, so a long query with many problems cost its length times their number. Now `outline_of` and `in_comment_or_string` take the tokens their caller already has, and `problem_range` takes the text's tokens, made once for all its problems, and finds the one under a problem by halving. --- crates/gq_language/src/complete.rs | 10 +++-- crates/gq_language/src/gq_language.rs | 4 +- crates/gq_language/src/lex.rs | 59 ++++++++++++++++++--------- crates/query_editor/src/analysis.rs | 11 +++-- 4 files changed, 55 insertions(+), 29 deletions(-) diff --git a/crates/gq_language/src/complete.rs b/crates/gq_language/src/complete.rs index 126c6435..8dff55ee 100644 --- a/crates/gq_language/src/complete.rs +++ b/crates/gq_language/src/complete.rs @@ -8,7 +8,7 @@ use std::ops::Range; -use crate::lex::{KEYWORDS, TokenKind, in_comment_or_string, outline, tokenize}; +use crate::lex::{KEYWORDS, TokenKind, in_comment_or_string, outline_of, tokenize}; use crate::schema::{SchemaModel, traversal_name}; #[derive(Debug, Clone, PartialEq, Eq)] @@ -43,14 +43,16 @@ pub fn completions(schema: &SchemaModel, source: &str, offset: usize) -> (usize, }; // A comment or a string is prose: nothing is offered there, so ↵ (which confirms a shown // completion) ends the line instead of putting a type name in it. - if in_comment_or_string(source, offset) { + // The source's tokens, read for that and for its outline. + let tokens = tokenize(source); + if in_comment_or_string(source, &tokens, offset) { return (start, Vec::new()); } let prefix = &source[start..offset]; let lower = prefix.to_lowercase(); let matches = |label: &str| label.to_lowercase().starts_with(&lower); - let declaration = outline(source) + let declaration = outline_of(source, &tokens) .declaration_at(offset) .map(|d| d.range.clone()) .unwrap_or(0..source.len()); @@ -336,7 +338,7 @@ pub fn hover(schema: &SchemaModel, source: &str, offset: usize) -> Option<(Range .position(|t| t.range.start <= offset && offset < t.range.end)?; let token = &tokens[index]; let word = &source[token.range.clone()]; - let declaration = outline(source) + let declaration = outline_of(source, &tokens) .declaration_at(offset) .map(|d| d.range.clone()) .unwrap_or(0..source.len()); diff --git a/crates/gq_language/src/gq_language.rs b/crates/gq_language/src/gq_language.rs index 3665e183..294c28b6 100644 --- a/crates/gq_language/src/gq_language.rs +++ b/crates/gq_language/src/gq_language.rs @@ -18,8 +18,8 @@ pub use builder::{BuildError, BuiltQuery, Filter, FilterOp, PageStart, Sort, exa pub use check::{Diagnostic, QueryCheck, QueryInfo, ResultColumn, check, duplicate_names}; pub use complete::{Completion, completions, hover}; pub use lex::{ - Declaration, Outline, Statement, Token, TokenKind, looks_like_mutation, outline, problem_range, - tokenize, + Declaration, Outline, Statement, Token, TokenKind, looks_like_mutation, outline, outline_of, + problem_range, tokenize, }; pub use params::param_value; pub use parts::{PartKind, QueryPart, operator_parts, query_parts}; diff --git a/crates/gq_language/src/lex.rs b/crates/gq_language/src/lex.rs index 3b0f3a86..a2b4609d 100644 --- a/crates/gq_language/src/lex.rs +++ b/crates/gq_language/src/lex.rs @@ -362,7 +362,11 @@ impl Statement { /// The declarations of a source and its settings prefix. pub fn outline(source: &str) -> Outline { - let tokens = tokenize(source); + outline_of(source, &tokenize(source)) +} + +/// [`outline`] from `source`'s tokens, [`tokenize`]d once by a caller that reads them too. +pub fn outline_of(source: &str, tokens: &[Token]) -> Outline { let text = |t: &Token| &source[t.range.clone()]; let mut outline = Outline::default(); @@ -542,10 +546,11 @@ impl Outline { } } -/// Whether text typed at `offset` goes into a comment or a string: inside one, or at the end of -/// one still open. A line comment runs to the end of its line, so its end is inside it. -pub fn in_comment_or_string(source: &str, offset: usize) -> bool { - tokenize(source).into_iter().any(|token| { +/// Whether text typed at `offset` of `source`, whose `tokens` these are, goes into a comment or +/// a string: inside one, or at the end of one still open. A line comment runs to the end of its +/// line, so its end is inside it. +pub fn in_comment_or_string(source: &str, tokens: &[Token], offset: usize) -> bool { + tokens.iter().any(|token| { let text = &source[token.range.clone()]; let open = match token.kind { TokenKind::Comment => { @@ -584,15 +589,16 @@ pub fn looks_like_mutation(source: &str) -> bool { }) } -/// The byte range to underline for a problem reported at `byte`: the token there, or one -/// character when no token starts or covers it. -pub fn problem_range(source: &str, byte: usize) -> Range { +/// The byte range to underline for a problem reported at `byte`: the token there among +/// `tokens` (`source`'s, [`tokenize`]d once for all its problems), or one character when no +/// token starts or covers it. +pub fn problem_range(source: &str, tokens: &[Token], byte: usize) -> Range { let byte = byte.min(source.len()); - if let Some(token) = tokenize(source) - .into_iter() - .find(|t| t.range.start <= byte && byte < t.range.end) - { - return token.range; + // `source`'s tokens, made once for all its problems, in order and apart: the one holding + // `byte` is found by halving. + let at = tokens.partition_point(|t| t.range.end <= byte); + if let Some(token) = tokens.get(at).filter(|t| t.range.start <= byte) { + return token.range.clone(); } let start = source.floor_char_boundary(byte); let end = source[start..] @@ -824,7 +830,7 @@ mod tests { /// typed after a closed one, it does not. #[test] fn comments_and_strings_hold_what_is_typed_in_them() { - let at_end = |source: &str| in_comment_or_string(source, source.len()); + let at_end = |source: &str| in_comment_or_string(source, &tokenize(source), source.len()); assert!(at_end("// see the docs.")); assert!(at_end("query q() {} // group by company")); assert!(at_end("/* open")); @@ -835,12 +841,16 @@ mod tests { assert!(!at_end("query q() { match { $p: Person { name: \"Ada\"")); assert!(!at_end("query q() { match { $p: Pe")); let source = "// note\nquery"; - assert!(in_comment_or_string(source, 3)); + assert!(in_comment_or_string(source, &tokenize(source), 3)); assert!( - !in_comment_or_string(source, 0), + !in_comment_or_string(source, &tokenize(source), 0), "before the comment starts" ); - assert!(!in_comment_or_string(source, source.len())); + assert!(!in_comment_or_string( + source, + &tokenize(source), + source.len() + )); } #[test] @@ -909,9 +919,18 @@ mod tests { fn problems_underline_a_token() { let source = "query q() { match { $p: Person } return { $p.nope } }"; let at = source.find("nope").unwrap(); - assert_eq!(&source[problem_range(source, at)], "nope"); - assert_eq!(&source[problem_range(source, source.len())], "}"); - assert_eq!(problem_range(source, at + 4), at + 4..at + 5); + assert_eq!( + &source[problem_range(source, &tokenize(source), at)], + "nope" + ); + assert_eq!( + &source[problem_range(source, &tokenize(source), source.len())], + "}" + ); + assert_eq!( + problem_range(source, &tokenize(source), at + 4), + at + 4..at + 5 + ); } /// A statement starts at the first word after the settings prefix, as `outline` reads it: diff --git a/crates/query_editor/src/analysis.rs b/crates/query_editor/src/analysis.rs index 5ba716ec..6069bb96 100644 --- a/crates/query_editor/src/analysis.rs +++ b/crates/query_editor/src/analysis.rs @@ -70,6 +70,8 @@ impl Analysis { ); checked_alone.push(declaration); } + // Made once for every problem's underline, rather than once each. + let tokens = gq_language::tokenize(&text); let mut problems: Vec<(Diagnostic, Option>)> = checked .diagnostics .into_iter() @@ -79,7 +81,7 @@ impl Analysis { .byte .map(|b| b as usize) .or_else(|| locate(&text, &outline, &diagnostic)); - let range = byte.map(|byte| gq_language::problem_range(&text, byte)); + let range = byte.map(|byte| gq_language::problem_range(&text, &tokens, byte)); (diagnostic, range) }) .filter(|(diagnostic, range)| { @@ -100,7 +102,7 @@ impl Analysis { .map(|diagnostic| { let range = diagnostic .byte - .map(|byte| gq_language::problem_range(&text, byte as usize)); + .map(|byte| gq_language::problem_range(&text, &tokens, byte as usize)); (diagnostic, range) }), ); @@ -196,7 +198,10 @@ fn placed( .map(|(range, with)| Some((moved(range.start)?..moved(range.end)?, with))) .collect::>>() .unwrap_or_default(); - let range = at.map(|at| gq_language::problem_range(&text[..end], at)); + let range = at.map(|at| { + let prefix = &text[..end]; + gq_language::problem_range(prefix, &gq_language::tokenize(prefix), at) + }); Some((diagnostic, range)) } From 646a57a804c3bad36c2ab6049e6636266f14303a Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Sat, 31 Oct 2026 15:43:24 -0700 Subject: [PATCH 2/3] perf(query): the check's outline shares its tokens with the underlines A review found the check still tokenized the text twice: once for its outline and once more for the underlines, so a valid query, which used to tokenize none for underlines, now did one more. The outline is now made from the same tokens (`outline_of`). --- crates/query_editor/src/analysis.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/query_editor/src/analysis.rs b/crates/query_editor/src/analysis.rs index 6069bb96..34bfa979 100644 --- a/crates/query_editor/src/analysis.rs +++ b/crates/query_editor/src/analysis.rs @@ -22,7 +22,9 @@ pub struct Analysis { impl Analysis { /// Checks `text` against `schema`. Runs off the main thread. pub fn new(schema: &SchemaModel, text: String) -> Self { - let outline = gq_language::outline(&text); + // Made once, for the outline and every problem's underline. + let tokens = gq_language::tokenize(&text); + let outline = gq_language::outline_of(&text, &tokens); let checked = gq_language::check(schema, &text); // The whole text's check describes the declarations it parsed in the order of the // text, so each is taken by its place: two may share a name, and the name would give @@ -70,8 +72,6 @@ impl Analysis { ); checked_alone.push(declaration); } - // Made once for every problem's underline, rather than once each. - let tokens = gq_language::tokenize(&text); let mut problems: Vec<(Diagnostic, Option>)> = checked .diagnostics .into_iter() From d11ac6c69736c5a962e109934c8aa96582e0ebb7 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Sat, 31 Oct 2026 16:13:44 -0700 Subject: [PATCH 3/3] perf(query): place a declaration's problems with the whole text's tokens A problem a declaration's own check found lexed the whole text up to the declaration's end, and counted lines from the start, once per problem: work that grew with the text times the problems. It now takes those tokens from the ones the analysis already made and finds lines once per check. The diff view and the gateway rule also lex their text once for both the tokens and the outline. --- crates/agent_panel/src/gateway_rules.rs | 2 +- crates/query_editor/src/analysis.rs | 108 ++++++++++++++++-------- crates/query_editor/src/results.rs | 5 +- 3 files changed, 78 insertions(+), 37 deletions(-) diff --git a/crates/agent_panel/src/gateway_rules.rs b/crates/agent_panel/src/gateway_rules.rs index 2f387757..8033724e 100644 --- a/crates/agent_panel/src/gateway_rules.rs +++ b/crates/agent_panel/src/gateway_rules.rs @@ -391,7 +391,7 @@ fn held_off_the_branch(query: &str, name: Option<&str>) -> bool { use gq_language::TokenKind; let tokens = gq_language::tokenize(query); let mut declarations: Vec> = Vec::new(); - let outline = gq_language::outline(query).declarations; + let outline = gq_language::outline_of(query, &tokens).declarations; let named = outline .iter() .any(|declaration| Some(declaration.name.as_str()) == name); diff --git a/crates/query_editor/src/analysis.rs b/crates/query_editor/src/analysis.rs index 34bfa979..4ae106e3 100644 --- a/crates/query_editor/src/analysis.rs +++ b/crates/query_editor/src/analysis.rs @@ -4,7 +4,7 @@ use std::ops::Range; -use gq_language::{Diagnostic, Outline, QueryInfo, SchemaModel}; +use gq_language::{Diagnostic, Outline, QueryInfo, SchemaModel, Token}; #[derive(Debug, Clone, Default)] pub struct Analysis { @@ -44,6 +44,7 @@ impl Analysis { let mut queries = Vec::new(); let mut own_problems = Vec::new(); let mut checked_alone: Vec<&gq_language::Declaration> = Vec::new(); + let mut lines = Lines::default(); for (index, declaration) in outline.declarations.iter().enumerate() { if let Some(query) = checked .queries @@ -55,12 +56,13 @@ impl Analysis { } let own_text = outline.runnable_text(&text, declaration); let own = gq_language::check(schema, &own_text); - let own_outline = gq_language::outline(&own_text); + let own_tokens = gq_language::tokenize(&own_text); + let own_outline = gq_language::outline_of(&own_text, &own_tokens); own_problems.extend(own.diagnostics.into_iter().filter_map(|diagnostic| { placed( diagnostic, - &text, - (&own_text, &own_outline), + (&text, &tokens, &mut lines), + (&own_text, &own_tokens, &own_outline), outline.prefix.end, &declaration.range, ) @@ -80,7 +82,7 @@ impl Analysis { let byte = diagnostic .byte .map(|b| b as usize) - .or_else(|| locate(&text, &outline, &diagnostic)); + .or_else(|| locate(&text, &tokens, &outline, &diagnostic)); let range = byte.map(|byte| gq_language::problem_range(&text, &tokens, byte)); (diagnostic, range) }) @@ -151,43 +153,64 @@ impl Analysis { } } +/// Where each line of a text starts, found on first use and then shared by every problem placed +/// in it. +#[derive(Default)] +struct Lines(Option>); + +impl Lines { + /// The line and column, from one, of byte `at` in `text`. + fn of(&mut self, text: &str, at: usize) -> (u32, u32) { + let starts = self.0.get_or_insert_with(|| { + std::iter::once(0) + .chain(text.match_indices('\n').map(|(at, _)| at + 1)) + .collect() + }); + let line = starts.partition_point(|start| *start <= at); + let column = text[starts[line - 1]..at].chars().count(); + (line as u32, column as u32 + 1) + } +} + +/// The tokens of `tokens` (a text's, in order) that lie in `range`, whose ends are token +/// boundaries: the tokens of that slice of the text, at their places in the whole. +fn tokens_in<'a>(tokens: &'a [Token], range: &Range) -> &'a [Token] { + let start = tokens.partition_point(|token| token.range.start < range.start); + let end = tokens.partition_point(|token| token.range.end <= range.end); + &tokens[start..end.max(start)] +} + /// A problem a declaration's own check found, moved from that check's text (`own`: the -/// settings prefix, `prefix` bytes, then the declaration) to where the declaration is in -/// `text`. One in the prefix is the whole text's check's to report. A problem at the end of -/// the declaration is placed on its last word, not on the declaration after it. +/// settings prefix, `prefix` bytes, then the declaration, with its tokens and outline) to +/// where the declaration is in `text` (`whole`: with its tokens, and its lines as found so +/// far). One in the prefix is the whole text's check's to report. A problem at the end of the +/// declaration is placed on its last word, not on the declaration after it. fn placed( mut diagnostic: Diagnostic, - text: &str, - own: (&str, &Outline), + whole: (&str, &[Token], &mut Lines), + own: (&str, &[Token], &Outline), prefix: usize, declaration: &Range, ) -> Option<(Diagnostic, Option>)> { + let (text, tokens, lines) = whole; // Its last word: an unclosed declaration runs on to the next, over blank lines and any // comment above it. - let end = gq_language::tokenize(&text[declaration.clone()]) + let end = tokens_in(tokens, declaration) .iter() .rev() .find(|token| token.kind != gq_language::TokenKind::Comment) - .map_or(declaration.start, |token| { - declaration.start + token.range.end - }); + .map_or(declaration.start, |token| token.range.end); let moved = |at: usize| (at >= prefix).then(|| (declaration.start + at - prefix).min(end)); let at = match diagnostic.byte { Some(byte) => { let at = moved(byte as usize)?; - let before = &text[..at]; + let (line, column) = lines.of(text, at); diagnostic.byte = Some(at as u32); - diagnostic.line = Some(before.matches('\n').count() as u32 + 1); - diagnostic.column = Some( - before - .rsplit('\n') - .next() - .map_or(0, |line| line.chars().count()) as u32 - + 1, - ); + diagnostic.line = Some(line); + diagnostic.column = Some(column); Some(at) } - None => match locate(own.0, own.1, &diagnostic) { + None => match locate(own.0, own.1, own.2, &diagnostic) { Some(at) => Some(moved(at)?), None => None, }, @@ -198,21 +221,25 @@ fn placed( .map(|(range, with)| Some((moved(range.start)?..moved(range.end)?, with))) .collect::>>() .unwrap_or_default(); - let range = at.map(|at| { - let prefix = &text[..end]; - gq_language::problem_range(prefix, &gq_language::tokenize(prefix), at) - }); + // Underlined within the text up to its last word, whose tokens are the whole text's up to + // there: `end` is a token's end. + let range = + at.map(|at| gq_language::problem_range(&text[..end], tokens_in(tokens, &(0..end)), at)); Some((diagnostic, range)) } /// Where a type error without a position probably is: the last name the message quotes /// (`` type `Person` has no property `nmae` ``) that appears as a token in the declaration the -/// error names. Only an underline hint; the message stays the authority. -fn locate(text: &str, outline: &Outline, diagnostic: &Diagnostic) -> Option { +/// error names (`text`'s, with its tokens and outline). Only an underline hint; the message +/// stays the authority. +fn locate( + text: &str, + tokens: &[Token], + outline: &Outline, + diagnostic: &Diagnostic, +) -> Option { let name = diagnostic.query.as_deref()?; let declaration = outline.declarations.iter().find(|d| d.name == name)?; - let body = &text[declaration.range.clone()]; - let tokens = gq_language::tokenize(body); let quoted: Vec<&str> = diagnostic .message .split('`') @@ -220,11 +247,12 @@ fn locate(text: &str, outline: &Outline, diagnostic: &Diagnostic) -> Option bool { /// The words of the `branch diff` statement `source` holds after its settings prefix, or /// `None` when it holds another. fn diff_tokens(source: &str) -> Option> { - let prefix = gq_language::outline(source).prefix.end; - let tokens: Vec = gq_language::tokenize(source) + let tokens = gq_language::tokenize(source); + let prefix = gq_language::outline_of(source, &tokens).prefix.end; + let tokens: Vec = tokens .into_iter() .filter(|t| t.range.start >= prefix && t.kind != TokenKind::Comment) .collect();