perf(query): tokenize a query once per completion, hover and check

crates/agent_panel/src/gateway_rules.rs

@@ -391,7 +391,7 @@ fn held_off_the_branch(query: &str, name: Option<&str>) -> bool {
     use gq_language::TokenKind;
     let tokens = gq_language::tokenize(query);
     let mut declarations: Vec<std::ops::Range<usize>> = Vec::new();
-    let outline = gq_language::outline(query).declarations;
+    let outline = gq_language::outline_of(query, &tokens).declarations;
     let named = outline
         .iter()
         .any(|declaration| Some(declaration.name.as_str()) == name);

crates/gq_language/src/complete.rs

@@ -8,7 +8,7 @@
 
 use std::ops::Range;
 
-use crate::lex::{KEYWORDS, TokenKind, in_comment_or_string, outline, tokenize};
+use crate::lex::{KEYWORDS, TokenKind, in_comment_or_string, outline_of, tokenize};
 use crate::schema::{SchemaModel, traversal_name};
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -43,14 +43,16 @@ pub fn completions(schema: &SchemaModel, source: &str, offset: usize) -> (usize,
     };
     // A comment or a string is prose: nothing is offered there, so ↵ (which confirms a shown
     // completion) ends the line instead of putting a type name in it.
-    if in_comment_or_string(source, offset) {
+    // The source's tokens, read for that and for its outline.
+    let tokens = tokenize(source);
+    if in_comment_or_string(source, &tokens, offset) {
         return (start, Vec::new());
     }
     let prefix = &source[start..offset];
     let lower = prefix.to_lowercase();
     let matches = |label: &str| label.to_lowercase().starts_with(&lower);
 
-    let declaration = outline(source)
+    let declaration = outline_of(source, &tokens)
         .declaration_at(offset)
         .map(|d| d.range.clone())
         .unwrap_or(0..source.len());
@@ -336,7 +338,7 @@ pub fn hover(schema: &SchemaModel, source: &str, offset: usize) -> Option<(Range
         .position(|t| t.range.start <= offset && offset < t.range.end)?;
     let token = &tokens[index];
     let word = &source[token.range.clone()];
-    let declaration = outline(source)
+    let declaration = outline_of(source, &tokens)
         .declaration_at(offset)
         .map(|d| d.range.clone())
         .unwrap_or(0..source.len());

crates/gq_language/src/gq_language.rs

@@ -18,8 +18,8 @@ pub use builder::{BuildError, BuiltQuery, Filter, FilterOp, PageStart, Sort, exa
 pub use check::{Diagnostic, QueryCheck, QueryInfo, ResultColumn, check, duplicate_names};
 pub use complete::{Completion, completions, hover};
 pub use lex::{
-    Declaration, Outline, Statement, Token, TokenKind, looks_like_mutation, outline, problem_range,
-    tokenize,
+    Declaration, Outline, Statement, Token, TokenKind, looks_like_mutation, outline, outline_of,
+    problem_range, tokenize,
 };
 pub use params::param_value;
 pub use parts::{PartKind, QueryPart, operator_parts, query_parts};

crates/gq_language/src/lex.rs

@@ -362,7 +362,11 @@ impl Statement {
 
 /// The declarations of a source and its settings prefix.
 pub fn outline(source: &str) -> Outline {
-    let tokens = tokenize(source);
+    outline_of(source, &tokenize(source))
+}
+
+/// [`outline`] from `source`'s tokens, [`tokenize`]d once by a caller that reads them too.
+pub fn outline_of(source: &str, tokens: &[Token]) -> Outline {
     let text = |t: &Token| &source[t.range.clone()];
     let mut outline = Outline::default();
 
@@ -542,10 +546,11 @@ impl Outline {
     }
 }
 
-/// Whether text typed at `offset` goes into a comment or a string: inside one, or at the end of
-/// one still open. A line comment runs to the end of its line, so its end is inside it.
-pub fn in_comment_or_string(source: &str, offset: usize) -> bool {
-    tokenize(source).into_iter().any(|token| {
+/// Whether text typed at `offset` of `source`, whose `tokens` these are, goes into a comment or
+/// a string: inside one, or at the end of one still open. A line comment runs to the end of its
+/// line, so its end is inside it.
+pub fn in_comment_or_string(source: &str, tokens: &[Token], offset: usize) -> bool {
+    tokens.iter().any(|token| {
         let text = &source[token.range.clone()];
         let open = match token.kind {
             TokenKind::Comment => {
@@ -584,15 +589,16 @@ pub fn looks_like_mutation(source: &str) -> bool {
     })
 }
 
-/// The byte range to underline for a problem reported at `byte`: the token there, or one
-/// character when no token starts or covers it.
-pub fn problem_range(source: &str, byte: usize) -> Range<usize> {
+/// The byte range to underline for a problem reported at `byte`: the token there among
+/// `tokens` (`source`'s, [`tokenize`]d once for all its problems), or one character when no
+/// token starts or covers it.
+pub fn problem_range(source: &str, tokens: &[Token], byte: usize) -> Range<usize> {
     let byte = byte.min(source.len());
-    if let Some(token) = tokenize(source)
-        .into_iter()
-        .find(|t| t.range.start <= byte && byte < t.range.end)
-    {
-        return token.range;
+    // `source`'s tokens, made once for all its problems, in order and apart: the one holding
+    // `byte` is found by halving.
+    let at = tokens.partition_point(|t| t.range.end <= byte);
+    if let Some(token) = tokens.get(at).filter(|t| t.range.start <= byte) {
+        return token.range.clone();
     }
     let start = source.floor_char_boundary(byte);
     let end = source[start..]
@@ -824,7 +830,7 @@ mod tests {
     /// typed after a closed one, it does not.
     #[test]
     fn comments_and_strings_hold_what_is_typed_in_them() {
-        let at_end = |source: &str| in_comment_or_string(source, source.len());
+        let at_end = |source: &str| in_comment_or_string(source, &tokenize(source), source.len());
         assert!(at_end("// see the docs."));
         assert!(at_end("query q() {} // group by company"));
         assert!(at_end("/* open"));
@@ -835,12 +841,16 @@ mod tests {
         assert!(!at_end("query q() { match { $p: Person { name: \"Ada\""));
         assert!(!at_end("query q() { match { $p: Pe"));
         let source = "// note\nquery";
-        assert!(in_comment_or_string(source, 3));
+        assert!(in_comment_or_string(source, &tokenize(source), 3));
         assert!(
-            !in_comment_or_string(source, 0),
+            !in_comment_or_string(source, &tokenize(source), 0),
             "before the comment starts"
         );
-        assert!(!in_comment_or_string(source, source.len()));
+        assert!(!in_comment_or_string(
+            source,
+            &tokenize(source),
+            source.len()
+        ));
     }
 
     #[test]
@@ -909,9 +919,18 @@ mod tests {
     fn problems_underline_a_token() {
         let source = "query q() { match { $p: Person } return { $p.nope } }";
         let at = source.find("nope").unwrap();
-        assert_eq!(&source[problem_range(source, at)], "nope");
-        assert_eq!(&source[problem_range(source, source.len())], "}");
-        assert_eq!(problem_range(source, at + 4), at + 4..at + 5);
+        assert_eq!(
+            &source[problem_range(source, &tokenize(source), at)],
+            "nope"
+        );
+        assert_eq!(
+            &source[problem_range(source, &tokenize(source), source.len())],
+            "}"
+        );
+        assert_eq!(
+            problem_range(source, &tokenize(source), at + 4),
+            at + 4..at + 5
+        );
     }
 
     /// A statement starts at the first word after the settings prefix, as `outline` reads it:

crates/query_editor/src/analysis.rs

@@ -4,7 +4,7 @@
 
 use std::ops::Range;
 
-use gq_language::{Diagnostic, Outline, QueryInfo, SchemaModel};
+use gq_language::{Diagnostic, Outline, QueryInfo, SchemaModel, Token};
 
 #[derive(Debug, Clone, Default)]
 pub struct Analysis {
@@ -22,7 +22,9 @@ pub struct Analysis {
 impl Analysis {
     /// Checks `text` against `schema`. Runs off the main thread.
     pub fn new(schema: &SchemaModel, text: String) -> Self {
-        let outline = gq_language::outline(&text);
+        // Made once, for the outline and every problem's underline.
+        let tokens = gq_language::tokenize(&text);
+        let outline = gq_language::outline_of(&text, &tokens);
         let checked = gq_language::check(schema, &text);
         // The whole text's check describes the declarations it parsed in the order of the
         // text, so each is taken by its place: two may share a name, and the name would give
@@ -42,6 +44,7 @@ impl Analysis {
         let mut queries = Vec::new();
         let mut own_problems = Vec::new();
         let mut checked_alone: Vec<&gq_language::Declaration> = Vec::new();
+        let mut lines = Lines::default();
         for (index, declaration) in outline.declarations.iter().enumerate() {
             if let Some(query) = checked
                 .queries
@@ -53,12 +56,13 @@ impl Analysis {
             }
             let own_text = outline.runnable_text(&text, declaration);
             let own = gq_language::check(schema, &own_text);
-            let own_outline = gq_language::outline(&own_text);
+            let own_tokens = gq_language::tokenize(&own_text);
+            let own_outline = gq_language::outline_of(&own_text, &own_tokens);
             own_problems.extend(own.diagnostics.into_iter().filter_map(|diagnostic| {
                 placed(
                     diagnostic,
-                    &text,
-                    (&own_text, &own_outline),
+                    (&text, &tokens, &mut lines),
+                    (&own_text, &own_tokens, &own_outline),
                     outline.prefix.end,
                     &declaration.range,
                 )
@@ -78,8 +82,8 @@ impl Analysis {
                 let byte = diagnostic
                     .byte
                     .map(|b| b as usize)
-                    .or_else(|| locate(&text, &outline, &diagnostic));
-                let range = byte.map(|byte| gq_language::problem_range(&text, byte));
+                    .or_else(|| locate(&text, &tokens, &outline, &diagnostic));
+                let range = byte.map(|byte| gq_language::problem_range(&text, &tokens, byte));
                 (diagnostic, range)
             })
             .filter(|(diagnostic, range)| {
@@ -100,7 +104,7 @@ impl Analysis {
                 .map(|diagnostic| {
                     let range = diagnostic
                         .byte
-                        .map(|byte| gq_language::problem_range(&text, byte as usize));
+                        .map(|byte| gq_language::problem_range(&text, &tokens, byte as usize));
                     (diagnostic, range)
                 }),
         );
@@ -149,43 +153,64 @@ impl Analysis {
     }
 }
 
+/// Where each line of a text starts, found on first use and then shared by every problem placed
+/// in it.
+#[derive(Default)]
+struct Lines(Option<Vec<usize>>);
+
+impl Lines {
+    /// The line and column, from one, of byte `at` in `text`.
+    fn of(&mut self, text: &str, at: usize) -> (u32, u32) {
+        let starts = self.0.get_or_insert_with(|| {
+            std::iter::once(0)
+                .chain(text.match_indices('\n').map(|(at, _)| at + 1))
+                .collect()
+        });
+        let line = starts.partition_point(|start| *start <= at);
+        let column = text[starts[line - 1]..at].chars().count();
+        (line as u32, column as u32 + 1)
+    }
+}
+
+/// The tokens of `tokens` (a text's, in order) that lie in `range`, whose ends are token
+/// boundaries: the tokens of that slice of the text, at their places in the whole.
+fn tokens_in<'a>(tokens: &'a [Token], range: &Range<usize>) -> &'a [Token] {
+    let start = tokens.partition_point(|token| token.range.start < range.start);
+    let end = tokens.partition_point(|token| token.range.end <= range.end);
+    &tokens[start..end.max(start)]
+}
+
 /// A problem a declaration's own check found, moved from that check's text (`own`: the
-/// settings prefix, `prefix` bytes, then the declaration) to where the declaration is in
-/// `text`. One in the prefix is the whole text's check's to report. A problem at the end of
-/// the declaration is placed on its last word, not on the declaration after it.
+/// settings prefix, `prefix` bytes, then the declaration, with its tokens and outline) to
+/// where the declaration is in `text` (`whole`: with its tokens, and its lines as found so
+/// far). One in the prefix is the whole text's check's to report. A problem at the end of the
+/// declaration is placed on its last word, not on the declaration after it.
 fn placed(
     mut diagnostic: Diagnostic,
-    text: &str,
-    own: (&str, &Outline),
+    whole: (&str, &[Token], &mut Lines),
+    own: (&str, &[Token], &Outline),
     prefix: usize,
     declaration: &Range<usize>,
 ) -> Option<(Diagnostic, Option<Range<usize>>)> {
+    let (text, tokens, lines) = whole;
     // Its last word: an unclosed declaration runs on to the next, over blank lines and any
     // comment above it.
-    let end = gq_language::tokenize(&text[declaration.clone()])
+    let end = tokens_in(tokens, declaration)
         .iter()
         .rev()
         .find(|token| token.kind != gq_language::TokenKind::Comment)
-        .map_or(declaration.start, |token| {
-            declaration.start + token.range.end
-        });
+        .map_or(declaration.start, |token| token.range.end);
     let moved = |at: usize| (at >= prefix).then(|| (declaration.start + at - prefix).min(end));
     let at = match diagnostic.byte {
         Some(byte) => {
             let at = moved(byte as usize)?;
-            let before = &text[..at];
+            let (line, column) = lines.of(text, at);
             diagnostic.byte = Some(at as u32);
-            diagnostic.line = Some(before.matches('\n').count() as u32 + 1);
-            diagnostic.column = Some(
-                before
-                    .rsplit('\n')
-                    .next()
-                    .map_or(0, |line| line.chars().count()) as u32
-                    + 1,
-            );
+            diagnostic.line = Some(line);
+            diagnostic.column = Some(column);
             Some(at)
         }
-        None => match locate(own.0, own.1, &diagnostic) {
+        None => match locate(own.0, own.1, own.2, &diagnostic) {
             Some(at) => Some(moved(at)?),
             None => None,
         },
@@ -196,30 +221,38 @@ fn placed(
         .map(|(range, with)| Some((moved(range.start)?..moved(range.end)?, with)))
         .collect::<Option<Vec<_>>>()
         .unwrap_or_default();
-    let range = at.map(|at| gq_language::problem_range(&text[..end], at));
+    // Underlined within the text up to its last word, whose tokens are the whole text's up to
+    // there: `end` is a token's end.
+    let range =
+        at.map(|at| gq_language::problem_range(&text[..end], tokens_in(tokens, &(0..end)), at));
     Some((diagnostic, range))
 }
 
 /// Where a type error without a position probably is: the last name the message quotes
 /// (`` type `Person` has no property `nmae` ``) that appears as a token in the declaration the
-/// error names. Only an underline hint; the message stays the authority.
-fn locate(text: &str, outline: &Outline, diagnostic: &Diagnostic) -> Option<usize> {
+/// error names (`text`'s, with its tokens and outline). Only an underline hint; the message
+/// stays the authority.
+fn locate(
+    text: &str,
+    tokens: &[Token],
+    outline: &Outline,
+    diagnostic: &Diagnostic,
+) -> Option<usize> {
     let name = diagnostic.query.as_deref()?;
     let declaration = outline.declarations.iter().find(|d| d.name == name)?;
-    let body = &text[declaration.range.clone()];
-    let tokens = gq_language::tokenize(body);
     let quoted: Vec<&str> = diagnostic
         .message
         .split('`')
         .skip(1)
         .step_by(2)
         .filter(|quoted| !quoted.is_empty())
         .collect();
+    let tokens = tokens_in(tokens, &declaration.range);
     quoted.into_iter().rev().find_map(|quoted| {
         tokens
             .iter()
-            .find(|token| &body[token.range.clone()] == quoted)
-            .map(|token| declaration.range.start + token.range.start)
+            .find(|token| &text[token.range.clone()] == quoted)
+            .map(|token| token.range.start)
     })
 }
 
@@ -263,6 +296,18 @@ mod tests {
         assert_eq!(at.len(), 2, "{:?}", analysis.problems);
         assert!(at[0] < text.find("query q").unwrap(), "{at:?}");
         assert_eq!(at[1], text.find("nope").unwrap());
+
+        // A problem its own check found is placed by line and column in the whole text, the
+        // column counted in characters.
+        let text = "query people() { match { $p: Person } return { $p.name } }\n\n// é\n\
+            query broken() { match { $p: Person { name: \"é\" } } return { $p.naem\n";
+        let analysis = Analysis::new(&schema, text.to_owned());
+        assert_eq!(analysis.problems.len(), 1, "{:?}", analysis.problems);
+        let (problem, range) = &analysis.problems[0];
+        let end = text.find("naem").unwrap() + "naem".len();
+        assert_eq!(problem.byte, Some(end as u32), "at its last word");
+        assert_eq!((problem.line, problem.column), (Some(4), Some(69)));
+        assert!(range.as_ref().is_some_and(|range| range.end == end));
     }
 
     /// A read copied and changed to a delete keeps the read's name: each declaration is still

crates/query_editor/src/results.rs

@@ -641,8 +641,9 @@ pub fn is_branch_diff(source: &str) -> bool {
 /// The words of the `branch diff` statement `source` holds after its settings prefix, or
 /// `None` when it holds another.
 fn diff_tokens(source: &str) -> Option<Vec<gq_language::Token>> {
-    let prefix = gq_language::outline(source).prefix.end;
-    let tokens: Vec<gq_language::Token> = gq_language::tokenize(source)
+    let tokens = gq_language::tokenize(source);
+    let prefix = gq_language::outline_of(source, &tokens).prefix.end;
+    let tokens: Vec<gq_language::Token> = tokens
         .into_iter()
         .filter(|t| t.range.start >= prefix && t.kind != TokenKind::Comment)
         .collect();