diff --git a/crates/agent_panel/src/gateway_rules.rs b/crates/agent_panel/src/gateway_rules.rs index 2f387757..8033724e 100644 --- a/crates/agent_panel/src/gateway_rules.rs +++ b/crates/agent_panel/src/gateway_rules.rs @@ -391,7 +391,7 @@ fn held_off_the_branch(query: &str, name: Option<&str>) -> bool { use gq_language::TokenKind; let tokens = gq_language::tokenize(query); let mut declarations: Vec> = Vec::new(); - let outline = gq_language::outline(query).declarations; + let outline = gq_language::outline_of(query, &tokens).declarations; let named = outline .iter() .any(|declaration| Some(declaration.name.as_str()) == name); diff --git a/crates/gq_language/src/complete.rs b/crates/gq_language/src/complete.rs index 126c6435..8dff55ee 100644 --- a/crates/gq_language/src/complete.rs +++ b/crates/gq_language/src/complete.rs @@ -8,7 +8,7 @@ use std::ops::Range; -use crate::lex::{KEYWORDS, TokenKind, in_comment_or_string, outline, tokenize}; +use crate::lex::{KEYWORDS, TokenKind, in_comment_or_string, outline_of, tokenize}; use crate::schema::{SchemaModel, traversal_name}; #[derive(Debug, Clone, PartialEq, Eq)] @@ -43,14 +43,16 @@ pub fn completions(schema: &SchemaModel, source: &str, offset: usize) -> (usize, }; // A comment or a string is prose: nothing is offered there, so ↵ (which confirms a shown // completion) ends the line instead of putting a type name in it. - if in_comment_or_string(source, offset) { + // The source's tokens, read for that and for its outline. + let tokens = tokenize(source); + if in_comment_or_string(source, &tokens, offset) { return (start, Vec::new()); } let prefix = &source[start..offset]; let lower = prefix.to_lowercase(); let matches = |label: &str| label.to_lowercase().starts_with(&lower); - let declaration = outline(source) + let declaration = outline_of(source, &tokens) .declaration_at(offset) .map(|d| d.range.clone()) .unwrap_or(0..source.len()); @@ -336,7 +338,7 @@ pub fn hover(schema: &SchemaModel, source: &str, offset: usize) -> Option<(Range .position(|t| t.range.start <= offset && offset < t.range.end)?; let token = &tokens[index]; let word = &source[token.range.clone()]; - let declaration = outline(source) + let declaration = outline_of(source, &tokens) .declaration_at(offset) .map(|d| d.range.clone()) .unwrap_or(0..source.len()); diff --git a/crates/gq_language/src/gq_language.rs b/crates/gq_language/src/gq_language.rs index 3665e183..294c28b6 100644 --- a/crates/gq_language/src/gq_language.rs +++ b/crates/gq_language/src/gq_language.rs @@ -18,8 +18,8 @@ pub use builder::{BuildError, BuiltQuery, Filter, FilterOp, PageStart, Sort, exa pub use check::{Diagnostic, QueryCheck, QueryInfo, ResultColumn, check, duplicate_names}; pub use complete::{Completion, completions, hover}; pub use lex::{ - Declaration, Outline, Statement, Token, TokenKind, looks_like_mutation, outline, problem_range, - tokenize, + Declaration, Outline, Statement, Token, TokenKind, looks_like_mutation, outline, outline_of, + problem_range, tokenize, }; pub use params::param_value; pub use parts::{PartKind, QueryPart, operator_parts, query_parts}; diff --git a/crates/gq_language/src/lex.rs b/crates/gq_language/src/lex.rs index 3b0f3a86..a2b4609d 100644 --- a/crates/gq_language/src/lex.rs +++ b/crates/gq_language/src/lex.rs @@ -362,7 +362,11 @@ impl Statement { /// The declarations of a source and its settings prefix. pub fn outline(source: &str) -> Outline { - let tokens = tokenize(source); + outline_of(source, &tokenize(source)) +} + +/// [`outline`] from `source`'s tokens, [`tokenize`]d once by a caller that reads them too. +pub fn outline_of(source: &str, tokens: &[Token]) -> Outline { let text = |t: &Token| &source[t.range.clone()]; let mut outline = Outline::default(); @@ -542,10 +546,11 @@ impl Outline { } } -/// Whether text typed at `offset` goes into a comment or a string: inside one, or at the end of -/// one still open. A line comment runs to the end of its line, so its end is inside it. -pub fn in_comment_or_string(source: &str, offset: usize) -> bool { - tokenize(source).into_iter().any(|token| { +/// Whether text typed at `offset` of `source`, whose `tokens` these are, goes into a comment or +/// a string: inside one, or at the end of one still open. A line comment runs to the end of its +/// line, so its end is inside it. +pub fn in_comment_or_string(source: &str, tokens: &[Token], offset: usize) -> bool { + tokens.iter().any(|token| { let text = &source[token.range.clone()]; let open = match token.kind { TokenKind::Comment => { @@ -584,15 +589,16 @@ pub fn looks_like_mutation(source: &str) -> bool { }) } -/// The byte range to underline for a problem reported at `byte`: the token there, or one -/// character when no token starts or covers it. -pub fn problem_range(source: &str, byte: usize) -> Range { +/// The byte range to underline for a problem reported at `byte`: the token there among +/// `tokens` (`source`'s, [`tokenize`]d once for all its problems), or one character when no +/// token starts or covers it. +pub fn problem_range(source: &str, tokens: &[Token], byte: usize) -> Range { let byte = byte.min(source.len()); - if let Some(token) = tokenize(source) - .into_iter() - .find(|t| t.range.start <= byte && byte < t.range.end) - { - return token.range; + // `source`'s tokens, made once for all its problems, in order and apart: the one holding + // `byte` is found by halving. + let at = tokens.partition_point(|t| t.range.end <= byte); + if let Some(token) = tokens.get(at).filter(|t| t.range.start <= byte) { + return token.range.clone(); } let start = source.floor_char_boundary(byte); let end = source[start..] @@ -824,7 +830,7 @@ mod tests { /// typed after a closed one, it does not. #[test] fn comments_and_strings_hold_what_is_typed_in_them() { - let at_end = |source: &str| in_comment_or_string(source, source.len()); + let at_end = |source: &str| in_comment_or_string(source, &tokenize(source), source.len()); assert!(at_end("// see the docs.")); assert!(at_end("query q() {} // group by company")); assert!(at_end("/* open")); @@ -835,12 +841,16 @@ mod tests { assert!(!at_end("query q() { match { $p: Person { name: \"Ada\"")); assert!(!at_end("query q() { match { $p: Pe")); let source = "// note\nquery"; - assert!(in_comment_or_string(source, 3)); + assert!(in_comment_or_string(source, &tokenize(source), 3)); assert!( - !in_comment_or_string(source, 0), + !in_comment_or_string(source, &tokenize(source), 0), "before the comment starts" ); - assert!(!in_comment_or_string(source, source.len())); + assert!(!in_comment_or_string( + source, + &tokenize(source), + source.len() + )); } #[test] @@ -909,9 +919,18 @@ mod tests { fn problems_underline_a_token() { let source = "query q() { match { $p: Person } return { $p.nope } }"; let at = source.find("nope").unwrap(); - assert_eq!(&source[problem_range(source, at)], "nope"); - assert_eq!(&source[problem_range(source, source.len())], "}"); - assert_eq!(problem_range(source, at + 4), at + 4..at + 5); + assert_eq!( + &source[problem_range(source, &tokenize(source), at)], + "nope" + ); + assert_eq!( + &source[problem_range(source, &tokenize(source), source.len())], + "}" + ); + assert_eq!( + problem_range(source, &tokenize(source), at + 4), + at + 4..at + 5 + ); } /// A statement starts at the first word after the settings prefix, as `outline` reads it: diff --git a/crates/query_editor/src/analysis.rs b/crates/query_editor/src/analysis.rs index 5ba716ec..4ae106e3 100644 --- a/crates/query_editor/src/analysis.rs +++ b/crates/query_editor/src/analysis.rs @@ -4,7 +4,7 @@ use std::ops::Range; -use gq_language::{Diagnostic, Outline, QueryInfo, SchemaModel}; +use gq_language::{Diagnostic, Outline, QueryInfo, SchemaModel, Token}; #[derive(Debug, Clone, Default)] pub struct Analysis { @@ -22,7 +22,9 @@ pub struct Analysis { impl Analysis { /// Checks `text` against `schema`. Runs off the main thread. pub fn new(schema: &SchemaModel, text: String) -> Self { - let outline = gq_language::outline(&text); + // Made once, for the outline and every problem's underline. + let tokens = gq_language::tokenize(&text); + let outline = gq_language::outline_of(&text, &tokens); let checked = gq_language::check(schema, &text); // The whole text's check describes the declarations it parsed in the order of the // text, so each is taken by its place: two may share a name, and the name would give @@ -42,6 +44,7 @@ impl Analysis { let mut queries = Vec::new(); let mut own_problems = Vec::new(); let mut checked_alone: Vec<&gq_language::Declaration> = Vec::new(); + let mut lines = Lines::default(); for (index, declaration) in outline.declarations.iter().enumerate() { if let Some(query) = checked .queries @@ -53,12 +56,13 @@ impl Analysis { } let own_text = outline.runnable_text(&text, declaration); let own = gq_language::check(schema, &own_text); - let own_outline = gq_language::outline(&own_text); + let own_tokens = gq_language::tokenize(&own_text); + let own_outline = gq_language::outline_of(&own_text, &own_tokens); own_problems.extend(own.diagnostics.into_iter().filter_map(|diagnostic| { placed( diagnostic, - &text, - (&own_text, &own_outline), + (&text, &tokens, &mut lines), + (&own_text, &own_tokens, &own_outline), outline.prefix.end, &declaration.range, ) @@ -78,8 +82,8 @@ impl Analysis { let byte = diagnostic .byte .map(|b| b as usize) - .or_else(|| locate(&text, &outline, &diagnostic)); - let range = byte.map(|byte| gq_language::problem_range(&text, byte)); + .or_else(|| locate(&text, &tokens, &outline, &diagnostic)); + let range = byte.map(|byte| gq_language::problem_range(&text, &tokens, byte)); (diagnostic, range) }) .filter(|(diagnostic, range)| { @@ -100,7 +104,7 @@ impl Analysis { .map(|diagnostic| { let range = diagnostic .byte - .map(|byte| gq_language::problem_range(&text, byte as usize)); + .map(|byte| gq_language::problem_range(&text, &tokens, byte as usize)); (diagnostic, range) }), ); @@ -149,43 +153,64 @@ impl Analysis { } } +/// Where each line of a text starts, found on first use and then shared by every problem placed +/// in it. +#[derive(Default)] +struct Lines(Option>); + +impl Lines { + /// The line and column, from one, of byte `at` in `text`. + fn of(&mut self, text: &str, at: usize) -> (u32, u32) { + let starts = self.0.get_or_insert_with(|| { + std::iter::once(0) + .chain(text.match_indices('\n').map(|(at, _)| at + 1)) + .collect() + }); + let line = starts.partition_point(|start| *start <= at); + let column = text[starts[line - 1]..at].chars().count(); + (line as u32, column as u32 + 1) + } +} + +/// The tokens of `tokens` (a text's, in order) that lie in `range`, whose ends are token +/// boundaries: the tokens of that slice of the text, at their places in the whole. +fn tokens_in<'a>(tokens: &'a [Token], range: &Range) -> &'a [Token] { + let start = tokens.partition_point(|token| token.range.start < range.start); + let end = tokens.partition_point(|token| token.range.end <= range.end); + &tokens[start..end.max(start)] +} + /// A problem a declaration's own check found, moved from that check's text (`own`: the -/// settings prefix, `prefix` bytes, then the declaration) to where the declaration is in -/// `text`. One in the prefix is the whole text's check's to report. A problem at the end of -/// the declaration is placed on its last word, not on the declaration after it. +/// settings prefix, `prefix` bytes, then the declaration, with its tokens and outline) to +/// where the declaration is in `text` (`whole`: with its tokens, and its lines as found so +/// far). One in the prefix is the whole text's check's to report. A problem at the end of the +/// declaration is placed on its last word, not on the declaration after it. fn placed( mut diagnostic: Diagnostic, - text: &str, - own: (&str, &Outline), + whole: (&str, &[Token], &mut Lines), + own: (&str, &[Token], &Outline), prefix: usize, declaration: &Range, ) -> Option<(Diagnostic, Option>)> { + let (text, tokens, lines) = whole; // Its last word: an unclosed declaration runs on to the next, over blank lines and any // comment above it. - let end = gq_language::tokenize(&text[declaration.clone()]) + let end = tokens_in(tokens, declaration) .iter() .rev() .find(|token| token.kind != gq_language::TokenKind::Comment) - .map_or(declaration.start, |token| { - declaration.start + token.range.end - }); + .map_or(declaration.start, |token| token.range.end); let moved = |at: usize| (at >= prefix).then(|| (declaration.start + at - prefix).min(end)); let at = match diagnostic.byte { Some(byte) => { let at = moved(byte as usize)?; - let before = &text[..at]; + let (line, column) = lines.of(text, at); diagnostic.byte = Some(at as u32); - diagnostic.line = Some(before.matches('\n').count() as u32 + 1); - diagnostic.column = Some( - before - .rsplit('\n') - .next() - .map_or(0, |line| line.chars().count()) as u32 - + 1, - ); + diagnostic.line = Some(line); + diagnostic.column = Some(column); Some(at) } - None => match locate(own.0, own.1, &diagnostic) { + None => match locate(own.0, own.1, own.2, &diagnostic) { Some(at) => Some(moved(at)?), None => None, }, @@ -196,18 +221,25 @@ fn placed( .map(|(range, with)| Some((moved(range.start)?..moved(range.end)?, with))) .collect::>>() .unwrap_or_default(); - let range = at.map(|at| gq_language::problem_range(&text[..end], at)); + // Underlined within the text up to its last word, whose tokens are the whole text's up to + // there: `end` is a token's end. + let range = + at.map(|at| gq_language::problem_range(&text[..end], tokens_in(tokens, &(0..end)), at)); Some((diagnostic, range)) } /// Where a type error without a position probably is: the last name the message quotes /// (`` type `Person` has no property `nmae` ``) that appears as a token in the declaration the -/// error names. Only an underline hint; the message stays the authority. -fn locate(text: &str, outline: &Outline, diagnostic: &Diagnostic) -> Option { +/// error names (`text`'s, with its tokens and outline). Only an underline hint; the message +/// stays the authority. +fn locate( + text: &str, + tokens: &[Token], + outline: &Outline, + diagnostic: &Diagnostic, +) -> Option { let name = diagnostic.query.as_deref()?; let declaration = outline.declarations.iter().find(|d| d.name == name)?; - let body = &text[declaration.range.clone()]; - let tokens = gq_language::tokenize(body); let quoted: Vec<&str> = diagnostic .message .split('`') @@ -215,11 +247,12 @@ fn locate(text: &str, outline: &Outline, diagnostic: &Diagnostic) -> Option bool { /// The words of the `branch diff` statement `source` holds after its settings prefix, or /// `None` when it holds another. fn diff_tokens(source: &str) -> Option> { - let prefix = gq_language::outline(source).prefix.end; - let tokens: Vec = gq_language::tokenize(source) + let tokens = gq_language::tokenize(source); + let prefix = gq_language::outline_of(source, &tokens).prefix.end; + let tokens: Vec = tokens .into_iter() .filter(|t| t.range.start >= prefix && t.kind != TokenKind::Comment) .collect();