diff --git a/Documentation/Queries.md b/Documentation/Queries.md index 341daf4..a267bf3 100644 --- a/Documentation/Queries.md +++ b/Documentation/Queries.md @@ -20,8 +20,26 @@ interactively (**t**/**T**/**Alt-t**). ## Filtering -**/** filters any query view with a `LIKE` pattern (matched against query, -user, query_id, ...); **-** shows everything again: +**/** filters any query view; the list narrows as you type (the rows already +loaded), and the server is asked in the background for more rows matching the +filter. Free text is a `LIKE` against query, user, query_id, database, host, +log_comment, ... (`%text%` unless it has a `%`), and `fieldvalue` +predicates can be combined (all must match), e.g. + +``` +user=default elapsed>10s mem>1G +q~insert kind=Insert host=ch1 +cancelled=1 exception~Timeout +``` + +Fields: `user`, `initial_user`, `host`, `db`, `query_id`, `initial_query_id`, +`hash`, `query` (`q`), `log_comment`, `exception`, `elapsed` (`500ms`, `10s`, +`2m`), `mem` (`100M`, `2G`), `cpu` (percent), `thr`, `cancelled`, `initial`, +`kind`. Operators: `=`, `!=`, `~` (LIKE), `!~`, `>`, `>=`, `<`, `<=`; quote +values with spaces. **Tab**/**Shift-Tab** cycle through the completions of the +field name, or of the value from the rows on screen (users, databases, hosts, +...); the hint line above the prompt lists the candidates. **-** clears the filter and +shows everything again: ![filter](images/filter.png) diff --git a/src/interpreter/clickhouse.rs b/src/interpreter/clickhouse.rs index a996e2f..1bfeffe 100644 --- a/src/interpreter/clickhouse.rs +++ b/src/interpreter/clickhouse.rs @@ -1,3 +1,4 @@ +use crate::interpreter::queries_filter::{Filter, FilterColumns}; use crate::{ common::RelativeDateTime, interpreter::{ @@ -125,8 +126,8 @@ pub enum TraceType { /// Filters of the queries views (processes/slow_query_log/last_query_log). #[derive(Debug, Clone, Default)] pub struct QueriesFilter { - /// The '/'-prompt LIKE pattern (matched against query, user, query_id, ...). - pub like: String, + /// The '/'-prompt filter (free text and predicates, see queries_filter). + pub filter: Filter, /// `query_kind IN (...)` restriction; empty = all kinds. pub query_kind: Vec, } @@ -571,6 +572,34 @@ impl ClickHouse { self.trace_type_cast_expr.is_some() } + /// Column expressions of the queries filter over system.query_log. + fn query_log_filter_columns<'a>(&self, peak_threads_usage: &'a str) -> FilterColumns<'a> { + FilterColumns { + host: self.get_log_hostname_column(), + elapsed: "(query_duration_ms / 1e3)", + memory: "memory_usage", + threads: peak_threads_usage, + cpu: "(ProfileEvents['OSCPUVirtualTimeMicroseconds'] / 1e6 / greatest(query_duration_ms / 1e3, 1e-9) * 100)", + hash: "toString(normalized_query_hash)", + log_comment: "log_comment", + exception: "exception", + cancelled: "(exception_code = 394)", + kind: Some("query_kind"), + text: &[ + "client_hostname", + "log_comment", + "os_user", + "user", + "initial_user", + "client_name", + "query_id", + "query", + "current_database", + "toString(normalized_query_hash)", + ], + } + } + pub async fn get_slow_query_log( &self, filter: &QueriesFilter, @@ -581,18 +610,6 @@ impl ClickHouse { ) -> Result { let dbtable = self.get_log_table_name("query_log"); let host_filter = self.get_log_host_filter_clause(selected_host); - let mut filter_clause = if !filter.like.is_empty() { - format!( - "AND (client_hostname LIKE '{0}' OR log_comment LIKE '{0}' OR os_user LIKE '{0}' OR user LIKE '{0}' OR initial_user LIKE '{0}' OR client_name LIKE '{0}' OR query_id LIKE '{0}' OR query LIKE '{0}' OR current_database LIKE '{0}' OR toString(normalized_query_hash) LIKE '{0}')", - &filter.like - ) - } else { - "".to_string() - }; - // The condition sits in the initial_query_id selection, so it is the - // *initial* query's kind: the whole query group (subqueries included) - // is still pulled by the outer query. - filter_clause.push_str(&filter.query_kind_clause()); let peak_threads_usage = if self .quirks .has(ClickHouseAvailableQuirks::QueryLogPeakThreadsUsage) @@ -601,6 +618,13 @@ impl ClickHouse { } else { "length(thread_ids)" }; + // The conditions sit in the initial_query_id selection, so they apply + // to the *initial* query (its kind, user, ...): the whole query group + // (subqueries included) is still pulled by the outer query. + let mut filter_clause = filter + .filter + .to_sql(&self.query_log_filter_columns(peak_threads_usage)); + filter_clause.push_str(&filter.query_kind_clause()); let internal = self.get_internal_filter_clause(); let (id_condition, ids_cte) = if self @@ -741,16 +765,6 @@ impl ClickHouse { // - distributed_group_by_no_merge=2 is broken for this query with WINDOW function let dbtable = self.get_log_table_name("query_log"); let host_filter = self.get_log_host_filter_clause(selected_host); - let mut filter_clause = if !filter.like.is_empty() { - format!( - "AND (client_hostname LIKE '{0}' OR log_comment LIKE '{0}' OR os_user LIKE '{0}' OR user LIKE '{0}' OR initial_user LIKE '{0}' OR client_name LIKE '{0}' OR query_id LIKE '{0}' OR query LIKE '{0}' OR current_database LIKE '{0}' OR toString(normalized_query_hash) LIKE '{0}')", - &filter.like - ) - } else { - "".to_string() - }; - // The initial query's kind (see get_slow_query_log). - filter_clause.push_str(&filter.query_kind_clause()); let peak_threads_usage = if self .quirks .has(ClickHouseAvailableQuirks::QueryLogPeakThreadsUsage) @@ -759,6 +773,11 @@ impl ClickHouse { } else { "length(thread_ids)" }; + // The initial query's conditions (see get_slow_query_log). + let mut filter_clause = filter + .filter + .to_sql(&self.query_log_filter_columns(peak_threads_usage)); + filter_clause.push_str(&filter.query_kind_clause()); let internal = self.get_internal_filter_clause(); let (id_condition, ids_cte) = if self @@ -911,11 +930,52 @@ impl ClickHouse { limit: u64, ) -> Result { let dbtable = self.get_live_table_name("processes"); - // system.processes has query_kind only since 23.2 - let query_kind_clause = if self + let has_query_kind = self .quirks - .has(ClickHouseAvailableQuirks::ProcessesQueryKind) + .has(ClickHouseAvailableQuirks::ProcessesQueryKind); + let elapsed = if self.quirks.has(ClickHouseAvailableQuirks::ProcessesElapsed) { + "(elapsed / 10)" + } else { + "elapsed" + }; + let peak_threads_usage = if self + .quirks + .has(ClickHouseAvailableQuirks::ProcessesPeakThreadsUsage) { + "peak_threads_usage" + } else { + "length(thread_ids)" + }; + let cpu = format!( + "(ProfileEvents['OSCPUVirtualTimeMicroseconds'] / 1e6 / greatest({}, 1e-9) * 100)", + elapsed + ); + let filter_clause = filter.filter.to_sql(&FilterColumns { + host: "hostName()", + elapsed, + memory: "memory_usage", + threads: peak_threads_usage, + cpu: &cpu, + hash: "toString(normalizedQueryHash(query))", + log_comment: "Settings['log_comment']", + exception: "''", + cancelled: "is_cancelled", + kind: has_query_kind.then_some("query_kind"), + text: &[ + "client_hostname", + "Settings['log_comment']", + "os_user", + "user", + "initial_user", + "client_name", + "query_id", + "query", + "current_database", + "toString(normalizedQueryHash(query))", + ], + }); + // system.processes has query_kind only since 23.2 + let query_kind_clause = if has_query_kind { filter.query_kind_clause() } else { if !filter.query_kind.is_empty() { @@ -970,7 +1030,10 @@ impl ClickHouse { } else { 1 }, - current_database = if self.quirks.has(ClickHouseAvailableQuirks::ProcessesCurrentDatabase) { + current_database = if self + .quirks + .has(ClickHouseAvailableQuirks::ProcessesCurrentDatabase) + { // This is required for EXPLAIN (available since 20.6), // so EXPLAIN with non-default current_database will be broken from processes view. "'default'" @@ -978,16 +1041,8 @@ impl ClickHouse { "current_database" }, internal = self.get_internal_filter_clause(), - filter = if !filter.like.is_empty() { - format!("AND (client_hostname LIKE '{0}' OR Settings['log_comment'] LIKE '{0}' OR os_user LIKE '{0}' OR user LIKE '{0}' OR initial_user LIKE '{0}' OR client_name LIKE '{0}' OR query_id LIKE '{0}' OR query LIKE '{0}' OR current_database LIKE '{0}' OR toString(normalizedQueryHash(query)) LIKE '{0}')", &filter.like) - } else { - "".to_string() - }, - peak_threads_usage = if self.quirks.has(ClickHouseAvailableQuirks::ProcessesPeakThreadsUsage) { - "peak_threads_usage" - } else { - "length(thread_ids)" - }, + filter = filter_clause, + peak_threads_usage = peak_threads_usage, id_filter = id_filter, host_filter = host_filter, ) diff --git a/src/interpreter/mod.rs b/src/interpreter/mod.rs index 576f838..3d3a4e1 100644 --- a/src/interpreter/mod.rs +++ b/src/interpreter/mod.rs @@ -10,6 +10,7 @@ mod worker; pub mod flamegraph; pub mod options; pub mod perfetto; +pub mod queries_filter; pub use clickhouse::ClickHouse; pub use clickhouse::TextLogArguments; diff --git a/src/interpreter/queries_filter.rs b/src/interpreter/queries_filter.rs new file mode 100644 index 0000000..3c7462e --- /dev/null +++ b/src/interpreter/queries_filter.rs @@ -0,0 +1,681 @@ +//! The queries views filter: free text and `fieldvalue` predicates. +//! +//! One filter string is evaluated twice: instantly against the queries already +//! loaded (matches()) and as SQL for the next server refresh (to_sql()), so +//! that the list narrows while typing and fills up with rows that were beyond +//! the LIMIT once the server answers. +//! +//! Syntax: whitespace separated tokens, quotes ('...' or "...") keep spaces. +//! A token with an operator (`=`, `!=`, `~`, `!~`, `>`, `>=`, `<`, `<=`) after +//! a known field name is a predicate, anything else is free text matched as +//! LIKE against the usual columns (`%...%` unless the text has a `%`). +//! `~` is LIKE too, `=` an exact match. Numbers take units: elapsed +//! `500ms 10s 2m 1h`, mem `100M 2G` (binary), cpu `50%`. + +use crate::interpreter::Query; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Field { + /// Free text (no field), matched against several columns + Text, + User, + InitialUser, + Host, + Database, + QueryId, + InitialQueryId, + Hash, + Query, + LogComment, + Exception, + Elapsed, + Memory, + Cpu, + Threads, + Cancelled, + Initial, + Kind, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Kind { + String, + Number, + Bool, +} + +impl Field { + /// Canonical name first, then aliases. + const NAMES: &'static [(Field, &'static [&'static str])] = &[ + (Field::User, &["user", "u"]), + (Field::InitialUser, &["initial_user", "iuser"]), + (Field::Host, &["host", "hostname"]), + (Field::Database, &["db", "database"]), + (Field::QueryId, &["query_id", "qid", "id"]), + (Field::InitialQueryId, &["initial_query_id", "iqid"]), + (Field::Hash, &["hash", "qhash", "normalized_query_hash"]), + (Field::Query, &["query", "q", "sql"]), + (Field::LogComment, &["log_comment", "comment"]), + (Field::Exception, &["exception", "error"]), + (Field::Elapsed, &["elapsed", "duration", "time"]), + (Field::Memory, &["mem", "memory"]), + (Field::Cpu, &["cpu"]), + (Field::Threads, &["thr", "threads"]), + (Field::Cancelled, &["cancelled", "killed"]), + (Field::Initial, &["initial", "is_initial"]), + (Field::Kind, &["kind", "query_kind"]), + ]; + + pub fn parse(name: &str) -> Option { + let name = name.to_ascii_lowercase(); + Self::NAMES + .iter() + .find(|(_, names)| names.contains(&name.as_str())) + .map(|(field, _)| *field) + } + + pub fn name(self) -> &'static str { + Self::NAMES + .iter() + .find(|(field, _)| *field == self) + .map(|(_, names)| names[0]) + .unwrap_or("") + } + + /// Canonical names of all fields. + pub fn all() -> impl Iterator { + Self::NAMES.iter().map(|(field, _)| *field) + } + + pub fn kind(self) -> Kind { + match self { + Field::Elapsed | Field::Memory | Field::Cpu | Field::Threads => Kind::Number, + Field::Cancelled | Field::Initial => Kind::Bool, + _ => Kind::String, + } + } + + /// Example values shown as a hint when there is nothing to complete from. + pub fn hint(self) -> &'static str { + match self { + Field::Elapsed => "10s, 500ms, 2m, 1h", + Field::Memory => "100M, 2G", + Field::Cpu => "50 (percent)", + Field::Threads => "8", + Field::Cancelled | Field::Initial => "1 or 0", + Field::Query | Field::Exception | Field::LogComment => "text (LIKE with ~ or %)", + _ => "", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Op { + Eq, + Ne, + Like, + NotLike, + Gt, + Ge, + Lt, + Le, +} + +impl Op { + /// Longest first, so that `!=`/`>=` win over `=`/`>`. + const ALL: &'static [(&'static str, Op)] = &[ + ("!=", Op::Ne), + ("!~", Op::NotLike), + (">=", Op::Ge), + ("<=", Op::Le), + ("=", Op::Eq), + ("~", Op::Like), + (">", Op::Gt), + ("<", Op::Lt), + ]; + + fn sql(self) -> &'static str { + match self { + Op::Eq => "=", + Op::Ne => "!=", + Op::Like => "LIKE", + Op::NotLike => "NOT LIKE", + Op::Gt => ">", + Op::Ge => ">=", + Op::Lt => "<", + Op::Le => "<=", + } + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Predicate { + pub field: Field, + pub op: Op, + pub value: String, + /// Parsed `value` for Number/Bool fields (None = not a number, never matches) + number: Option, +} + +#[derive(Debug, Clone, Default, PartialEq)] +pub struct Filter { + predicates: Vec, +} + +/// Splits on whitespace, keeping quoted parts ('...' or "...") together +/// (quotes are removed). +pub fn tokenize(text: &str) -> Vec { + let mut tokens = Vec::new(); + let mut current = String::new(); + let mut quote: Option = None; + let mut in_token = false; + for c in text.chars() { + match quote { + Some(q) if c == q => quote = None, + Some(_) => current.push(c), + // A quote opens only at the start of a token or of a value + // (`user='a b'`), an apostrophe inside a word is literal + None if (c == '\'' || c == '"') + && (current.is_empty() || current.ends_with(['=', '~', '>', '<'])) => + { + quote = Some(c); + in_token = true; + } + None if c.is_whitespace() => { + if in_token { + tokens.push(std::mem::take(&mut current)); + in_token = false; + } + } + None => { + current.push(c); + in_token = true; + } + } + } + if in_token { + tokens.push(current); + } + tokens +} + +/// `(field, op, value)` of a predicate token, None for free text. +pub fn split_predicate(token: &str) -> Option<(Field, Op, &str)> { + let op_pos = token.find(['=', '!', '~', '>', '<'])?; + let field = Field::parse(&token[..op_pos])?; + let rest = &token[op_pos..]; + let (op_str, op) = Op::ALL.iter().find(|(s, _)| rest.starts_with(s))?; + Some((field, *op, &rest[op_str.len()..])) +} + +fn parse_number(field: Field, value: &str) -> Option { + let value = value.trim(); + if field.kind() == Kind::Bool { + return match value.to_ascii_lowercase().as_str() { + "1" | "true" | "yes" | "y" => Some(1.), + "0" | "false" | "no" | "n" => Some(0.), + _ => None, + }; + } + let split = value + .find(|c: char| !(c.is_ascii_digit() || c == '.')) + .unwrap_or(value.len()); + let number: f64 = value[..split].parse().ok()?; + let unit = value[split..].trim().to_ascii_lowercase(); + let multiplier = match field { + Field::Elapsed => match unit.as_str() { + "" | "s" | "sec" => 1., + "ms" => 1e-3, + "us" => 1e-6, + "m" | "min" => 60., + "h" => 3600., + "d" => 86400., + _ => return None, + }, + Field::Memory => match unit.as_str() { + "" | "b" => 1., + "k" | "kb" | "kib" => 1024., + "m" | "mb" | "mib" => 1024f64.powi(2), + "g" | "gb" | "gib" => 1024f64.powi(3), + "t" | "tb" | "tib" => 1024f64.powi(4), + _ => return None, + }, + Field::Cpu => match unit.as_str() { + "" | "%" => 1., + _ => return None, + }, + _ => match unit.as_str() { + "" => 1., + "k" => 1e3, + _ => return None, + }, + }; + Some(number * multiplier) +} + +/// SQL LIKE (`%` any, `_` one char) over chars. +pub fn like_match(pattern: &str, text: &str) -> bool { + let p: Vec = pattern.chars().collect(); + let t: Vec = text.chars().collect(); + // dp over the pattern with the classic two-pointer + backtrack to the last % + let (mut pi, mut ti) = (0, 0); + let mut star: Option<(usize, usize)> = None; + while ti < t.len() { + if pi < p.len() && (p[pi] == '_' || p[pi] == t[ti]) { + pi += 1; + ti += 1; + } else if pi < p.len() && p[pi] == '%' { + star = Some((pi, ti)); + pi += 1; + } else if let Some((sp, st)) = star { + pi = sp + 1; + ti = st + 1; + star = Some((sp, st + 1)); + } else { + return false; + } + } + while pi < p.len() && p[pi] == '%' { + pi += 1; + } + pi == p.len() +} + +/// The LIKE pattern of a text value: as is with a `%`, `%text%` otherwise. +fn like_pattern(value: &str) -> String { + if value.contains('%') { + value.to_string() + } else { + format!("%{}%", value) + } +} + +fn sql_quote(value: &str) -> String { + format!("'{}'", value.replace('\\', "\\\\").replace('\'', "\\'")) +} + +/// Column expressions of the table the filter is translated for. +pub struct FilterColumns<'a> { + pub host: &'a str, + pub elapsed: &'a str, + pub memory: &'a str, + pub threads: &'a str, + pub cpu: &'a str, + pub hash: &'a str, + pub log_comment: &'a str, + pub exception: &'a str, + pub cancelled: &'a str, + /// None = the table has no query_kind (the predicate is skipped) + pub kind: Option<&'a str>, + /// Columns matched by the free text + pub text: &'a [&'a str], +} + +impl Predicate { + fn string_matches(&self, actual: &str) -> bool { + match self.op { + Op::Eq => actual == self.value, + Op::Ne => actual != self.value, + Op::Like => like_match(&like_pattern(&self.value), actual), + Op::NotLike => !like_match(&like_pattern(&self.value), actual), + Op::Gt => actual > self.value.as_str(), + Op::Ge => actual >= self.value.as_str(), + Op::Lt => actual < self.value.as_str(), + Op::Le => actual <= self.value.as_str(), + } + } + + fn number_matches(&self, actual: f64) -> bool { + let Some(expected) = self.number else { + return false; + }; + match self.op { + Op::Eq | Op::Like => actual == expected, + Op::Ne | Op::NotLike => actual != expected, + Op::Gt => actual > expected, + Op::Ge => actual >= expected, + Op::Lt => actual < expected, + Op::Le => actual <= expected, + } + } + + pub fn matches(&self, query: &Query) -> bool { + let log_comment = || { + query + .settings + .get("log_comment") + .map(String::as_str) + .unwrap_or("") + }; + match self.field { + Field::Text => { + let pattern = like_pattern(&self.value); + let hash = query.normalized_query_hash.to_string(); + [ + query.user.as_str(), + query.initial_user.as_str(), + query.query_id.as_str(), + query.original_query.as_str(), + query.current_database.as_str(), + query.host_name.as_str(), + log_comment(), + hash.as_str(), + ] + .iter() + .any(|s| like_match(&pattern, s)) + } + Field::User => self.string_matches(&query.user), + Field::InitialUser => self.string_matches(&query.initial_user), + Field::Host => self.string_matches(&query.host_name), + Field::Database => self.string_matches(&query.current_database), + Field::QueryId => self.string_matches(&query.query_id), + Field::InitialQueryId => self.string_matches(&query.initial_query_id), + Field::Hash => self.string_matches(&query.normalized_query_hash.to_string()), + Field::Query => self.string_matches(&query.original_query), + Field::LogComment => self.string_matches(log_comment()), + Field::Exception => self.string_matches(&query.exception), + Field::Elapsed => self.number_matches(query.elapsed), + Field::Memory => self.number_matches(query.memory as f64), + Field::Cpu => self.number_matches(query.cpu()), + Field::Threads => self.number_matches(query.threads as f64), + Field::Cancelled => self.number_matches(query.is_cancelled as u8 as f64), + Field::Initial => self.number_matches(query.is_initial_query as u8 as f64), + // Not available in the rows, the server filters by it + Field::Kind => true, + } + } + + /// The SQL condition, None when the table cannot evaluate it. + fn to_sql(&self, columns: &FilterColumns<'_>) -> Option { + let string_condition = |column: &str| -> String { + let value = match self.op { + Op::Like | Op::NotLike => like_pattern(&self.value), + _ => self.value.clone(), + }; + format!("{} {} {}", column, self.op.sql(), sql_quote(&value)) + }; + let number_condition = |column: &str| -> Option { + let number = self.number?; + let op = match self.op { + Op::Like => "=", + Op::NotLike => "!=", + op => op.sql(), + }; + Some(format!("{} {} {}", column, op, number)) + }; + let condition = match self.field { + Field::Text => { + let pattern = sql_quote(&like_pattern(&self.value)); + columns + .text + .iter() + .map(|c| format!("{} LIKE {}", c, pattern)) + .collect::>() + .join(" OR ") + } + Field::User => string_condition("user"), + Field::InitialUser => string_condition("initial_user"), + Field::Host => string_condition(columns.host), + Field::Database => string_condition("current_database"), + Field::QueryId => string_condition("query_id"), + Field::InitialQueryId => string_condition("initial_query_id"), + Field::Hash => string_condition(columns.hash), + Field::Query => string_condition("query"), + Field::LogComment => string_condition(columns.log_comment), + Field::Exception => string_condition(columns.exception), + Field::Elapsed => number_condition(columns.elapsed)?, + Field::Memory => number_condition(columns.memory)?, + Field::Cpu => number_condition(columns.cpu)?, + Field::Threads => number_condition(columns.threads)?, + Field::Cancelled => number_condition(columns.cancelled)?, + Field::Initial => number_condition("is_initial_query")?, + Field::Kind => { + let column = columns.kind?; + let value = self.value.to_ascii_lowercase(); + match self.op { + Op::Ne | Op::NotLike => { + format!("lower({}) != {}", column, sql_quote(&value)) + } + _ => format!("lower({}) = {}", column, sql_quote(&value)), + } + } + }; + Some(format!("({})", condition)) + } +} + +impl Filter { + pub fn parse(text: &str) -> Filter { + let predicates = tokenize(text) + .iter() + .filter_map(|token| { + let (field, op, value) = match split_predicate(token) { + Some(p) => p, + None => (Field::Text, Op::Like, token.as_str()), + }; + // Incomplete while typing (`user=`), do not filter everything out + if value.is_empty() { + return None; + } + let number = match field.kind() { + Kind::String => None, + _ => parse_number(field, value), + }; + Some(Predicate { + field, + op, + value: value.to_string(), + number, + }) + }) + .collect(); + Filter { predicates } + } + + pub fn is_empty(&self) -> bool { + self.predicates.is_empty() + } + + pub fn matches(&self, query: &Query) -> bool { + self.predicates.iter().all(|p| p.matches(query)) + } + + /// ` AND (...) AND (...)` for the WHERE of `columns`' table (empty for an + /// empty filter). + pub fn to_sql(&self, columns: &FilterColumns<'_>) -> String { + self.predicates + .iter() + .filter_map(|p| p.to_sql(columns)) + .map(|c| format!(" AND {}", c)) + .collect() + } +} + +/// Completion of the token under the cursor (see the prompt): the candidates +/// replace the whole token. `values(field)` yields the values seen so far for +/// a string field, most frequent first. +pub fn suggest( + text: &str, + cursor: usize, + mut values: impl FnMut(Field) -> Vec, +) -> Vec { + let cursor = cursor.min(text.len()); + let before = &text[..cursor]; + let token_start = before + .rfind(char::is_whitespace) + .map(|i| i + 1) + .unwrap_or(0); + let token = &before[token_start..]; + + if let Some((field, op, value)) = split_predicate(token) { + let prefix = format!("{}{}", &token[..token.len() - value.len()], ""); + let _ = op; + return match field.kind() { + Kind::String => { + let value_lower = value.to_ascii_lowercase(); + values(field) + .into_iter() + .filter(|v| v.to_ascii_lowercase().starts_with(&value_lower)) + .map(|v| { + if v.contains(char::is_whitespace) { + format!("{}'{}'", prefix, v) + } else { + format!("{}{}", prefix, v) + } + }) + .collect() + } + _ => { + if value.is_empty() { + vec![format!("{}<{}>", prefix, field.hint())] + } else { + Vec::new() + } + } + }; + } + + // A field name (with its first operator); free text has no completion + let token_lower = token.to_ascii_lowercase(); + Field::all() + .filter(|f| f.name().starts_with(&token_lower)) + .map(|f| { + let op = if f.kind() == Kind::Number { ">" } else { "=" }; + format!("{}{}", f.name(), op) + }) + .collect() +} + +/// `text` with the token under `cursor` replaced by `completion`; returns the +/// new text and cursor. +pub fn complete(text: &str, cursor: usize, completion: &str) -> (String, usize) { + let cursor = cursor.min(text.len()); + let before = &text[..cursor]; + let token_start = before + .rfind(char::is_whitespace) + .map(|i| i + 1) + .unwrap_or(0); + let mut result = String::new(); + result.push_str(&text[..token_start]); + result.push_str(completion); + let new_cursor = result.len(); + result.push_str(&text[cursor..]); + (result, new_cursor) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_tokenize() { + assert_eq!(tokenize("a b"), vec!["a", "b"]); + assert_eq!(tokenize("user='a b' x"), vec!["user=a b", "x"]); + assert_eq!(tokenize("q~\"select 1\""), vec!["q~select 1"]); + assert_eq!(tokenize("it's q~it's"), vec!["it's", "q~it's"]); + assert_eq!(tokenize(""), Vec::::new()); + } + + #[test] + fn test_split_predicate() { + assert_eq!( + split_predicate("user!=default"), + Some((Field::User, Op::Ne, "default")) + ); + assert_eq!( + split_predicate("elapsed>=10s"), + Some((Field::Elapsed, Op::Ge, "10s")) + ); + assert_eq!(split_predicate("nosuch=1"), None); + assert_eq!(split_predicate("plain"), None); + assert_eq!(split_predicate("user="), Some((Field::User, Op::Eq, ""))); + } + + #[test] + fn test_numbers() { + assert_eq!(parse_number(Field::Elapsed, "500ms"), Some(0.5)); + assert_eq!(parse_number(Field::Elapsed, "2m"), Some(120.)); + assert_eq!( + parse_number(Field::Memory, "2G"), + Some(2. * 1024f64.powi(3)) + ); + assert_eq!(parse_number(Field::Cpu, "50%"), Some(50.)); + assert_eq!(parse_number(Field::Cancelled, "yes"), Some(1.)); + assert_eq!(parse_number(Field::Elapsed, "10x"), None); + } + + #[test] + fn test_like_match() { + assert!(like_match("%sel%", "SELECT select")); + assert!(like_match("a_c", "abc")); + assert!(!like_match("a_c", "abbc")); + assert!(like_match("%", "")); + assert!(like_match("abc%", "abcdef")); + assert!(!like_match("abc", "abcdef")); + } + + fn columns() -> FilterColumns<'static> { + FilterColumns { + host: "hostName()", + elapsed: "elapsed", + memory: "memory_usage", + threads: "length(thread_ids)", + cpu: "cpu_", + hash: "toString(normalizedQueryHash(query))", + log_comment: "Settings['log_comment']", + exception: "''", + cancelled: "is_cancelled", + kind: None, + text: &["user", "query"], + } + } + + #[test] + fn test_to_sql() { + let filter = Filter::parse("user=default elapsed>10s q~insert kind=select foo it's"); + assert_eq!( + filter.to_sql(&columns()), + " AND (user = 'default') AND (elapsed > 10) AND (query LIKE '%insert%') \ + AND (user LIKE '%foo%' OR query LIKE '%foo%') \ + AND (user LIKE '%it\\'s%' OR query LIKE '%it\\'s%')" + ); + // Incomplete predicate and empty filter + assert_eq!(Filter::parse("user=").to_sql(&columns()), ""); + assert!(Filter::parse(" ").is_empty()); + // A raw LIKE pattern stays as is + assert_eq!( + Filter::parse("it-proc-%").to_sql(&columns()), + " AND (user LIKE 'it-proc-%' OR query LIKE 'it-proc-%')" + ); + } + + #[test] + fn test_suggest() { + let values = |field: Field| match field { + Field::User => vec!["default".to_string(), "dev ops".to_string()], + _ => Vec::new(), + }; + assert_eq!(suggest("us", 2, values), vec!["user="]); + assert_eq!( + suggest("user=d", 6, values), + vec!["user=default", "user='dev ops'"] + ); + assert_eq!( + suggest("user=D", 6, values), + vec!["user=default", "user='dev ops'"] + ); + assert_eq!( + suggest("x elapsed>", 10, values), + vec!["elapsed><10s, 500ms, 2m, 1h>"] + ); + assert_eq!(suggest("plain", 5, values), Vec::::new()); + assert_eq!( + complete("x user=d", 8, "user=default"), + ("x user=default".to_string(), 14) + ); + assert_eq!( + complete("us more", 2, "user="), + ("user= more".to_string(), 5) + ); + } +} diff --git a/src/tui/dialog.rs b/src/tui/dialog.rs index 619c209..21f2596 100644 --- a/src/tui/dialog.rs +++ b/src/tui/dialog.rs @@ -190,6 +190,10 @@ impl Component for Dialog { self.focus = Focus::Button(0); EventResult::consumed() } + Event::Key(Key::BackTab) if !self.buttons.is_empty() => { + self.focus = Focus::Button(self.buttons.len() - 1); + EventResult::consumed() + } _ => EventResult::Ignored, } } @@ -213,6 +217,16 @@ impl Component for Dialog { } EventResult::consumed() } + Event::Key(Key::BackTab) => { + if i > 0 { + self.focus = Focus::Button(i - 1); + } else if self.content.take_focus() { + self.focus = Focus::Content; + } else { + self.focus = Focus::Button(self.buttons.len() - 1); + } + EventResult::consumed() + } Event::Key(Key::Up) if self.content.take_focus() => { self.focus = Focus::Content; EventResult::consumed() diff --git a/src/tui/event.rs b/src/tui/event.rs index 418f137..632bcc8 100644 --- a/src/tui/event.rs +++ b/src/tui/event.rs @@ -7,6 +7,8 @@ use super::app::App; pub enum Key { Enter, Tab, + /// Shift-Tab + BackTab, Backspace, Esc, Left, @@ -89,7 +91,8 @@ fn convert_key_code(code: crossterm::event::KeyCode) -> Option { use crossterm::event::KeyCode; Some(match code { KeyCode::Enter => Key::Enter, - KeyCode::Tab | KeyCode::BackTab => Key::Tab, + KeyCode::Tab => Key::Tab, + KeyCode::BackTab => Key::BackTab, KeyCode::Backspace => Key::Backspace, KeyCode::Esc => Key::Esc, KeyCode::Left => Key::Left, @@ -137,6 +140,10 @@ impl Event { }); } let k = convert_key_code(key.code)?; + // Terminals report Shift-Tab as BackTab with or without SHIFT + if k == Key::BackTab && !ctrl && !alt { + return Some(Event::Key(k)); + } Some(if ctrl { Event::Ctrl(k) } else if alt { diff --git a/src/tui/linear.rs b/src/tui/linear.rs index e64790b..54a65c0 100644 --- a/src/tui/linear.rs +++ b/src/tui/linear.rs @@ -235,6 +235,7 @@ impl Component for LinearLayout { (Orientation::Vertical, Event::Key(Key::Up)) => self.move_focus(false), (Orientation::Vertical, Event::Key(Key::Down)) => self.move_focus(true), (_, Event::Key(Key::Tab)) => self.move_focus(true), + (_, Event::Key(Key::BackTab)) => self.move_focus(false), _ => EventResult::Ignored, } } diff --git a/src/tui/prompt.rs b/src/tui/prompt.rs index 0efaa7a..0c44a24 100644 --- a/src/tui/prompt.rs +++ b/src/tui/prompt.rs @@ -4,9 +4,14 @@ use super::edit::EditView; use super::event::{EventResult, Key}; use super::linear::LinearLayout; use super::resize::Resizable; -use super::style::{Style, StyledString}; +use super::style::{Color, Modifier, Style, StyledString}; use super::text::TextView; use crate::interpreter::ContextArc; +use std::sync::{Arc, Mutex}; + +/// Completion callback of show_bottom_prompt_with_suggestions(): candidates +/// for the token under the cursor (each replaces that whole token). +pub type Suggest = Arc Vec + Send + Sync>; /// Wraps a form (dialog content) so that Enter anywhere inside it submits the /// form, instead of requiring to Tab to the submit button. Wrap only the @@ -26,6 +31,107 @@ where pub fn show_bottom_prompt(app: &mut App, prefix: &'static str, on_submit: F) where F: Fn(&mut App, &str) + Send + Sync + 'static, +{ + show_bottom_prompt_impl(app, prefix, String::new(), None, None, on_submit); +} + +/// show_bottom_prompt() pre-filled with `initial`, notifying `on_edit` on +/// every change and completing with Tab: `suggest` yields the candidates for +/// the token under the cursor, shown on a hint line above the prompt. +pub fn show_bottom_prompt_with_suggestions( + app: &mut App, + prefix: &'static str, + initial: String, + on_edit: E, + suggest: Suggest, + on_submit: F, +) where + E: Fn(&mut App, &str) + Send + Sync + 'static, + F: Fn(&mut App, &str) + Send + Sync + 'static, +{ + show_bottom_prompt_impl( + app, + prefix, + initial, + Some(Arc::new(on_edit)), + Some(suggest), + on_submit, + ); +} + +const HINT_VIEW: &str = "bottom_prompt_hint"; + +/// Tab completion state: the candidates for the token that was under the +/// cursor before the first Tab (frozen while cycling, since every accepted +/// candidate narrows what suggest() would return for the new text). +#[derive(Default)] +struct Completion { + candidates: Vec, + /// Text and cursor the candidates were computed for + base: (String, usize), + /// The candidate currently applied (cycling position) + index: Option, + /// Text produced by the last Tab, to recognize its own on_edit + applied: Option, +} + +impl Completion { + fn reset(&mut self, candidates: Vec, text: &str, cursor: usize) { + self.candidates = candidates; + self.base = (text.to_string(), cursor); + self.index = None; + self.applied = None; + } + + /// The next (or previous) candidate applied to the base text: (text, cursor). + fn step(&mut self, forward: bool) -> Option<(String, usize)> { + // Hints (<...>) are not completions + if self.candidates.is_empty() || self.candidates[0].starts_with('<') { + return None; + } + let len = self.candidates.len(); + let index = match (self.index, forward) { + (None, true) => 0, + (None, false) => len - 1, + (Some(i), true) => (i + 1) % len, + (Some(i), false) => (i + len - 1) % len, + }; + self.index = Some(index); + let (text, cursor) = crate::interpreter::queries_filter::complete( + &self.base.0, + self.base.1, + &self.candidates[index], + ); + self.applied = Some(text.clone()); + Some((text, cursor)) + } + + fn hint(&self) -> StyledString { + let mut hint = StyledString::new(); + for (i, candidate) in self.candidates.iter().enumerate() { + if i > 0 { + hint.append_plain(" "); + } + let style = if self.index == Some(i) { + Style::default().add_modifier(Modifier::REVERSED) + } else { + Style::default().fg(Color::DarkGray) + }; + hint.append_styled(candidate.clone(), style); + } + hint + } +} + +fn show_bottom_prompt_impl( + app: &mut App, + prefix: &'static str, + initial: String, + on_edit: Option>, + suggest: Option, + on_submit: F, +) where + F: Fn(&mut App, &str) + Send + Sync + 'static, { let search_history = app .user_data::() @@ -41,17 +147,70 @@ where let history_up = search_history.clone(); let history_down = search_history; - let edit_view = - EditView::new() - .style(Style::default()) - .on_submit(move |app: &mut App, text: &str| { - if let Some(history) = &history_submit { - history.add_entry(text.to_string()); + let completion = Arc::new(Mutex::new(Completion::default())); + if let Some(suggest) = &suggest { + let candidates = suggest(app, &initial, initial.len()); + completion + .lock() + .unwrap() + .reset(candidates, &initial, initial.len()); + } + let initial_hint = completion.lock().unwrap().hint(); + + let edit_completion = completion.clone(); + let edit_suggest = suggest.clone(); + let mut edit_view = EditView::new() + .content(initial) + .style(Style::default()) + .on_submit(move |app: &mut App, text: &str| { + if let Some(history) = &history_submit { + history.add_entry(text.to_string()); + } + on_submit(app, text); + }); + if on_edit.is_some() || suggest.is_some() { + edit_view = edit_view.on_edit(move |app: &mut App, text: &str, cursor: usize| { + if let Some(suggest) = &edit_suggest { + // Not for the text a Tab has just produced: the cycle goes on + let own = edit_completion.lock().unwrap().applied.as_deref() == Some(text); + if !own { + let candidates = suggest(app, text, cursor); + edit_completion + .lock() + .unwrap() + .reset(candidates, text, cursor); } - on_submit(app, text); - }); + let hint = edit_completion.lock().unwrap().hint(); + app.call_on_name(HINT_VIEW, |view: &mut TextView| { + view.set_content(hint); + }); + } + if let Some(on_edit) = &on_edit { + on_edit(app, text); + } + }); + } + // Tab/Shift-Tab cycle through the candidates + let tab_completion = completion.clone(); + let backtab_completion = completion; let edit_with_history = OnEventView::new(edit_view) + .on_pre_event_inner(Key::Tab, move |edit: &mut EditView, _event| { + let Some((text, cursor)) = tab_completion.lock().unwrap().step(true) else { + return Some(EventResult::consumed()); + }; + let cb = edit.set_content(text); + edit.set_cursor(cursor); + Some(EventResult::Consumed(Some(cb))) + }) + .on_pre_event_inner(Key::BackTab, move |edit: &mut EditView, _event| { + let Some((text, cursor)) = backtab_completion.lock().unwrap().step(false) else { + return Some(EventResult::consumed()); + }; + let cb = edit.set_content(text); + edit.set_cursor(cursor); + Some(EventResult::Consumed(Some(cb))) + }) .on_pre_event_inner(Key::Up, move |edit: &mut EditView, _event| { if let Some(history) = &history_up { let current = edit.get_content(); @@ -76,6 +235,17 @@ where .full_width() .fixed_height(1); - let y = app.screen_size().height.saturating_sub(1); - app.add_layer_at(0, y, filter_bar); + if suggest.is_some() { + let hint = TextView::new(initial_hint).no_wrap().with_name(HINT_VIEW); + let layer = LinearLayout::vertical() + .child(hint.full_width().fixed_height(1)) + .child(filter_bar) + .full_width() + .fixed_height(2); + let y = app.screen_size().height.saturating_sub(2); + app.add_layer_at(0, y, layer); + } else { + let y = app.screen_size().height.saturating_sub(1); + app.add_layer_at(0, y, filter_bar); + } } diff --git a/src/tui/views/flamelens_view.rs b/src/tui/views/flamelens_view.rs index dc6842d..b4bc77d 100644 --- a/src/tui/views/flamelens_view.rs +++ b/src/tui/views/flamelens_view.rs @@ -37,6 +37,7 @@ fn key_code(key: Key) -> Option { Some(match key { Key::Enter => KeyCode::Enter, Key::Tab => KeyCode::Tab, + Key::BackTab => KeyCode::BackTab, Key::Backspace => KeyCode::Backspace, Key::Esc => KeyCode::Esc, Key::Left => KeyCode::Left, diff --git a/src/tui/views/queries_view.rs b/src/tui/views/queries_view.rs index bd0d3dc..3551097 100644 --- a/src/tui/views/queries_view.rs +++ b/src/tui/views/queries_view.rs @@ -13,6 +13,7 @@ use std::mem::take; use std::sync::{Arc, Mutex}; use crate::common::RelativeDateTime; +use crate::interpreter::queries_filter::{self, Field as FilterField, Filter}; use crate::interpreter::{ BackgroundRunner, ContextArc, Query, TextLogArguments, WorkerEvent, clickhouse::{Columns, QueriesFilter, TraceType}, @@ -24,7 +25,7 @@ use crate::tui::dialog::Dialog; use crate::tui::event::{Event, EventResult}; use crate::tui::linear::LinearLayout; use crate::tui::navigation::Navigation; -use crate::tui::prompt::show_bottom_prompt; +use crate::tui::prompt::show_bottom_prompt_with_suggestions; use crate::tui::resize::{Resizable, SizeConstraint}; use crate::tui::scroll::Scrollable; use crate::tui::style::{Color, Modifier, Style, StyledString}; @@ -405,8 +406,11 @@ pub struct QueriesView { options: ViewOptions, // Is this running processes, or queries from system.query_log? is_system_processes: bool, - // Used to filter queries + // Used to filter queries: the text (shared with the settings dialog and + // the refresh callback, which translates it to SQL) and its parsed form + // applied to the loaded rows filter: Arc>, + parsed_filter: Filter, // Number of queries to render limit: Arc>, // Keep clipboard alive so X11 clipboard manager can persist the data @@ -483,6 +487,11 @@ impl QueriesView { continue; } } + // The server applies the same filter, this narrows the rows + // loaded before the filter changed (instantly while typing) + if !self.parsed_filter.matches(query) { + continue; + } items.push(query.clone()); } } @@ -978,10 +987,88 @@ impl QueriesView { fn action_show_all_queries(&mut self) -> Result> { self.query_id = None; - self.update_view(); + self.set_filter(""); Ok(Some(EventResult::consumed())) } + /// Applies the '/'-filter text: to the loaded rows right away, and to the + /// server side with a refresh (rows beyond the LIMIT that match). + pub fn set_filter(&mut self, text: &str) { + let parsed = Filter::parse(text); + let changed = { + let mut current = self.filter.lock().unwrap(); + let changed = *current != text; + *current = text.to_string(); + changed + }; + self.parsed_filter = parsed; + self.update_view(); + if changed { + log::info!("Set filter to '{}'", text); + self.bg_runner.schedule(); + } + } + + pub fn filter_text(&self) -> String { + self.filter.lock().unwrap().clone() + } + + /// Values of a string filter field among the loaded rows, most frequent + /// first (the completion candidates). + pub fn filter_values(&self, field: FilterField) -> Vec { + const KINDS: &[&str] = &[ + "Select", + "Insert", + "Create", + "Drop", + "Alter", + "Rename", + "System", + "Show", + "Describe", + "Explain", + "Use", + "Set", + "KillQuery", + "Optimize", + "Truncate", + "Detach", + "Attach", + "Grant", + "Revoke", + "Backup", + "Restore", + "Undrop", + ]; + if field == FilterField::Kind { + return KINDS.iter().map(|k| k.to_string()).collect(); + } + let mut counts = HashMap::::new(); + for query in self.items.values() { + let value = match field { + FilterField::User => query.user.clone(), + FilterField::InitialUser => query.initial_user.clone(), + FilterField::Host => query.host_name.clone(), + FilterField::Database => query.current_database.clone(), + FilterField::QueryId => query.query_id.clone(), + FilterField::InitialQueryId => query.initial_query_id.clone(), + FilterField::Hash => query.normalized_query_hash.to_string(), + FilterField::LogComment => query + .settings + .get("log_comment") + .cloned() + .unwrap_or_default(), + _ => return Vec::new(), + }; + if !value.is_empty() { + *counts.entry(value).or_default() += 1; + } + } + let mut values: Vec<(String, usize)> = counts.into_iter().collect(); + values.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + values.into_iter().map(|(v, _)| v).collect() + } + fn action_show_queries_on_shards(&mut self) -> Result> { let selected_query = self.get_selected_query()?; let query_id = selected_query.query_id.clone(); @@ -1558,6 +1645,7 @@ impl QueriesView { let is_system_processes = matches!(processes_type, Type::ProcessList); let filter = context.lock().unwrap().queries_filter(&view_name); + let parsed_filter = Filter::parse(&filter.lock().unwrap()); let limit = context.lock().unwrap().queries_limit.clone(); let event_owner = context.lock().unwrap().worker.event_owner(); @@ -1570,7 +1658,7 @@ impl QueriesView { let view_name = &update_callback_view_name; let mut context = update_callback_context.lock().unwrap(); let filter = QueriesFilter { - like: update_callback_filter.lock().unwrap().clone(), + filter: Filter::parse(&update_callback_filter.lock().unwrap()), query_kind: context.view_query_kind(view_name), }; let limit = context.view_limit(view_name, *update_callback_limit.lock().unwrap()); @@ -1729,6 +1817,7 @@ impl QueriesView { limit, clipboard: None, view_name: view_name.clone(), + parsed_filter, bg_runner, }; @@ -1765,22 +1854,37 @@ impl QueriesView { context.add_view_action(&mut event_view, view_name.clone(), "Filter", '/', move |_v| { let view_name = filter_view_name.clone(); return Ok(Some(EventResult::with_cb(move |app: &mut App| { - let view_name = view_name.clone(); - let filter_cb = move |app: &mut App, text: &str| { - app.call_on_name(&view_name, |v: &mut OnEventView| { - let v = v.get_inner_mut(); - log::info!("Set filter to '{}'", text); - *v.filter.lock().unwrap() = text.to_string(); - // Trigger update - // - // NOTE: It will require first summary view and only after - // processes view, and this may be slow in case of cluster mode, and - // should be addressed. - v.bg_runner.schedule(); + let initial = app + .call_on_name(&view_name, |v: &mut OnEventView| { + v.get_inner_mut().filter_text() + }) + .unwrap_or_default(); + let edit_view_name = view_name.clone(); + let on_edit = move |app: &mut App, text: &str| { + app.call_on_name(&edit_view_name, |v: &mut OnEventView| { + v.get_inner_mut().set_filter(text); + }); + }; + let suggest_view_name = view_name.clone(); + let suggest = Arc::new(move |app: &mut App, text: &str, cursor: usize| { + let view_name = suggest_view_name.clone(); + // The candidates come from the rows loaded so far + let values = |field| { + app.call_on_name(&view_name, |v: &mut OnEventView| { + v.get_inner_mut().filter_values(field) + }) + .unwrap_or_default() + }; + queries_filter::suggest(text, cursor, values) + }); + let submit_view_name = view_name.clone(); + let on_submit = move |app: &mut App, text: &str| { + app.call_on_name(&submit_view_name, |v: &mut OnEventView| { + v.get_inner_mut().set_filter(text); }); app.pop_layer(); }; - show_bottom_prompt(app, "/", filter_cb); + show_bottom_prompt_with_suggestions(app, "/", initial, on_edit, suggest, on_submit); }))); }); add_action!(context, &mut event_view, "Select", ' ', action_select); diff --git a/tests/integration.rs b/tests/integration.rs index d5591a0..b2a0a30 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -7,6 +7,7 @@ use chdig::interpreter::clickhouse::{ }; use chdig::interpreter::options::ClickHouseOptions; use chdig::interpreter::perfetto::PerfettoTraceBuilder; +use chdig::interpreter::queries_filter::Filter; use chdig::interpreter::{ClickHouse, ClickHouseQuirks, TextLogArguments}; use chrono::{DateTime, Local, TimeDelta}; use perfetto_protos::trace::Trace; @@ -21,7 +22,7 @@ use std::collections::HashMap; fn like(pattern: &str) -> QueriesFilter { QueriesFilter { - like: pattern.to_string(), + filter: Filter::parse(pattern), ..Default::default() } } @@ -167,7 +168,7 @@ async fn test_last_query_log_query_kind() { let (start, end) = window(); let kinds = |kinds: &[&str]| QueriesFilter { - like: "it-kind-%".to_string(), + filter: Filter::parse("it-kind-%"), query_kind: kinds.iter().map(|kind| kind.to_string()).collect(), };