diff --git a/CHANGELOG.md b/CHANGELOG.md index c98a10d24..0c48f668f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,113 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- **`sum by (detected_level)` over a JSON pipeline follows the level Loki's + ingest gave each line (#760).** Loki sets detected_level once per line + (`distributor/field_detection.go`): a level stream label or structured + metadata, else the first string value of a level key (`level`, `LEVEL`, + `Level`, `log.level`, `severity`, `lvl`, ...) at depth 0 or 1 in document + order, counted only when it is a known spelling (`info`, `INFO`, `inf` and + `information` are one level), else the first level word of a keyword scan of + the whole line, else `unknown`. The row evaluator (plain `| json`, + `__error__!=""`, every pushdown fallback) grouped the stream's spelling as + stored and read only a body `level` key, and the stats pushdown read only + that key as well: pino's numeric levels (`{"level":30}` was `30`, Loki scans + the line and says `unknown`, or `error` for a line with an `Error` object), + `notice`, a padded ` info`, `severity` or `Level` keys and a line without a + level key that mentions `error` each made a series of their own or were + `unknown`. The row evaluator now derives the level with the derivation the + log lines use, and a body `detected_level` key is renamed + `detected_level_extracted` as in Loki; a `detected_level` stream label keeps + its stored spelling. A filter on detected_level + (`| json | detected_level="info"`, Logs Drilldown's detected_level + breakdown) compares that value too, where it compared the stream's level or + the body key; the breakdown, which main answered from raw rows (2.4 s at 1h, + 502 at 6h and 24h), is served by the stats pushdown in 0.02 s to 0.30 s. + The stats pushdown first groups the buckets by copies of the stored + `detected_level` and `level` fields (taken before any unpack) and gives each + bucket the level Loki's ingest gives it, a known spelling in any case as its + level and any other word as stored: main's query, with no chain and no level + guard probe, for every window whose rows all carry a stored level. A filter on + detected_level runs in VictoriaLogs on the stored spellings; Loki's series + limit is applied to the answer's series, so `INFO`, `info` and `Information` + count as one, and a query whose VictoriaLogs groups pass twice the limit is + asked again with the chain, whose groups are its series, instead of failing + with a 400 Loki does not give. A + bucket without a stored level, or with a stored `detected_level` (a stream + label keeps its spelling), asks again with the pipes of the native metric + route, built from the level fields and body keys the window holds (a presence + probe: one more VictoriaLogs query of counts, over the whole window when it is + not remembered and on every request whose window reaches into the last five + minutes, because lines may still arrive there; a failed probe keeps the full + chain; the narrowed query also flags every row holding what the probe left + out, for a line that arrives late inside the remembered part, and is asked + again with the full chain), and probes the window for lines those pipes read + differently from Loki: two level keys, a level value with an escape (Loki + reads a JSON value's raw bytes), a line that starts with a brace but does not + end with one (logfmt for Loki), a level key VictoriaLogs does not unpack, a + `detected_level` stream label. One such line, and a line that only looks + like one (a body with both `level` and `severity`, as audit logs have), sends + the whole query to the raw-row evaluator: exact, but bounded by the row limit, + so a long range answers 502 where the pushdown answered 200. Each probe stops + at its first match, which bounds what it returns, not the work. On one hour + of the A/B harness's backfill (about 290,000 lines, a level stream label on + every row) with VictoriaLogs capped at 2 CPUs, warm p50, Explore's volume over + `| json` takes 98 ms where main takes 79 ms and a volume behind a + `detected_level` filter before the parser (which unpacks logfmt for every line + first) 279 ms where main takes 181 ms (the builds measured in the other order: + 76 ms against 101 ms and 245 ms against 259 ms), and 6h and 24h stay within + 8%. Rows with no stored level (the level only in the line) pay the chain's + own query, which reads every line; main answers 4 series there where Loki + answers 8. The chain parses each line once for the plan's labels and for the + level (one `unpack_json`, where the plan unpacked its labels itself and the + pipes the level keys again) and runs its keyword regexp only on lines that + hold one of the nine words as a whole word (VictoriaLogs' `i()` filter, a + superset of what the regexp matches), and the escape check of the presence + probe is a substring filter. On one hour of that data, VictoriaLogs alone + and capped at 2 CPUs, Explore's volume takes 105 ms where main takes 84 ms + (102 ms against 87 ms in the other order, 15 runs each) and was 282 ms + against 38 ms in the previous revision; at 6h and 24h it takes 0.69 s + against 0.54 s and 2.75 s against 2.39 s (was 1.5 s and 6.1 s). On the + VictoriaLogs side, per query: main's 102 ms and 225 CPU-ms, the previous + chain 290 ms and 550 CPU-ms, this one 107 ms and 260 CPU-ms. The first view + of a window (nothing remembered) still asks the stored-level query, the + level probes and the presence probe before the chain, about 0.5 s at 1h + where main takes 0.08 s. + With + `-detected-level-body-scan=false` the raw-row evaluator answers. A line that + is JSON for Loki (a brace first and last) is never read as logfmt by the + pipes either, and the severity VictoriaLogs derives for an OTLP record + without one (`Unspecified`, `Info2`, from v1.50 in `severity_text` with + `severity_number`, before in `severity`) is no level for them, as for Loki, + which reads the number or the line. A few lines stay different on the stats + pushdown and the native route, recorded in + `semantics/detected-level-pipes-known-differences` (open): a stream level + field outranked by a metadata one, a logfmt level key in another case, a + logfmt syntax error before the level key, and a row VictoriaLogs unpacked + from a JSON body. + The plain and `| logfmt` logs volume shapes + (`sum by (level, detected_level)`) take the same pipes, so a stream without a + level label now gets the level of its body (a `level=warn` key or a keyword) + as in Loki, where it was `unknown`. The native metric route for a plain + selector still reads level keys in the pipes' fixed order and overwrites a + `detected_level` stream label, with no raw-row fallback. + The verdicts of the probes of the JSON and logfmt pushdowns are remembered + per request scope (tenant, routing and forwarded identity, where the key held + the tenant only), and extending a remembered window no longer renews the + expiry of the part checked earlier, so a window kept alive by a refreshing + dashboard is checked again in full every five minutes (a refresh checks only + the part younger than five minutes, as before). + (`semantics/json-metric-detected-level-derivation`) +- **The in-memory cache's byte budget (`-cache-max-bytes`) counts the key of an + entry as well as its value (#760).** Entries keyed by a long query (the + window verdicts of the pushdown probes, whose key holds the whole base, 8 to + 14 KB) were counted as their one-byte value. An entry whose key and value + together pass a tenth of the budget is no longer stored, and a cache full of + long keys evicts sooner; the cache size metrics grow by the length of the + keys. + ## [2.15.1] - 2026-10-14 ### Fixed diff --git a/bench/ab/README.md b/bench/ab/README.md index 7da531f9d..464498b08 100644 --- a/bench/ab/README.md +++ b/bench/ab/README.md @@ -84,6 +84,20 @@ python3 bench/ab/report.py compare bench/ab/results/2026-09-23-pr611-control.jso It lists only the shapes that changed beyond noise and prints one summary line. +### Rows with no stored level + +The generator puts a `level` stream label on every row, so a run answers +`detected_level` from the stored fields. To measure the routes that read the +level from the line (JSON `level`, logfmt, plain text), seed a stack whose +rows have no stored level and run the same sets on it: + +```bash +python3 bench/ab/pr_smoke.py --base origin/main --no-level-label --out /tmp/ab-body-only +``` + +(`stack.py seed --no-level-label` does the same for a hand-run stack; the +generator option is `LOG_NO_LEVEL_LABEL=1`.) + ## Reading the results - Timings are warm p50 over the runs after the first; the first run is kept as diff --git a/bench/ab/results/2026-10-14-detlevel-1h.json b/bench/ab/results/2026-10-14-detlevel-1h.json new file mode 100644 index 000000000..9a2741cb9 --- /dev/null +++ b/bench/ab/results/2026-10-14-detlevel-1h.json @@ -0,0 +1,232 @@ +{ + "label": "detlevel-1h", + "date": "2026-10-14", + "set": "json-detected-level", + "description": "sum by (detected_level) over a JSON pipeline (row evaluator, and the stats pushdown with the level computed in VictoriaLogs and its guard probes).", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 5, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 5, + "faster": 1 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "DLV1 drilldown logs volume, json", + "range": "1h", + "p50": { + "base": 0.486, + "pr": 0.5, + "loki": 0.012 + }, + "p95": { + "base": 0.495, + "pr": 0.511, + "loki": 0.015 + }, + "cold": { + "base": 0.504, + "pr": 0.52, + "loki": 0.184 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 62.99, + "pr": 62.99, + "loki": 62.99 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.97 + }, + { + "shape": "DLV2 json error filter, detected_level", + "range": "1h", + "p50": { + "base": 2.079, + "pr": 2.228, + "loki": 0.025 + }, + "p95": { + "base": 2.124, + "pr": 2.244, + "loki": 0.034 + }, + "cold": { + "base": 2.059, + "pr": 2.312, + "loki": 0.372 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 34.19, + "pr": 37.49, + "loki": 30.93 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.93 + }, + { + "shape": "DLV3 plain json, detected_level", + "range": "1h", + "p50": { + "base": 0.508, + "pr": 0.53, + "loki": 0.008 + }, + "p95": { + "base": 0.549, + "pr": 0.549, + "loki": 0.009 + }, + "cold": { + "base": 0.474, + "pr": 0.526, + "loki": 0.087 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 30.89, + "pr": 23.32, + "loki": 23.32 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.96 + }, + { + "shape": "DLV4 explore json volume, level and detected_level", + "range": "1h", + "p50": { + "base": 0.036, + "pr": 0.048, + "loki": 0.005 + }, + "p95": { + "base": 0.048, + "pr": 0.063, + "loki": 0.01 + }, + "cold": { + "base": 0.027, + "pr": 0.024, + "loki": 0.106 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 24.12, + "pr": 24.12, + "loki": 24.12 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.75 + }, + { + "shape": "DLV5 json error dropped, detected_level", + "range": "1h", + "p50": { + "base": 0.055, + "pr": 0.008, + "loki": 0.021 + }, + "p95": { + "base": 0.059, + "pr": 0.011, + "loki": 0.021 + }, + "cold": { + "base": 0.07, + "pr": 0.007, + "loki": 0.424 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 28.32, + "pr": 28.32, + "loki": 28.32 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 6.88 + }, + { + "shape": "DLV6 drilldown detected_level breakdown, json", + "range": "1h", + "p50": { + "base": 2.29, + "pr": 0.009, + "loki": 0.017 + }, + "p95": { + "base": 2.336, + "pr": 0.011, + "loki": 0.019 + }, + "cold": { + "base": 2.265, + "pr": 0.009, + "loki": 0.047 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 36.32, + "pr": 29.07, + "loki": 29.07 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "faster", + "speedup": 254.44 + } + ] +} diff --git a/bench/ab/results/2026-10-14-detlevel-control-1h.json b/bench/ab/results/2026-10-14-detlevel-control-1h.json new file mode 100644 index 000000000..0763ca618 --- /dev/null +++ b/bench/ab/results/2026-10-14-detlevel-control-1h.json @@ -0,0 +1,707 @@ +{ + "label": "detlevel-control-1h", + "date": "2026-10-14", + "set": "control", + "description": "Shapes a change must not slow down or alter. Run on every performance-relevant PR.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 5, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 20 + }, + "result_differs_from_reference": 1, + "result_differs_preexisting": 1, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "logs plain", + "range": "1h", + "p50": { + "base": 0.024, + "pr": 0.019, + "loki": 0.028 + }, + "p95": { + "base": 0.025, + "pr": 0.022, + "loki": 0.034 + }, + "cold": { + "base": 0.028, + "pr": 0.018, + "loki": 0.039 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 13.43, + "pr": 13.43, + "loki": 13.43 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.26 + }, + { + "shape": "logs | json", + "range": "1h", + "p50": { + "base": 0.028, + "pr": 0.03, + "loki": 0.041 + }, + "p95": { + "base": 0.04, + "pr": 0.035, + "loki": 0.045 + }, + "cold": { + "base": 0.029, + "pr": 0.027, + "loki": 0.042 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 25.27, + "pr": 25.27, + "loki": 25.27 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.93 + }, + { + "shape": "logs | logfmt", + "range": "1h", + "p50": { + "base": 0.044, + "pr": 0.042, + "loki": 0.044 + }, + "p95": { + "base": 0.078, + "pr": 0.074, + "loki": 0.052 + }, + "cold": { + "base": 0.037, + "pr": 0.052, + "loki": 0.033 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 35.51, + "pr": 35.51, + "loki": 47.32 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.05 + }, + { + "shape": "json volume no filters", + "range": "1h", + "p50": { + "base": 0.043, + "pr": 0.105, + "loki": 0.021 + }, + "p95": { + "base": 0.064, + "pr": 0.114, + "loki": 0.026 + }, + "cold": { + "base": 0.049, + "pr": 0.093, + "loki": 0.451 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 54.99, + "pr": 54.99, + "loki": 54.99 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.41 + }, + { + "shape": "logfmt volume", + "range": "1h", + "p50": { + "base": 0.126, + "pr": 0.111, + "loki": 0.035 + }, + "p95": { + "base": 0.143, + "pr": 0.186, + "loki": 0.039 + }, + "cold": { + "base": 0.078, + "pr": 0.101, + "loki": 0.603 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 62.62, + "pr": 62.62, + "loki": 62.62 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.14 + }, + { + "shape": "plain volume", + "range": "1h", + "p50": { + "base": 0.009, + "pr": 0.008, + "loki": 0.02 + }, + "p95": { + "base": 0.017, + "pr": 0.011, + "loki": 0.03 + }, + "cold": { + "base": 0.007, + "pr": 0.008, + "loki": 0.346 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 56.16, + "pr": 56.16, + "loki": 56.16 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.12 + }, + { + "shape": "stream-label count by app", + "range": "1h", + "p50": { + "base": 0.013, + "pr": 0.012, + "loki": 0.023 + }, + "p95": { + "base": 0.017, + "pr": 0.015, + "loki": 0.024 + }, + "cold": { + "base": 0.013, + "pr": 0.011, + "loki": 0.325 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 9.85, + "pr": 9.85, + "loki": 9.85 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.08 + }, + { + "shape": "ungrouped rate", + "range": "1h", + "p50": { + "base": 0.005, + "pr": 0.007, + "loki": 0.04 + }, + "p95": { + "base": 0.006, + "pr": 0.013, + "loki": 0.052 + }, + "cold": { + "base": 0.007, + "pr": 0.006, + "loki": 0.335 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 8.56, + "pr": 8.56, + "loki": 8.56 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.71 + }, + { + "shape": "bytes_over_time by app", + "range": "1h", + "p50": { + "base": 0.026, + "pr": 0.025, + "loki": 0.044 + }, + "p95": { + "base": 0.027, + "pr": 0.028, + "loki": 0.049 + }, + "cold": { + "base": 0.033, + "pr": 0.025, + "loki": 0.345 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 22.12, + "pr": 22.12, + "loki": 22.12 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.04 + }, + { + "shape": "topk rate by app", + "range": "1h", + "p50": { + "base": 0.019, + "pr": 0.019, + "loki": 0.038 + }, + "p95": { + "base": 0.019, + "pr": 0.02, + "loki": 0.043 + }, + "cold": { + "base": 0.017, + "pr": 0.018, + "loki": 0.346 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 22.12, + "pr": 12.58, + "loki": 12.58 + }, + "parity_vs_reference": "differs", + "baseline_parity_vs_reference": "differs", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.0 + }, + { + "shape": "detected_level volume", + "range": "1h", + "p50": { + "base": 0.012, + "pr": 0.02, + "loki": 0.018 + }, + "p95": { + "base": 0.013, + "pr": 0.022, + "loki": 0.02 + }, + "cold": { + "base": 0.009, + "pr": 0.027, + "loki": 0.333 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 20.54, + "pr": 20.54, + "loki": 20.54 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.6 + }, + { + "shape": "by service_name", + "range": "1h", + "p50": { + "base": 0.021, + "pr": 0.016, + "loki": 0.025 + }, + "p95": { + "base": 0.024, + "pr": 0.023, + "loki": 0.03 + }, + "cold": { + "base": 0.024, + "pr": 0.025, + "loki": 0.333 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 20.1, + "pr": 20.1, + "loki": 20.1 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.31 + }, + { + "shape": "drilldown level volume", + "range": "1h", + "p50": { + "base": 0.007, + "pr": 0.007, + "loki": 0.013 + }, + "p95": { + "base": 0.009, + "pr": 0.009, + "loki": 0.019 + }, + "cold": { + "base": 0.014, + "pr": 0.007, + "loki": 0.32 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 9.03, + "pr": 9.03, + "loki": 8.47 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.0 + }, + { + "shape": "drilldown service landing", + "range": "1h", + "p50": { + "base": 0.018, + "pr": 0.014, + "loki": 0.017 + }, + "p95": { + "base": 0.022, + "pr": 0.017, + "loki": 0.02 + }, + "cold": { + "base": 0.019, + "pr": 0.015, + "loki": 0.342 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 13.89, + "pr": 13.89, + "loki": 13.89 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.29 + }, + { + "shape": "drilldown detected_level", + "range": "1h", + "p50": { + "base": 0.022, + "pr": 0.018, + "loki": 0.012 + }, + "p95": { + "base": 0.027, + "pr": 0.019, + "loki": 0.014 + }, + "cold": { + "base": 0.025, + "pr": 0.023, + "loki": 0.347 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 22.05, + "pr": 24.2, + "loki": 24.2 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.22 + }, + { + "shape": "json rate by level (no filter, no drop)", + "range": "1h", + "p50": { + "base": 0.002, + "pr": 0.002, + "loki": 0.06 + }, + "p95": { + "base": 0.005, + "pr": 0.005, + "loki": 0.061 + }, + "cold": { + "base": 0.002, + "pr": 0.004, + "loki": 0.064 + }, + "status": { + "base": "400", + "pr": "400", + "loki": "400" + }, + "vl_cpu_max": { + "base": 24.2, + "pr": 24.2, + "loki": 24.2 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "n/a", + "verdict": "same", + "speedup": null + }, + { + "shape": "logfmt count by level, filter, drop", + "range": "1h", + "p50": { + "base": 0.17, + "pr": 0.182, + "loki": 0.025 + }, + "p95": { + "base": 0.224, + "pr": 0.184, + "loki": 0.028 + }, + "cold": { + "base": 0.171, + "pr": 0.287, + "loki": 0.57 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 90.24, + "pr": 110.49, + "loki": 110.49 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.93 + }, + { + "shape": "instant ungrouped rate", + "range": "instant", + "p50": { + "base": 0.004, + "pr": 0.006, + "loki": 0.037 + }, + "p95": { + "base": 0.005, + "pr": 0.008, + "loki": 0.042 + }, + "cold": { + "base": 0.008, + "pr": 0.004, + "loki": 0.034 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 110.49, + "pr": 110.49, + "loki": 110.49 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.67 + }, + { + "shape": "instant json volume", + "range": "instant", + "p50": { + "base": 0.177, + "pr": 0.189, + "loki": 0.049 + }, + "p95": { + "base": 0.189, + "pr": 0.202, + "loki": 0.05 + }, + "cold": { + "base": 0.195, + "pr": 0.208, + "loki": 0.049 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 16.3, + "pr": 16.3, + "loki": 16.3 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.94 + }, + { + "shape": "instant json filtered", + "range": "instant", + "p50": { + "base": 0.157, + "pr": 0.161, + "loki": 0.054 + }, + "p95": { + "base": 0.171, + "pr": 0.17, + "loki": 0.055 + }, + "cold": { + "base": 0.157, + "pr": 0.163, + "loki": 0.042 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 17.14, + "pr": 17.14, + "loki": 17.14 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.98 + } + ] +} diff --git a/bench/ab/results/2026-10-14-detlevel-d-capped-1h.json b/bench/ab/results/2026-10-14-detlevel-d-capped-1h.json new file mode 100644 index 000000000..f25fbc7f3 --- /dev/null +++ b/bench/ab/results/2026-10-14-detlevel-d-capped-1h.json @@ -0,0 +1,61 @@ +{ + "label": "detlevel-d-capped-1h", + "date": "2026-10-14", + "set": "json-filter-pushdown", + "description": "| json / | logfmt range metrics with label filters on parsed fields (PR #611). Grafana Explore log volume and Logs Drilldown field breakdowns over a JSON stream.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 5, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 1 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "D volume, no filters", + "range": "1h", + "p50": { + "base": 0.079, + "pr": 0.098, + "loki": 0.029 + }, + "p95": { + "base": 0.105, + "pr": 0.106, + "loki": 0.06 + }, + "cold": { + "base": 0.064, + "pr": 0.096, + "loki": 0.909 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 65.81, + "pr": 65.81, + "loki": 65.81 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.81 + } + ] +} diff --git a/bench/ab/results/2026-10-14-detlevel-d-capped-long.json b/bench/ab/results/2026-10-14-detlevel-d-capped-long.json new file mode 100644 index 000000000..cc8963b9c --- /dev/null +++ b/bench/ab/results/2026-10-14-detlevel-d-capped-long.json @@ -0,0 +1,85 @@ +{ + "label": "detlevel-d-capped-long", + "date": "2026-10-14", + "set": "json-filter-pushdown", + "description": "| json / | logfmt range metrics with label filters on parsed fields (PR #611). Grafana Explore log volume and Logs Drilldown field breakdowns over a JSON stream.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 3, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 2 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "D volume, no filters", + "range": "6h", + "p50": { + "base": 0.574, + "pr": 0.608 + }, + "p95": { + "base": 0.575, + "pr": 0.617 + }, + "cold": { + "base": 0.934, + "pr": 0.611 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 186.12, + "pr": 186.12 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.94 + }, + { + "shape": "D volume, no filters", + "range": "24h", + "p50": { + "base": 2.372, + "pr": 2.402 + }, + "p95": { + "base": 2.373, + "pr": 2.404 + }, + "cold": { + "base": 2.583, + "pr": 2.41 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 203.76, + "pr": 220.03 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.99 + } + ] +} diff --git a/bench/ab/results/2026-10-14-detlevel-dl6-capped-1h.json b/bench/ab/results/2026-10-14-detlevel-dl6-capped-1h.json new file mode 100644 index 000000000..e52cefce6 --- /dev/null +++ b/bench/ab/results/2026-10-14-detlevel-dl6-capped-1h.json @@ -0,0 +1,61 @@ +{ + "label": "detlevel-dl6-capped-1h", + "date": "2026-10-14", + "set": "metric-latency-gap", + "description": "Shapes the metric latency work targets: the parse-risk probes of the | json stats pushdown (breakdowns and volumes by underscore labels), label filters before the parser (Logs Drilldown pages with a level filter selected) and Explore volumes with a level filter. Run with the json-filter-pushdown set.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 5, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 1 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "DL6 explore volume, level filter before json", + "range": "1h", + "p50": { + "base": 0.181, + "pr": 0.279, + "loki": 0.017 + }, + "p95": { + "base": 0.2, + "pr": 0.28, + "loki": 0.021 + }, + "cold": { + "base": 0.54, + "pr": 0.617, + "loki": 0.331 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 140.37, + "pr": 140.37, + "loki": 140.37 + }, + "parity_vs_reference": "same series, sum \u00b10.27%", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.65 + } + ] +} diff --git a/bench/ab/results/2026-10-14-detlevel-dl6-capped-long.json b/bench/ab/results/2026-10-14-detlevel-dl6-capped-long.json new file mode 100644 index 000000000..04ebb6675 --- /dev/null +++ b/bench/ab/results/2026-10-14-detlevel-dl6-capped-long.json @@ -0,0 +1,85 @@ +{ + "label": "detlevel-dl6-capped-long", + "date": "2026-10-14", + "set": "metric-latency-gap", + "description": "Shapes the metric latency work targets: the parse-risk probes of the | json stats pushdown (breakdowns and volumes by underscore labels), label filters before the parser (Logs Drilldown pages with a level filter selected) and Explore volumes with a level filter. Run with the json-filter-pushdown set.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 3, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 2 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "DL6 explore volume, level filter before json", + "range": "6h", + "p50": { + "base": 1.547, + "pr": 1.598 + }, + "p95": { + "base": 1.585, + "pr": 1.611 + }, + "cold": { + "base": 3.377, + "pr": 3.805 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 200.11, + "pr": 199.19 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.97 + }, + { + "shape": "DL6 explore volume, level filter before json", + "range": "24h", + "p50": { + "base": 6.377, + "pr": 6.904 + }, + "p95": { + "base": 6.381, + "pr": 6.908 + }, + "cold": { + "base": 11.992, + "pr": 12.493 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 206.93, + "pr": 208.81 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.92 + } + ] +} diff --git a/bench/ab/results/2026-10-14-detlevel-long.json b/bench/ab/results/2026-10-14-detlevel-long.json new file mode 100644 index 000000000..fe47dfbd1 --- /dev/null +++ b/bench/ab/results/2026-10-14-detlevel-long.json @@ -0,0 +1,377 @@ +{ + "label": "detlevel-long", + "date": "2026-10-14", + "set": "json-detected-level", + "description": "sum by (detected_level) over a JSON pipeline (row evaluator, and the stats pushdown with the level computed in VictoriaLogs and its guard probes).", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 3, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 8, + "faster": 2, + "fixed": 2 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "DLV1 drilldown logs volume, json", + "range": "6h", + "p50": { + "base": 2.943, + "pr": 3.159 + }, + "p95": { + "base": 2.957, + "pr": 3.172 + }, + "cold": { + "base": 3.086, + "pr": 3.266 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 80.38, + "pr": 55.04 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.93 + }, + { + "shape": "DLV1 drilldown logs volume, json", + "range": "24h", + "p50": { + "base": 9.034, + "pr": 9.633 + }, + "p95": { + "base": 9.043, + "pr": 9.634 + }, + "cold": { + "base": 9.096, + "pr": 9.675 + }, + "status": { + "base": "502", + "pr": "502" + }, + "vl_cpu_max": { + "base": 84.09, + "pr": 80.33 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "same", + "speedup": null + }, + { + "shape": "DLV2 json error filter, detected_level", + "range": "6h", + "p50": { + "base": 7.345, + "pr": 7.965 + }, + "p95": { + "base": 7.402, + "pr": 7.973 + }, + "cold": { + "base": 7.026, + "pr": 7.811 + }, + "status": { + "base": "502", + "pr": "502" + }, + "vl_cpu_max": { + "base": 59.34, + "pr": 39.91 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "same", + "speedup": null + }, + { + "shape": "DLV2 json error filter, detected_level", + "range": "24h", + "p50": { + "base": 7.479, + "pr": 7.987 + }, + "p95": { + "base": 7.565, + "pr": 8.025 + }, + "cold": { + "base": 7.395, + "pr": 7.927 + }, + "status": { + "base": "502", + "pr": "502" + }, + "vl_cpu_max": { + "base": 39.65, + "pr": 47.51 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "same", + "speedup": null + }, + { + "shape": "DLV3 plain json, detected_level", + "range": "6h", + "p50": { + "base": 2.957, + "pr": 3.212 + }, + "p95": { + "base": 2.966, + "pr": 3.262 + }, + "cold": { + "base": 3.072, + "pr": 3.234 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 39.1, + "pr": 39.74 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.92 + }, + { + "shape": "DLV3 plain json, detected_level", + "range": "24h", + "p50": { + "base": 9.167, + "pr": 9.57 + }, + "p95": { + "base": 9.198, + "pr": 9.604 + }, + "cold": { + "base": 9.036, + "pr": 9.774 + }, + "status": { + "base": "502", + "pr": "502" + }, + "vl_cpu_max": { + "base": 104.71, + "pr": 182.22 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "same", + "speedup": null + }, + { + "shape": "DLV4 explore json volume, level and detected_level", + "range": "6h", + "p50": { + "base": 0.143, + "pr": 0.106 + }, + "p95": { + "base": 0.188, + "pr": 0.116 + }, + "cold": { + "base": 0.195, + "pr": 0.187 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 90.22, + "pr": 90.22 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.35 + }, + { + "shape": "DLV4 explore json volume, level and detected_level", + "range": "24h", + "p50": { + "base": 0.689, + "pr": 0.7 + }, + "p95": { + "base": 0.695, + "pr": 0.705 + }, + "cold": { + "base": 0.78, + "pr": 0.804 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 155.72, + "pr": 157.03 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.98 + }, + { + "shape": "DLV5 json error dropped, detected_level", + "range": "6h", + "p50": { + "base": 0.503, + "pr": 0.028 + }, + "p95": { + "base": 0.541, + "pr": 0.031 + }, + "cold": { + "base": 0.65, + "pr": 0.027 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 114.94, + "pr": 114.94 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "faster", + "speedup": 17.96 + }, + { + "shape": "DLV5 json error dropped, detected_level", + "range": "24h", + "p50": { + "base": 2.407, + "pr": 0.174 + }, + "p95": { + "base": 2.422, + "pr": 0.179 + }, + "cold": { + "base": 2.362, + "pr": 0.185 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 199.81, + "pr": 199.1 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "faster", + "speedup": 13.83 + }, + { + "shape": "DLV6 drilldown detected_level breakdown, json", + "range": "6h", + "p50": { + "base": 7.787, + "pr": 0.025 + }, + "p95": { + "base": 7.865, + "pr": 0.026 + }, + "cold": { + "base": 7.947, + "pr": 0.027 + }, + "status": { + "base": "502", + "pr": "200" + }, + "vl_cpu_max": { + "base": 198.58, + "pr": 196.28 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "fixed", + "speedup": null + }, + { + "shape": "DLV6 drilldown detected_level breakdown, json", + "range": "24h", + "p50": { + "base": 7.697, + "pr": 0.178 + }, + "p95": { + "base": 7.723, + "pr": 0.214 + }, + "cold": { + "base": 7.85, + "pr": 0.142 + }, + "status": { + "base": "502", + "pr": "200" + }, + "vl_cpu_max": { + "base": 62.27, + "pr": 54.9 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "fixed", + "speedup": null + } + ] +} diff --git a/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-1h.json b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-1h.json new file mode 100644 index 000000000..b4ef36de2 --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-1h.json @@ -0,0 +1,232 @@ +{ + "label": "detlevel-bodyonly-1h", + "date": "2026-10-14", + "set": "json-detected-level", + "description": "sum by (detected_level) over a JSON pipeline (row evaluator, and the stats pushdown with the level computed in VictoriaLogs and its guard probes).", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 5, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 5, + "faster": 1 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "DLV1 drilldown logs volume, json", + "range": "1h", + "p50": { + "base": 0.483, + "pr": 0.507, + "loki": 0.014 + }, + "p95": { + "base": 0.495, + "pr": 0.519, + "loki": 0.017 + }, + "cold": { + "base": 0.503, + "pr": 0.521, + "loki": 0.14 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 65.57, + "pr": 65.57, + "loki": 65.57 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.95 + }, + { + "shape": "DLV2 json error filter, detected_level", + "range": "1h", + "p50": { + "base": 2.044, + "pr": 2.294, + "loki": 0.018 + }, + "p95": { + "base": 2.082, + "pr": 2.382, + "loki": 0.019 + }, + "cold": { + "base": 1.963, + "pr": 2.26, + "loki": 0.28 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 34.74, + "pr": 37.59, + "loki": 32.09 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "differs", + "parity_vs_baseline": "differs", + "verdict": "same", + "speedup": 0.89 + }, + { + "shape": "DLV3 plain json, detected_level", + "range": "1h", + "p50": { + "base": 0.498, + "pr": 0.543, + "loki": 0.008 + }, + "p95": { + "base": 0.509, + "pr": 0.56, + "loki": 0.011 + }, + "cold": { + "base": 0.491, + "pr": 0.574, + "loki": 0.071 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 31.29, + "pr": 24.0, + "loki": 24.0 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.92 + }, + { + "shape": "DLV4 explore json volume, level and detected_level", + "range": "1h", + "p50": { + "base": 0.036, + "pr": 0.04, + "loki": 0.007 + }, + "p95": { + "base": 0.038, + "pr": 0.05, + "loki": 0.01 + }, + "cold": { + "base": 0.021, + "pr": 0.027, + "loki": 0.071 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 22.58, + "pr": 22.22, + "loki": 22.22 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.9 + }, + { + "shape": "DLV5 json error dropped, detected_level", + "range": "1h", + "p50": { + "base": 0.068, + "pr": 0.088, + "loki": 0.021 + }, + "p95": { + "base": 0.1, + "pr": 0.103, + "loki": 0.028 + }, + "cold": { + "base": 0.037, + "pr": 0.307, + "loki": 0.29 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 52.61, + "pr": 52.61, + "loki": 52.61 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "differs", + "parity_vs_baseline": "differs", + "verdict": "same", + "speedup": 0.77 + }, + { + "shape": "DLV6 drilldown detected_level breakdown, json", + "range": "1h", + "p50": { + "base": 2.248, + "pr": 0.106, + "loki": 0.013 + }, + "p95": { + "base": 2.356, + "pr": 0.119, + "loki": 0.015 + }, + "cold": { + "base": 2.292, + "pr": 0.039, + "loki": 0.011 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 41.01, + "pr": 34.24, + "loki": 34.24 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "differs", + "parity_vs_baseline": "differs", + "verdict": "faster", + "speedup": 21.21 + } + ] +} diff --git a/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-d-1h.json b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-d-1h.json new file mode 100644 index 000000000..4a3149c4b --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-d-1h.json @@ -0,0 +1,61 @@ +{ + "label": "detlevel-bodyonly-d-1h", + "date": "2026-10-14", + "set": "json-filter-pushdown", + "description": "| json / | logfmt range metrics with label filters on parsed fields (PR #611). Grafana Explore log volume and Logs Drilldown field breakdowns over a JSON stream.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 5, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 1 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "D volume, no filters", + "range": "1h", + "p50": { + "base": 0.118, + "pr": 0.097, + "loki": 0.067 + }, + "p95": { + "base": 0.136, + "pr": 0.107, + "loki": 0.071 + }, + "cold": { + "base": 0.068, + "pr": 0.591, + "loki": 0.665 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 158.63, + "pr": 158.63, + "loki": 158.63 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "differs", + "parity_vs_baseline": "differs", + "verdict": "same", + "speedup": 1.22 + } + ] +} diff --git a/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-d-long.json b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-d-long.json new file mode 100644 index 000000000..60684d697 --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-d-long.json @@ -0,0 +1,85 @@ +{ + "label": "detlevel-bodyonly-d-long", + "date": "2026-10-14", + "set": "json-filter-pushdown", + "description": "| json / | logfmt range metrics with label filters on parsed fields (PR #611). Grafana Explore log volume and Logs Drilldown field breakdowns over a JSON stream.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 3, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 2 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "D volume, no filters", + "range": "6h", + "p50": { + "base": 0.541, + "pr": 0.692 + }, + "p95": { + "base": 0.589, + "pr": 0.706 + }, + "cold": { + "base": 0.868, + "pr": 2.709 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 198.96, + "pr": 198.96 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "differs", + "verdict": "same", + "speedup": 0.78 + }, + { + "shape": "D volume, no filters", + "range": "24h", + "p50": { + "base": 2.388, + "pr": 2.748 + }, + "p95": { + "base": 2.463, + "pr": 2.806 + }, + "cold": { + "base": 2.765, + "pr": 6.754 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 206.62, + "pr": 206.62 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "differs", + "verdict": "same", + "speedup": 0.87 + } + ] +} diff --git a/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-dl6-1h.json b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-dl6-1h.json new file mode 100644 index 000000000..f9d087e8f --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-dl6-1h.json @@ -0,0 +1,61 @@ +{ + "label": "detlevel-bodyonly-dl6-1h", + "date": "2026-10-14", + "set": "metric-latency-gap", + "description": "Shapes the metric latency work targets: the parse-risk probes of the | json stats pushdown (breakdowns and volumes by underscore labels), label filters before the parser (Logs Drilldown pages with a level filter selected) and Explore volumes with a level filter. Run with the json-filter-pushdown set.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 5, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 1 + }, + "result_differs_from_reference": 1, + "result_differs_preexisting": 1, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "DL6 explore volume, level filter before json", + "range": "1h", + "p50": { + "base": 0.196, + "pr": 0.296, + "loki": 0.019 + }, + "p95": { + "base": 0.264, + "pr": 0.357, + "loki": 0.021 + }, + "cold": { + "base": 0.62, + "pr": 0.502, + "loki": 0.496 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 117.64, + "pr": 117.64, + "loki": 117.64 + }, + "parity_vs_reference": "differs", + "baseline_parity_vs_reference": "differs", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.66 + } + ] +} diff --git a/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-dl6-long.json b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-dl6-long.json new file mode 100644 index 000000000..ad97e0ac6 --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-dl6-long.json @@ -0,0 +1,85 @@ +{ + "label": "detlevel-bodyonly-dl6-long", + "date": "2026-10-14", + "set": "metric-latency-gap", + "description": "Shapes the metric latency work targets: the parse-risk probes of the | json stats pushdown (breakdowns and volumes by underscore labels), label filters before the parser (Logs Drilldown pages with a level filter selected) and Explore volumes with a level filter. Run with the json-filter-pushdown set.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 3, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 2 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "DL6 explore volume, level filter before json", + "range": "6h", + "p50": { + "base": 1.445, + "pr": 1.589 + }, + "p95": { + "base": 1.484, + "pr": 1.664 + }, + "cold": { + "base": 3.281, + "pr": 3.373 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 199.01, + "pr": 199.44 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.91 + }, + { + "shape": "DL6 explore volume, level filter before json", + "range": "24h", + "p50": { + "base": 6.045, + "pr": 6.237 + }, + "p95": { + "base": 6.257, + "pr": 6.369 + }, + "cold": { + "base": 11.38, + "pr": 12.507 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 202.31, + "pr": 205.74 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.97 + } + ] +} diff --git a/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-long.json b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-long.json new file mode 100644 index 000000000..703104156 --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-long.json @@ -0,0 +1,376 @@ +{ + "label": "detlevel-bodyonly-long", + "date": "2026-10-14", + "set": "json-detected-level", + "description": "sum by (detected_level) over a JSON pipeline (row evaluator, and the stats pushdown with the level computed in VictoriaLogs and its guard probes).", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 3, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 10, + "fixed": 2 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "DLV1 drilldown logs volume, json", + "range": "6h", + "p50": { + "base": 2.923, + "pr": 3.149 + }, + "p95": { + "base": 2.931, + "pr": 3.217 + }, + "cold": { + "base": 2.975, + "pr": 3.134 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 66.91, + "pr": 38.28 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.93 + }, + { + "shape": "DLV1 drilldown logs volume, json", + "range": "24h", + "p50": { + "base": 9.602, + "pr": 10.348 + }, + "p95": { + "base": 9.655, + "pr": 10.423 + }, + "cold": { + "base": 9.407, + "pr": 10.162 + }, + "status": { + "base": "502", + "pr": "502" + }, + "vl_cpu_max": { + "base": 100.11, + "pr": 88.5 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "same", + "speedup": null + }, + { + "shape": "DLV2 json error filter, detected_level", + "range": "6h", + "p50": { + "base": 7.556, + "pr": 8.253 + }, + "p95": { + "base": 7.723, + "pr": 8.491 + }, + "cold": { + "base": 7.172, + "pr": 8.327 + }, + "status": { + "base": "502", + "pr": "502" + }, + "vl_cpu_max": { + "base": 49.05, + "pr": 49.05 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "same", + "speedup": null + }, + { + "shape": "DLV2 json error filter, detected_level", + "range": "24h", + "p50": { + "base": 7.442, + "pr": 8.147 + }, + "p95": { + "base": 7.525, + "pr": 8.285 + }, + "cold": { + "base": 7.938, + "pr": 8.293 + }, + "status": { + "base": "502", + "pr": "502" + }, + "vl_cpu_max": { + "base": 34.46, + "pr": 138.74 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "same", + "speedup": null + }, + { + "shape": "DLV3 plain json, detected_level", + "range": "6h", + "p50": { + "base": 2.917, + "pr": 3.155 + }, + "p95": { + "base": 2.92, + "pr": 3.197 + }, + "cold": { + "base": 2.929, + "pr": 3.202 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 87.43, + "pr": 52.04 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.92 + }, + { + "shape": "DLV3 plain json, detected_level", + "range": "24h", + "p50": { + "base": 9.195, + "pr": 9.773 + }, + "p95": { + "base": 9.286, + "pr": 9.777 + }, + "cold": { + "base": 9.139, + "pr": 9.959 + }, + "status": { + "base": "502", + "pr": "502" + }, + "vl_cpu_max": { + "base": 208.83, + "pr": 87.02 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "same", + "speedup": null + }, + { + "shape": "DLV4 explore json volume, level and detected_level", + "range": "6h", + "p50": { + "base": 0.15, + "pr": 0.144 + }, + "p95": { + "base": 0.194, + "pr": 0.167 + }, + "cold": { + "base": 0.188, + "pr": 0.217 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 71.54, + "pr": 88.56 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.04 + }, + { + "shape": "DLV4 explore json volume, level and detected_level", + "range": "24h", + "p50": { + "base": 0.691, + "pr": 0.734 + }, + "p95": { + "base": 0.694, + "pr": 0.779 + }, + "cold": { + "base": 0.87, + "pr": 0.794 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 155.17, + "pr": 155.17 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.94 + }, + { + "shape": "DLV5 json error dropped, detected_level", + "range": "6h", + "p50": { + "base": 0.585, + "pr": 0.641 + }, + "p95": { + "base": 0.596, + "pr": 0.682 + }, + "cold": { + "base": 0.693, + "pr": 1.902 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 156.26, + "pr": 156.26 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "differs", + "verdict": "same", + "speedup": 0.91 + }, + { + "shape": "DLV5 json error dropped, detected_level", + "range": "24h", + "p50": { + "base": 2.242, + "pr": 2.739 + }, + "p95": { + "base": 2.289, + "pr": 2.787 + }, + "cold": { + "base": 2.574, + "pr": 6.205 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 199.14, + "pr": 208.36 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "differs", + "verdict": "same", + "speedup": 0.82 + }, + { + "shape": "DLV6 drilldown detected_level breakdown, json", + "range": "6h", + "p50": { + "base": 7.493, + "pr": 0.666 + }, + "p95": { + "base": 7.521, + "pr": 0.687 + }, + "cold": { + "base": 7.47, + "pr": 0.618 + }, + "status": { + "base": "502", + "pr": "200" + }, + "vl_cpu_max": { + "base": 134.56, + "pr": 138.98 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "fixed", + "speedup": null + }, + { + "shape": "DLV6 drilldown detected_level breakdown, json", + "range": "24h", + "p50": { + "base": 7.514, + "pr": 2.612 + }, + "p95": { + "base": 7.52, + "pr": 2.66 + }, + "cold": { + "base": 7.416, + "pr": 2.666 + }, + "status": { + "base": "502", + "pr": "200" + }, + "vl_cpu_max": { + "base": 196.63, + "pr": 199.3 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "fixed", + "speedup": null + } + ] +} diff --git a/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-swapped-1h.json b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-swapped-1h.json new file mode 100644 index 000000000..3f2515731 --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-swapped-1h.json @@ -0,0 +1,232 @@ +{ + "label": "detlevel-bodyonly-swapped-1h", + "date": "2026-10-14", + "set": "json-detected-level", + "description": "sum by (detected_level) over a JSON pipeline (row evaluator, and the stats pushdown with the level computed in VictoriaLogs and its guard probes).", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 5, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 5, + "faster": 1 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "DLV1 drilldown logs volume, json", + "range": "1h", + "p50": { + "pr": 0.498, + "base": 0.472, + "loki": 0.014 + }, + "p95": { + "pr": 0.514, + "base": 0.484, + "loki": 0.02 + }, + "cold": { + "pr": 0.518, + "base": 0.474, + "loki": 0.161 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 58.95, + "base": 58.95, + "loki": 58.95 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.95 + }, + { + "shape": "DLV2 json error filter, detected_level", + "range": "1h", + "p50": { + "pr": 2.256, + "base": 1.971, + "loki": 0.018 + }, + "p95": { + "pr": 2.307, + "base": 2.002, + "loki": 0.027 + }, + "cold": { + "pr": 2.204, + "base": 1.95, + "loki": 0.64 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 32.48, + "base": 34.53, + "loki": 34.53 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "differs", + "parity_vs_baseline": "differs", + "verdict": "same", + "speedup": 0.87 + }, + { + "shape": "DLV3 plain json, detected_level", + "range": "1h", + "p50": { + "pr": 0.506, + "base": 0.482, + "loki": 0.009 + }, + "p95": { + "pr": 0.537, + "base": 0.501, + "loki": 0.009 + }, + "cold": { + "pr": 0.509, + "base": 0.479, + "loki": 0.085 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 34.18, + "base": 34.18, + "loki": 34.18 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.95 + }, + { + "shape": "DLV4 explore json volume, level and detected_level", + "range": "1h", + "p50": { + "pr": 0.041, + "base": 0.036, + "loki": 0.006 + }, + "p95": { + "pr": 0.045, + "base": 0.038, + "loki": 0.007 + }, + "cold": { + "pr": 0.026, + "base": 0.021, + "loki": 0.104 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 22.32, + "base": 22.32, + "loki": 22.32 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.88 + }, + { + "shape": "DLV5 json error dropped, detected_level", + "range": "1h", + "p50": { + "pr": 0.06, + "base": 0.091, + "loki": 0.019 + }, + "p95": { + "pr": 0.11, + "base": 0.1, + "loki": 0.027 + }, + "cold": { + "pr": 0.316, + "base": 0.088, + "loki": 0.367 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 80.81, + "base": 80.81, + "loki": 80.81 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "differs", + "parity_vs_baseline": "differs", + "verdict": "same", + "speedup": 1.52 + }, + { + "shape": "DLV6 drilldown detected_level breakdown, json", + "range": "1h", + "p50": { + "pr": 0.052, + "base": 2.138, + "loki": 0.013 + }, + "p95": { + "pr": 0.129, + "base": 2.179, + "loki": 0.018 + }, + "cold": { + "pr": 0.061, + "base": 2.137, + "loki": 0.018 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 30.38, + "base": 46.69, + "loki": 38.85 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "differs", + "parity_vs_baseline": "differs", + "verdict": "faster", + "speedup": 41.12 + } + ] +} diff --git a/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-swapped-d-1h.json b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-swapped-d-1h.json new file mode 100644 index 000000000..9cb339ebb --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-swapped-d-1h.json @@ -0,0 +1,61 @@ +{ + "label": "detlevel-bodyonly-swapped-d-1h", + "date": "2026-10-14", + "set": "json-filter-pushdown", + "description": "| json / | logfmt range metrics with label filters on parsed fields (PR #611). Grafana Explore log volume and Logs Drilldown field breakdowns over a JSON stream.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 5, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 1 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "D volume, no filters", + "range": "1h", + "p50": { + "pr": 0.169, + "base": 0.095, + "loki": 0.062 + }, + "p95": { + "pr": 0.21, + "base": 0.104, + "loki": 0.07 + }, + "cold": { + "pr": 0.458, + "base": 0.18, + "loki": 0.863 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 153.13, + "base": 153.13, + "loki": 153.13 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "differs", + "parity_vs_baseline": "differs", + "verdict": "same", + "speedup": 0.56 + } + ] +} diff --git a/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-swapped-d-long.json b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-swapped-d-long.json new file mode 100644 index 000000000..540fd9b51 --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-swapped-d-long.json @@ -0,0 +1,85 @@ +{ + "label": "detlevel-bodyonly-swapped-d-long", + "date": "2026-10-14", + "set": "json-filter-pushdown", + "description": "| json / | logfmt range metrics with label filters on parsed fields (PR #611). Grafana Explore log volume and Logs Drilldown field breakdowns over a JSON stream.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 3, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 2 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "D volume, no filters", + "range": "6h", + "p50": { + "pr": 0.702, + "base": 0.57 + }, + "p95": { + "pr": 0.706, + "base": 0.576 + }, + "cold": { + "pr": 2.931, + "base": 0.673 + }, + "status": { + "pr": "200", + "base": "200" + }, + "vl_cpu_max": { + "pr": 213.31, + "base": 185.06 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "differs", + "verdict": "same", + "speedup": 0.81 + }, + { + "shape": "D volume, no filters", + "range": "24h", + "p50": { + "pr": 2.842, + "base": 2.3 + }, + "p95": { + "pr": 2.882, + "base": 2.306 + }, + "cold": { + "pr": 6.918, + "base": 2.676 + }, + "status": { + "pr": "200", + "base": "200" + }, + "vl_cpu_max": { + "pr": 200.09, + "base": 200.3 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "differs", + "verdict": "same", + "speedup": 0.81 + } + ] +} diff --git a/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-swapped-dl6-1h.json b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-swapped-dl6-1h.json new file mode 100644 index 000000000..739b6c818 --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-swapped-dl6-1h.json @@ -0,0 +1,61 @@ +{ + "label": "detlevel-bodyonly-swapped-dl6-1h", + "date": "2026-10-14", + "set": "metric-latency-gap", + "description": "Shapes the metric latency work targets: the parse-risk probes of the | json stats pushdown (breakdowns and volumes by underscore labels), label filters before the parser (Logs Drilldown pages with a level filter selected) and Explore volumes with a level filter. Run with the json-filter-pushdown set.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 5, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 1 + }, + "result_differs_from_reference": 1, + "result_differs_preexisting": 1, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "DL6 explore volume, level filter before json", + "range": "1h", + "p50": { + "pr": 0.269, + "base": 0.216, + "loki": 0.072 + }, + "p95": { + "pr": 0.299, + "base": 0.272, + "loki": 0.086 + }, + "cold": { + "pr": 0.615, + "base": 0.582, + "loki": 0.638 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 146.85, + "base": 146.85, + "loki": 146.85 + }, + "parity_vs_reference": "differs", + "baseline_parity_vs_reference": "differs", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.8 + } + ] +} diff --git a/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-swapped-dl6-long.json b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-swapped-dl6-long.json new file mode 100644 index 000000000..ff5b3f658 --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-swapped-dl6-long.json @@ -0,0 +1,85 @@ +{ + "label": "detlevel-bodyonly-swapped-dl6-long", + "date": "2026-10-14", + "set": "metric-latency-gap", + "description": "Shapes the metric latency work targets: the parse-risk probes of the | json stats pushdown (breakdowns and volumes by underscore labels), label filters before the parser (Logs Drilldown pages with a level filter selected) and Explore volumes with a level filter. Run with the json-filter-pushdown set.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 3, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 2 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "DL6 explore volume, level filter before json", + "range": "6h", + "p50": { + "pr": 1.504, + "base": 1.44 + }, + "p95": { + "pr": 1.505, + "base": 1.482 + }, + "cold": { + "pr": 3.473, + "base": 3.293 + }, + "status": { + "pr": "200", + "base": "200" + }, + "vl_cpu_max": { + "pr": 200.65, + "base": 199.85 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.96 + }, + { + "shape": "DL6 explore volume, level filter before json", + "range": "24h", + "p50": { + "pr": 6.266, + "base": 6.002 + }, + "p95": { + "pr": 6.279, + "base": 6.081 + }, + "cold": { + "pr": 11.874, + "base": 11.452 + }, + "status": { + "pr": "200", + "base": "200" + }, + "vl_cpu_max": { + "pr": 219.89, + "base": 206.54 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.96 + } + ] +} diff --git a/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-swapped-long.json b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-swapped-long.json new file mode 100644 index 000000000..6381c38b1 --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-swapped-long.json @@ -0,0 +1,376 @@ +{ + "label": "detlevel-bodyonly-swapped-long", + "date": "2026-10-14", + "set": "json-detected-level", + "description": "sum by (detected_level) over a JSON pipeline (row evaluator, and the stats pushdown with the level computed in VictoriaLogs and its guard probes).", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 3, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 10, + "fixed": 2 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "DLV1 drilldown logs volume, json", + "range": "6h", + "p50": { + "pr": 3.213, + "base": 3.08 + }, + "p95": { + "pr": 3.276, + "base": 3.127 + }, + "cold": { + "pr": 3.146, + "base": 2.862 + }, + "status": { + "pr": "200", + "base": "200" + }, + "vl_cpu_max": { + "pr": 79.71, + "base": 54.34 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.96 + }, + { + "shape": "DLV1 drilldown logs volume, json", + "range": "24h", + "p50": { + "pr": 10.196, + "base": 9.513 + }, + "p95": { + "pr": 10.21, + "base": 9.518 + }, + "cold": { + "pr": 10.39, + "base": 9.521 + }, + "status": { + "pr": "502", + "base": "502" + }, + "vl_cpu_max": { + "pr": 100.16, + "base": 38.87 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "same", + "speedup": null + }, + { + "shape": "DLV2 json error filter, detected_level", + "range": "6h", + "p50": { + "pr": 8.04, + "base": 7.316 + }, + "p95": { + "pr": 8.058, + "base": 7.337 + }, + "cold": { + "pr": 8.212, + "base": 7.329 + }, + "status": { + "pr": "502", + "base": "502" + }, + "vl_cpu_max": { + "pr": 36.78, + "base": 46.06 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "same", + "speedup": null + }, + { + "shape": "DLV2 json error filter, detected_level", + "range": "24h", + "p50": { + "pr": 7.972, + "base": 7.189 + }, + "p95": { + "pr": 8.012, + "base": 7.211 + }, + "cold": { + "pr": 8.078, + "base": 7.195 + }, + "status": { + "pr": "502", + "base": "502" + }, + "vl_cpu_max": { + "pr": 38.06, + "base": 41.61 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "same", + "speedup": null + }, + { + "shape": "DLV3 plain json, detected_level", + "range": "6h", + "p50": { + "pr": 3.268, + "base": 3.067 + }, + "p95": { + "pr": 3.389, + "base": 3.084 + }, + "cold": { + "pr": 3.209, + "base": 2.929 + }, + "status": { + "pr": "200", + "base": "200" + }, + "vl_cpu_max": { + "pr": 201.75, + "base": 125.2 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.94 + }, + { + "shape": "DLV3 plain json, detected_level", + "range": "24h", + "p50": { + "pr": 9.976, + "base": 9.296 + }, + "p95": { + "pr": 10.152, + "base": 9.358 + }, + "cold": { + "pr": 10.023, + "base": 9.219 + }, + "status": { + "pr": "502", + "base": "502" + }, + "vl_cpu_max": { + "pr": 203.71, + "base": 81.62 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "same", + "speedup": null + }, + { + "shape": "DLV4 explore json volume, level and detected_level", + "range": "6h", + "p50": { + "pr": 0.157, + "base": 0.142 + }, + "p95": { + "pr": 0.196, + "base": 0.166 + }, + "cold": { + "pr": 0.146, + "base": 0.182 + }, + "status": { + "pr": "200", + "base": "200" + }, + "vl_cpu_max": { + "pr": 80.75, + "base": 80.75 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.9 + }, + { + "shape": "DLV4 explore json volume, level and detected_level", + "range": "24h", + "p50": { + "pr": 0.743, + "base": 0.699 + }, + "p95": { + "pr": 0.768, + "base": 0.712 + }, + "cold": { + "pr": 0.798, + "base": 0.767 + }, + "status": { + "pr": "200", + "base": "200" + }, + "vl_cpu_max": { + "pr": 160.52, + "base": 160.52 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.94 + }, + { + "shape": "DLV5 json error dropped, detected_level", + "range": "6h", + "p50": { + "pr": 0.698, + "base": 0.589 + }, + "p95": { + "pr": 0.704, + "base": 0.591 + }, + "cold": { + "pr": 2.005, + "base": 0.669 + }, + "status": { + "pr": "200", + "base": "200" + }, + "vl_cpu_max": { + "pr": 175.65, + "base": 159.04 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "differs", + "verdict": "same", + "speedup": 0.84 + }, + { + "shape": "DLV5 json error dropped, detected_level", + "range": "24h", + "p50": { + "pr": 2.785, + "base": 2.248 + }, + "p95": { + "pr": 2.801, + "base": 2.288 + }, + "cold": { + "pr": 6.488, + "base": 2.587 + }, + "status": { + "pr": "200", + "base": "200" + }, + "vl_cpu_max": { + "pr": 200.1, + "base": 200.26 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "differs", + "verdict": "same", + "speedup": 0.81 + }, + { + "shape": "DLV6 drilldown detected_level breakdown, json", + "range": "6h", + "p50": { + "pr": 0.711, + "base": 7.777 + }, + "p95": { + "pr": 0.724, + "base": 7.88 + }, + "cold": { + "pr": 0.698, + "base": 7.655 + }, + "status": { + "pr": "200", + "base": "502" + }, + "vl_cpu_max": { + "pr": 200.26, + "base": 146.93 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "fixed", + "speedup": null + }, + { + "shape": "DLV6 drilldown detected_level breakdown, json", + "range": "24h", + "p50": { + "pr": 2.841, + "base": 7.911 + }, + "p95": { + "pr": 2.887, + "base": 8.159 + }, + "cold": { + "pr": 3.171, + "base": 8.5 + }, + "status": { + "pr": "200", + "base": "502" + }, + "vl_cpu_max": { + "pr": 202.76, + "base": 199.68 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "fixed", + "speedup": null + } + ] +} diff --git a/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-vlonly-d-1h.json b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-vlonly-d-1h.json new file mode 100644 index 000000000..98da74793 --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-vlonly-d-1h.json @@ -0,0 +1,56 @@ +{ + "label": "detlevel-bodyonly-vlonly-d-1h", + "date": "2026-10-14", + "set": "json-filter-pushdown", + "description": "| json / | logfmt range metrics with label filters on parsed fields (PR #611). Grafana Explore log volume and Logs Drilldown field breakdowns over a JSON stream.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 15, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 1 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "D volume, no filters", + "range": "1h", + "p50": { + "base": 0.084, + "pr": 0.105 + }, + "p95": { + "base": 0.101, + "pr": 0.149 + }, + "cold": { + "base": 0.067, + "pr": 0.499 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 143.39, + "pr": 143.39 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "differs", + "verdict": "same", + "speedup": 0.8 + } + ] +} diff --git a/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-vlonly-dlv5-1h.json b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-vlonly-dlv5-1h.json new file mode 100644 index 000000000..0b65a96c9 --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-vlonly-dlv5-1h.json @@ -0,0 +1,56 @@ +{ + "label": "detlevel-bodyonly-vlonly-dlv5-1h", + "date": "2026-10-14", + "set": "json-detected-level", + "description": "sum by (detected_level) over a JSON pipeline (row evaluator, and the stats pushdown with the level computed in VictoriaLogs and its guard probes).", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 15, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 1 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "DLV5 json error dropped, detected_level", + "range": "1h", + "p50": { + "base": 0.088, + "pr": 0.099 + }, + "p95": { + "base": 0.103, + "pr": 0.141 + }, + "cold": { + "base": 0.069, + "pr": 0.218 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 103.77, + "pr": 103.77 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "differs", + "verdict": "same", + "speedup": 0.89 + } + ] +} diff --git a/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-vlonly-swapped-d-1h.json b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-vlonly-swapped-d-1h.json new file mode 100644 index 000000000..b15b7ec3c --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-vlonly-swapped-d-1h.json @@ -0,0 +1,56 @@ +{ + "label": "detlevel-bodyonly-vlonly-swapped-d-1h", + "date": "2026-10-14", + "set": "json-filter-pushdown", + "description": "| json / | logfmt range metrics with label filters on parsed fields (PR #611). Grafana Explore log volume and Logs Drilldown field breakdowns over a JSON stream.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 15, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 1 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "D volume, no filters", + "range": "1h", + "p50": { + "pr": 0.102, + "base": 0.087 + }, + "p95": { + "pr": 0.191, + "base": 0.103 + }, + "cold": { + "pr": 0.436, + "base": 0.081 + }, + "status": { + "pr": "200", + "base": "200" + }, + "vl_cpu_max": { + "pr": 132.67, + "base": 132.67 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "differs", + "verdict": "same", + "speedup": 0.85 + } + ] +} diff --git a/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-vlonly-swapped-dlv5-1h.json b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-vlonly-swapped-dlv5-1h.json new file mode 100644 index 000000000..c23fad26e --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-14-detlevel-bodyonly-vlonly-swapped-dlv5-1h.json @@ -0,0 +1,56 @@ +{ + "label": "detlevel-bodyonly-vlonly-swapped-dlv5-1h", + "date": "2026-10-14", + "set": "json-detected-level", + "description": "sum by (detected_level) over a JSON pipeline (row evaluator, and the stats pushdown with the level computed in VictoriaLogs and its guard probes).", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 15, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 1 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "DLV5 json error dropped, detected_level", + "range": "1h", + "p50": { + "pr": 0.101, + "base": 0.085 + }, + "p95": { + "pr": 0.111, + "base": 0.095 + }, + "cold": { + "pr": 0.196, + "base": 0.081 + }, + "status": { + "pr": "200", + "base": "200" + }, + "vl_cpu_max": { + "pr": 94.43, + "base": 94.43 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "differs", + "verdict": "same", + "speedup": 0.84 + } + ] +} diff --git a/bench/ab/results/oneoff/2026-10-14-detlevel-swapped-1h.json b/bench/ab/results/oneoff/2026-10-14-detlevel-swapped-1h.json new file mode 100644 index 000000000..42adc1344 --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-14-detlevel-swapped-1h.json @@ -0,0 +1,232 @@ +{ + "label": "detlevel-swapped-1h", + "date": "2026-10-14", + "set": "json-detected-level", + "description": "sum by (detected_level) over a JSON pipeline (row evaluator, and the stats pushdown with the level computed in VictoriaLogs and its guard probes).", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 5, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 5, + "faster": 1 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "DLV1 drilldown logs volume, json", + "range": "1h", + "p50": { + "pr": 0.526, + "base": 0.477, + "loki": 0.007 + }, + "p95": { + "pr": 0.531, + "base": 0.516, + "loki": 0.008 + }, + "cold": { + "pr": 0.52, + "base": 0.478, + "loki": 0.065 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 66.58, + "base": 66.58, + "loki": 66.58 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.91 + }, + { + "shape": "DLV2 json error filter, detected_level", + "range": "1h", + "p50": { + "pr": 2.193, + "base": 2.058, + "loki": 0.02 + }, + "p95": { + "pr": 2.208, + "base": 2.102, + "loki": 0.033 + }, + "cold": { + "pr": 2.182, + "base": 2.009, + "loki": 0.225 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 34.58, + "base": 39.34, + "loki": 39.34 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.94 + }, + { + "shape": "DLV3 plain json, detected_level", + "range": "1h", + "p50": { + "pr": 0.51, + "base": 0.487, + "loki": 0.011 + }, + "p95": { + "pr": 0.549, + "base": 0.513, + "loki": 0.017 + }, + "cold": { + "pr": 0.507, + "base": 0.488, + "loki": 0.072 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 38.45, + "base": 28.97, + "loki": 28.97 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.95 + }, + { + "shape": "DLV4 explore json volume, level and detected_level", + "range": "1h", + "p50": { + "pr": 0.043, + "base": 0.036, + "loki": 0.007 + }, + "p95": { + "pr": 0.045, + "base": 0.052, + "loki": 0.008 + }, + "cold": { + "pr": 0.021, + "base": 0.028, + "loki": 0.055 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 23.82, + "base": 23.82, + "loki": 23.82 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.84 + }, + { + "shape": "DLV5 json error dropped, detected_level", + "range": "1h", + "p50": { + "pr": 0.007, + "base": 0.067, + "loki": 0.019 + }, + "p95": { + "pr": 0.009, + "base": 0.108, + "loki": 0.02 + }, + "cold": { + "pr": 0.01, + "base": 0.064, + "loki": 0.252 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 30.52, + "base": 30.52, + "loki": 30.52 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 9.57 + }, + { + "shape": "DLV6 drilldown detected_level breakdown, json", + "range": "1h", + "p50": { + "pr": 0.006, + "base": 2.216, + "loki": 0.013 + }, + "p95": { + "pr": 0.012, + "base": 2.276, + "loki": 0.019 + }, + "cold": { + "pr": 0.014, + "base": 2.246, + "loki": 0.02 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 33.98, + "base": 42.79, + "loki": 36.68 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "faster", + "speedup": 369.33 + } + ] +} diff --git a/bench/ab/results/oneoff/2026-10-14-detlevel-swapped-control-1h.json b/bench/ab/results/oneoff/2026-10-14-detlevel-swapped-control-1h.json new file mode 100644 index 000000000..4c8980a5f --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-14-detlevel-swapped-control-1h.json @@ -0,0 +1,707 @@ +{ + "label": "detlevel-swapped-control-1h", + "date": "2026-10-14", + "set": "control", + "description": "Shapes a change must not slow down or alter. Run on every performance-relevant PR.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 5, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 20 + }, + "result_differs_from_reference": 1, + "result_differs_preexisting": 1, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "logs plain", + "range": "1h", + "p50": { + "pr": 0.025, + "base": 0.022, + "loki": 0.021 + }, + "p95": { + "pr": 0.035, + "base": 0.023, + "loki": 0.03 + }, + "cold": { + "pr": 0.036, + "base": 0.025, + "loki": 0.029 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 15.51, + "base": 15.51, + "loki": 15.51 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.88 + }, + { + "shape": "logs | json", + "range": "1h", + "p50": { + "pr": 0.036, + "base": 0.041, + "loki": 0.037 + }, + "p95": { + "pr": 0.045, + "base": 0.046, + "loki": 0.046 + }, + "cold": { + "pr": 0.032, + "base": 0.036, + "loki": 0.043 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 35.28, + "base": 35.28, + "loki": 35.28 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.14 + }, + { + "shape": "logs | logfmt", + "range": "1h", + "p50": { + "pr": 0.061, + "base": 0.077, + "loki": 0.024 + }, + "p95": { + "pr": 0.069, + "base": 0.103, + "loki": 0.026 + }, + "cold": { + "pr": 0.102, + "base": 0.09, + "loki": 0.03 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 54.1, + "base": 54.1, + "loki": 54.1 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.26 + }, + { + "shape": "json volume no filters", + "range": "1h", + "p50": { + "pr": 0.06, + "base": 0.081, + "loki": 0.021 + }, + "p95": { + "pr": 0.065, + "base": 0.088, + "loki": 0.031 + }, + "cold": { + "pr": 0.061, + "base": 0.102, + "loki": 0.255 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 52.11, + "base": 52.11, + "loki": 52.11 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.35 + }, + { + "shape": "logfmt volume", + "range": "1h", + "p50": { + "pr": 0.159, + "base": 0.128, + "loki": 0.029 + }, + "p95": { + "pr": 0.185, + "base": 0.157, + "loki": 0.033 + }, + "cold": { + "pr": 0.104, + "base": 0.087, + "loki": 0.374 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 69.27, + "base": 69.27, + "loki": 69.27 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.81 + }, + { + "shape": "plain volume", + "range": "1h", + "p50": { + "pr": 0.008, + "base": 0.007, + "loki": 0.018 + }, + "p95": { + "pr": 0.009, + "base": 0.01, + "loki": 0.024 + }, + "cold": { + "pr": 0.006, + "base": 0.008, + "loki": 0.204 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 30.44, + "base": 30.44, + "loki": 30.44 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.88 + }, + { + "shape": "stream-label count by app", + "range": "1h", + "p50": { + "pr": 0.013, + "base": 0.011, + "loki": 0.02 + }, + "p95": { + "pr": 0.022, + "base": 0.012, + "loki": 0.022 + }, + "cold": { + "pr": 0.016, + "base": 0.01, + "loki": 0.202 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 5.75, + "base": 5.75, + "loki": 5.75 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.85 + }, + { + "shape": "ungrouped rate", + "range": "1h", + "p50": { + "pr": 0.006, + "base": 0.005, + "loki": 0.034 + }, + "p95": { + "pr": 0.007, + "base": 0.009, + "loki": 0.034 + }, + "cold": { + "pr": 0.008, + "base": 0.007, + "loki": 0.201 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 9.09, + "base": 9.09, + "loki": 9.09 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.83 + }, + { + "shape": "bytes_over_time by app", + "range": "1h", + "p50": { + "pr": 0.025, + "base": 0.024, + "loki": 0.038 + }, + "p95": { + "pr": 0.034, + "base": 0.034, + "loki": 0.04 + }, + "cold": { + "pr": 0.024, + "base": 0.022, + "loki": 0.232 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 21.33, + "base": 21.33, + "loki": 32.3 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.96 + }, + { + "shape": "topk rate by app", + "range": "1h", + "p50": { + "pr": 0.026, + "base": 0.026, + "loki": 0.046 + }, + "p95": { + "pr": 0.033, + "base": 0.035, + "loki": 0.055 + }, + "cold": { + "pr": 0.024, + "base": 0.021, + "loki": 0.233 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 32.3, + "base": 32.3, + "loki": 32.3 + }, + "parity_vs_reference": "differs", + "baseline_parity_vs_reference": "differs", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.0 + }, + { + "shape": "detected_level volume", + "range": "1h", + "p50": { + "pr": 0.023, + "base": 0.01, + "loki": 0.021 + }, + "p95": { + "pr": 0.028, + "base": 0.015, + "loki": 0.024 + }, + "cold": { + "pr": 0.078, + "base": 0.014, + "loki": 0.212 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 31.41, + "base": 31.41, + "loki": 31.41 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.43 + }, + { + "shape": "by service_name", + "range": "1h", + "p50": { + "pr": 0.015, + "base": 0.014, + "loki": 0.019 + }, + "p95": { + "pr": 0.02, + "base": 0.018, + "loki": 0.024 + }, + "cold": { + "pr": 0.018, + "base": 0.024, + "loki": 0.215 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 14.84, + "base": 14.84, + "loki": 14.84 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.93 + }, + { + "shape": "drilldown level volume", + "range": "1h", + "p50": { + "pr": 0.007, + "base": 0.009, + "loki": 0.014 + }, + "p95": { + "pr": 0.017, + "base": 0.012, + "loki": 0.024 + }, + "cold": { + "pr": 0.007, + "base": 0.007, + "loki": 0.214 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 8.58, + "base": 8.58, + "loki": 8.58 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.29 + }, + { + "shape": "drilldown service landing", + "range": "1h", + "p50": { + "pr": 0.013, + "base": 0.011, + "loki": 0.013 + }, + "p95": { + "pr": 0.015, + "base": 0.011, + "loki": 0.015 + }, + "cold": { + "pr": 0.011, + "base": 0.011, + "loki": 0.194 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 9.41, + "base": 9.41, + "loki": 9.41 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.85 + }, + { + "shape": "drilldown detected_level", + "range": "1h", + "p50": { + "pr": 0.017, + "base": 0.018, + "loki": 0.011 + }, + "p95": { + "pr": 0.021, + "base": 0.022, + "loki": 0.019 + }, + "cold": { + "pr": 0.016, + "base": 0.019, + "loki": 0.196 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 23.39, + "base": 23.39, + "loki": 23.39 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.06 + }, + { + "shape": "json rate by level (no filter, no drop)", + "range": "1h", + "p50": { + "pr": 0.002, + "base": 0.003, + "loki": 0.048 + }, + "p95": { + "pr": 0.003, + "base": 0.004, + "loki": 0.056 + }, + "cold": { + "pr": 0.003, + "base": 0.003, + "loki": 0.069 + }, + "status": { + "pr": "400", + "base": "400", + "loki": "400" + }, + "vl_cpu_max": { + "pr": 21.22, + "base": 21.22, + "loki": 21.22 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "n/a", + "verdict": "same", + "speedup": null + }, + { + "shape": "logfmt count by level, filter, drop", + "range": "1h", + "p50": { + "pr": 0.152, + "base": 0.205, + "loki": 0.026 + }, + "p95": { + "pr": 0.167, + "base": 0.271, + "loki": 0.028 + }, + "cold": { + "pr": 0.161, + "base": 0.2, + "loki": 0.34 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 94.9, + "base": 94.9, + "loki": 94.9 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.35 + }, + { + "shape": "instant ungrouped rate", + "range": "instant", + "p50": { + "pr": 0.005, + "base": 0.005, + "loki": 0.044 + }, + "p95": { + "pr": 0.007, + "base": 0.005, + "loki": 0.058 + }, + "cold": { + "pr": 0.005, + "base": 0.005, + "loki": 0.031 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 71.16, + "base": 71.16, + "loki": 71.16 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.0 + }, + { + "shape": "instant json volume", + "range": "instant", + "p50": { + "pr": 0.2, + "base": 0.2, + "loki": 0.041 + }, + "p95": { + "pr": 0.209, + "base": 0.212, + "loki": 0.043 + }, + "cold": { + "pr": 0.199, + "base": 0.185, + "loki": 0.042 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 17.39, + "base": 17.39, + "loki": 17.39 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.0 + }, + { + "shape": "instant json filtered", + "range": "instant", + "p50": { + "pr": 0.165, + "base": 0.184, + "loki": 0.032 + }, + "p95": { + "pr": 0.169, + "base": 0.186, + "loki": 0.033 + }, + "cold": { + "pr": 0.15, + "base": 0.167, + "loki": 0.036 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 19.98, + "base": 19.98, + "loki": 19.98 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.12 + } + ] +} diff --git a/bench/ab/results/oneoff/2026-10-14-detlevel-swapped-d-1h.json b/bench/ab/results/oneoff/2026-10-14-detlevel-swapped-d-1h.json new file mode 100644 index 000000000..9f25c35b1 --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-14-detlevel-swapped-d-1h.json @@ -0,0 +1,61 @@ +{ + "label": "detlevel-swapped-d-1h", + "date": "2026-10-14", + "set": "json-filter-pushdown", + "description": "| json / | logfmt range metrics with label filters on parsed fields (PR #611). Grafana Explore log volume and Logs Drilldown field breakdowns over a JSON stream.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 5, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 1 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "D volume, no filters", + "range": "1h", + "p50": { + "pr": 0.076, + "base": 0.101, + "loki": 0.021 + }, + "p95": { + "pr": 0.112, + "base": 0.105, + "loki": 0.032 + }, + "cold": { + "pr": 0.05, + "base": 0.087, + "loki": 0.285 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 63.67, + "base": 63.67, + "loki": 63.67 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.33 + } + ] +} diff --git a/bench/ab/results/oneoff/2026-10-14-detlevel-swapped-d-long.json b/bench/ab/results/oneoff/2026-10-14-detlevel-swapped-d-long.json new file mode 100644 index 000000000..ea4008626 --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-14-detlevel-swapped-d-long.json @@ -0,0 +1,85 @@ +{ + "label": "detlevel-swapped-d-long", + "date": "2026-10-14", + "set": "json-filter-pushdown", + "description": "| json / | logfmt range metrics with label filters on parsed fields (PR #611). Grafana Explore log volume and Logs Drilldown field breakdowns over a JSON stream.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 3, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 2 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "D volume, no filters", + "range": "6h", + "p50": { + "pr": 0.589, + "base": 0.596 + }, + "p95": { + "pr": 0.59, + "base": 0.597 + }, + "cold": { + "pr": 0.731, + "base": 0.669 + }, + "status": { + "pr": "200", + "base": "200" + }, + "vl_cpu_max": { + "pr": 162.32, + "base": 162.32 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.01 + }, + { + "shape": "D volume, no filters", + "range": "24h", + "p50": { + "pr": 2.331, + "base": 2.25 + }, + "p95": { + "pr": 2.383, + "base": 2.284 + }, + "cold": { + "pr": 2.691, + "base": 2.485 + }, + "status": { + "pr": "200", + "base": "200" + }, + "vl_cpu_max": { + "pr": 202.43, + "base": 198.98 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.97 + } + ] +} diff --git a/bench/ab/results/oneoff/2026-10-14-detlevel-swapped-dl6-1h.json b/bench/ab/results/oneoff/2026-10-14-detlevel-swapped-dl6-1h.json new file mode 100644 index 000000000..29d09b597 --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-14-detlevel-swapped-dl6-1h.json @@ -0,0 +1,61 @@ +{ + "label": "detlevel-swapped-dl6-1h", + "date": "2026-10-14", + "set": "metric-latency-gap", + "description": "Shapes the metric latency work targets: the parse-risk probes of the | json stats pushdown (breakdowns and volumes by underscore labels), label filters before the parser (Logs Drilldown pages with a level filter selected) and Explore volumes with a level filter. Run with the json-filter-pushdown set.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 5, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 1 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "DL6 explore volume, level filter before json", + "range": "1h", + "p50": { + "pr": 0.245, + "base": 0.259, + "loki": 0.023 + }, + "p95": { + "pr": 0.333, + "base": 0.303, + "loki": 0.031 + }, + "cold": { + "pr": 0.635, + "base": 0.51, + "loki": 0.22 + }, + "status": { + "pr": "200", + "base": "200", + "loki": "200" + }, + "vl_cpu_max": { + "pr": 150.58, + "base": 150.58, + "loki": 150.58 + }, + "parity_vs_reference": "same series, sum \u00b10.31%", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.06 + } + ] +} diff --git a/bench/ab/results/oneoff/2026-10-14-detlevel-swapped-dl6-long.json b/bench/ab/results/oneoff/2026-10-14-detlevel-swapped-dl6-long.json new file mode 100644 index 000000000..4269de8f0 --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-14-detlevel-swapped-dl6-long.json @@ -0,0 +1,85 @@ +{ + "label": "detlevel-swapped-dl6-long", + "date": "2026-10-14", + "set": "metric-latency-gap", + "description": "Shapes the metric latency work targets: the parse-risk probes of the | json stats pushdown (breakdowns and volumes by underscore labels), label filters before the parser (Logs Drilldown pages with a level filter selected) and Explore volumes with a level filter. Run with the json-filter-pushdown set.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 3, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 2 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "DL6 explore volume, level filter before json", + "range": "6h", + "p50": { + "pr": 1.586, + "base": 1.5 + }, + "p95": { + "pr": 1.586, + "base": 1.502 + }, + "cold": { + "pr": 3.357, + "base": 3.209 + }, + "status": { + "pr": "200", + "base": "200" + }, + "vl_cpu_max": { + "pr": 198.65, + "base": 199.03 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.95 + }, + { + "shape": "DL6 explore volume, level filter before json", + "range": "24h", + "p50": { + "pr": 6.445, + "base": 5.984 + }, + "p95": { + "pr": 6.483, + "base": 6.072 + }, + "cold": { + "pr": 11.999, + "base": 11.885 + }, + "status": { + "pr": "200", + "base": "200" + }, + "vl_cpu_max": { + "pr": 215.31, + "base": 208.49 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.93 + } + ] +} diff --git a/bench/ab/results/oneoff/2026-10-14-detlevel-swapped-long.json b/bench/ab/results/oneoff/2026-10-14-detlevel-swapped-long.json new file mode 100644 index 000000000..b1301cea7 --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-14-detlevel-swapped-long.json @@ -0,0 +1,377 @@ +{ + "label": "detlevel-swapped-long", + "date": "2026-10-14", + "set": "json-detected-level", + "description": "sum by (detected_level) over a JSON pipeline (row evaluator, and the stats pushdown with the level computed in VictoriaLogs and its guard probes).", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 3, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 8, + "faster": 2, + "fixed": 2 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "DLV1 drilldown logs volume, json", + "range": "6h", + "p50": { + "pr": 3.26, + "base": 3.089 + }, + "p95": { + "pr": 3.297, + "base": 3.103 + }, + "cold": { + "pr": 3.241, + "base": 3.088 + }, + "status": { + "pr": "200", + "base": "200" + }, + "vl_cpu_max": { + "pr": 74.88, + "base": 40.17 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.95 + }, + { + "shape": "DLV1 drilldown logs volume, json", + "range": "24h", + "p50": { + "pr": 10.154, + "base": 9.651 + }, + "p95": { + "pr": 10.212, + "base": 9.743 + }, + "cold": { + "pr": 10.327, + "base": 9.465 + }, + "status": { + "pr": "502", + "base": "502" + }, + "vl_cpu_max": { + "pr": 52.52, + "base": 83.39 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "same", + "speedup": null + }, + { + "shape": "DLV2 json error filter, detected_level", + "range": "6h", + "p50": { + "pr": 8.146, + "base": 7.527 + }, + "p95": { + "pr": 8.168, + "base": 7.577 + }, + "cold": { + "pr": 8.221, + "base": 7.536 + }, + "status": { + "pr": "502", + "base": "502" + }, + "vl_cpu_max": { + "pr": 45.01, + "base": 36.45 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "same", + "speedup": null + }, + { + "shape": "DLV2 json error filter, detected_level", + "range": "24h", + "p50": { + "pr": 8.042, + "base": 7.476 + }, + "p95": { + "pr": 8.066, + "base": 7.498 + }, + "cold": { + "pr": 8.075, + "base": 7.386 + }, + "status": { + "pr": "502", + "base": "502" + }, + "vl_cpu_max": { + "pr": 33.09, + "base": 204.51 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "same", + "speedup": null + }, + { + "shape": "DLV3 plain json, detected_level", + "range": "6h", + "p50": { + "pr": 3.29, + "base": 3.087 + }, + "p95": { + "pr": 3.364, + "base": 3.124 + }, + "cold": { + "pr": 3.452, + "base": 3.066 + }, + "status": { + "pr": "200", + "base": "200" + }, + "vl_cpu_max": { + "pr": 204.51, + "base": 57.31 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.94 + }, + { + "shape": "DLV3 plain json, detected_level", + "range": "24h", + "p50": { + "pr": 9.858, + "base": 9.191 + }, + "p95": { + "pr": 9.872, + "base": 9.21 + }, + "cold": { + "pr": 9.913, + "base": 9.338 + }, + "status": { + "pr": "502", + "base": "502" + }, + "vl_cpu_max": { + "pr": 203.7, + "base": 203.7 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "same", + "speedup": null + }, + { + "shape": "DLV4 explore json volume, level and detected_level", + "range": "6h", + "p50": { + "pr": 0.184, + "base": 0.157 + }, + "p95": { + "pr": 0.195, + "base": 0.199 + }, + "cold": { + "pr": 0.186, + "base": 0.284 + }, + "status": { + "pr": "200", + "base": "200" + }, + "vl_cpu_max": { + "pr": 85.83, + "base": 85.83 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.85 + }, + { + "shape": "DLV4 explore json volume, level and detected_level", + "range": "24h", + "p50": { + "pr": 0.732, + "base": 0.746 + }, + "p95": { + "pr": 0.773, + "base": 0.781 + }, + "cold": { + "pr": 0.783, + "base": 0.811 + }, + "status": { + "pr": "200", + "base": "200" + }, + "vl_cpu_max": { + "pr": 159.49, + "base": 159.49 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.02 + }, + { + "shape": "DLV5 json error dropped, detected_level", + "range": "6h", + "p50": { + "pr": 0.026, + "base": 0.595 + }, + "p95": { + "pr": 0.026, + "base": 0.613 + }, + "cold": { + "pr": 0.035, + "base": 0.57 + }, + "status": { + "pr": "200", + "base": "200" + }, + "vl_cpu_max": { + "pr": 137.16, + "base": 137.16 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "faster", + "speedup": 22.88 + }, + { + "shape": "DLV5 json error dropped, detected_level", + "range": "24h", + "p50": { + "pr": 0.184, + "base": 2.446 + }, + "p95": { + "pr": 0.189, + "base": 2.49 + }, + "cold": { + "pr": 0.182, + "base": 2.498 + }, + "status": { + "pr": "200", + "base": "200" + }, + "vl_cpu_max": { + "pr": 198.65, + "base": 200.92 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "faster", + "speedup": 13.29 + }, + { + "shape": "DLV6 drilldown detected_level breakdown, json", + "range": "6h", + "p50": { + "pr": 0.026, + "base": 7.832 + }, + "p95": { + "pr": 0.026, + "base": 7.84 + }, + "cold": { + "pr": 0.09, + "base": 7.793 + }, + "status": { + "pr": "200", + "base": "502" + }, + "vl_cpu_max": { + "pr": 203.33, + "base": 40.16 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "fixed", + "speedup": null + }, + { + "shape": "DLV6 drilldown detected_level breakdown, json", + "range": "24h", + "p50": { + "pr": 0.182, + "base": 7.854 + }, + "p95": { + "pr": 0.222, + "base": 7.94 + }, + "cold": { + "pr": 0.206, + "base": 7.948 + }, + "status": { + "pr": "200", + "base": "502" + }, + "vl_cpu_max": { + "pr": 32.69, + "base": 56.31 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "fixed", + "speedup": null + } + ] +} diff --git a/bench/ab/shapes.json b/bench/ab/shapes.json index 0daddaecc..6f635ab76 100644 --- a/bench/ab/shapes.json +++ b/bench/ab/shapes.json @@ -1724,6 +1724,76 @@ ] } ] + }, + "json-detected-level": { + "description": "sum by (detected_level) over a JSON pipeline (row evaluator, and the stats pushdown with the level computed in VictoriaLogs and its guard probes).", + "ranges": [ + "1h", + "6h", + "24h" + ], + "shapes": [ + { + "name": "DLV1 drilldown logs volume, json", + "query": "sum by (detected_level) (count_over_time({env=\"production\", app=\"api-gateway\"} | json [$__auto]))", + "headers": { + "X-Query-Tags": "Source=grafana-lokiexplore-app" + }, + "covers": [ + "semantics/json-metric-detected-level-derivation", + "severity-detected-level-derivation", + "loki_api_v1_query_range" + ] + }, + { + "name": "DLV2 json error filter, detected_level", + "query": "sum by (detected_level) (count_over_time({env=\"production\"} | json | __error__!=\"\" [1m]))", + "covers": [ + "semantics/json-metric-detected-level-derivation", + "severity-detected-level-derivation", + "loki_api_v1_query_range" + ] + }, + { + "name": "DLV3 plain json, detected_level", + "query": "sum by (detected_level) (count_over_time({env=\"production\", app=\"api-gateway\"} | json [1m]))", + "covers": [ + "semantics/json-metric-detected-level-derivation", + "severity-detected-level-derivation", + "loki_api_v1_query_range" + ] + }, + { + "name": "DLV4 explore json volume, level and detected_level", + "query": "sum by (level, detected_level) (count_over_time({env=\"production\", app=\"api-gateway\"} | json | drop __error__ [$__auto]))", + "covers": [ + "semantics/json-metric-detected-level-derivation", + "severity-detected-level-derivation", + "loki_api_v1_query_range" + ] + }, + { + "name": "DLV5 json error dropped, detected_level", + "query": "sum by (detected_level) (count_over_time({env=\"production\"} | json | drop __error__ [1m]))", + "covers": [ + "semantics/json-metric-detected-level-derivation", + "severity-detected-level-derivation", + "loki_api_v1_query_range" + ] + }, + { + "name": "DLV6 drilldown detected_level breakdown, json", + "query": "sum by (detected_level) (count_over_time({env=\"production\"} | json | drop __error__, __error_details__ | detected_level != \"\" [$__auto]))", + "headers": { + "X-Query-Tags": "Source=grafana-lokiexplore-app" + }, + "covers": [ + "semantics/json-metric-detected-level-derivation", + "severity-detected-level-derivation", + "loki_api_v1_query_range" + ] + } + ] } } } diff --git a/bench/ab/stack.py b/bench/ab/stack.py index f1cdffa79..6d5eca52c 100644 --- a/bench/ab/stack.py +++ b/bench/ab/stack.py @@ -81,8 +81,10 @@ def port_free(port): class Stack: - def __init__(self, project, loki_port=23101, vl_port=29428, loki_mem=None, vl_mem=None, root=ROOT, overlays=()): + def __init__(self, project, loki_port=23101, vl_port=29428, loki_mem=None, vl_mem=None, root=ROOT, overlays=(), + no_level_label=False): self.project, self.loki_port, self.vl_port, self.root = project, loki_port, vl_port, root + self.no_level_label = no_level_label # seed rows whose level lives only in the line self.overlays = list(overlays) # further compose files after OVERLAY (relative to root) self.env = dict(os.environ, AB_PROJECT=project, AB_LOKI_PORT=str(loki_port), AB_VL_PORT=str(vl_port)) if loki_mem: @@ -124,6 +126,8 @@ def seed(self, seconds, end, timeout=900, batch=BACKFILL_BATCH, interval=BACKFIL env = dict(os.environ, LOKI_URL=self.loki, VL_URL=self.vl, LOG_BATCH=str(batch), LOG_BACKFILL_SECONDS=str(seconds), LOG_BACKFILL_INTERVAL=str(interval), LOG_BACKFILL_END=str(end), LOG_BACKFILL_ONLY="1", LOG_BACKFILL_SEED=str(end)) + if self.no_level_label: + env["LOG_NO_LEVEL_LABEL"] = "1" t0 = time.time() subprocess.run([sys.executable, GENERATOR], cwd=self.root, env=env, check=True, stdout=sys.stderr) try: @@ -334,10 +338,13 @@ def stack_args(ap): ap.add_argument("--vl-port", type=int, default=29428) ap.add_argument("--loki-mem", default="", help="Loki memory limit, e.g. 4g (default: the base stack's)") ap.add_argument("--vl-mem", default="", help="VictoriaLogs memory limit (default: the base stack's)") + ap.add_argument("--no-level-label", action="store_true", + help="seed rows with no stored level (the level only in the line), for the detected_level routes") def stack_from(args): - return Stack(args.project, args.loki_port, args.vl_port, args.loki_mem or None, args.vl_mem or None) + return Stack(args.project, args.loki_port, args.vl_port, args.loki_mem or None, args.vl_mem or None, + no_level_label=args.no_level_label) def main(): diff --git a/conformance/registry/cases/semantics/detected-level-pipes-known-differences.yaml b/conformance/registry/cases/semantics/detected-level-pipes-known-differences.yaml new file mode 100644 index 000000000..280ca8da8 --- /dev/null +++ b/conformance/registry/cases/semantics/detected-level-pipes-known-differences.yaml @@ -0,0 +1,89 @@ +id: semantics/detected-level-pipes-known-differences +title: The detected_level pipes read level keys in a fixed order, do not decode logfmt like Loki and overwrite a detected_level stream label (open) +proves: + - loki_api_v1_query_range +consumer: explore +why: | + The native metric route (`sum by (detected_level) (count_over_time({...} + [1m]))` over a plain selector, which Logs Drilldown sends, and Explore's + plain-selector volume shape) computes detected_level in VictoriaLogs with + translator.BuildDetectedLevelChain. Those pipes read the level keys of a body + in a fixed order, do not search nested objects other than `log.level`, stop + at nothing in a line VictoriaLogs cannot parse as a whole, fold only ASCII + in the keyword scan, and write the derived level over a stored + `detected_level`, where Loki takes the first level key in document order, + searches objects at depth 1, keeps the keys read before a syntax error, folds + Unicode and keeps a detected_level stream label as stored. + + The pipes also read the stored level fields in one list order without + telling a stream label from structured metadata, where Loki and the raw-row + evaluator read the stream's before the metadata: a stream `severity=error` + with metadata `level=info` is error in Loki and info on the pipes. The JSON + pipeline guard does not see it (an exact probe is an OR over pairs of names, + and a cheap one would send every stream labelled severity to the raw-row + evaluator), so it is recorded here too; the e2e and unit probe + `stream severity before stored level metadata` assert the raw-row + evaluator's answer only. + + Three more lines the pipes read differently, which no guard probe sees + (an exact one would need a logfmt decoder per line): + - A logfmt level key in another case, `msg="error x" LvL=warn`: Loki's + decoder compares keys without case and reads `warn`, `unpack_logfmt` is + case-sensitive and the keyword scan says `error`. Loki answers `warn` + (measured on v3.7.7); the pushdown and this route answer `error`. + - A logfmt syntax error before the level key, + `msg="unterminated error level=info`: Loki's decoder stops at the error + and the line falls to the keyword scan, `error` (measured); the pipes skip + the error and read `level=info`. + - A row VictoriaLogs unpacked from a JSON body (stored without a message, + `{"level":"notice","msg":"error later"}` pushed with message parsing on), + whose stored `level=notice` the pipes keep as the word `notice`, where Loki + keeps the line and gives `error` (measured; the same on main). + The raw-row evaluator reads the first two like Loki + (TestOrderedJSONDetectedLevelKnownDifferences); the pushdown and the native + route keep the pipes' value. + + Measured against Loki v3.7.7 on VictoriaLogs v1.52.0 with the e2e probe + fixtures: `{"msg":"x","severity":"error","level":"info"}` is error in Loki and + info in the proxy; a stream with the label `detected_level="INFO"` is INFO in + Loki and info in the proxy; a stream `severity=error` with metadata + `level=info` is error in Loki and under `| json`, and info under + `| json | drop __error__`. Logs Drilldown's detected_level breakdown + (`| json | drop __error__, __error_details__ | detected_level != ""`) is not + on this route: the label translator maps detected_level to level, so the + native histogram is never exact for it and the JSON pipeline route answers. + The JSON pipeline route (`| json`, #760) probes + for these lines and answers them with the raw-row evaluator, which is exact; + the native route has no raw-row fallback for them. +request: + method: GET + path: /loki/api/v1/query_range + params: + query: 'sum by (detected_level) (count_over_time({app="json-level-probe"}[1m]))' + start: now-10m + end: now + step: "60" + headers: + X-Scope-OrgID: "0" + precondition: | + A line holding `severity` before `level` with different values, a stream + with a detected_level label, a stream `severity` with metadata `level`, a + logfmt line with `LvL=warn` or a syntax error before its level key, or a + row VictoriaLogs unpacked from a JSON body with a stored `level`. +compare: + mode: exact + fields: [series_labels, values] + expect: | + Loki: error for the line with severity first; INFO for the labelled stream; + warn for `LvL=warn`; error for the unterminated line and for the unpacked + row. Proxy: info for the first two, error for `LvL=warn`, info for the + unterminated line, notice for the unpacked row. +gap: + status: open + impact: edge + effort: M + area: 'internal/translator detected_level_logsql.go BuildDetectedLevelChain; internal/proxy native metric route' + loki: 'pkg/distributor/field_detection.go getLevelUsingJSONParser (document order, depth < 2), labels.go (detected_level_extracted)' + proxy: 'BuildDetectedLevelChain reads keys in its list order and overwrites detected_level; the JSON pipeline route guards these lines instead' + planned_test: 'an e2e on plain-selector probe lines once the route probes and falls back to a raw-row or keyed evaluation' + clusters: [] diff --git a/conformance/registry/cases/semantics/json-metric-detected-level-derivation.yaml b/conformance/registry/cases/semantics/json-metric-detected-level-derivation.yaml new file mode 100644 index 000000000..34bbdb765 --- /dev/null +++ b/conformance/registry/cases/semantics/json-metric-detected-level-derivation.yaml @@ -0,0 +1,237 @@ +id: semantics/json-metric-detected-level-derivation +title: sum by (detected_level) over a JSON pipeline follows the level Loki's ingest gave each line, by level key in document order, by keyword scan, with a detected_level stream label kept as stored (fixed; a few lines stay different, see semantics/detected-level-pipes-known-differences) +proves: + - loki_api_v1_query_range +consumer: api +why: | + Loki sets detected_level once per line at ingest (pkg/distributor/ + field_detection.go extractLogLevelFromLogLine, discover_log_levels on). A + level stream label or structured metadata decides first. Otherwise a line + whose first non-space character is `{` and last is `}` is read as JSON: the + first string value of an allowed key (level, LEVEL, Level, log.level, + severity, SEVERITY, Severity, SeverityText, lvl, LVL, Lvl, severity_text, + Severity_Text, SEVERITY_TEXT) at depth 0 or 1, in document order, counts only + when it is a known spelling (info, INFO, inf and information are `info`). + Anything else (a number, a boolean, `notice`, no such key, a line that is not + JSON) falls to a keyword scan of the whole line, which finds the first + bounded level word; `unknown` is the answer only when the scan finds none. + The value is structured metadata present on every line, so a body key named + detected_level is detected_level_extracted, and a detected_level stream label + keeps its stored value (the derived one moves to detected_level_extracted). + + Measured against Loki v3.7.7 on 50 probe lines (the e2e test pins each): + `{"level":30,"msg":"hi"}` is unknown; a pino error line + (`{"level":50,...,"err":{"type":"Error"}}`) is error; `{"level":"notice"}` is + unknown but `{"level":"notice","msg":"error connecting"}` is error; + `{"level":" info"}` is info; `{"severity":"ERROR"}`, `{"Level":"Information"}`, + `{"LEVEL":"warn"}` and `{"log":{"level":"warn"}}` are error, info, warn and + warn; `{"msg":"x","severity":"error","level":"info"}` is error (document + order) and `{"level":"info","severity":"error"}` is info; a line without a + level key that says "error" is error. A JSON line is read for keys only: the + logfmt-looking text in `{"msg":"error: cannot set level=debug now"}` is no + level, so it is error. A line that starts with a brace but does not end with + one is logfmt (`{ error level=info` is info). Loki compares a JSON value's raw + bytes, so `{"level":"\u0069nfo"}` is unknown, and decodes keys, so + `{"\u006cevel":"warn"}` is warn. For logfmt the first level key in document + order wins within a spelling family (`Level=info LEVEL=error` is info). An + OTLP record is its severity number's level, or its text when the text is its + own: `Unspecified` or no number is unknown (or the keyword scan of the line), + number 10 is info, `notice` is notice. The proxy took the stream's spelling as + stored, read only a body `level` key and made every other value a series of + its own (`30`, `notice`, ` info`, `true`) or `unknown`. + + VictoriaLogs stores an OTLP record's severity in two shapes: from v1.50 + severity_text (the record's text, or a name it derives from the number: + `Unspecified` for 0, `Info2` for 10) with severity_number, and before one + severity field with no number. The derived text is no level field of Loki's, + so the pipes skip it and the number decides (a record without a severity + falls to the line, as in Loki). The e2e probes run on v1.47.0 and v1.52.0 and + cover both shapes; the pairing of a derived name with its number is checked + by family (a record sending `Info2` with number 12 is read as derived). + + Both routes of `sum by (detected_level) (... | json ...)` now give Loki's + value for every probe line above, bar the differences recorded in + semantics/detected-level-pipes-known-differences. The row evaluator (plain `| json`, `__error__!=""`, every pushdown + fallback) gives each row Loki's structured metadata before any stage runs, + using the derivation the log lines use (deriveRowLevel), so a body + detected_level key is renamed and a filter on detected_level + (`| json | detected_level="info"`) compares the derived value. The stats + pushdown first groups the buckets by copies of the stored detected_level and + level fields, taken before any unpack, and gives each bucket the level Loki's + ingest gives it: the first non-empty of the two, a known spelling in any case + as its level and any other word as stored (translator.NormalizeStoredLevel, tested against + the chain's own spelling regexps and against a table of Loki v3.7.7's + normalizeLogLevel answers). That is main's query with + no chain and no level guard probe, and it answers every window whose rows + all carry a stored level (a stream label or structured metadata, as Loki + pipelines send it); the proxy reads a stored detected_level, then level, and + a bucket with a level needs no other field of the proxy's list. Loki takes any + allowed stream label before any structured metadata, so a stream `severity` + with metadata `level` is error in Loki and info here: a recorded known + difference (semantics/detected-level-pipes-known-differences), not part of + this check. A filter on detected_level runs in VictoriaLogs on the stored + spellings (translator.BuildStoredLevelCopyFilter, which keeps every row the + stored fields do not decide so that those reach the chain) and again in Go on + the normalised level. Loki's series limit is applied to the answer's series: + the spellings of one level (`INFO`, `info`, `Information`) are one series, + and a read of more than twice the limit of VictoriaLogs groups + (`errStatsGroupGuard`, a memory guard) is asked again with the chain, whose + groups are its series. A + bucket without a stored level, or with a stored detected_level (a stream + label keeps its spelling, structured metadata is normalised, and the stats + cannot tell them apart), sends the query again with the pipes of the native + metric route (translator.BuildDetectedLevelChain), built from the level + fields and body keys the window holds (a presence probe: one stats query of + counts per stored field and per body key word, over the same base and + window, run on every request over the part of the window younger than five + minutes; an escape sequence in a body makes every key present, a failed probe + keeps the full chain; the narrowed query flags each row holding a field or key + the probe left out, a line that arrived late inside the remembered part, and + is then asked again with the full chain), placed before the unpack of any other label, and a + guard (detectedLevelStreamRisk, + detectedLevelKeysRisk, detectedLevelUnpackRisk, cached with the other + pushdown probes) hands the query to the raw-row evaluator when the window + holds a line those pipes read differently from Loki: a detected_level stream + label, two mentions of a level key (the pipes read keys in a fixed order), + a level key whose JSON value has an escape (Loki reads raw bytes), + logfmt keys in a line that starts with a brace but does not end with one + (Loki reads that as logfmt, the pipes do not unpack it), or a level key + unpack_json yields no value for (nested other than log.level, leading + whitespace, text after the closing brace, a syntax error). Differences it + does not see are recorded as open in the known-differences row: a stream + level field outranked in the pipes' list order by a stored metadata field + (stream `severity` with metadata `level`), a logfmt level key in another case + (`LvL=warn`), a logfmt syntax error before the level key, and a row + VictoriaLogs unpacked from a JSON body. A filter + on detected_level is applied after the pipes (`!= ""` always holds), also for + Logs Drilldown's detected_level breakdown, which is not the native histogram + (the label translator maps detected_level to level, so + orderedJSONNativeGroupingIsExact is false for it). With + -detected-level-body-scan=false the raw-row evaluator answers, as only the + stored fields decide. + + The plain and `| logfmt` logs volume shapes (`sum by (level, + detected_level)`) take the same plan, and the same pipes when a row has no + stored level: a stream without a level label gets + the level of its body (a `level=warn` key or a keyword), as in Loki, where + main answered `unknown`. The plain shape has no raw-row fallback, so it runs + without the guard (see semantics/detected-level-pipes-known-differences). + + Cost: a window whose rows all carry a stored level costs what main's query + costs (two more grouped columns and one copy pipe). Measured against main on + one hour of the A/B harness's backfill (about 290,000 lines, a level stream + label on every row) with VictoriaLogs capped at 2 CPUs as on a CI runner, + warm p50 of 5 to 7 runs, the two builds measured in each order (the second + target of the harness runs slower by up to 30% even for the same binary): + Explore's JSON volume 79 ms to 98 ms (101 ms to 76 ms in the other order), + the volume behind a `detected_level="info"` filter before the parser + (VictoriaLogs first unpacks logfmt for every line) 181 ms to 279 ms (259 ms to + 245 ms); at 6h and 24h 574 ms to 608 ms and 2.37 s to 2.40 s (596 ms to + 589 ms and 2.25 s to 2.33 s), 1.55 s to 1.60 s and 6.38 s to 6.90 s (1.50 s to + 1.59 s and 5.98 s to 6.45 s). + + A row with no stored level (the level only in the line: the same backfill + with the level left out of the stream labels, which the harness seeds with + `--no-level-label`) is read by the chain, which reads every line; main + answers 4 series for Explore's volume there where Loki answers 8. The chain + parses each line once, for the plan's labels and for the level: the plan's + own `unpack_json` and the level keys' are one (translator.SharedUnpack: the + chain's unpack reads the level keys and the plan's labels with + result_prefix "__j_", the level keys are read only for rows no stored level + decides, and before the closing delete each label is copied from its + prefixed field where the field is not stored, which is what the plan's + unpack with keep_original_fields wrote; the unpack parses a row only where a + label is missing as a stored field or the chain reads the line). The keyword + regexp runs only on lines that hold one of its nine words as a whole word of + VictoriaLogs' tokenizer (`_msg:(i(trace) OR ... OR i(info))`, a superset of + the regexp's matches: every byte Loki accepts around a keyword is a token + separator and the folding of (?i) maps no non-ASCII rune onto the letters of + the words), and the escape check of the presence probe and its witness is a + substring filter (`_msg:*"\\u"*`), not a regexp. Measured on one hour of + that data, VictoriaLogs capped at 2 CPUs, warm p50, VictoriaLogs alone, 15 + runs in each measurement order at 1h: Explore's volume 84 ms to 105 ms (87 ms + to 102 ms with the branch measured first), the detected_level breakdown with + the error rows dropped 88 ms to 99 ms (85 ms to 101 ms); at 6h and 24h (3 + runs, both orders) Explore's volume 541 ms to 692 ms and 2.39 s to 2.75 s + (570 ms to 702 ms and 2.30 s to 2.84 s), the + breakdown 585 ms to 641 ms and 2.24 s to 2.74 s (589 ms to 698 ms and 2.25 s + to 2.79 s). The previous revision took 38 ms to 282 ms, 0.55 s to 1.5 s and + 2.3 s to 6.1 s for Explore's volume and 56 ms to 99 ms and 2.2 s to 3.8 s for + the breakdown. On VictoriaLogs alone, one hour, per query: main's query 102 + ms and 225 CPU-ms (cgroup CPU time; the wall time at 2 CPUs moves in steps of + the 100 ms quota period), the stored-level plan 100 ms, the previous chain + 290 ms and 550 CPU-ms, this one 107 ms and 260 CPU-ms. Where the CPU of the + chain went (one group of pipes left out at a time): the plan's own unpack 194 + CPU-ms, the keyword regexp 144, the witness 102 before; the witness 87, the + keyword regexp 48 and the json unpack 20 now. The detected_level breakdown of + Logs Drilldown (a filter, `| json | drop __error__`), which main answered + from raw rows, goes on that data from 2.2 s to 0.05-0.11 s at 1h and from + 7.5 s to 0.67 s (6h) and 2.6 s (24h), and on the data with a level stream + label from 2.3 s to 7 ms at 1h. The first view of a window (nothing + remembered: the stored-level attempt, the level probes, the presence probe, + then the chain) takes about 0.5 s at 1h for Explore's volume where main takes + 0.08 s (0.55-0.68 s in the previous revision, both builds restarted on one + stack): the probes scan the window and share the 2 CPUs with the chain, the + heaviest of them (detectedLevelKeysRisk, 260-300 ms: two level keys in a line, + an escape in a level value, a brace without its closing brace) one this change + adds. A window with a row that has no stored level asks + twice, the stored-level attempt first (it costs what main's query costs; its + verdict is remembered for the base for five minutes so the next ask goes to + the chain at once), and pays the chain with the three guard probes and the + presence probe. Each guard probe stops at its first matching line (`limit 1`), + which bounds what it returns, not the work: on a clean window it scans the + lines that hold a level-key word (VictoriaLogs' word index narrows them) and + the rows without a stored level. The guard is a superset of the lines the + pipes read differently: a body with both `level` and `severity` (audit logs) + trips it although the pipes might agree. One such line sends the whole query + to the raw-row evaluator, which is exact and bounded by the row limit, so a + long range answers 502 where the pushdown answered 200. The chain is about + 7.1 KB of LogsQL unpruned and 1.5 KB for a window with one stored level field + and no body key, plus the witness (up to 1.5 KB, none when the window holds + every field and key): with a `service_name` selector Explore's volume sends + 4.2 KB (9.8 KB unpruned) and a regexp selector of seven long names about + 9.6 KB with the witness (14.4 KB unpruned; those names match no row, so the + chain is at its smallest), against VictoriaLogs' 16 KiB -search.maxQueryLen (a + longer query answers 502; a window whose presence cannot be told keeps the + full chain and its size; a unit test pins the witnessed request under the + limit). The remembered verdicts (guard probes and presence) are kept per + request scope, cover only the part of a window older than five minutes, and + expire five minutes after the check of their oldest part, so a refreshing + dashboard checks the last five minutes on every refresh and the whole window + again every five minutes. +request: + method: GET + path: /loki/api/v1/query_range + params: + query: 'sum by (detected_level) (count_over_time({app="json-level-probe"} | json | drop __error__ [1m]))' + start: now-10m + end: now + step: "60" + headers: + X-Scope-OrgID: "0" + precondition: | + One stream per probe line: pino numeric levels, notice, a padded level, the + severity, Level, LEVEL and nested log.level keys, two level keys in both + orders, a line with the word error and no level key, a detected_level + stream label, a body detected_level key, structured metadata level and + detected_level. +compare: + mode: exact + fields: [status, series_labels, values] + expect: | + The series Loki answers for each probe, on `| json` (raw-row evaluator) and + `| json | drop __error__` (pushdown or its guarded fallback). +gap: + status: fixed + fixed_by: "#760" + impact: explore-visible + visual: none + visual_reason: 'the generator data carries one lowercase spelling, the same in the stream label and the body, so main and the branch render the same panel; the e2e probes against Loki are the proof (v1.47 and v1.52)' + effort: M + area: 'internal/proxy ordered_json_metric.go (withDerivedLevel, setStatsPushdown, detectedLevel*Risk), stream_processing.go stringRow' + loki: 'pkg/distributor/field_detection.go extractLogLevelFromLogLine, detectLevelFromLogLine, normalizeLogLevel; pkg/distributor/labels.go (detected_level_extracted)' + proxy: 'the row evaluator and the stats pushdown read the stream level or a body level key and normalized it' + planned_test: 'TestOrderedJSONDetectedLevelLikeLoki, TestOrderedJSONDetectedLevelMixedRows, TestOrderedJSONDetectedLevelKnownDifferences (proxy); TestRangeMetricCompatibilityJSONDetectedLevelProbes, TestRangeMetricCompatibilityJSONDetectedLevelSharedUnpack, TestRangeMetricCompatibilityDetectedLevelPipeVariants, TestRangeMetricCompatibilityEscapeSubstringFilter and TestRangeMetricCompatibilityJSONDetectedLevel (test/e2e-compat, group semantics); TestKeywordScanLikeLoki, TestKeywordGateIsASupersetOfTheRegexp, TestBuildDetectedLevelChainShared (translator); TestSharedUnpackKeepsThePlansLabelsLikeLoki, TestSharedLevelUnpackEligibility (proxy)' + note: | + Measured on VictoriaLogs v1.52.0 and v1.47.0 against Loki v3.7.7. + clusters: [] diff --git a/conformance/registry/generated/perf-evidence.json b/conformance/registry/generated/perf-evidence.json index d2f9a7a39..a845fc262 100644 --- a/conformance/registry/generated/perf-evidence.json +++ b/conformance/registry/generated/perf-evidence.json @@ -843,12 +843,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.015, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.025, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.271, - "loki_warm_p50": 0.019, + "loki_cold": 0.333, + "loki_warm_p50": 0.025, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -857,7 +857,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.013 + "warm_p50": 0.016 }, { "baseline": "base", @@ -919,12 +919,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.013, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.015, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.271, - "loki_warm_p50": 0.013, + "loki_cold": 0.342, + "loki_warm_p50": 0.017, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -933,7 +933,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.01 + "warm_p50": 0.014 }, { "baseline": "base", @@ -1071,13 +1071,13 @@ "warm_p50": 0.051 }, { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 0.037, - "date": "2026-10-01", - "label": "ordered-json-json-filter-pushdown-1h", + "cold": 0.096, + "date": "2026-10-14", + "label": "detlevel-d-capped-1h", "loki_cache_served": false, - "loki_cold": 0.37, + "loki_cold": 0.909, "loki_warm_p50": 0.029, "parity_vs_loki": "same", "range": "1h", @@ -1087,14 +1087,14 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.043 + "warm_p50": 0.098 }, { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 0.704, - "date": "2026-10-01", - "label": "ordered-json-json-filter-pushdown-long", + "cold": 2.41, + "date": "2026-10-14", + "label": "detlevel-d-capped-long", "loki_cache_served": false, "loki_cold": null, "loki_warm_p50": null, @@ -1106,7 +1106,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.406 + "warm_p50": 2.402 }, { "baseline": "main", @@ -1128,11 +1128,11 @@ "warm_p50": 0.354 }, { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 0.139, - "date": "2026-10-01", - "label": "ordered-json-json-filter-pushdown-long", + "cold": 0.611, + "date": "2026-10-14", + "label": "detlevel-d-capped-long", "loki_cache_served": false, "loki_cold": null, "loki_warm_p50": null, @@ -1144,7 +1144,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.12 + "warm_p50": 0.608 }, { "baseline": "main", @@ -1225,11 +1225,11 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.037, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.093, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.335, + "loki_cold": 0.451, "loki_warm_p50": 0.021, "parity_vs_loki": "same", "range": "1h", @@ -1239,7 +1239,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.044 + "warm_p50": 0.105 }, { "baseline": "base", @@ -1301,12 +1301,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.2, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.208, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.043, - "loki_warm_p50": 0.044, + "loki_cold": 0.049, + "loki_warm_p50": 0.049, "parity_vs_loki": "same", "range": "instant", "set": "control", @@ -1315,33 +1315,33 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.191 + "warm_p50": 0.189 }, { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 0.172, - "date": "2026-10-01", - "label": "ordered-json-metric-latency-gap-1h", + "cold": 0.617, + "date": "2026-10-14", + "label": "detlevel-dl6-capped-1h", "loki_cache_served": false, - "loki_cold": 0.348, - "loki_warm_p50": 0.021, - "parity_vs_loki": "same series, sum \u00b10.22%", + "loki_cold": 0.331, + "loki_warm_p50": 0.017, + "parity_vs_loki": "same series, sum \u00b10.27%", "range": "1h", "set": "metric-latency-gap", "shape": "DL6 explore volume, level filter before json", - "slower_than_loki_cold": false, + "slower_than_loki_cold": true, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.058 + "warm_p50": 0.279 }, { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 3.329, - "date": "2026-10-01", - "label": "ordered-json-metric-latency-gap-long", + "cold": 12.493, + "date": "2026-10-14", + "label": "detlevel-dl6-capped-long", "loki_cache_served": false, "loki_cold": null, "loki_warm_p50": null, @@ -1353,7 +1353,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 1.082 + "warm_p50": 6.904 }, { "baseline": "main", @@ -1375,11 +1375,11 @@ "warm_p50": 0.139 }, { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 1.211, - "date": "2026-10-01", - "label": "ordered-json-metric-latency-gap-long", + "cold": 3.805, + "date": "2026-10-14", + "label": "detlevel-dl6-capped-long", "loki_cache_served": false, "loki_cold": null, "loki_warm_p50": null, @@ -1391,7 +1391,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 1.665 + "warm_p50": 1.598 }, { "baseline": "base", @@ -5770,11 +5770,11 @@ "baseline": "base", "baseline_status": "200", "cold": 0.004, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.032, - "loki_warm_p50": 0.033, + "loki_cold": 0.034, + "loki_warm_p50": 0.037, "parity_vs_loki": "same", "range": "instant", "set": "control", @@ -5783,17 +5783,17 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.005 + "warm_p50": 0.006 }, { "baseline": "base", "baseline_status": "200", - "cold": 0.2, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.208, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.043, - "loki_warm_p50": 0.044, + "loki_cold": 0.049, + "loki_warm_p50": 0.049, "parity_vs_loki": "same", "range": "instant", "set": "control", @@ -5802,17 +5802,17 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.191 + "warm_p50": 0.189 }, { "baseline": "base", "baseline_status": "200", - "cold": 0.17, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.163, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.036, - "loki_warm_p50": 0.033, + "loki_cold": 0.042, + "loki_warm_p50": 0.054, "parity_vs_loki": "same", "range": "instant", "set": "control", @@ -5821,7 +5821,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.166 + "warm_p50": 0.161 }, { "baseline": "main", @@ -6206,13 +6206,13 @@ "warm_p50": 0.044 }, { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 0.037, - "date": "2026-10-01", - "label": "ordered-json-json-filter-pushdown-1h", + "cold": 0.096, + "date": "2026-10-14", + "label": "detlevel-d-capped-1h", "loki_cache_served": false, - "loki_cold": 0.37, + "loki_cold": 0.909, "loki_warm_p50": 0.029, "parity_vs_loki": "same", "range": "1h", @@ -6222,14 +6222,14 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.043 + "warm_p50": 0.098 }, { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 0.704, - "date": "2026-10-01", - "label": "ordered-json-json-filter-pushdown-long", + "cold": 2.41, + "date": "2026-10-14", + "label": "detlevel-d-capped-long", "loki_cache_served": false, "loki_cold": null, "loki_warm_p50": null, @@ -6241,7 +6241,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.406 + "warm_p50": 2.402 }, { "baseline": "main", @@ -6263,11 +6263,11 @@ "warm_p50": 0.354 }, { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 0.139, - "date": "2026-10-01", - "label": "ordered-json-json-filter-pushdown-long", + "cold": 0.611, + "date": "2026-10-14", + "label": "detlevel-d-capped-long", "loki_cache_served": false, "loki_cold": null, "loki_warm_p50": null, @@ -6279,7 +6279,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.12 + "warm_p50": 0.608 }, { "baseline": "main", @@ -6664,12 +6664,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.025, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.018, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.02, - "loki_warm_p50": 0.023, + "loki_cold": 0.039, + "loki_warm_p50": 0.028, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -6678,7 +6678,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.024 + "warm_p50": 0.019 }, { "baseline": "base", @@ -6740,12 +6740,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.046, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.027, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.031, - "loki_warm_p50": 0.033, + "loki_cold": 0.042, + "loki_warm_p50": 0.041, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -6754,7 +6754,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.034 + "warm_p50": 0.03 }, { "baseline": "base", @@ -6816,12 +6816,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.047, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.052, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.031, - "loki_warm_p50": 0.027, + "loki_cold": 0.033, + "loki_warm_p50": 0.044, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -6830,7 +6830,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.047 + "warm_p50": 0.042 }, { "baseline": "base", @@ -6892,11 +6892,11 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.037, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.093, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.335, + "loki_cold": 0.451, "loki_warm_p50": 0.021, "parity_vs_loki": "same", "range": "1h", @@ -6906,7 +6906,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.044 + "warm_p50": 0.105 }, { "baseline": "base", @@ -6968,12 +6968,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.057, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.101, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.472, - "loki_warm_p50": 0.027, + "loki_cold": 0.603, + "loki_warm_p50": 0.035, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -6982,7 +6982,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.093 + "warm_p50": 0.111 }, { "baseline": "base", @@ -7044,12 +7044,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.007, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.008, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.274, - "loki_warm_p50": 0.018, + "loki_cold": 0.346, + "loki_warm_p50": 0.02, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -7058,7 +7058,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.007 + "warm_p50": 0.008 }, { "baseline": "base", @@ -7121,11 +7121,11 @@ "baseline": "base", "baseline_status": "200", "cold": 0.011, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.288, - "loki_warm_p50": 0.018, + "loki_cold": 0.325, + "loki_warm_p50": 0.023, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -7134,7 +7134,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.011 + "warm_p50": 0.012 }, { "baseline": "base", @@ -7196,12 +7196,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.009, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.006, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.262, - "loki_warm_p50": 0.036, + "loki_cold": 0.335, + "loki_warm_p50": 0.04, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -7210,7 +7210,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.008 + "warm_p50": 0.007 }, { "baseline": "base", @@ -7272,12 +7272,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.035, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.025, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.312, - "loki_warm_p50": 0.045, + "loki_cold": 0.345, + "loki_warm_p50": 0.044, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -7286,7 +7286,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.023 + "warm_p50": 0.025 }, { "baseline": "base", @@ -7348,12 +7348,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.021, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.018, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.287, - "loki_warm_p50": 0.039, + "loki_cold": 0.346, + "loki_warm_p50": 0.038, "parity_vs_loki": "differs", "range": "1h", "set": "control", @@ -7362,7 +7362,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.02 + "warm_p50": 0.019 }, { "baseline": "base", @@ -7424,12 +7424,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.008, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.027, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.282, - "loki_warm_p50": 0.017, + "loki_cold": 0.333, + "loki_warm_p50": 0.018, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -7438,7 +7438,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.006 + "warm_p50": 0.02 }, { "baseline": "base", @@ -7500,12 +7500,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.015, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.025, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.271, - "loki_warm_p50": 0.019, + "loki_cold": 0.333, + "loki_warm_p50": 0.025, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -7514,7 +7514,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.013 + "warm_p50": 0.016 }, { "baseline": "base", @@ -7577,11 +7577,11 @@ "baseline": "base", "baseline_status": "200", "cold": 0.007, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.274, - "loki_warm_p50": 0.011, + "loki_cold": 0.32, + "loki_warm_p50": 0.013, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -7652,12 +7652,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.013, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.015, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.271, - "loki_warm_p50": 0.013, + "loki_cold": 0.342, + "loki_warm_p50": 0.017, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -7666,7 +7666,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.01 + "warm_p50": 0.014 }, { "baseline": "base", @@ -7728,12 +7728,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.027, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.023, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.281, - "loki_warm_p50": 0.01, + "loki_cold": 0.347, + "loki_warm_p50": 0.012, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -7742,7 +7742,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.02 + "warm_p50": 0.018 }, { "baseline": "base", @@ -7804,12 +7804,12 @@ { "baseline": "base", "baseline_status": "400", - "cold": 0.003, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.004, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.063, - "loki_warm_p50": 0.062, + "loki_cold": 0.064, + "loki_warm_p50": 0.06, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -7818,7 +7818,7 @@ "status": "400", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.003 + "warm_p50": 0.002 }, { "baseline": "base", @@ -7880,12 +7880,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.075, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.287, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.462, - "loki_warm_p50": 0.034, + "loki_cold": 0.57, + "loki_warm_p50": 0.025, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -7894,7 +7894,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.073 + "warm_p50": 0.182 }, { "baseline": "base", @@ -7954,30 +7954,30 @@ "warm_p50": 0.214 }, { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 0.172, - "date": "2026-10-01", - "label": "ordered-json-metric-latency-gap-1h", + "cold": 0.617, + "date": "2026-10-14", + "label": "detlevel-dl6-capped-1h", "loki_cache_served": false, - "loki_cold": 0.348, - "loki_warm_p50": 0.021, - "parity_vs_loki": "same series, sum \u00b10.22%", + "loki_cold": 0.331, + "loki_warm_p50": 0.017, + "parity_vs_loki": "same series, sum \u00b10.27%", "range": "1h", "set": "metric-latency-gap", "shape": "DL6 explore volume, level filter before json", - "slower_than_loki_cold": false, + "slower_than_loki_cold": true, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.058 + "warm_p50": 0.279 }, { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 3.329, - "date": "2026-10-01", - "label": "ordered-json-metric-latency-gap-long", + "cold": 12.493, + "date": "2026-10-14", + "label": "detlevel-dl6-capped-long", "loki_cache_served": false, "loki_cold": null, "loki_warm_p50": null, @@ -7989,7 +7989,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 1.082 + "warm_p50": 6.904 }, { "baseline": "main", @@ -8011,11 +8011,11 @@ "warm_p50": 0.139 }, { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 1.211, - "date": "2026-10-01", - "label": "ordered-json-metric-latency-gap-long", + "cold": 3.805, + "date": "2026-10-14", + "label": "detlevel-dl6-capped-long", "loki_cache_served": false, "loki_cold": null, "loki_warm_p50": null, @@ -8027,7 +8027,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 1.665 + "warm_p50": 1.598 }, { "baseline": "main", @@ -13310,51 +13310,393 @@ "valid": true, "verdict_vs_baseline": "same", "warm_p50": 0.143 - } - ], - "metric-series-identity": [ + }, { "baseline": "base", "baseline_status": "200", - "cold": 0.011, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.52, + "date": "2026-10-14", + "label": "detlevel-1h", "loki_cache_served": false, - "loki_cold": 0.288, - "loki_warm_p50": 0.018, + "loki_cold": 0.184, + "loki_warm_p50": 0.012, "parity_vs_loki": "same", "range": "1h", - "set": "control", - "shape": "stream-label count by app", - "slower_than_loki_cold": false, + "set": "json-detected-level", + "shape": "DLV1 drilldown logs volume, json", + "slower_than_loki_cold": true, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.011 + "warm_p50": 0.5 }, { "baseline": "base", - "baseline_status": "200", - "cold": 0.161, - "date": "2026-10-12", - "label": "patanchor-control-24h", + "baseline_status": "502", + "cold": 9.675, + "date": "2026-10-14", + "label": "detlevel-long", "loki_cache_served": false, "loki_cold": null, "loki_warm_p50": null, "parity_vs_loki": "n/a", "range": "24h", - "set": "control", - "shape": "stream-label count by app", + "set": "json-detected-level", + "shape": "DLV1 drilldown logs volume, json", "slower_than_loki_cold": false, - "status": "200", + "status": "502", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.095 + "warm_p50": 9.633 }, { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 0.019, + "cold": 3.266, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-detected-level", + "shape": "DLV1 drilldown logs volume, json", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 3.159 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 2.312, + "date": "2026-10-14", + "label": "detlevel-1h", + "loki_cache_served": false, + "loki_cold": 0.372, + "loki_warm_p50": 0.025, + "parity_vs_loki": "same", + "range": "1h", + "set": "json-detected-level", + "shape": "DLV2 json error filter, detected_level", + "slower_than_loki_cold": true, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 2.228 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 7.927, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-detected-level", + "shape": "DLV2 json error filter, detected_level", + "slower_than_loki_cold": false, + "status": "502", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 7.987 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 7.811, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-detected-level", + "shape": "DLV2 json error filter, detected_level", + "slower_than_loki_cold": false, + "status": "502", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 7.965 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.526, + "date": "2026-10-14", + "label": "detlevel-1h", + "loki_cache_served": false, + "loki_cold": 0.087, + "loki_warm_p50": 0.008, + "parity_vs_loki": "same", + "range": "1h", + "set": "json-detected-level", + "shape": "DLV3 plain json, detected_level", + "slower_than_loki_cold": true, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.53 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 9.774, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-detected-level", + "shape": "DLV3 plain json, detected_level", + "slower_than_loki_cold": false, + "status": "502", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 9.57 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 3.234, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-detected-level", + "shape": "DLV3 plain json, detected_level", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 3.212 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.024, + "date": "2026-10-14", + "label": "detlevel-1h", + "loki_cache_served": false, + "loki_cold": 0.106, + "loki_warm_p50": 0.005, + "parity_vs_loki": "same", + "range": "1h", + "set": "json-detected-level", + "shape": "DLV4 explore json volume, level and detected_level", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.048 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.804, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-detected-level", + "shape": "DLV4 explore json volume, level and detected_level", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.7 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.187, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-detected-level", + "shape": "DLV4 explore json volume, level and detected_level", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.106 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.007, + "date": "2026-10-14", + "label": "detlevel-1h", + "loki_cache_served": false, + "loki_cold": 0.424, + "loki_warm_p50": 0.021, + "parity_vs_loki": "same", + "range": "1h", + "set": "json-detected-level", + "shape": "DLV5 json error dropped, detected_level", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.008 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.185, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-detected-level", + "shape": "DLV5 json error dropped, detected_level", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 0.174 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.027, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-detected-level", + "shape": "DLV5 json error dropped, detected_level", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 0.028 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.009, + "date": "2026-10-14", + "label": "detlevel-1h", + "loki_cache_served": false, + "loki_cold": 0.047, + "loki_warm_p50": 0.017, + "parity_vs_loki": "same", + "range": "1h", + "set": "json-detected-level", + "shape": "DLV6 drilldown detected_level breakdown, json", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 0.009 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 0.142, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-detected-level", + "shape": "DLV6 drilldown detected_level breakdown, json", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 0.178 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 0.027, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-detected-level", + "shape": "DLV6 drilldown detected_level breakdown, json", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 0.025 + } + ], + "metric-series-identity": [ + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.011, + "date": "2026-10-14", + "label": "detlevel-control-1h", + "loki_cache_served": false, + "loki_cold": 0.325, + "loki_warm_p50": 0.023, + "parity_vs_loki": "same", + "range": "1h", + "set": "control", + "shape": "stream-label count by app", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.012 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.161, + "date": "2026-10-12", + "label": "patanchor-control-24h", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "control", + "shape": "stream-label count by app", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.095 + }, + { + "baseline": "main", + "baseline_status": "200", + "cold": 0.019, "date": "2026-10-06", "label": "plain-name-control", "loki_cache_served": false, @@ -13908,12 +14250,12 @@ "baseline": "base", "baseline_status": "200", "cold": 0.011, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.288, - "loki_warm_p50": 0.018, - "parity_vs_loki": "same", + "loki_cold": 0.325, + "loki_warm_p50": 0.023, + "parity_vs_loki": "same", "range": "1h", "set": "control", "shape": "stream-label count by app", @@ -13921,7 +14263,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.011 + "warm_p50": 0.012 }, { "baseline": "base", @@ -13983,12 +14325,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.009, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.006, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.262, - "loki_warm_p50": 0.036, + "loki_cold": 0.335, + "loki_warm_p50": 0.04, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -13997,7 +14339,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.008 + "warm_p50": 0.007 }, { "baseline": "base", @@ -14059,12 +14401,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.035, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.025, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.312, - "loki_warm_p50": 0.045, + "loki_cold": 0.345, + "loki_warm_p50": 0.044, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -14073,7 +14415,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.023 + "warm_p50": 0.025 }, { "baseline": "base", @@ -14137,12 +14479,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.021, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.018, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.287, - "loki_warm_p50": 0.039, + "loki_cold": 0.346, + "loki_warm_p50": 0.038, "parity_vs_loki": "differs", "range": "1h", "set": "control", @@ -14151,7 +14493,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.02 + "warm_p50": 0.019 }, { "baseline": "base", @@ -14901,12 +15243,12 @@ { "baseline": "base", "baseline_status": "400", - "cold": 0.003, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.004, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.063, - "loki_warm_p50": 0.062, + "loki_cold": 0.064, + "loki_warm_p50": 0.06, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -14915,7 +15257,7 @@ "status": "400", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.003 + "warm_p50": 0.002 }, { "baseline": "base", @@ -16518,12 +16860,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.046, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.027, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.031, - "loki_warm_p50": 0.033, + "loki_cold": 0.042, + "loki_warm_p50": 0.041, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -16532,7 +16874,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.034 + "warm_p50": 0.03 }, { "baseline": "base", @@ -16594,12 +16936,12 @@ { "baseline": "base", "baseline_status": "400", - "cold": 0.003, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.004, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.063, - "loki_warm_p50": 0.062, + "loki_cold": 0.064, + "loki_warm_p50": 0.06, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -16608,7 +16950,7 @@ "status": "400", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.003 + "warm_p50": 0.002 }, { "baseline": "base", @@ -16957,12 +17299,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.047, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.052, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.031, - "loki_warm_p50": 0.027, + "loki_cold": 0.033, + "loki_warm_p50": 0.044, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -16971,7 +17313,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.047 + "warm_p50": 0.042 }, { "baseline": "base", @@ -17033,12 +17375,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.057, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.101, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.472, - "loki_warm_p50": 0.027, + "loki_cold": 0.603, + "loki_warm_p50": 0.035, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -17047,7 +17389,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.093 + "warm_p50": 0.111 }, { "baseline": "base", @@ -17109,12 +17451,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.075, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.287, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.462, - "loki_warm_p50": 0.034, + "loki_cold": 0.57, + "loki_warm_p50": 0.025, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -17123,7 +17465,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.073 + "warm_p50": 0.182 }, { "baseline": "base", @@ -18508,12 +18850,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.035, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.025, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.312, - "loki_warm_p50": 0.045, + "loki_cold": 0.345, + "loki_warm_p50": 0.044, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -18522,7 +18864,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.023 + "warm_p50": 0.025 }, { "baseline": "base", @@ -18587,11 +18929,11 @@ "baseline": "base", "baseline_status": "200", "cold": 0.011, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.288, - "loki_warm_p50": 0.018, + "loki_cold": 0.325, + "loki_warm_p50": 0.023, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -18600,7 +18942,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.011 + "warm_p50": 0.012 }, { "baseline": "base", @@ -18664,12 +19006,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.009, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.006, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.262, - "loki_warm_p50": 0.036, + "loki_cold": 0.335, + "loki_warm_p50": 0.04, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -18678,7 +19020,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.008 + "warm_p50": 0.007 }, { "baseline": "base", @@ -18740,12 +19082,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.021, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.018, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.287, - "loki_warm_p50": 0.039, + "loki_cold": 0.346, + "loki_warm_p50": 0.038, "parity_vs_loki": "differs", "range": "1h", "set": "control", @@ -18754,7 +19096,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.02 + "warm_p50": 0.019 }, { "baseline": "base", @@ -18817,11 +19159,11 @@ "baseline": "base", "baseline_status": "200", "cold": 0.004, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.032, - "loki_warm_p50": 0.033, + "loki_cold": 0.034, + "loki_warm_p50": 0.037, "parity_vs_loki": "same", "range": "instant", "set": "control", @@ -18830,7 +19172,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.005 + "warm_p50": 0.006 } ], "semantics/extracted-suffix-collision": [ @@ -20101,12 +20443,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.17, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.163, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.036, - "loki_warm_p50": 0.033, + "loki_cold": 0.042, + "loki_warm_p50": 0.054, "parity_vs_loki": "same", "range": "instant", "set": "control", @@ -20115,7 +20457,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.166 + "warm_p50": 0.161 } ], "semantics/json-label-spelling-probe": [ @@ -20272,147 +20614,491 @@ "warm_p50": 0.056 } ], - "semantics/label-browser-latency-cold-and-refresh": [ + "semantics/json-metric-detected-level-derivation": [ { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 0.02, - "date": "2026-10-01", - "label": "backfilled-empty-buckets-metadata-inventory", - "loki_cache_served": true, - "loki_cold": 0.006, - "loki_warm_p50": 0.003, + "cold": 0.52, + "date": "2026-10-14", + "label": "detlevel-1h", + "loki_cache_served": false, + "loki_cold": 0.184, + "loki_warm_p50": 0.012, "parity_vs_loki": "same", "range": "1h", - "set": "metadata-inventory", - "shape": "explore labels (label browser)", - "slower_than_loki_cold": false, + "set": "json-detected-level", + "shape": "DLV1 drilldown logs volume, json", + "slower_than_loki_cold": true, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.003 + "warm_p50": 0.5 }, { - "baseline": "main", - "baseline_status": "200", - "cold": 0.209, - "date": "2026-10-01", - "label": "backfilled-empty-buckets-metadata-inventory", - "loki_cache_served": true, - "loki_cold": 0.005, - "loki_warm_p50": 0.004, - "parity_vs_loki": "same", + "baseline": "base", + "baseline_status": "502", + "cold": 9.675, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", "range": "24h", - "set": "metadata-inventory", - "shape": "explore labels (label browser)", + "set": "json-detected-level", + "shape": "DLV1 drilldown logs volume, json", "slower_than_loki_cold": false, - "status": "200", + "status": "502", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.001 + "warm_p50": 9.633 }, { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 0.07, - "date": "2026-10-01", - "label": "backfilled-empty-buckets-metadata-inventory", - "loki_cache_served": true, - "loki_cold": 0.004, - "loki_warm_p50": 0.003, - "parity_vs_loki": "same", + "cold": 3.266, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", "range": "6h", - "set": "metadata-inventory", - "shape": "explore labels (label browser)", + "set": "json-detected-level", + "shape": "DLV1 drilldown logs volume, json", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.003 + "warm_p50": 3.159 }, { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 0.226, - "date": "2026-10-01", - "label": "backfilled-empty-buckets-metadata-inventory", - "loki_cache_served": true, - "loki_cold": 0.005, - "loki_warm_p50": 0.004, + "cold": 2.312, + "date": "2026-10-14", + "label": "detlevel-1h", + "loki_cache_served": false, + "loki_cold": 0.372, + "loki_warm_p50": 0.025, "parity_vs_loki": "same", - "range": "7d", - "set": "metadata-inventory", - "shape": "explore labels (label browser)", - "slower_than_loki_cold": false, + "range": "1h", + "set": "json-detected-level", + "shape": "DLV2 json error filter, detected_level", + "slower_than_loki_cold": true, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.001 + "warm_p50": 2.228 }, { - "baseline": "main", + "baseline": "base", + "baseline_status": "502", + "cold": 7.927, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-detected-level", + "shape": "DLV2 json error filter, detected_level", + "slower_than_loki_cold": false, + "status": "502", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 7.987 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 7.811, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-detected-level", + "shape": "DLV2 json error filter, detected_level", + "slower_than_loki_cold": false, + "status": "502", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 7.965 + }, + { + "baseline": "base", "baseline_status": "200", - "cold": 0.018, - "date": "2026-10-01", - "label": "backfilled-empty-buckets-metadata-inventory", - "loki_cache_served": true, - "loki_cold": 0.022, - "loki_warm_p50": 0.014, + "cold": 0.526, + "date": "2026-10-14", + "label": "detlevel-1h", + "loki_cache_served": false, + "loki_cold": 0.087, + "loki_warm_p50": 0.008, "parity_vs_loki": "same", "range": "1h", - "set": "metadata-inventory", - "shape": "explore labels with selector", - "slower_than_loki_cold": false, + "set": "json-detected-level", + "shape": "DLV3 plain json, detected_level", + "slower_than_loki_cold": true, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.003 + "warm_p50": 0.53 }, { - "baseline": "main", - "baseline_status": "200", - "cold": 0.241, - "date": "2026-10-01", - "label": "backfilled-empty-buckets-metadata-inventory", - "loki_cache_served": true, - "loki_cold": 0.013, - "loki_warm_p50": 0.014, - "parity_vs_loki": "same", + "baseline": "base", + "baseline_status": "502", + "cold": 9.774, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", "range": "24h", - "set": "metadata-inventory", - "shape": "explore labels with selector", + "set": "json-detected-level", + "shape": "DLV3 plain json, detected_level", "slower_than_loki_cold": false, - "status": "200", + "status": "502", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.001 + "warm_p50": 9.57 }, { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 0.061, - "date": "2026-10-01", - "label": "backfilled-empty-buckets-metadata-inventory", - "loki_cache_served": true, - "loki_cold": 0.014, - "loki_warm_p50": 0.012, - "parity_vs_loki": "same", + "cold": 3.234, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", "range": "6h", - "set": "metadata-inventory", - "shape": "explore labels with selector", + "set": "json-detected-level", + "shape": "DLV3 plain json, detected_level", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.003 + "warm_p50": 3.212 }, { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 0.256, - "date": "2026-10-01", - "label": "backfilled-empty-buckets-metadata-inventory", - "loki_cache_served": true, + "cold": 0.024, + "date": "2026-10-14", + "label": "detlevel-1h", + "loki_cache_served": false, + "loki_cold": 0.106, + "loki_warm_p50": 0.005, + "parity_vs_loki": "same", + "range": "1h", + "set": "json-detected-level", + "shape": "DLV4 explore json volume, level and detected_level", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.048 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.804, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-detected-level", + "shape": "DLV4 explore json volume, level and detected_level", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.7 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.187, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-detected-level", + "shape": "DLV4 explore json volume, level and detected_level", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.106 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.007, + "date": "2026-10-14", + "label": "detlevel-1h", + "loki_cache_served": false, + "loki_cold": 0.424, + "loki_warm_p50": 0.021, + "parity_vs_loki": "same", + "range": "1h", + "set": "json-detected-level", + "shape": "DLV5 json error dropped, detected_level", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.008 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.185, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-detected-level", + "shape": "DLV5 json error dropped, detected_level", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 0.174 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.027, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-detected-level", + "shape": "DLV5 json error dropped, detected_level", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 0.028 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.009, + "date": "2026-10-14", + "label": "detlevel-1h", + "loki_cache_served": false, + "loki_cold": 0.047, + "loki_warm_p50": 0.017, + "parity_vs_loki": "same", + "range": "1h", + "set": "json-detected-level", + "shape": "DLV6 drilldown detected_level breakdown, json", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 0.009 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 0.142, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-detected-level", + "shape": "DLV6 drilldown detected_level breakdown, json", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 0.178 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 0.027, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-detected-level", + "shape": "DLV6 drilldown detected_level breakdown, json", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 0.025 + } + ], + "semantics/label-browser-latency-cold-and-refresh": [ + { + "baseline": "main", + "baseline_status": "200", + "cold": 0.02, + "date": "2026-10-01", + "label": "backfilled-empty-buckets-metadata-inventory", + "loki_cache_served": true, + "loki_cold": 0.006, + "loki_warm_p50": 0.003, + "parity_vs_loki": "same", + "range": "1h", + "set": "metadata-inventory", + "shape": "explore labels (label browser)", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.003 + }, + { + "baseline": "main", + "baseline_status": "200", + "cold": 0.209, + "date": "2026-10-01", + "label": "backfilled-empty-buckets-metadata-inventory", + "loki_cache_served": true, + "loki_cold": 0.005, + "loki_warm_p50": 0.004, + "parity_vs_loki": "same", + "range": "24h", + "set": "metadata-inventory", + "shape": "explore labels (label browser)", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.001 + }, + { + "baseline": "main", + "baseline_status": "200", + "cold": 0.07, + "date": "2026-10-01", + "label": "backfilled-empty-buckets-metadata-inventory", + "loki_cache_served": true, + "loki_cold": 0.004, + "loki_warm_p50": 0.003, + "parity_vs_loki": "same", + "range": "6h", + "set": "metadata-inventory", + "shape": "explore labels (label browser)", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.003 + }, + { + "baseline": "main", + "baseline_status": "200", + "cold": 0.226, + "date": "2026-10-01", + "label": "backfilled-empty-buckets-metadata-inventory", + "loki_cache_served": true, + "loki_cold": 0.005, + "loki_warm_p50": 0.004, + "parity_vs_loki": "same", + "range": "7d", + "set": "metadata-inventory", + "shape": "explore labels (label browser)", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.001 + }, + { + "baseline": "main", + "baseline_status": "200", + "cold": 0.018, + "date": "2026-10-01", + "label": "backfilled-empty-buckets-metadata-inventory", + "loki_cache_served": true, + "loki_cold": 0.022, + "loki_warm_p50": 0.014, + "parity_vs_loki": "same", + "range": "1h", + "set": "metadata-inventory", + "shape": "explore labels with selector", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.003 + }, + { + "baseline": "main", + "baseline_status": "200", + "cold": 0.241, + "date": "2026-10-01", + "label": "backfilled-empty-buckets-metadata-inventory", + "loki_cache_served": true, + "loki_cold": 0.013, + "loki_warm_p50": 0.014, + "parity_vs_loki": "same", + "range": "24h", + "set": "metadata-inventory", + "shape": "explore labels with selector", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.001 + }, + { + "baseline": "main", + "baseline_status": "200", + "cold": 0.061, + "date": "2026-10-01", + "label": "backfilled-empty-buckets-metadata-inventory", + "loki_cache_served": true, + "loki_cold": 0.014, + "loki_warm_p50": 0.012, + "parity_vs_loki": "same", + "range": "6h", + "set": "metadata-inventory", + "shape": "explore labels with selector", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.003 + }, + { + "baseline": "main", + "baseline_status": "200", + "cold": 0.256, + "date": "2026-10-01", + "label": "backfilled-empty-buckets-metadata-inventory", + "loki_cache_served": true, "loki_cold": 0.044, "loki_warm_p50": 0.019, "parity_vs_loki": "same", @@ -20884,30 +21570,30 @@ ], "semantics/label-filter-before-parser-pushdown": [ { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 0.172, - "date": "2026-10-01", - "label": "ordered-json-metric-latency-gap-1h", + "cold": 0.617, + "date": "2026-10-14", + "label": "detlevel-dl6-capped-1h", "loki_cache_served": false, - "loki_cold": 0.348, - "loki_warm_p50": 0.021, - "parity_vs_loki": "same series, sum \u00b10.22%", + "loki_cold": 0.331, + "loki_warm_p50": 0.017, + "parity_vs_loki": "same series, sum \u00b10.27%", "range": "1h", "set": "metric-latency-gap", "shape": "DL6 explore volume, level filter before json", - "slower_than_loki_cold": false, + "slower_than_loki_cold": true, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.058 + "warm_p50": 0.279 }, { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 3.329, - "date": "2026-10-01", - "label": "ordered-json-metric-latency-gap-long", + "cold": 12.493, + "date": "2026-10-14", + "label": "detlevel-dl6-capped-long", "loki_cache_served": false, "loki_cold": null, "loki_warm_p50": null, @@ -20919,7 +21605,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 1.082 + "warm_p50": 6.904 }, { "baseline": "main", @@ -20941,11 +21627,11 @@ "warm_p50": 0.139 }, { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 1.211, - "date": "2026-10-01", - "label": "ordered-json-metric-latency-gap-long", + "cold": 3.805, + "date": "2026-10-14", + "label": "detlevel-dl6-capped-long", "loki_cache_served": false, "loki_cold": null, "loki_warm_p50": null, @@ -20957,7 +21643,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 1.665 + "warm_p50": 1.598 }, { "baseline": "main", @@ -26880,12 +27566,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.015, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.025, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.271, - "loki_warm_p50": 0.019, + "loki_cold": 0.333, + "loki_warm_p50": 0.025, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -26894,7 +27580,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.013 + "warm_p50": 0.016 }, { "baseline": "base", @@ -26956,12 +27642,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.013, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.015, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.271, - "loki_warm_p50": 0.013, + "loki_cold": 0.342, + "loki_warm_p50": 0.017, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -26970,7 +27656,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.01 + "warm_p50": 0.014 }, { "baseline": "base", @@ -27032,13 +27718,13 @@ ], "severity-detected-level-derivation": [ { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 0.037, - "date": "2026-10-01", - "label": "ordered-json-json-filter-pushdown-1h", + "cold": 0.096, + "date": "2026-10-14", + "label": "detlevel-d-capped-1h", "loki_cache_served": false, - "loki_cold": 0.37, + "loki_cold": 0.909, "loki_warm_p50": 0.029, "parity_vs_loki": "same", "range": "1h", @@ -27048,14 +27734,14 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.043 + "warm_p50": 0.098 }, { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 0.704, - "date": "2026-10-01", - "label": "ordered-json-json-filter-pushdown-long", + "cold": 2.41, + "date": "2026-10-14", + "label": "detlevel-d-capped-long", "loki_cache_served": false, "loki_cold": null, "loki_warm_p50": null, @@ -27067,7 +27753,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.406 + "warm_p50": 2.402 }, { "baseline": "main", @@ -27089,11 +27775,11 @@ "warm_p50": 0.354 }, { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 0.139, - "date": "2026-10-01", - "label": "ordered-json-json-filter-pushdown-long", + "cold": 0.611, + "date": "2026-10-14", + "label": "detlevel-d-capped-long", "loki_cache_served": false, "loki_cold": null, "loki_warm_p50": null, @@ -27105,16 +27791,16 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.12 + "warm_p50": 0.608 }, { "baseline": "base", "baseline_status": "200", - "cold": 0.037, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.093, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.335, + "loki_cold": 0.451, "loki_warm_p50": 0.021, "parity_vs_loki": "same", "range": "1h", @@ -27124,7 +27810,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.044 + "warm_p50": 0.105 }, { "baseline": "base", @@ -27186,12 +27872,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.057, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.101, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.472, - "loki_warm_p50": 0.027, + "loki_cold": 0.603, + "loki_warm_p50": 0.035, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -27200,7 +27886,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.093 + "warm_p50": 0.111 }, { "baseline": "base", @@ -27262,12 +27948,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.007, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.008, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.274, - "loki_warm_p50": 0.018, + "loki_cold": 0.346, + "loki_warm_p50": 0.02, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -27276,7 +27962,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.007 + "warm_p50": 0.008 }, { "baseline": "base", @@ -27338,12 +28024,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.008, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.027, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.282, - "loki_warm_p50": 0.017, + "loki_cold": 0.333, + "loki_warm_p50": 0.018, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -27352,7 +28038,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.006 + "warm_p50": 0.02 }, { "baseline": "base", @@ -27415,11 +28101,11 @@ "baseline": "base", "baseline_status": "200", "cold": 0.007, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.274, - "loki_warm_p50": 0.011, + "loki_cold": 0.32, + "loki_warm_p50": 0.013, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -27490,12 +28176,12 @@ { "baseline": "base", "baseline_status": "200", - "cold": 0.027, - "date": "2026-10-12", - "label": "patfilter-control-1h", + "cold": 0.023, + "date": "2026-10-14", + "label": "detlevel-control-1h", "loki_cache_served": false, - "loki_cold": 0.281, - "loki_warm_p50": 0.01, + "loki_cold": 0.347, + "loki_warm_p50": 0.012, "parity_vs_loki": "same", "range": "1h", "set": "control", @@ -27504,7 +28190,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.02 + "warm_p50": 0.018 }, { "baseline": "base", @@ -27564,30 +28250,30 @@ "warm_p50": 0.065 }, { - "baseline": "main", - "baseline_status": "200", - "cold": 0.172, - "date": "2026-10-01", - "label": "ordered-json-metric-latency-gap-1h", + "baseline": "base", + "baseline_status": "200", + "cold": 0.617, + "date": "2026-10-14", + "label": "detlevel-dl6-capped-1h", "loki_cache_served": false, - "loki_cold": 0.348, - "loki_warm_p50": 0.021, - "parity_vs_loki": "same series, sum \u00b10.22%", + "loki_cold": 0.331, + "loki_warm_p50": 0.017, + "parity_vs_loki": "same series, sum \u00b10.27%", "range": "1h", "set": "metric-latency-gap", "shape": "DL6 explore volume, level filter before json", - "slower_than_loki_cold": false, + "slower_than_loki_cold": true, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.058 + "warm_p50": 0.279 }, { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 3.329, - "date": "2026-10-01", - "label": "ordered-json-metric-latency-gap-long", + "cold": 12.493, + "date": "2026-10-14", + "label": "detlevel-dl6-capped-long", "loki_cache_served": false, "loki_cold": null, "loki_warm_p50": null, @@ -27599,7 +28285,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 1.082 + "warm_p50": 6.904 }, { "baseline": "main", @@ -27621,11 +28307,11 @@ "warm_p50": 0.139 }, { - "baseline": "main", + "baseline": "base", "baseline_status": "200", - "cold": 1.211, - "date": "2026-10-01", - "label": "ordered-json-metric-latency-gap-long", + "cold": 3.805, + "date": "2026-10-14", + "label": "detlevel-dl6-capped-long", "loki_cache_served": false, "loki_cold": null, "loki_warm_p50": null, @@ -27637,7 +28323,7 @@ "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 1.665 + "warm_p50": 1.598 }, { "baseline": "main", @@ -27885,6 +28571,348 @@ "valid": true, "verdict_vs_baseline": "same", "warm_p50": 0.129 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.52, + "date": "2026-10-14", + "label": "detlevel-1h", + "loki_cache_served": false, + "loki_cold": 0.184, + "loki_warm_p50": 0.012, + "parity_vs_loki": "same", + "range": "1h", + "set": "json-detected-level", + "shape": "DLV1 drilldown logs volume, json", + "slower_than_loki_cold": true, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.5 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 9.675, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-detected-level", + "shape": "DLV1 drilldown logs volume, json", + "slower_than_loki_cold": false, + "status": "502", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 9.633 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 3.266, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-detected-level", + "shape": "DLV1 drilldown logs volume, json", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 3.159 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 2.312, + "date": "2026-10-14", + "label": "detlevel-1h", + "loki_cache_served": false, + "loki_cold": 0.372, + "loki_warm_p50": 0.025, + "parity_vs_loki": "same", + "range": "1h", + "set": "json-detected-level", + "shape": "DLV2 json error filter, detected_level", + "slower_than_loki_cold": true, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 2.228 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 7.927, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-detected-level", + "shape": "DLV2 json error filter, detected_level", + "slower_than_loki_cold": false, + "status": "502", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 7.987 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 7.811, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-detected-level", + "shape": "DLV2 json error filter, detected_level", + "slower_than_loki_cold": false, + "status": "502", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 7.965 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.526, + "date": "2026-10-14", + "label": "detlevel-1h", + "loki_cache_served": false, + "loki_cold": 0.087, + "loki_warm_p50": 0.008, + "parity_vs_loki": "same", + "range": "1h", + "set": "json-detected-level", + "shape": "DLV3 plain json, detected_level", + "slower_than_loki_cold": true, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.53 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 9.774, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-detected-level", + "shape": "DLV3 plain json, detected_level", + "slower_than_loki_cold": false, + "status": "502", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 9.57 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 3.234, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-detected-level", + "shape": "DLV3 plain json, detected_level", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 3.212 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.024, + "date": "2026-10-14", + "label": "detlevel-1h", + "loki_cache_served": false, + "loki_cold": 0.106, + "loki_warm_p50": 0.005, + "parity_vs_loki": "same", + "range": "1h", + "set": "json-detected-level", + "shape": "DLV4 explore json volume, level and detected_level", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.048 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.804, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-detected-level", + "shape": "DLV4 explore json volume, level and detected_level", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.7 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.187, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-detected-level", + "shape": "DLV4 explore json volume, level and detected_level", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.106 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.007, + "date": "2026-10-14", + "label": "detlevel-1h", + "loki_cache_served": false, + "loki_cold": 0.424, + "loki_warm_p50": 0.021, + "parity_vs_loki": "same", + "range": "1h", + "set": "json-detected-level", + "shape": "DLV5 json error dropped, detected_level", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.008 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.185, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-detected-level", + "shape": "DLV5 json error dropped, detected_level", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 0.174 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.027, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-detected-level", + "shape": "DLV5 json error dropped, detected_level", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 0.028 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.009, + "date": "2026-10-14", + "label": "detlevel-1h", + "loki_cache_served": false, + "loki_cold": 0.047, + "loki_warm_p50": 0.017, + "parity_vs_loki": "same", + "range": "1h", + "set": "json-detected-level", + "shape": "DLV6 drilldown detected_level breakdown, json", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 0.009 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 0.142, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-detected-level", + "shape": "DLV6 drilldown detected_level breakdown, json", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 0.178 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 0.027, + "date": "2026-10-14", + "label": "detlevel-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-detected-level", + "shape": "DLV6 drilldown detected_level breakdown, json", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 0.025 } ] } diff --git a/conformance/registry/generated/wiring.json b/conformance/registry/generated/wiring.json index 8f695a3fe..37ca347cb 100644 --- a/conformance/registry/generated/wiring.json +++ b/conformance/registry/generated/wiring.json @@ -131,6 +131,7 @@ "internal/proxy/drilldown_breakdown_matrix_test.go", "internal/proxy/series_limit_routes_test.go", "internal/proxy/tumbling_axis_parity_test.go", + "test/e2e-compat/json_detected_level_series_limit_test.go", "test/e2e-compat/range_metric_offset_series_limit_test.go" ], "limits/stats-row-limit-per-grid": [ @@ -377,6 +378,10 @@ "semantics/client-cancellation-propagates": [ "internal/proxy/backend_heavy_test.go" ], + "semantics/detected-level-pipes-known-differences": [ + "internal/proxy/detected_level_presence_test.go", + "internal/proxy/ordered_json_detected_level_test.go" + ], "semantics/extracted-metric-known-differences": [ "internal/proxy/ordered_json_extracted_differential_test.go" ], @@ -420,6 +425,16 @@ "internal/proxy/ordered_json_filter_pushdown_test.go", "test/e2e-compat/json_filter_pushdown_test.go" ], + "semantics/json-metric-detected-level-derivation": [ + "internal/proxy/detected_level_presence_sweep_test.go", + "internal/proxy/detected_level_presence_test.go", + "internal/proxy/detected_level_shared_unpack_test.go", + "internal/proxy/ordered_json_detected_level_test.go", + "internal/proxy/stored_level_series_limit_test.go", + "test/e2e-compat/detected_level_variants_test.go", + "test/e2e-compat/json_detected_level_series_limit_test.go", + "test/e2e-compat/json_detected_level_test.go" + ], "semantics/label-browser-latency-cold-and-refresh": [ "internal/proxy/metadata_inventory_test.go" ], @@ -545,6 +560,7 @@ ], "severity-detected-level-derivation": [ "internal/proxy/detected_level_test.go", + "internal/translator/detected_level_keyword_test.go", "internal/translator/detected_level_logsql_test.go", "test/e2e-compat/detected_level_corpus_test.go", "test/e2e-compat/detected_level_test.go" @@ -580,7 +596,7 @@ "test/e2e-compat/unwrap_ungrouped_sum_test.go" ] }, - "registry_items": 303, + "registry_items": 305, "routes_without_registry_item": [], "test_to_items": { "cmd/proxy/main_test.go": [ @@ -638,9 +654,19 @@ "internal/proxy/compat_cache_freshness_test.go": [ "semantics/metadata-answers-include-last-24h-like-loki" ], + "internal/proxy/detected_level_presence_sweep_test.go": [ + "semantics/json-metric-detected-level-derivation" + ], + "internal/proxy/detected_level_presence_test.go": [ + "semantics/detected-level-pipes-known-differences", + "semantics/json-metric-detected-level-derivation" + ], "internal/proxy/detected_level_response_test.go": [ "severity-exposure-surfaces" ], + "internal/proxy/detected_level_shared_unpack_test.go": [ + "semantics/json-metric-detected-level-derivation" + ], "internal/proxy/detected_level_test.go": [ "severity-detected-level-derivation" ], @@ -771,6 +797,10 @@ "semantics/json-label-spelling-probe", "semantics/label-filter-before-parser-pushdown" ], + "internal/proxy/ordered_json_detected_level_test.go": [ + "semantics/detected-level-pipes-known-differences", + "semantics/json-metric-detected-level-derivation" + ], "internal/proxy/ordered_json_drilldown_test.go": [ "parser-error-and-label-collision", "semantics/json-filter-pushdown-translated-label" @@ -851,6 +881,9 @@ "profiles/parsed-fields-without-parser", "profiles/stage-field-exposure" ], + "internal/proxy/stored_level_series_limit_test.go": [ + "semantics/json-metric-detected-level-derivation" + ], "internal/proxy/tail_pipeline_test.go": [ "loki_api_v1_tail" ], @@ -922,6 +955,9 @@ "internal/rulesmigrate/migrate_test.go": [ "semantics/parenthesized-log-range" ], + "internal/translator/detected_level_keyword_test.go": [ + "severity-detected-level-derivation" + ], "internal/translator/detected_level_logsql_test.go": [ "severity-detected-level-derivation", "severity-exposure-surfaces" @@ -984,6 +1020,9 @@ "severity-detected-level-derivation", "severity-exposure-surfaces" ], + "test/e2e-compat/detected_level_variants_test.go": [ + "semantics/json-metric-detected-level-derivation" + ], "test/e2e-compat/drilldown_field_breakdown_test.go": [ "limits/drilldown-breakdown-exact", "loki_api_v1_query_range", @@ -1022,6 +1061,13 @@ "semantics/instant-result-type-missing", "semantics/scalar-sample-time-units" ], + "test/e2e-compat/json_detected_level_series_limit_test.go": [ + "limits/series-limit-error", + "semantics/json-metric-detected-level-derivation" + ], + "test/e2e-compat/json_detected_level_test.go": [ + "semantics/json-metric-detected-level-derivation" + ], "test/e2e-compat/json_filter_pushdown_test.go": [ "parser-error-and-label-collision", "semantics/json-filter-pushdown-translated-label", @@ -1154,7 +1200,7 @@ "profiles/structured-metadata-keys-per-profile" ] }, - "tests_with_markers": 107, + "tests_with_markers": 116, "unknown_ids": {}, "unwired_items": [ "api_prom_label", @@ -1261,5 +1307,5 @@ "versions/capabilities-floor-vs-matrix", "versions/rejected-stats-query-wording-vl-1.47" ], - "wired_items": 111 + "wired_items": 113 } diff --git a/conformance/reports/gaps.md b/conformance/reports/gaps.md index 11332b8ff..0dcb70e01 100644 --- a/conformance/reports/gaps.md +++ b/conformance/reports/gaps.md @@ -73,7 +73,7 @@ Semantics, severity, identity and data-quality behaviour the proxy must reproduc | semantics | `parser-error-and-label-collision` — Parser errors, __error__ and _extracted collisions | 21 | 10 | partial | | limits | `series-limits-and-partial-results` — Series limits: error, or partial result with a warning | 9 | 9 | partial | | identity | `service-name-derivation` — service_name follows Loki's discovery order | 9 | 0 | partial | -| severity | `severity-detected-level-derivation` — detected_level is derived on the read path | 10 | 4 | partial | +| severity | `severity-detected-level-derivation` — detected_level is derived on the read path | 10 | 5 | partial | | severity | `severity-exposure-surfaces` — Where detected_level and level must appear | 7 | 3 | partial | | limits | `tenant-query-limits` — Loki's per-tenant query limits, enforced as published | 13 | 3 | proven | | semantics | `window-bounds-and-step-alignment` — Range windows, bucket edges and step alignment | 8 | 5 | partial | @@ -125,9 +125,10 @@ cold request (bench/ab runs; details in [performance.md](performance.md)). | `semantics/unwrap-conversion-error` | 15 | 13.13s (E3 logfmt latency_ms rate, production, 7d) | 0.01s | | `semantics/extracted-suffix-collision` | 14 | 2.65s (X1 json filter level_extracted, by level, 6h) | 0.01s | | `semantics/json-extracted-metric-pushdown` | 13 | 2.65s (X1 json filter level_extracted, by level, 6h) | 0.01s | -| `explore-logs-volume-json` | 6 | 0.57s (DL6 explore volume, level filter before json, 3h) | 0.01s | -| `semantics/label-filter-before-parser-pushdown` | 2 | 0.57s (DL6 explore volume, level filter before json, 3h) | 0.01s | -| `severity-detected-level-derivation` | 5 | 0.57s (DL6 explore volume, level filter before json, 3h) | 0.01s | +| `semantics/json-metric-detected-level-derivation` | 3 | 2.31s (DLV2 json error filter, detected_level, 1h) | 0.37s | +| `severity-detected-level-derivation` | 9 | 2.31s (DLV2 json error filter, detected_level, 1h) | 0.37s | +| `explore-logs-volume-json` | 7 | 0.57s (DL6 explore volume, level filter before json, 3h) | 0.01s | +| `semantics/label-filter-before-parser-pushdown` | 3 | 0.57s (DL6 explore volume, level filter before json, 3h) | 0.01s | | `parsed-label-series-identity` | 5 | 0.49s (E field breakdown pipeline (explore), 3h) | 0.03s | | `parser-json` | 2 | 0.49s (E field breakdown pipeline (explore), 3h) | 0.03s | | `loki-compatible-profile` | 3 | 0.52s (E7 json count by level_extracted, 1h) | 0.09s | diff --git a/conformance/reports/parity-gaps.md b/conformance/reports/parity-gaps.md index 26fdc7c65..9724fc099 100644 --- a/conformance/reports/parity-gaps.md +++ b/conformance/reports/parity-gaps.md @@ -69,33 +69,34 @@ Last run: 2026-10-05T20:16:15Z on `fcaccfcc`, window [1791220800, 1791224400]. | 48 | `profiles/json-extraction-list-missing-key` — A json extraction-list key absent from the line is dropped; Loki keeps it in the metric series (open) | open | api-only | 2 | 2 | S | internal/proxy stage-field exposure (extraction lists, metric path) | pkg/logql/log/parser.go:671-720 JSONExpressionParser.Process | TestCompat_JSONExtractionListMissingKey (test/e2e-compat, group semantics) | | 49 | `semantics/unpack-non-string-values` — `\| unpack` extracts non-string JSON values that Loki ignores (open) | open | api-only | 2 | 2 | S | internal/translator unpack (string values only) | pkg/logql/log/parser.go:789-820 UnpackParser.unpack | TestCompat_UnpackStringValuesOnly (test/e2e-compat, group semantics) | | 50 | `semantics/pattern-line-filter-invalid-captures` — A pattern line filter with consecutive or named captures is a 400 on Loki; the proxy answers it (open) | open | api-only | — | — | S | internal/translator/translator.go planPatternLineFilter (patterns are translated without validation) | pkg/logql/log/pattern/pattern.go:36-52 (ParseLineFilter, validateNoConsecutiveCaptures, validateNoNamedCaptures) | TestCompat_PatternLineFilterInvalidCaptures (test/e2e-compat, group core) | -| 51 | `semantics/extracted-derived-labels-raw-evaluator` — service_name_extracted and detected_level_extracted metrics keep the raw-row evaluator (open) | open | edge | — | — | S | internal/proxy orderedJSONPushdownLabelOK (service_name, detected_level) | pkg/logql/log/parser.go duplicateSuffix; service_name set by discover_service_name at ingest | TestCompat_ExtractedLabelMetricPushdown rows for service_name_extracted (test/e2e-compat, group core) | -| 52 | `semantics/extracted-metric-known-differences` — Two name_extracted differences against Loki remain in the metric pushdown and the raw evaluator (open) | open | edge | — | — | M | internal/proxy raw evaluator parsed-label naming (processWithStreamLabels) and translator.ExtractedLabelPipes | pkg/logql/log/parser.go duplicateSuffix and ParserHint.Extracted (a key an earlier extraction produced is skipped) | TestCompat_ExtractedBothKeysInLineOrder (test/e2e-compat, group core) | -| 53 | `semantics/unwrap-conversion-error-parser-divergence` — Where VictoriaLogs' logfmt parser reads another value than Loki's, an unwrap misses Loki's SampleExtractionErr or drops samples Loki keep... | open | edge | — | — | L | internal/translator logfmt unpacking (unpack_logfmt keeps the last value); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:379-438 LogfmtParser.Process; pkg/logql/log/logfmt/decode.go | TestUnwrapConversionErrorDifferential counts the first case; TestUnwrapLokiLabels pins the decoder; TestCompat_UnwrapConversionErrorParit... | -| 54 | `semantics/unwrap-conversion-error-postfilter-forms` — An unwrap whose post filter keeps only the error, compares a label, or combines filters with and/or answers 200 where Loki may fail with ... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorPostFilters; the translator's post-filter placement | pkg/logql/log/label_filter.go:90-100 BinaryLabelFilter.Process, 306-340 NumericLabelFilter.Process, 418-424 labelValue | TestUnwrapErrorProbePlans and TestUnwrapConversionErrorDifferential (internal/proxy) assert the detection stays off for these forms; flip... | -| 55 | `semantics/unwrap-conversion-error-stored-value-overwritten` — An unwrap over a label stored as structured metadata and also parsed from the line misses Loki's SampleExtractionErr (open) | open | edge | — | — | M | internal/translator parsed-key collision with stored fields (unpack keep_original_fields); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:151,809 duplicateSuffix | TestUnwrapConversionErrorDifferential counts the first case; TestCompat_UnwrapConversionErrorParity asserts the 200 side | -| 56 | `semantics/unwrap-conversion-error-undetected-shapes` — A few query shapes answer 200 where Loki fails with SampleExtractionErr, because no VictoriaLogs query of theirs carries the conversion c... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorProbes (opaque expressions), unwrapCountingQuery (pipes after stats), unwrapLokiLabe... | pkg/logql/evaluator.go:726-733 | TestCompat_UnwrapConversionErrorParity open/* asserts Loki's 400 and skips naming this case (absent_over_time, label_replace); TestUnwrap... | -| 57 | `semantics/unwrap-gate-parsefloat-divergences` — The unwrap sample gate drops a few values Loki's conversions accept or reject (open) | open | edge | — | — | S | internal/translator/unwrap_gate.go (UnwrapNumberPattern, UnwrapDurationPattern, UnwrapBytesPattern) | pkg/logql/log/metrics_extraction.go:316-338 convertFloat, convertDuration, convertBytes | TestUnwrapNumberPatternMatchesParseFloat (internal/translator) pins the known differences | -| 58 | `versions/stats-bucket-label-v1.45` — Stats-bucket metrics put a line on a bucket edge one bucket late on VictoriaLogs before v1.45 (open) | open | edge | — | — | M | stats-bucket relabel (internal/proxy range_metric_compat.go setSlidingStatsRangeParams / slidingStatsBucket; sliding_window_stats.go) | pkg/logql range vector (T-range, T] | TestRangeMetricCompatibilityJSONFilterPushdown on VictoriaLogs v1.40-v1.44 (version matrix); fix by relabelling edge lines or declining t... | -| 59 | `metric-identity/structured-metadata-splits-series` — Loki splits a stream's metric series per structured-metadata set; the proxy keeps the stream (owner-accepted deviation) | owner-kept | explore-visible | 173 | 37 | L | internal/translator series identity (stats by stream); VictoriaLogs cannot group by a row's remaining fields | pkg/logql/log/metrics_extraction.go:102-120 builder.Add(StructuredMetadataLabel) -> GroupedLabels | differential run cluster count (bench/parity) | -| 60 | `parsed-identity/dynamic-key-parsers` — `\| json` and `\| logfmt` metric series keep the stream identity by default (owner-accepted deviation, opt-in flag) | owner-kept | explore-visible | 95 | 29 | L | internal/translator parser series identity; -exact-parser-series-identity | pkg/logql/log/parser.go label extraction into the sample labels | differential run cluster count (bench/parity) | -| 61 | `profiles/label-filter-on-line-field` — A label filter on a line key no earlier stage exposes matches like Loki | owner-kept | explore-visible | — | — | M | internal/proxy label-filter refill (-label-filter-refill-max-pages); metric queries on keys no stage exposes | pkg/logql/log/label_filter.go (filter reads only labels the pipeline set) | TestCompat_LabelFilterOnLineField (exists, profile tests) | -| 62 | `profiles/detected-fields-no-extracted-suffix` — detected_fields lists service / service.name without Loki's _extracted suffix (owner decision) | owner-kept | drilldown-visible | — | — | S | internal/proxy/label_handlers.go handleDetectedFields | pkg/logql/log/parser.go:25 duplicateSuffix | bench/parity diff rule (documented facet, service fields only) | -| 63 | `profiles/json-line-stored-without-line` — A JSON line pushed without _msg is stored without its line (documented) | documented | explore-visible | — | — | L | client push configuration (disable_message_parsing=1); docs/KNOWN_ISSUES.md | pkg/distributor push keeps the line verbatim | cases/profiles/stage-field-exposure with disable_message_parsing pushes | -| 64 | `drilldown/detected-labels-sampled-cardinality` — detected_labels cardinality is sampled from the window's last minutes; Loki counts every stream its ingesters hold (documented) | documented | drilldown-visible | 20 | 2 | M | internal/proxy/patterns.go handleDetectedLabels | pkg/ingester/instance.go LabelsWithValues | bench/parity diff rule (documented facet) | -| 65 | `profiles/volume-detected-level-extension` — index/volume answers targetLabels=detected_level; Loki returns no volume for it (documented proxy extension) | documented | drilldown-visible | 17 | 2 | S | internal/proxy/volume.go | pkg/storage volume index (stream labels only) | differential run row volume by detected_level | -| 66 | `drilldown/patterns-mined-differently` — Patterns come from different miners on each side (documented) | documented | drilldown-visible | 15 | 3 | L | internal/proxy/patterns.go | pkg/pattern (ingester-side mining) | bench/visual history-dependent comparison (exists) | -| 67 | `profiles/volume-bytes-accounting` — Volume values count line bytes; Loki counts chunk bytes with structured metadata (documented) | documented | drilldown-visible | 4 | 3 | L | internal/proxy/volume.go; docs/KNOWN_ISSUES.md | pkg/storage volume index (chunk bytes incl. structured metadata) | bench/parity diff rule (documented facet) | -| 68 | `limits/drilldown-breakdown-keeps-busiest` — Over max_query_series a Drilldown breakdown keeps the busiest series; Loki keeps the first it meets (documented) | documented | drilldown-visible | 1 | 1 | S | internal/proxy series limit ranking | pkg/logql engine series limit | cases/limits/drilldown-partial-with-warning (exists) | -| 69 | `quality/index-stats-bytes-and-chunks` — index/stats bytes and chunks have no VictoriaLogs equivalent (documented) | documented | drilldown-visible | — | — | L | internal/proxy/label_handlers.go handleIndexStats; docs/KNOWN_ISSUES.md | pkg/storage index stats from chunk refs (bytes, chunks) | bench/parity diff rule (documented facet) | -| 70 | `profiles/grafana-stats-error-as-warning` — A failed stats query from Grafana answers 200 with warnings; Loki returns the error (documented) | documented | grafana-datasource | — | — | S | internal/proxy Grafana-sourced stats error handling | pkg/querier/queryrange error propagation (no warnings conversion) | TestCompat_GrafanaStatsErrorStatus (test/e2e-compat, fault injection) | -| 71 | `profiles/metadata-default-lookback` — Label endpoints without start/end default to a different lookback than Loki's 1h (documented) | documented | api-only | — | — | S | internal/config flag default -metadata-default-lookback | pkg/loghttp/params.go defaultSince = 1h | TestCompat_MetadataDefaultLookback (test/e2e-compat, group core) | -| 72 | `profiles/multi-tenant-volume-merge` — Multi-tenant volume returns per-tenant series with __tenant_id__; Loki sums the tenants' volumes (documented) | documented | api-only | — | — | S | internal/proxy/volume.go multi-tenant merge | pkg/querier/queryrange volume merge | TestCompat_MultiTenantVolume (test/e2e-compat, group tail-multitenancy) | -| 73 | `semantics/count-values-unsupported` — count_values() is rejected with 400 where Loki answers (documented) | documented | api-only | — | — | M | internal/translator + proxy-side post-aggregation of a stats matrix | pkg/logql/syntax OpTypeCountValues; vector evaluator | TestCompat_CountValues (test/e2e-compat, group semantics) | -| 74 | `semantics/eager-pipeline-validation` — Invalid pipeline stages fail with 400 regardless of data; Loki answers 200 empty over a range with no data (documented) | documented | edge | — | — | S | internal/translator validation | pkg/logql/log/fmt.go template build at pipeline construction | TestCompat_EagerValidationEmptyRange (test/e2e-compat, group semantics) | -| 75 | `semantics/ip-filter-precision` — ip() filters are approximated with regular expressions (documented) | documented | edge | — | — | M | internal/proxy/postprocess.go:84 ip filtering | pkg/logql/log/ip.go | TestCompat_IPFilterForms (test/e2e-compat, group semantics) | -| 76 | `drilldown/plugin-legend-sync-race` — Logs Drilldown crashes a breakdown page when a query answers before the panel module loads (upstream, not the proxy) | upstream | drilldown-visible | — | — | S | none (grafana-lokiexplore-app) | n/a (client race) | bench/visual capture waits for the panel module (already) | -| 77 | `quality/index-stats-loki-chunk-accounting` — Loki's index/stats counts chunks it holds twice; the proxy's counts equal Loki's own count_over_time (upstream) | upstream | drilldown-visible | — | — | S | none (Loki chunk index accounting) | pkg/storage stats from chunk refs (flushed and retained chunks both counted) | bench/parity cross-check (run.py loki_counts_itself) | +| 51 | `semantics/detected-level-pipes-known-differences` — The detected_level pipes read level keys in a fixed order, do not decode logfmt like Loki and overwrite a detected_level stream label (open) | open | edge | — | — | M | internal/translator detected_level_logsql.go BuildDetectedLevelChain; internal/proxy native metric route | pkg/distributor/field_detection.go getLevelUsingJSONParser (document order, depth < 2), labels.go (detected_level_extracted) | an e2e on plain-selector probe lines once the route probes and falls back to a raw-row or keyed evaluation | +| 52 | `semantics/extracted-derived-labels-raw-evaluator` — service_name_extracted and detected_level_extracted metrics keep the raw-row evaluator (open) | open | edge | — | — | S | internal/proxy orderedJSONPushdownLabelOK (service_name, detected_level) | pkg/logql/log/parser.go duplicateSuffix; service_name set by discover_service_name at ingest | TestCompat_ExtractedLabelMetricPushdown rows for service_name_extracted (test/e2e-compat, group core) | +| 53 | `semantics/extracted-metric-known-differences` — Two name_extracted differences against Loki remain in the metric pushdown and the raw evaluator (open) | open | edge | — | — | M | internal/proxy raw evaluator parsed-label naming (processWithStreamLabels) and translator.ExtractedLabelPipes | pkg/logql/log/parser.go duplicateSuffix and ParserHint.Extracted (a key an earlier extraction produced is skipped) | TestCompat_ExtractedBothKeysInLineOrder (test/e2e-compat, group core) | +| 54 | `semantics/unwrap-conversion-error-parser-divergence` — Where VictoriaLogs' logfmt parser reads another value than Loki's, an unwrap misses Loki's SampleExtractionErr or drops samples Loki keep... | open | edge | — | — | L | internal/translator logfmt unpacking (unpack_logfmt keeps the last value); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:379-438 LogfmtParser.Process; pkg/logql/log/logfmt/decode.go | TestUnwrapConversionErrorDifferential counts the first case; TestUnwrapLokiLabels pins the decoder; TestCompat_UnwrapConversionErrorParit... | +| 55 | `semantics/unwrap-conversion-error-postfilter-forms` — An unwrap whose post filter keeps only the error, compares a label, or combines filters with and/or answers 200 where Loki may fail with ... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorPostFilters; the translator's post-filter placement | pkg/logql/log/label_filter.go:90-100 BinaryLabelFilter.Process, 306-340 NumericLabelFilter.Process, 418-424 labelValue | TestUnwrapErrorProbePlans and TestUnwrapConversionErrorDifferential (internal/proxy) assert the detection stays off for these forms; flip... | +| 56 | `semantics/unwrap-conversion-error-stored-value-overwritten` — An unwrap over a label stored as structured metadata and also parsed from the line misses Loki's SampleExtractionErr (open) | open | edge | — | — | M | internal/translator parsed-key collision with stored fields (unpack keep_original_fields); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:151,809 duplicateSuffix | TestUnwrapConversionErrorDifferential counts the first case; TestCompat_UnwrapConversionErrorParity asserts the 200 side | +| 57 | `semantics/unwrap-conversion-error-undetected-shapes` — A few query shapes answer 200 where Loki fails with SampleExtractionErr, because no VictoriaLogs query of theirs carries the conversion c... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorProbes (opaque expressions), unwrapCountingQuery (pipes after stats), unwrapLokiLabe... | pkg/logql/evaluator.go:726-733 | TestCompat_UnwrapConversionErrorParity open/* asserts Loki's 400 and skips naming this case (absent_over_time, label_replace); TestUnwrap... | +| 58 | `semantics/unwrap-gate-parsefloat-divergences` — The unwrap sample gate drops a few values Loki's conversions accept or reject (open) | open | edge | — | — | S | internal/translator/unwrap_gate.go (UnwrapNumberPattern, UnwrapDurationPattern, UnwrapBytesPattern) | pkg/logql/log/metrics_extraction.go:316-338 convertFloat, convertDuration, convertBytes | TestUnwrapNumberPatternMatchesParseFloat (internal/translator) pins the known differences | +| 59 | `versions/stats-bucket-label-v1.45` — Stats-bucket metrics put a line on a bucket edge one bucket late on VictoriaLogs before v1.45 (open) | open | edge | — | — | M | stats-bucket relabel (internal/proxy range_metric_compat.go setSlidingStatsRangeParams / slidingStatsBucket; sliding_window_stats.go) | pkg/logql range vector (T-range, T] | TestRangeMetricCompatibilityJSONFilterPushdown on VictoriaLogs v1.40-v1.44 (version matrix); fix by relabelling edge lines or declining t... | +| 60 | `metric-identity/structured-metadata-splits-series` — Loki splits a stream's metric series per structured-metadata set; the proxy keeps the stream (owner-accepted deviation) | owner-kept | explore-visible | 173 | 37 | L | internal/translator series identity (stats by stream); VictoriaLogs cannot group by a row's remaining fields | pkg/logql/log/metrics_extraction.go:102-120 builder.Add(StructuredMetadataLabel) -> GroupedLabels | differential run cluster count (bench/parity) | +| 61 | `parsed-identity/dynamic-key-parsers` — `\| json` and `\| logfmt` metric series keep the stream identity by default (owner-accepted deviation, opt-in flag) | owner-kept | explore-visible | 95 | 29 | L | internal/translator parser series identity; -exact-parser-series-identity | pkg/logql/log/parser.go label extraction into the sample labels | differential run cluster count (bench/parity) | +| 62 | `profiles/label-filter-on-line-field` — A label filter on a line key no earlier stage exposes matches like Loki | owner-kept | explore-visible | — | — | M | internal/proxy label-filter refill (-label-filter-refill-max-pages); metric queries on keys no stage exposes | pkg/logql/log/label_filter.go (filter reads only labels the pipeline set) | TestCompat_LabelFilterOnLineField (exists, profile tests) | +| 63 | `profiles/detected-fields-no-extracted-suffix` — detected_fields lists service / service.name without Loki's _extracted suffix (owner decision) | owner-kept | drilldown-visible | — | — | S | internal/proxy/label_handlers.go handleDetectedFields | pkg/logql/log/parser.go:25 duplicateSuffix | bench/parity diff rule (documented facet, service fields only) | +| 64 | `profiles/json-line-stored-without-line` — A JSON line pushed without _msg is stored without its line (documented) | documented | explore-visible | — | — | L | client push configuration (disable_message_parsing=1); docs/KNOWN_ISSUES.md | pkg/distributor push keeps the line verbatim | cases/profiles/stage-field-exposure with disable_message_parsing pushes | +| 65 | `drilldown/detected-labels-sampled-cardinality` — detected_labels cardinality is sampled from the window's last minutes; Loki counts every stream its ingesters hold (documented) | documented | drilldown-visible | 20 | 2 | M | internal/proxy/patterns.go handleDetectedLabels | pkg/ingester/instance.go LabelsWithValues | bench/parity diff rule (documented facet) | +| 66 | `profiles/volume-detected-level-extension` — index/volume answers targetLabels=detected_level; Loki returns no volume for it (documented proxy extension) | documented | drilldown-visible | 17 | 2 | S | internal/proxy/volume.go | pkg/storage volume index (stream labels only) | differential run row volume by detected_level | +| 67 | `drilldown/patterns-mined-differently` — Patterns come from different miners on each side (documented) | documented | drilldown-visible | 15 | 3 | L | internal/proxy/patterns.go | pkg/pattern (ingester-side mining) | bench/visual history-dependent comparison (exists) | +| 68 | `profiles/volume-bytes-accounting` — Volume values count line bytes; Loki counts chunk bytes with structured metadata (documented) | documented | drilldown-visible | 4 | 3 | L | internal/proxy/volume.go; docs/KNOWN_ISSUES.md | pkg/storage volume index (chunk bytes incl. structured metadata) | bench/parity diff rule (documented facet) | +| 69 | `limits/drilldown-breakdown-keeps-busiest` — Over max_query_series a Drilldown breakdown keeps the busiest series; Loki keeps the first it meets (documented) | documented | drilldown-visible | 1 | 1 | S | internal/proxy series limit ranking | pkg/logql engine series limit | cases/limits/drilldown-partial-with-warning (exists) | +| 70 | `quality/index-stats-bytes-and-chunks` — index/stats bytes and chunks have no VictoriaLogs equivalent (documented) | documented | drilldown-visible | — | — | L | internal/proxy/label_handlers.go handleIndexStats; docs/KNOWN_ISSUES.md | pkg/storage index stats from chunk refs (bytes, chunks) | bench/parity diff rule (documented facet) | +| 71 | `profiles/grafana-stats-error-as-warning` — A failed stats query from Grafana answers 200 with warnings; Loki returns the error (documented) | documented | grafana-datasource | — | — | S | internal/proxy Grafana-sourced stats error handling | pkg/querier/queryrange error propagation (no warnings conversion) | TestCompat_GrafanaStatsErrorStatus (test/e2e-compat, fault injection) | +| 72 | `profiles/metadata-default-lookback` — Label endpoints without start/end default to a different lookback than Loki's 1h (documented) | documented | api-only | — | — | S | internal/config flag default -metadata-default-lookback | pkg/loghttp/params.go defaultSince = 1h | TestCompat_MetadataDefaultLookback (test/e2e-compat, group core) | +| 73 | `profiles/multi-tenant-volume-merge` — Multi-tenant volume returns per-tenant series with __tenant_id__; Loki sums the tenants' volumes (documented) | documented | api-only | — | — | S | internal/proxy/volume.go multi-tenant merge | pkg/querier/queryrange volume merge | TestCompat_MultiTenantVolume (test/e2e-compat, group tail-multitenancy) | +| 74 | `semantics/count-values-unsupported` — count_values() is rejected with 400 where Loki answers (documented) | documented | api-only | — | — | M | internal/translator + proxy-side post-aggregation of a stats matrix | pkg/logql/syntax OpTypeCountValues; vector evaluator | TestCompat_CountValues (test/e2e-compat, group semantics) | +| 75 | `semantics/eager-pipeline-validation` — Invalid pipeline stages fail with 400 regardless of data; Loki answers 200 empty over a range with no data (documented) | documented | edge | — | — | S | internal/translator validation | pkg/logql/log/fmt.go template build at pipeline construction | TestCompat_EagerValidationEmptyRange (test/e2e-compat, group semantics) | +| 76 | `semantics/ip-filter-precision` — ip() filters are approximated with regular expressions (documented) | documented | edge | — | — | M | internal/proxy/postprocess.go:84 ip filtering | pkg/logql/log/ip.go | TestCompat_IPFilterForms (test/e2e-compat, group semantics) | +| 77 | `drilldown/plugin-legend-sync-race` — Logs Drilldown crashes a breakdown page when a query answers before the panel module loads (upstream, not the proxy) | upstream | drilldown-visible | — | — | S | none (grafana-lokiexplore-app) | n/a (client race) | bench/visual capture waits for the panel module (already) | +| 78 | `quality/index-stats-loki-chunk-accounting` — Loki's index/stats counts chunks it holds twice; the proxy's counts equal Loki's own count_over_time (upstream) | upstream | drilldown-visible | — | — | S | none (Loki chunk index accounting) | pkg/storage stats from chunk refs (flushed and retained chunks both counted) | bench/parity cross-check (run.py loki_counts_itself) | ## Fixed, awaiting the next run @@ -109,6 +110,7 @@ Last run: 2026-10-05T20:16:15Z on `fcaccfcc`, window [1791220800, 1791224400]. | `semantics/plain-name-after-parser-reads-stream-label` — A filter or grouping on a name a parser also reads reads the parsed value; Loki reads the stream label (fixed) | #676 | 4 | 6 | | `semantics/pattern-line-filter` — An empty pattern line filter (`\|> ""`, `!> ""`) matches only the empty line, like Loki (fixed) | #743 | 2 | 2 | | `semantics/json-extracted-metric-pushdown` — A metric filtered or grouped on a name_extracted label after `\| json` is answered from VictoriaLogs stats (fixed) | #690 | 0 | 0 | +| `semantics/json-metric-detected-level-derivation` — sum by (detected_level) over a JSON pipeline follows the level Loki's ingest gave each line, by level key in document order, by keyword s... | #760 | 0 | 0 | | `semantics/labels-end-inclusive-like-loki` — /labels and /label/{name}/values include data at the request's end, so a zero-width request (Grafana "Show context") lists the row's labe... | #703 | 0 | 0 | | `semantics/unwrap-conversion-error` — An unwrap over a value its conversion rejects fails like Loki, with a 400 SampleExtractionErr (fixed) | #704 | 0 | 0 | | `semantics/unwrap-duration-seconds` — '`unwrap duration_seconds(label)` is converted like duration(), as Loki does (fixed)' | #729 | 0 | 0 | diff --git a/conformance/reports/performance.md b/conformance/reports/performance.md index 7906a9ffe..47f9c6438 100644 --- a/conformance/reports/performance.md +++ b/conformance/reports/performance.md @@ -7,14 +7,14 @@ Do not edit by hand. Timings are the candidate build's; Loki is compared on cold Loki's first run can still hit split-level caches warmed by earlier requests, so a Loki cold time near zero understates what a truly cold Loki takes. -Runs: pr611-latency-gap-json-filter-pushdown (2026-09-23), pr611-latency-gap-metric-latency-gap (2026-09-23), pr624-final-drilldown-breakdown-exact (2026-09-30), pr624-final-loki-profile-otel-metadata (2026-09-30), backfilled-empty-buckets-metadata-freshness (2026-10-01), backfilled-empty-buckets-metadata-inventory (2026-10-01), loki-parity-pages-loki-profile-otel-metadata (2026-10-01), ordered-json-control-long (2026-10-01), ordered-json-drilldown-breakdown-exact-long (2026-10-01), ordered-json-json-filter-pushdown-1h (2026-10-01), ordered-json-json-filter-pushdown-long (2026-10-01), ordered-json-metric-latency-gap-1h (2026-10-01), ordered-json-metric-latency-gap-long (2026-10-01), stage-field-exposure (2026-10-02), extracted-suffix (2026-10-06), paren-or-filters-parenthesized-range-or-filters (2026-10-06), plain-name-after-parser (2026-10-06), plain-name-control (2026-10-06), plain-name-drilldown-breakdown-exact (2026-10-06), unwrap-validity (2026-10-06), unwrap-validity-24h (2026-10-06), extmetric-extracted (2026-10-07), unwraperr-conversion (2026-10-08), unwrapsum-1h (2026-10-11), unwrapsum-long (2026-10-11), dursec-1h (2026-10-12), dursec-24h (2026-10-12), patanchor-1h (2026-10-12), patanchor-24h (2026-10-12), patanchor-24h-multi (2026-10-12), patanchor-control-24h (2026-10-12), patfilter-1h (2026-10-12), patfilter-control-1h (2026-10-12). +Runs: pr611-latency-gap-json-filter-pushdown (2026-09-23), pr611-latency-gap-metric-latency-gap (2026-09-23), pr624-final-drilldown-breakdown-exact (2026-09-30), pr624-final-loki-profile-otel-metadata (2026-09-30), backfilled-empty-buckets-metadata-freshness (2026-10-01), backfilled-empty-buckets-metadata-inventory (2026-10-01), loki-parity-pages-loki-profile-otel-metadata (2026-10-01), ordered-json-control-long (2026-10-01), ordered-json-drilldown-breakdown-exact-long (2026-10-01), ordered-json-json-filter-pushdown-1h (2026-10-01), ordered-json-json-filter-pushdown-long (2026-10-01), ordered-json-metric-latency-gap-1h (2026-10-01), ordered-json-metric-latency-gap-long (2026-10-01), stage-field-exposure (2026-10-02), extracted-suffix (2026-10-06), paren-or-filters-parenthesized-range-or-filters (2026-10-06), plain-name-after-parser (2026-10-06), plain-name-control (2026-10-06), plain-name-drilldown-breakdown-exact (2026-10-06), unwrap-validity (2026-10-06), unwrap-validity-24h (2026-10-06), extmetric-extracted (2026-10-07), unwraperr-conversion (2026-10-08), unwrapsum-1h (2026-10-11), unwrapsum-long (2026-10-11), dursec-1h (2026-10-12), dursec-24h (2026-10-12), patanchor-1h (2026-10-12), patanchor-24h (2026-10-12), patanchor-24h-multi (2026-10-12), patanchor-control-24h (2026-10-12), patfilter-1h (2026-10-12), detlevel-1h (2026-10-14), detlevel-control-1h (2026-10-14), detlevel-d-capped-1h (2026-10-14), detlevel-d-capped-long (2026-10-14), detlevel-dl6-capped-1h (2026-10-14), detlevel-dl6-capped-long (2026-10-14), detlevel-long (2026-10-14). ## Slower than Loki on first load | shape | range | proxy cold | Loki cold | proxy warm | Loki warm | registry items | |---|---|---|---|---|---|---| -| instant json filtered | instant | 0.17s | 0.04s | 0.17s | 0.03s | `semantics/json-filter-pushdown-without-error-drop` | -| instant json volume | instant | 0.20s | 0.04s | 0.19s | 0.04s | `explore-logs-volume-json` | +| instant json filtered | instant | 0.16s | 0.04s | 0.16s | 0.05s | `semantics/json-filter-pushdown-without-error-drop` | +| instant json volume | instant | 0.21s | 0.05s | 0.19s | 0.05s | `explore-logs-volume-json` | | X1 json filter level_extracted, by level | 1h | 0.50s | 0.01s | 0.07s | 0.00s | `parser-error-and-label-collision`, `semantics/extracted-suffix-collision`, `semantics/json-extracted-metric-pushdown` | | X1 json filter level_extracted, by level | 6h | 2.65s | 0.01s | 0.40s | 0.00s | `parser-error-and-label-collision`, `semantics/extracted-suffix-collision`, `semantics/json-extracted-metric-pushdown` | | X2 json group by level_extracted | 1h | 0.07s | 0.00s | 0.07s | 0.00s | `parser-error-and-label-collision`, `semantics/extracted-suffix-collision`, `semantics/json-extracted-metric-pushdown` | @@ -29,6 +29,9 @@ Runs: pr611-latency-gap-json-filter-pushdown (2026-09-23), pr611-latency-gap-met | X5 logfmt filter level_extracted, by level | 24h | 0.74s | 0.00s | 0.71s | 0.00s | `parser-error-and-label-collision`, `semantics/extracted-suffix-collision`, `semantics/json-extracted-metric-pushdown` | | X5 logfmt filter level_extracted, by level | 6h | 0.68s | 0.01s | 0.74s | 0.01s | `parser-error-and-label-collision`, `semantics/extracted-suffix-collision`, `semantics/json-extracted-metric-pushdown` | | E7 json count by level_extracted | 1h | 0.52s | 0.09s | 0.52s | 0.01s | `loki-compatible-profile`, `profiles/structured-metadata-label-collision`, `semantics/extracted-suffix-collision` | +| DLV1 drilldown logs volume, json | 1h | 0.52s | 0.18s | 0.50s | 0.01s | `semantics/json-metric-detected-level-derivation`, `severity-detected-level-derivation` | +| DLV2 json error filter, detected_level | 1h | 2.31s | 0.37s | 2.23s | 0.03s | `semantics/json-metric-detected-level-derivation`, `severity-detected-level-derivation` | +| DLV3 plain json, detected_level | 1h | 0.53s | 0.09s | 0.53s | 0.01s | `semantics/json-metric-detected-level-derivation`, `severity-detected-level-derivation` | | A explore volume, 2 filters, drop | 3h | 0.37s | 0.00s | 0.11s | 0.01s | `explore-logs-volume-json`, `parser-error-and-label-collision`, `semantics/json-filter-pushdown-translated-label`, `semantics/json-filter-pushdown-underscore-label` | | B grouped sum, filter, no drop | 3h | 0.30s | 0.03s | 0.10s | 0.01s | `parser-error-and-label-collision`, `semantics/json-filter-pushdown-without-error-drop` | | C ungrouped sum, filter, no drop | 3h | 0.11s | 0.00s | 0.07s | 0.00s | `parser-error-and-label-collision`, `semantics/json-filter-pushdown-ungrouped-sum` | @@ -43,6 +46,7 @@ Runs: pr611-latency-gap-json-filter-pushdown (2026-09-23), pr611-latency-gap-met | DL1 drilldown field breakdown json-expr, level filter | 3h | 0.06s | 0.01s | 0.00s | 0.01s | `parsed-label-series-identity`, `parser-json`, `severity-detected-level-derivation` | | DL2 drilldown field breakdown, level + stream-label filter before json | 3h | 0.06s | 0.00s | 0.04s | 0.00s | `parsed-label-series-identity`, `semantics/label-filter-before-parser-pushdown`, `severity-detected-level-derivation` | | DL3 drilldown logfmt field breakdown, level filter | 3h | 0.34s | 0.12s | 0.33s | 0.13s | `parser-logfmt`, `severity-detected-level-derivation` | +| DL6 explore volume, level filter before json | 1h | 0.62s | 0.33s | 0.28s | 0.02s | `explore-logs-volume-json`, `semantics/label-filter-before-parser-pushdown`, `severity-detected-level-derivation` | | DL6 explore volume, level filter before json | 3h | 0.57s | 0.01s | 0.14s | 0.00s | `explore-logs-volume-json`, `semantics/label-filter-before-parser-pushdown`, `severity-detected-level-derivation` | | P8 json count by a name that is no stream label | 3h | 0.06s | 0.00s | 0.03s | 0.00s | `loki-compatible-profile`, `semantics/plain-name-after-parser-reads-stream-label` | | E1 json duration_ms sum by service | 24h | 0.61s | 0.01s | 0.65s | 0.01s | `parser-error-and-label-collision`, `semantics/unwrap-conversion-error` | @@ -62,7 +66,7 @@ Runs: pr611-latency-gap-json-filter-pushdown (2026-09-23), pr611-latency-gap-met | E9 logfmt latency_ms sum by service, namespace prod | 7d | 9.86s | 0.03s | 11.61s | 0.01s | `parser-error-and-label-collision`, `semantics/unwrap-conversion-error` | | U13 logfmt latency_ms over every service | 1h | 0.10s | 0.00s | 0.09s | 0.00s | `parser-error-and-label-collision`, `semantics/unwrap-sample-validity` | -48 shape×range slower than Loki beyond noise (25% and 50 ms). +52 shape×range slower than Loki beyond noise (25% and 50 ms). ## By registry item @@ -71,7 +75,7 @@ Runs: pr611-latency-gap-json-filter-pushdown (2026-09-23), pr611-latency-gap-met | `backend-admission-and-heavy-query-queueing` | 7 × 4 ranges | 28 same | worst 0.45s / 0.00s (7d) | 0.02s / 0.01s | differs; 2 slower than Loki cold | | `data-probing-and-freshness` | 4 × 4 ranges | 16 same | worst 0.02s / 0.00s (1h) | 0.00s / 0.00s | differs | | `drilldown-service-landing` | 2 × 4 ranges | 8 same | worst 0.13s / 0.17s (24h) | — / — | same | -| `explore-logs-volume-json` | 7 × 5 ranges | 4 faster, 20 same, 1 slower | worst 3.33s / 1.08s (24h) | — / — | differs; 6 slower than Loki cold | +| `explore-logs-volume-json` | 7 × 5 ranges | 4 faster, 20 same, 1 slower | worst 12.49s / 6.90s (24h) | — / — | differs; 7 slower than Loki cold | | `limits/concurrent-full-retention-scans-exhaust-backend` | 8 × 4 ranges | 32 same | worst 0.45s / 0.00s (7d) | 0.02s / 0.01s | same | | `limits/drilldown-breakdown-exact` | 4 × 5 ranges | 20 same | worst **502** / **502** (7d) | 180.00s / 41.84s | differs | | `limits/metadata-scan-adaptive-limit` | 4 × 4 ranges | 16 same | worst 0.00s / 0.01s (7d) | 0.02s / 0.02s | same | @@ -83,7 +87,7 @@ Runs: pr611-latency-gap-json-filter-pushdown (2026-09-23), pr611-latency-gap-met | `loki_api_v1_label_name_values` | 6 × 4 ranges | 24 same | worst 0.45s / 0.00s (7d) | 0.02s / 0.01s | differs | | `loki_api_v1_labels` | 6 × 4 ranges | 24 same | worst 0.26s / 0.00s (7d) | 0.04s / 0.02s | differs | | `loki_api_v1_query` | 11 × 1 ranges | 2 fixed, 9 same | worst 0.25s / 0.13s (instant) | — / — | same; 2 slower than Loki cold | -| `loki_api_v1_query_range` | 125 × 6 ranges | 32 faster, 62 fixed, 290 same, 2 slower | worst 13.13s / 13.44s (7d) | 0.01s / 0.00s | differs; 44 slower than Loki cold | +| `loki_api_v1_query_range` | 131 × 6 ranges | 35 faster, 64 fixed, 303 same, 2 slower | worst 13.13s / 13.44s (7d) | 0.01s / 0.00s | differs; 48 slower than Loki cold | | `metric-series-identity` | 10 × 6 ranges | 8 faster, 6 fixed, 17 same | worst 1.18s / 1.06s (7d) | — / — | differs | | `operator-sum` | 3 × 4 ranges | 12 same | worst 0.27s / 0.29s (24h) | — / — | same | | `operator-topk` | 1 × 4 ranges | 4 same | worst 0.18s / 0.16s (24h) | — / — | differs | @@ -106,8 +110,9 @@ Runs: pr611-latency-gap-json-filter-pushdown (2026-09-23), pr611-latency-gap-met | `semantics/json-filter-pushdown-ungrouped-sum` | 1 × 4 ranges | 2 faster, 2 same | worst 0.31s / 0.20s (24h) | — / — | same; 1 slower than Loki cold | | `semantics/json-filter-pushdown-without-error-drop` | 3 × 5 ranges | 3 faster, 5 same, 1 slower | worst 0.44s / 0.24s (24h) | — / — | same; 3 slower than Loki cold | | `semantics/json-label-spelling-probe` | 2 × 4 ranges | 3 faster, 5 same | worst 4.52s / 0.15s (24h) | — / — | same; 2 slower than Loki cold | +| `semantics/json-metric-detected-level-derivation` | 6 × 3 ranges | 3 faster, 2 fixed, 13 same | worst **502** / **502** (24h) | — / — | same; 3 slower than Loki cold | | `semantics/label-browser-latency-cold-and-refresh` | 8 × 4 ranges | 32 same | worst 0.45s / 0.00s (7d) | 0.02s / 0.01s | same | -| `semantics/label-filter-before-parser-pushdown` | 2 × 4 ranges | 2 faster, 6 same | worst 3.33s / 1.08s (24h) | — / — | differs; 2 slower than Loki cold | +| `semantics/label-filter-before-parser-pushdown` | 2 × 4 ranges | 2 faster, 6 same | worst 12.49s / 6.90s (24h) | — / — | differs; 3 slower than Loki cold | | `semantics/label-inventory-exact-and-incremental` | 12 × 4 ranges | 48 same | worst 0.45s / 0.00s (7d) | 0.02s / 0.01s | differs | | `semantics/labels-end-inclusive-like-loki` | 12 × 4 ranges | 48 same | worst 0.45s / 0.00s (7d) | 0.02s / 0.01s | differs | | `semantics/line-filter-or-alternatives` | 6 × 5 ranges | 21 fixed | worst 0.06s / 0.04s (3h) | 0.01s / 0.02s | same | @@ -122,4 +127,4 @@ Runs: pr611-latency-gap-json-filter-pushdown (2026-09-23), pr611-latency-gap-met | `semantics/unwrap-sample-validity` | 14 × 3 ranges | 1 faster, 24 same | worst 2.09s / 2.17s (24h) | — / — | differs; 1 slower than Loki cold | | `series-limits-and-partial-results` | 4 × 5 ranges | 20 same | worst **502** / **502** (7d) | 180.00s / 41.84s | differs | | `service-name-derivation` | 2 × 4 ranges | 8 same | worst 0.13s / 0.17s (24h) | — / — | same | -| `severity-detected-level-derivation` | 12 × 5 ranges | 3 faster, 41 same, 1 slower | worst 4.12s / 4.07s (24h) | — / — | differs; 5 slower than Loki cold | +| `severity-detected-level-derivation` | 18 × 5 ranges | 6 faster, 2 fixed, 54 same, 1 slower | worst 12.49s / 6.90s (24h) | — / — | differs; 9 slower than Loki cold | diff --git a/conformance/reports/roadmap.md b/conformance/reports/roadmap.md index a832e6103..0954aa322 100644 --- a/conformance/reports/roadmap.md +++ b/conformance/reports/roadmap.md @@ -33,7 +33,7 @@ Proven in part; the rest is named in the item's cases. | `parser-error-and-label-collision` — Parser errors, __error__ and _extracted collisions | — | 10 | The filtered-volume pushdown cases (underscore labels, filters without `drop __error__`, ungrouped sums, translated labe | | `series-limits-and-partial-results` — Series limits: error, or partial result with a warning | — | 9 | limits/series-limit-error and limits/drilldown-partial-with-warning are written and wired: a metric query above -max-sta | | `service-name-derivation` — service_name follows Loki's discovery order | — | 0 | The derivation matches Loki's 13-entry discover_service_name order and is used by selectors, label filters, grouping, la | -| `severity-detected-level-derivation` — detected_level is derived on the read path | — | 4 | detected_level is derived per returned row from what VictoriaLogs stored: a stored detected_level field, then the first | +| `severity-detected-level-derivation` — detected_level is derived on the read path | — | 5 | detected_level is derived per returned row from what VictoriaLogs stored: a stored detected_level field, then the first | | `severity-exposure-surfaces` — Where detected_level and level must appear | — | 3 | Log responses, /tail, /patterns and /detected_fields carry the derived detected_level; categorize-labels puts it and a s | | `window-bounds-and-step-alignment` — Range windows, bucket edges and step alignment | — | 5 | semantics/offset-sample-timestamps is written and wired: an offset modifier moves the window and leaves the sample times | diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 3a5540836..73bdbcdfc 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -464,7 +464,7 @@ func (c *Cache) setLocalOnly(key string, value []byte, ttl time.Duration, stampe if c == nil || c.disabled { return } - size := len(value) + size := entrySize(key, value) if size > c.maxBytes/10 { return } @@ -502,7 +502,7 @@ func (c *Cache) SetLocalAndDiskWithTTL(key string, value []byte, ttl time.Durati if c == nil || c.disabled { return } - size := len(value) + size := entrySize(key, value) if size > c.maxBytes/10 { return } @@ -525,7 +525,7 @@ func (c *Cache) SetWithTTL(key string, value []byte, ttl time.Duration) { } func (c *Cache) setWithTTL(key string, value []byte, ttl time.Duration, propagateL3, stamped bool) { - size := len(value) + size := entrySize(key, value) // Don't cache entries larger than 10% of max bytes if size > c.maxBytes/10 { return @@ -590,6 +590,11 @@ func (c *Cache) storeLocalLocked(key string, value []byte, ttl time.Duration, si } } +// entrySize is what an entry holds in memory for the byte budget: its value +// and its key, which for the helper entries keyed by a query is as long as +// the query. +func entrySize(key string, value []byte) int { return len(key) + len(value) } + func skipL2WriteForKey(key string) bool { return len(key) >= len("patterns:") && key[:len("patterns:")] == "patterns:" } diff --git a/internal/cache/cache_coverage_test.go b/internal/cache/cache_coverage_test.go index 8d26ca43b..2f338fabd 100644 --- a/internal/cache/cache_coverage_test.go +++ b/internal/cache/cache_coverage_test.go @@ -371,21 +371,21 @@ func TestSize_TracksCorrectly(t *testing.T) { c := New(60*time.Second, 100) defer c.Close() - c.Set("k1", []byte("hello")) // 5 bytes - c.Set("k2", []byte("world")) // 5 bytes + c.Set("k1", []byte("hello")) // 2 + 5 bytes: the key counts + c.Set("k2", []byte("world")) // 2 + 5 bytes entries, bytes := c.Size() if entries != 2 { t.Errorf("expected 2 entries, got %d", entries) } - if bytes != 10 { - t.Errorf("expected 10 bytes, got %d", bytes) + if bytes != 14 { + t.Errorf("expected 14 bytes, got %d", bytes) } c.Invalidate("k1") entries, bytes = c.Size() - if entries != 1 || bytes != 5 { - t.Errorf("after invalidate: entries=%d bytes=%d (expected 1, 5)", entries, bytes) + if entries != 1 || bytes != 7 { + t.Errorf("after invalidate: entries=%d bytes=%d (expected 1, 7)", entries, bytes) } } @@ -447,7 +447,7 @@ func TestSet_ReplaceExisting(t *testing.T) { } _, bytes := c.Size() - if bytes != len("v2-longer") { + if bytes != len("k")+len("v2-longer") { t.Errorf("bytes should reflect replacement, got %d", bytes) } } diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index b5f1c8054..87acc67be 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -2,6 +2,7 @@ package cache import ( "fmt" + "strings" "testing" "time" ) @@ -135,8 +136,8 @@ func TestCache_Size(t *testing.T) { if entries != 2 { t.Errorf("expected 2 entries, got %d", entries) } - if bytes != 10 { - t.Errorf("expected 10 bytes, got %d", bytes) + if bytes != 12 { // two keys of one byte and two values of five + t.Errorf("expected 12 bytes, got %d", bytes) } } @@ -226,8 +227,8 @@ func TestCache_ReplaceExistingKey(t *testing.T) { if entries != 1 { t.Errorf("expected 1 entry (replaced), got %d", entries) } - if bytes != 3 { - t.Errorf("expected 3 bytes (len('new')), got %d", bytes) + if want := len("new") + len("key"); bytes != want { + t.Errorf("expected %d bytes (the key and 'new'), got %d", want, bytes) } } @@ -318,3 +319,19 @@ func TestCache_Close(t *testing.T) { // Double close should not panic c.Close() } + +// A key as long as a query is memory the entry holds: it counts toward the +// byte budget, so a helper entry keyed by a long query cannot hide behind a +// one-byte value, and the budget evicts it. +func TestCache_SizeCountsTheKey(t *testing.T) { + c := NewWithMaxBytes(time.Hour, 100, 1000) + long := strings.Repeat("q", 60) + c.Set(long, []byte("1")) + if _, bytes := c.Size(); bytes != 61 { + t.Fatalf("a 60-byte key and a 1-byte value: %d bytes, want 61", bytes) + } + c.Set(strings.Repeat("r", 150), []byte("1")) // over a tenth of the budget: not stored + if entries, _ := c.Size(); entries != 1 { + t.Fatalf("an entry whose key alone is over the per-entry limit was stored: %d entries", entries) + } +} diff --git a/internal/proxy/backend.go b/internal/proxy/backend.go index cc5a007d3..d71369505 100644 --- a/internal/proxy/backend.go +++ b/internal/proxy/backend.go @@ -764,7 +764,9 @@ func (p *Proxy) vlGet(ctx context.Context, path string, params url.Values) (*htt // responsible for Allow() checks and RecordFailure/RecordSuccess calls. func (p *Proxy) vlPostHTTP(ctx context.Context, path string, params url.Values) (*http.Response, error) { // A metric checked for unwrap conversion errors counts them in its own stats. + sentParams := params params, stripUnwrapCounter, _ := unwrapCheckFrom(ctx).countingStatsQuery(path, params) + params, witness := p.pruneLevelChain(ctx, path, params) params = p.withBackendTimeoutArg(ctx, path, p.scopedTenantParams(ctx, params)) u := *p.backend u.Path = path @@ -805,6 +807,17 @@ func (p *Proxy) vlPostHTTP(ctx context.Context, path string, params url.Values) return nil, err } } + if witness != nil { + flagged, err := p.levelWitnessResponse(resp) + if err != nil { + return nil, err + } + if flagged { + // A row holds what the pruned chain does not read: the mask is stale. + p.dropLevelPresence(ctx, witness.base) + return p.vlPostHTTP(context.WithValue(ctx, levelFullChainKey{}, true), path, sentParams) + } + } return resp, nil } diff --git a/internal/proxy/detected_level_presence.go b/internal/proxy/detected_level_presence.go new file mode 100644 index 000000000..b347fe523 --- /dev/null +++ b/internal/proxy/detected_level_presence.go @@ -0,0 +1,554 @@ +package proxy + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "time" + + "github.com/ReliablyObserve/Loki-VL-proxy/internal/cache" + "github.com/ReliablyObserve/Loki-VL-proxy/internal/translator" +) + +// The pipes that derive detected_level (translator.BuildDetectedLevelChain) +// each pass every block of a window, so the full chain, which knows 16 stored +// level fields and 14 body keys, costs far more than a window that holds one +// field and one key needs. One VictoriaLogs stats query tells which of them +// the window holds, and the chain is built from those alone. +// +// The answer is a mask that may only over-report. A field or key reported +// present that is not costs a pipe; one reported absent that a line holds +// would be a wrong answer. So: +// - the probe runs over the stats query's own base and window; +// - a stored field counts as present when any row has it non-empty; +// - a body key counts as present when its words occur in the line (the same +// word index the guard probes use), and only rows that no definite stored +// level field decides are looked at, because the chain unpacks no other; +// - an escape sequence (\u) in such a line may spell a key (unpack_json +// decodes `level`), which the word test cannot see, so one makes +// every body key present; +// - a probe error, or a response that is not the expected row, yields the +// full chain. +// +// The mask of a window is cached per tenant and base like the guard probes' +// verdicts (coveredWindowProbe): only the part older than statsPushdownRiskSettle +// is remembered, the rest is probed on every request, so a key that first +// appears in recent lines is seen at once. + +// levelPresenceUndecided lists the stored fields that do not decide a row's +// level by themselves; every other level field, when non-empty, does (the +// chain's stored pick for it has no further condition). +var levelPresenceUndecided = []string{"severity", "severity_text", "severity_number"} + +// levelPresenceLayout is the order of the mask: the stored fields, the body +// keys, and the escape flag. +var ( + levelPresenceStored = translator.DetectedLevelFieldNames + levelPresenceBody = translator.DetectedLevelBodyKeys() +) + +func levelPresenceMaskLen() int { return len(levelPresenceStored) + len(levelPresenceBody) + 1 } + +// notLevelWordRune is true for the runes that end a word of VictoriaLogs' +// tokenizer in a level key (ASCII letters, digits and the underscore continue it). +func notLevelWordRune(r rune) bool { + return r != '_' && (r < '0' || r > '9') && (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') +} + +// levelPresenceDecided is the pipe flagging the rows a definite stored level +// field decides (every other field of the mask when non-empty), which the body +// predicates leave out because the chain unpacks no body key for them. +func levelPresenceDecided() string { + var decided []string + for _, name := range levelPresenceStored { + if !containsString(levelPresenceUndecided, name) { + decided = append(decided, quoteLogsQLIdent(name)+":*") + } + } + return ` | format if (` + strings.Join(decided, " OR ") + `) "1" as __p_h` +} + +// levelPresencePredicate is the filter a row satisfies when it holds what mask +// position i stands for: a stored field non-empty, the words of a body key in +// the line of a row no definite stored field decides, or an escape sequence in +// such a line. The probe counts the rows satisfying it and, in the pruned +// query, the witness flags them (levelWitnessPipes), so the two cannot drift. +func levelPresencePredicate(i int) string { + switch { + case i < len(levelPresenceStored): + return quoteLogsQLIdent(levelPresenceStored[i]) + ":*" + case i < len(levelPresenceStored)+len(levelPresenceBody): + words := strings.FieldsFunc(levelPresenceBody[i-len(levelPresenceStored)], notLevelWordRune) + for j, w := range words { + words[j] = "_msg:" + strconv.Quote(w) + } + return "-__p_h:* " + strings.Join(words, " ") + } + // A substring filter, not a regexp for the two bytes backslash and u: the + // same lines, without the regexp engine over every undecided row, which + // was most of what the witness cost. + return `-__p_h:* _msg:*"\\u"*` +} + +// levelPresenceQuery returns the probe: one row of counts, one per stored +// field, per body key and for the escape flag, in mask order. +func levelPresenceQuery(base string) string { + counts := make([]string, levelPresenceMaskLen()) + for i := range counts { + counts[i] = "count() if (" + levelPresencePredicate(i) + ") as p" + strconv.Itoa(i) + } + return base + levelPresenceDecided() + " | stats " + strings.Join(counts, ", ") +} + +// levelPresenceMask runs the probe over [start, end] and returns its mask. +func (p *Proxy) levelPresenceMask(ctx context.Context, base string, start, end time.Time) (string, error) { + params := url.Values{"query": {levelPresenceQuery(base)}, "start": {start.UTC().Format(time.RFC3339Nano)}, "end": {end.Add(time.Nanosecond).UTC().Format(time.RFC3339Nano)}} + resp, err := p.vlPost(ctx, "/select/logsql/query", params) + if err != nil { + return "", err + } + defer resp.Body.Close() + body, err := readBodyLimited(resp.Body, maxUpstreamErrorBodyBytes) + if err != nil { + return "", err + } + if resp.StatusCode >= 400 { + return "", p.redactedBackendStatusError("backend returned", resp.StatusCode, body) + } + var row map[string]string + line, _, _ := strings.Cut(strings.TrimSpace(string(body)), "\n") + if err := json.Unmarshal([]byte(line), &row); err != nil { + return "", fmt.Errorf("level presence probe: %w", err) + } + mask := make([]byte, levelPresenceMaskLen()) + for i := range mask { + n, ok := row["p"+strconv.Itoa(i)] + if !ok { + return "", fmt.Errorf("level presence probe: no count p%d", i) + } + mask[i] = '0' + if v, err := strconv.ParseInt(n, 10, 64); err != nil || v > 0 { + mask[i] = '1' // an unreadable count over-reports + } + } + return string(mask), nil +} + +// orMasks returns the union of two masks; an empty one is the identity. +func orMasks(a, b string) string { + if a == "" || len(a) != len(b) { + return b + } + out := []byte(a) + for i := range out { + if b[i] == '1' { + out[i] = '1' + } + } + return string(out) +} + +// levelPresenceFromMask turns a mask into the chain's presence set. +func levelPresenceFromMask(mask string) *translator.DetectedLevelPresence { + present := &translator.DetectedLevelPresence{} + escaped := mask[len(mask)-1] == '1' + for i, name := range levelPresenceStored { + if mask[i] == '1' { + present.Stored = append(present.Stored, name) + } + } + for i, key := range levelPresenceBody { + if escaped || mask[len(levelPresenceStored)+i] == '1' { + present.BodyKeys = append(present.BodyKeys, key) + } + } + return present +} + +// fullLevelChain is the chain every route puts into a query that groups by or +// filters on detected_level; vlPostHTTP narrows it (pruneLevelChain). +var fullLevelChain = sync.OnceValue(func() string { + return translator.BuildDetectedLevelChain(translator.DetectedLevelFieldNames, translator.DetectedLevelChainGrouping, nil) +}) + +type levelSharedUnpackKey struct{} + +// withLevelSharedUnpack tells pruneLevelChain that the chain of the requests +// under ctx also unpacks the labels of shared (see translator.SharedUnpack): +// it finds that chain by its text and builds the narrowed one with the same +// labels, which no presence mask may drop. +func withLevelSharedUnpack(ctx context.Context, shared *translator.SharedUnpack) context.Context { + return context.WithValue(ctx, levelSharedUnpackKey{}, shared) +} + +// levelChains returns the full chain of a request and the unpack it shares +// with its plan, if any: the chain that holds the plan's labels when the query +// carries it, otherwise the plain full chain. +func levelChains(ctx context.Context, query string) (chain string, shared *translator.SharedUnpack) { + if shared, _ = ctx.Value(levelSharedUnpackKey{}).(*translator.SharedUnpack); shared != nil { + chain = strings.TrimSuffix(translator.BuildDetectedLevelChainShared(translator.DetectedLevelFieldNames, translator.DetectedLevelChainGrouping, nil, shared), levelChainDelete) + if strings.Contains(query, chain) { + return chain, shared + } + } + return strings.TrimSuffix(fullLevelChain(), levelChainDelete), nil +} + +// levelChainDelete ends the chain; the native route drops it from its query, +// so the chain is matched and replaced without it. +const levelChainDelete = " | delete __dl, __dl_*, __j_*, __l_*" + +// levelWitness marks a stats_query_range request whose chain was pruned and +// which therefore carries the witness (see pruneLevelChain). +type levelWitness struct{ base string } + +type levelFullChainKey struct{} + +// pruneLevelChain narrows the detected_level chain of a stats_query_range +// request to the fields and keys its window holds. The pipes before the chain +// are the rows the chain reads, so they are the probe's base; the request's +// start and end are its window. Other requests (an instant query has no +// window to probe), a probe that fails, and a query without the chain keep +// the query as it is. +// +// The mask the probe gave may be older than the data: a line that arrives +// late for a timestamp the cache covers, holding a field or key the pruned +// chain no longer reads, would be misread. So the pruned query also carries a +// witness: pipes flagging every row that holds what the mask left out (the +// probe's own predicates, levelWitnessPipes) as a field the stats group by, +// which levelWitnessResponse reads back; a flagged row sends the request again +// with the full chain, which is exact. The witness costs one more predicate +// pass over the rows, on the same scan. +func (p *Proxy) pruneLevelChain(ctx context.Context, path string, params url.Values) (url.Values, *levelWitness) { + if full, _ := ctx.Value(levelFullChainKey{}).(bool); full { + return params, nil + } + if done, _ := ctx.Value(levelPrunedKey{}).(*levelPruned); done != nil && done.path == path && sameStatsParams(done.sent, params) { + return done.out, done.witness // pruned before the caller took its stats slot + } + query := params.Get("query") + chain, shared := levelChains(ctx, query) + // One chain only: its base is the probe's, and a second chain (a query of + // several segments) may sit on a different one. + at := strings.Index(query, chain) + if path != "/select/logsql/stats_query_range" || at < 0 || strings.Count(query, chain) != 1 { + return params, nil + } + from, okFrom := parseFlexibleUnixNanos(params.Get("start")) + to, okTo := parseFlexibleUnixNanos(params.Get("end")) + if !okFrom || !okTo { + return params, nil + } + base := strings.TrimSpace(query[:at]) + mask := p.detectedLevelMask(ctx, base, time.Unix(0, from), time.Unix(0, to)) + if mask == "" { + return params, nil + } + pruned := translator.BuildDetectedLevelChainShared(translator.DetectedLevelFieldNames, translator.DetectedLevelChainGrouping, levelPresenceFromMask(mask), shared) + pruned = strings.TrimSuffix(pruned, levelChainDelete) + // The witness precedes the chain, whose last pipe overwrites the stored + // detected_level field: the rows are as the probe read them. + rest := query[at+len(chain):] + rest, deleted := strings.CutPrefix(rest, levelChainDelete) + witness := levelWitnessPipes(mask) + if witness != "" { + var ok bool + if rest, ok = withWitnessGroup(rest); !ok { + return params, nil // a stats pipe the witness cannot ride: the full chain + } + } + if deleted { + pruned += levelChainDelete + } + out := cloneURLValues(params) + out.Set("query", query[:at]+witness+" "+pruned+rest) + if witness == "" { + return out, nil + } + return out, &levelWitness{base: base} +} + +type levelPrunedKey struct{} + +// levelPruned is a request pruned (pruneLevelChain) ahead of vlPostHTTP. +type levelPruned struct { + path string + sent, out url.Values + witness *levelWitness +} + +func sameStatsParams(a, b url.Values) bool { + return a.Get("query") == b.Get("query") && a.Get("start") == b.Get("start") && a.Get("end") == b.Get("end") +} + +// pruneBeforeSlot prunes a stats_query_range request before its caller takes a +// stats_query_range slot (-stats-query-range-concurrency): the presence probe +// is a request of its own that may queue for backend admission, and a slot +// must not wait for it. The returned context makes vlPostHTTP use the result. +// A request that unwrap checks rewrite on its way is left to vlPostHTTP. +func (p *Proxy) pruneBeforeSlot(ctx context.Context, params url.Values) context.Context { + const path = "/select/logsql/stats_query_range" + if p.statsQueryRangeSem == nil || unwrapCheckFrom(ctx) != nil { + return ctx + } + out, witness := p.pruneLevelChain(ctx, path, params) + // Kept even when nothing was narrowed (no chain, or a probe that failed): + // vlPostHTTP must not probe again. + return context.WithValue(ctx, levelPrunedKey{}, &levelPruned{path: path, sent: params, out: out, witness: witness}) +} + +// levelWitnessField is the field the witness pipes set to 1 on a row holding +// what the pruned chain no longer reads; the stats group by it. +const levelWitnessField = "__lw" + +// levelWitnessPipes returns the pipes flagging a row that holds a stored field +// or a body key the mask left out, or "" when it left out nothing. +func levelWitnessPipes(mask string) string { + var left []string + for i := 0; i < len(mask); i++ { + if mask[i] != '1' { + left = append(left, "("+levelPresencePredicate(i)+")") + } + } + if len(left) == 0 { + return "" + } + return strings.TrimPrefix(levelPresenceDecided()+` | format if (`+strings.Join(left, " OR ")+`) "1" as `+levelWitnessField, " ") +} + +// withWitnessGroup adds the witness field to the group-by of the first (and +// only) stats pipe of the pipes after the chain, or reports false when they +// hold no stats pipe it can extend. +func withWitnessGroup(rest string) (string, bool) { + const stats = " | stats " + at := strings.Index(rest, stats) + if at < 0 || strings.Count(rest, stats) != 1 { + return "", false + } + tail := rest[at+len(stats):] + if !strings.HasPrefix(tail, "by (") { + return rest[:at] + stats + "by (" + levelWitnessField + ") " + tail, true + } + end := strings.IndexByte(tail, ')') + if end < 0 || strings.ContainsAny(tail[len("by ("):end], "\"`'(") { + return "", false + } + return rest[:at] + stats + tail[:end] + ", " + levelWitnessField + tail[end:], true +} + +// levelWitnessResponse reads the answer to a request carrying the witness and +// reports whether a row was flagged (or the answer is too large to check), in +// which case the request is sent again with the full chain. Otherwise the +// witness field, empty in every group, is taken out of the labels. +func (p *Proxy) levelWitnessResponse(resp *http.Response) (flagged bool, err error) { + if resp.StatusCode != http.StatusOK { + return false, nil + } + body, err := readBodyLimited(resp.Body, maxStatsBucketResponseBytes) + _ = resp.Body.Close() + if errors.Is(err, errBodyTooLarge) || bytes.Contains(body, []byte(`"`+levelWitnessField+`":"1"`)) { + return true, nil + } + if err != nil { + return false, err + } + empty := []byte(`"` + levelWitnessField + `":""`) + body = bytes.ReplaceAll(body, append([]byte(","), empty...), nil) + body = bytes.ReplaceAll(body, append(append([]byte(nil), empty...), ','), nil) + body = bytes.ReplaceAll(body, empty, nil) + resp.Body = io.NopCloser(bytes.NewReader(body)) + resp.ContentLength = int64(len(body)) + resp.Header.Del("Content-Length") + return false, nil +} + +// detectedLevelPresence returns the level fields and body keys the stats query +// over [start, end] on base can read, or nil (the full chain) when they +// cannot be told. +func (p *Proxy) detectedLevelPresence(ctx context.Context, base string, start, end time.Time) *translator.DetectedLevelPresence { + mask := p.detectedLevelMask(ctx, base, start, end) + if mask == "" { + return nil + } + return levelPresenceFromMask(mask) +} + +// detectedLevelMask is the presence mask of the window, or "" when it cannot +// be told. +func (p *Proxy) detectedLevelMask(ctx context.Context, base string, start, end time.Time) string { + mask, _, err := p.coveredWindowProbe(ctx, levelPresenceKey(ctx, base), start, end, func(from, to time.Time, acc string) (string, bool, error) { + found, err := p.levelPresenceMask(ctx, base, from, to) + return orMasks(acc, found), false, err + }) + if err != nil || len(mask) != levelPresenceMaskLen() { + return "" + } + return mask +} + +func levelPresenceKey(ctx context.Context, base string) string { + return "level-presence:v1:" + getOrgID(ctx) + ":" + base +} + +// dropLevelPresence forgets the mask of base: a row it did not account for +// appeared. +func (p *Proxy) dropLevelPresence(ctx context.Context, base string) { + if p.cache != nil { + p.cache.Invalidate(p.windowProbeKey(ctx, levelPresenceKey(ctx, base))) + } +} + +// windowProbeKey is the key coveredWindowProbe stores key under: what a +// window holds depends on what the caller may see, the tenant, the routing and +// the forwarded identity, as for every cached response. +func (p *Proxy) windowProbeKey(ctx context.Context, key string) string { + return key + ":scope:" + p.contextScopeFingerprint(ctx) +} + +// coveredWindowProbe runs check over the parts of [start, end] that the entry +// under key does not already cover, passing it what the entry (and the parts +// checked before) found, and returns what the last check returned; a check that +// stops, or fails, ends the run and nothing is remembered. The entry covers a +// window per tenant and key, so a refresh or a wider range checks only the +// uncovered part. It never covers time later than statsPushdownRiskSettle +// before now, whatever the window, so lines arriving late for recent +// timestamps are checked on every request, and it keeps the age of the check +// that covered each part: extending an entry leaves its expiry where it was, +// because a part checked long ago must not look checked now. +func (p *Proxy) coveredWindowProbe(ctx context.Context, key string, start, end time.Time, check func(from, to time.Time, acc string) (string, bool, error)) (string, bool, error) { + if p.cache == nil { + return check(start, end, "") + } + key = p.windowProbeKey(ctx, key) + type window struct{ from, to time.Time } + todo := []window{{start, end}} + coveredFrom, coveredTo := start, end + acc, ttl := "", statsPushdownRiskCacheTTL + if raw, remaining, ok := p.cache.GetWithTTL(key); ok { + parts := strings.SplitN(string(raw), " ", 3) + var fromNs, toNs int64 + var err error + if len(parts) == 3 { + fromNs, err = strconv.ParseInt(parts[0], 10, 64) + if err == nil { + toNs, err = strconv.ParseInt(parts[1], 10, 64) + } + } + cachedFrom, cachedTo := time.Unix(0, fromNs), time.Unix(0, toNs) + if len(parts) == 3 && err == nil && !start.After(cachedTo) && !end.Before(cachedFrom) { + acc = parts[2] + if remaining < ttl { + ttl = remaining + } + todo = todo[:0] + if start.Before(cachedFrom) { + todo = append(todo, window{start, cachedFrom}) + } else { + coveredFrom = cachedFrom + } + if end.After(cachedTo) { + todo = append(todo, window{cachedTo, end}) + } else { + coveredTo = cachedTo + } + } + } + for _, w := range todo { + var stop bool + var err error + if acc, stop, err = check(w.from, w.to, acc); err != nil || stop { + return acc, stop, err + } + } + if settled := cache.Now().Add(-statsPushdownRiskSettle); coveredTo.After(settled) { + coveredTo = settled + } + if len(todo) > 0 && coveredTo.After(coveredFrom) { + // Only a window that was actually checked refreshes the entry (see above). + p.cache.SetLocalOnlyWithTTL(key, []byte(fmt.Sprintf("%d %d %s", coveredFrom.UnixNano(), coveredTo.UnixNano(), acc)), ttl) + } + return acc, false, nil +} + +// The fields a plan reading detected_level groups by when the stored level +// fields alone decide it (orderedJSONStatsBucketsWithReason): copies of the +// stored detected_level and level, taken before any unpack so that a body key +// never looks stored. +const ( + storedDetectedLevelField = "__dl_d" + storedLevelField = "__dl_l" +) + +// storedLevelSeries gives each series of a stats query grouped by the stored +// level fields its detected_level label, applies the filters on it, and reports +// false when a series is one the stored fields do not decide exactly: no +// stored level (the line decides), or a stored detected_level (a stream label +// keeps its stored spelling, structured metadata is normalised; the stats +// cannot tell them apart). A stored detected_level outranks level, which +// outranks every later field, so a series with a level needs no other field. +func storedLevelSeries(series map[string]manualSeriesSamples, filters []translator.DropCondition) (map[string]manualSeriesSamples, bool) { + out := make(map[string]manualSeriesSamples, len(series)) + for key, entry := range series { + stored := entry.Metric[storedLevelField] + if stored == "" || entry.Metric[storedDetectedLevelField] != "" { + return nil, false + } + level := translator.NormalizeStoredLevel(stored) + keep := true + for _, filter := range filters { + keep = keep && filter.Matches(level) + } + if !keep { + continue + } + metric := make(map[string]string, len(entry.Metric)+1) + for k, v := range entry.Metric { + metric[k] = v + } + metric["detected_level"] = level + entry.Metric = metric + out[key] = entry + } + return out, true +} + +// levelNeedsChain reports whether a window of base held rows the stored level +// fields do not decide, so the next ask goes to the chain at once; the verdict +// expires with the guard verdicts, so data that changes is tried again. +func (p *Proxy) levelNeedsChain(ctx context.Context, base string) bool { + if p.cache == nil { + return false + } + _, _, ok := p.cache.GetWithTTL(levelNeedsChainKey(p, ctx, base)) + return ok +} + +func (p *Proxy) markLevelNeedsChain(ctx context.Context, base string) { + if p.cache != nil { + p.cache.SetLocalOnlyWithTTL(levelNeedsChainKey(p, ctx, base), []byte("1"), statsPushdownRiskCacheTTL) + } +} + +// levelShapeSeparator follows the base in the verdict of one query shape, which +// a read past the group guard leaves (the same rows may answer a query with +// fewer groups from the stored fields). +const levelShapeSeparator = "\x00" + +func levelNeedsChainKey(p *Proxy, ctx context.Context, base string) string { + if at := strings.Index(base, levelShapeSeparator); at >= 0 { + // the shape is long: its digest stands for it + sum := sha256.Sum256([]byte(base[at+1:])) + base = base[:at+1] + hex.EncodeToString(sum[:8]) + } + return "level-needs-chain:v1:" + getOrgID(ctx) + ":" + p.contextScopeFingerprint(ctx) + ":" + base +} diff --git a/internal/proxy/detected_level_presence_sweep_test.go b/internal/proxy/detected_level_presence_sweep_test.go new file mode 100644 index 000000000..7110d9055 --- /dev/null +++ b/internal/proxy/detected_level_presence_sweep_test.go @@ -0,0 +1,102 @@ +package proxy + +import ( + "net/http" + "reflect" + "slices" + "strings" + "testing" + "time" +) + +// Every position of the presence mask is exercised alone: a window holding +// only a stored field (16 of them) or only a body key (14), or only an escape +// sequence, must get exactly that field or key in its pruned chain, from the +// one probe and with no second request. A probe that under-reports one +// position (a wrong word, a wrong index, a mask forced to 0 there) makes the +// pruned chain miss it; the witness then flags the row and the request is +// asked again with the full chain, which is the right answer but a second +// chain query, so both the extra query and the changed chain fail the test. +// The expected answers are Loki v3.7.7's (a level key or field of the allowed +// list, any of its spellings, is read: "dbg" is debug), written out here, and +// the fake VictoriaLogs counts with its own copies of the probe's tables +// (fakePresenceStored and friends). +// conformance: semantics/json-metric-detected-level-derivation +func TestLevelPresenceSweepOnePositionAtATime(t *testing.T) { + type position struct { + name string + vl map[string]string + msg string + // stored, body: the pruned chain's fields and keys; answer: Loki's + // detected_level ("" when the line is no Loki answer to check) + stored, body []string + answer string + } + var cases []position + for _, name := range []string{ + "detected_level", "level", "LEVEL", "Level", "log.level", "severity", "SEVERITY", "Severity", "SeverityText", + "lvl", "LVL", "Lvl", "severity_text", "Severity_Text", "SEVERITY_TEXT", "severity_number", + } { + value := "dbg" + if name == "severity_number" { + value = "5" // OTLP DEBUG + } + cases = append(cases, position{name: "stored " + name, vl: map[string]string{name: value}, msg: `{"n":1}`, stored: []string{name}, answer: "debug"}) + } + for _, key := range []string{ + "level", "LEVEL", "Level", "severity", "SEVERITY", "Severity", "SeverityText", + "lvl", "LVL", "Lvl", "severity_text", "Severity_Text", "SEVERITY_TEXT", + } { + cases = append(cases, position{name: "body " + key, msg: `{"` + key + `":"dbg"}`, body: []string{key}, answer: "debug"}) + } + // a nested key holds the words of both log and level + cases = append(cases, position{name: "body log.level", msg: `{"log":{"level":"dbg"}}`, body: []string{"level", "log.level"}, answer: "debug"}) + // an escape can spell any key, so every key is read + cases = append(cases, position{name: "escape", msg: `{"l\u0065vel":"dbg"}`, + body: []string{"level", "LEVEL", "Level", "log.level", "severity", "SEVERITY", "Severity", "SeverityText", "lvl", "LVL", "Lvl", "severity_text", "Severity_Text", "SEVERITY_TEXT"}}) + if len(cases) != 16+14+1 { + t.Fatalf("%d positions, want 31", len(cases)) + } + s0 := time.Unix(1700000400, 0).UTC() + end := s0.Add(2 * time.Minute) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + row := pushdownRow{ts: s0.Add(100 * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, vl: tc.vl, msg: tc.msg} + srv, fake := newPushdownFakeVL(t, []pushdownRow{row}, nil) + fake.presence = true + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + code, got, body := detectedLevelQuery(t, p, `sum by (detected_level) (count_over_time({env="prod"}[5m]))`, end, end, time.Minute) + if code != http.StatusOK { + t.Fatalf("status %d %.200s", code, body) + } + if tc.answer != "" { + want := map[string]map[int64]string{canonicalLabelsKey(map[string]string{"detected_level": tc.answer}): {end.Unix(): "1"}} + if !reflect.DeepEqual(got, want) { + t.Fatalf("got %v, want %v", got, want) + } + } + fake.mu.Lock() + chain := chainStatsCalls(fake.stats) + fake.mu.Unlock() + if len(chain) != 1 { + t.Fatalf("%d chain queries, want 1 (the probe left something out and the witness asked again)", len(chain)) + } + present := chainPresence(chain[0]) + stored := slices.DeleteFunc(slices.Clone(present.Stored), func(s string) bool { return s == noSeverityNamePipes }) + if !sameSet(stored, tc.stored) || !sameSet(present.BodyKeys, tc.body) { + t.Fatalf("pruned chain reads stored %v and body keys %v, want %v and %v", stored, present.BodyKeys, tc.stored, tc.body) + } + if strings.Contains(chain[0], levelChain) { + t.Fatalf("the full chain was sent") + } + }) + } +} + +func sameSet(a, b []string) bool { + a, b = slices.Clone(a), slices.Clone(b) + slices.Sort(a) + slices.Sort(b) + return slices.Equal(a, b) +} diff --git a/internal/proxy/detected_level_presence_test.go b/internal/proxy/detected_level_presence_test.go new file mode 100644 index 000000000..c8fc38f00 --- /dev/null +++ b/internal/proxy/detected_level_presence_test.go @@ -0,0 +1,661 @@ +package proxy + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "github.com/ReliablyObserve/Loki-VL-proxy/internal/cache" + "github.com/ReliablyObserve/Loki-VL-proxy/internal/translator" +) + +// The presence probe is one stats query with a count per stored level field, +// per body key and for the escape flag, in mask order; the body counts look +// only at rows no definite stored level decides. +func TestLevelPresenceQuery(t *testing.T) { + query := levelPresenceQuery(`env:="production"`) + if n := strings.Count(query, "count() if ("); n != levelPresenceMaskLen() || n != 31 { + t.Fatalf("%d counts, want 31 (16 stored fields, 14 body keys, 1 escape flag)", n) + } + for _, want := range []string{ + `env:="production" | format if (detected_level:* OR level:* OR LEVEL:* OR Level:* OR ` + "`log.level`" + `:* OR SEVERITY:* OR Severity:* OR SeverityText:* OR lvl:* OR LVL:* OR Lvl:* OR Severity_Text:* OR SEVERITY_TEXT:*) "1" as __p_h | stats `, + `count() if (detected_level:*) as p0, count() if (level:*) as p1,`, + `count() if (severity_number:*) as p15, count() if (-__p_h:* _msg:"level") as p16,`, + `count() if (-__p_h:* _msg:"log" _msg:"level") as p19,`, + `count() if (-__p_h:* _msg:"severity_text") as p27,`, + `count() if (-__p_h:* _msg:*"\\u"*) as p30`, + } { + if !strings.Contains(query, want) { + t.Errorf("probe lacks %s:\n%s", want, query) + } + } + // severity, severity_text and severity_number do not decide a row by + // themselves (a synthesised value does not), so they stay out of the filter + // that restricts the body counts. + for _, name := range []string{"severity:*", "severity_text:*", "severity_number:*"} { + if strings.Contains(query[:strings.Index(query, " | stats ")], " "+name) || strings.Contains(query[:strings.Index(query, " | stats ")], "("+name) { + t.Errorf("the deciding filter holds %s", name) + } + } + if len(query) > 1500 { + t.Errorf("probe is %d bytes", len(query)) + } +} + +func TestLevelPresenceFromMask(t *testing.T) { + mask := func(stored, body []string, escaped bool) string { + m := []byte(strings.Repeat("0", levelPresenceMaskLen())) + for _, name := range stored { + m[indexOf(levelPresenceStored, name)] = '1' + } + for _, key := range body { + m[len(levelPresenceStored)+indexOf(levelPresenceBody, key)] = '1' + } + if escaped { + m[len(m)-1] = '1' + } + return string(m) + } + for _, tc := range []struct { + name string + mask string + stored, body []string + }{ + {"nothing", mask(nil, nil, false), nil, nil}, + {"level field and key", mask([]string{"level"}, []string{"level"}, false), []string{"level"}, []string{"level"}}, + {"priority order", mask([]string{"severity_number", "level", "detected_level"}, []string{"lvl", "severity"}, false), + []string{"detected_level", "level", "severity_number"}, []string{"severity", "lvl"}}, + // unpack_json decodes an escaped key, which no word test sees. + {"escape makes every key present", mask([]string{"level"}, nil, true), []string{"level"}, translator.DetectedLevelBodyKeys()}, + } { + got := levelPresenceFromMask(tc.mask) + if !reflect.DeepEqual(got.Stored, tc.stored) || !reflect.DeepEqual(got.BodyKeys, tc.body) { + t.Errorf("%s: got %v / %v, want %v / %v", tc.name, got.Stored, got.BodyKeys, tc.stored, tc.body) + } + } + if got := orMasks("0101", "0011"); got != "0111" { + t.Errorf("orMasks = %s", got) + } + if got := orMasks("", "0011"); got != "0011" { + t.Errorf("orMasks with the empty mask = %s", got) + } +} + +func indexOf(list []string, name string) int { + for i, item := range list { + if item == name { + return i + } + } + panic(name) +} + +// A key that occurs on a single late line of the window is still read: the +// pruned chain lists every key the window holds, so the answer is Loki's. +// conformance: semantics/json-metric-detected-level-derivation +func TestDetectedLevelPresenceKeepsAKeyOfOneLateLine(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + var rows []pushdownRow + for i := 0; i < 40; i++ { + rows = append(rows, pushdownRow{ts: s0.Add(6*time.Minute + time.Duration(i)*time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: `{"level":"info","n":` + strconv.Itoa(i) + `}`}) + } + rows = append(rows, pushdownRow{ts: s0.Add(9*time.Minute + 30*time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: `{"severity":"ERROR","n":41}`}) + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.presence = true + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + end := s0.Add(10 * time.Minute) + code, got, body := detectedLevelQuery(t, p, `sum by (detected_level) (count_over_time({env="prod"} | json | drop __error__ [5m]))`, end, end, time.Minute) + want := map[string]map[int64]string{ + canonicalLabelsKey(map[string]string{"detected_level": "info"}): {end.Unix(): "40"}, + canonicalLabelsKey(map[string]string{"detected_level": "error"}): {end.Unix(): "1"}, + } + if code != http.StatusOK || !reflect.DeepEqual(got, want) { + t.Fatalf("status %d %.200s, got %v, want %v", code, body, got, want) + } + fake.mu.Lock() + defer fake.mu.Unlock() + // the stored-level attempt first (no row has a stored level), then the chain + if chain := chainStatsCalls(fake.stats); len(chain) != 1 || !strings.Contains(chain[0], `fields (level, severity) result_prefix "__j_"`) { + t.Fatalf("expected the chain to unpack exactly level and severity: %v", fake.stats) + } + if fake.raw != 0 { + t.Fatalf("the raw-row evaluator answered (%d)", fake.raw) + } +} + +// A probe that fails, or answers with something other than its counts, leaves +// the full chain (correct, slower), and nothing is remembered. +// conformance: semantics/json-metric-detected-level-derivation +func TestDetectedLevelPresenceProbeFailureKeepsFullChain(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + end := s0.Add(2 * time.Minute) + row := pushdownRow{ts: s0.Add(100 * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: `{"msg":"x","severity":"error"}`} + want := map[string]map[int64]string{canonicalLabelsKey(map[string]string{"detected_level": "error"}): {end.Unix(): "1"}} + for _, tc := range []struct { + name string + set func(f *pushdownFakeVL) + }{ + {"backend error", func(f *pushdownFakeVL) { f.presence, f.presenceFail = true, true }}, + {"no counts", func(f *pushdownFakeVL) {}}, + {"short row", func(f *pushdownFakeVL) { + f.presence, f.presenceLine = true, func(string) string { return `{"p0":"1"}` + "\n" } + }}, + } { + t.Run(tc.name, func(t *testing.T) { + srv, fake := newPushdownFakeVL(t, []pushdownRow{row}, nil) + tc.set(fake) + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + if code, got, body := detectedLevelQuery(t, p, `sum by (detected_level) (count_over_time({env="prod"} | json | drop __error__ [5m]))`, end, end, time.Minute); code != http.StatusOK || !reflect.DeepEqual(got, want) { + t.Fatalf("status %d %.200s, got %v, want %v", code, body, got, want) + } + calls := func() (probes int, statsQueries []string) { + fake.mu.Lock() + defer fake.mu.Unlock() + return len(fake.presenceCalls), append([]string(nil), fake.stats...) + } + probes, stats := calls() + for _, q := range chainStatsCalls(stats) { + if !strings.Contains(q, levelChain) { + t.Fatalf("a stats query without the full chain:\n%s", q) + } + } + if probes != 1 { + t.Fatalf("%d presence probes, want 1", probes) + } + // Nothing was remembered: asking again probes again. + fake.presenceFail = false + p.detectedLevelPresence(context.Background(), presenceBase(levelPresenceQuery(`app:="api"`)), s0, end) + if probes, _ = calls(); probes != 2 { + t.Fatalf("%d presence probes after the second request, want 2", probes) + } + }) + } +} + +// A mask that says a key is absent is only as good as the part of the window +// it was measured over: lines for recent timestamps may still arrive, so the +// part of a window younger than statsPushdownRiskSettle is probed on every +// request and never remembered. A line that appears there after an earlier +// request is seen by the next. +// conformance: semantics/json-metric-detected-level-derivation +func TestDetectedLevelPresenceNeverTrustsRecentAbsence(t *testing.T) { + now := time.Now() + var lines []jsonVolumeLine + for i := 0; i < 10; i++ { + lines = append(lines, jsonVolumeLine{ts: now.Add(-9*time.Minute + time.Duration(i)*time.Second), msg: `{"level":"info","n":` + strconv.Itoa(i) + `}`}) + } + srv, fake := newJSONVolumeFakeVL(t, lines) + fake.presence = true + p := newSlidingTestProxy(t, srv.URL) + start, end := now.Add(-10*time.Minute), now.Add(-time.Minute) + ask := func() *translator.DetectedLevelPresence { + t.Helper() + got := p.detectedLevelPresence(context.Background(), `app:="api"`, start, end) + if got == nil { + t.Fatal("no presence") + } + return got + } + if got := ask(); !reflect.DeepEqual(got.BodyKeys, []string{"level"}) || len(got.Stored) != 0 { + t.Fatalf("first request: %v / %v", got.Stored, got.BodyKeys) + } + // A line for a recent timestamp inside the window, with a key none had. + fake.mu.Lock() + fake.lines = append(fake.lines, jsonVolumeLine{ts: now.Add(-3 * time.Minute), msg: `{"severity":"error"}`}) + fake.mu.Unlock() + if got := ask(); !reflect.DeepEqual(got.BodyKeys, []string{"level", "severity"}) { + t.Fatalf("a key that first appeared in recent lines was missed: %v", got.BodyKeys) + } +} + +// The part of a window that was checked long ago must not look checked now: +// extending a remembered window leaves the entry's expiry where the first +// check set it, so a window kept alive by a refreshing dashboard is still +// checked again in full once the entry ages out. +func TestCoveredWindowProbeKeepsTheAgeOfTheCheck(t *testing.T) { + srv, _ := newJSONVolumeFakeVL(t, nil) + p := newSlidingTestProxy(t, srv.URL) + s0 := time.Unix(1700000400, 0).UTC() + var checked []time.Duration // the length of each window a check ran over + probe := func(to time.Duration) { + t.Helper() + if _, _, err := p.coveredWindowProbe(context.Background(), "age-test", s0, s0.Add(to), func(from, end time.Time, acc string) (string, bool, error) { + checked = append(checked, end.Sub(from)) + return acc, false, nil + }); err != nil { + t.Fatal(err) + } + } + age := func(d time.Duration) { t.Cleanup(cache.AdvanceClockForTesting(d)) } + probe(30 * time.Minute) + age(2 * time.Minute) + probe(40 * time.Minute) // extends: only the new ten minutes are checked + age(4 * time.Minute) + probe(40 * time.Minute) // the entry is six minutes old, past its five: checked again in full + want := []time.Duration{30 * time.Minute, 10 * time.Minute, 40 * time.Minute} + if !reflect.DeepEqual(checked, want) { + t.Fatalf("checked windows %v, want %v", checked, want) + } +} + +// The native route (a plain selector grouped by detected_level, what Logs +// Drilldown sends) gets its chain from the translator in full; the request +// that carries it is narrowed to the window's fields and keys on its way to +// VictoriaLogs, and the answer does not change. +// conformance: semantics/detected-level-pipes-known-differences +func TestDetectedLevelPresenceNativeRoute(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + query := `sum by (detected_level) (count_over_time({app="api"}[1m]))` + run := func(presence bool) (map[string]map[int64]string, []string, int) { + srv, fake := newJSONVolumeFakeVL(t, jsonVolumeFixture(s0)) + fake.presence = presence + p := newSlidingTestProxy(t, srv.URL) + got := runJSONVolumeQueryRange(t, p, query, s0.Add(time.Minute), s0.Add(5*time.Minute), time.Minute) + fake.mu.Lock() + defer fake.mu.Unlock() + return got, append([]string(nil), fake.statsCalls...), len(fake.presenceCalls) + } + full, fullStats, _ := run(false) + pruned, prunedStats, probes := run(true) + if !reflect.DeepEqual(pruned, full) { + t.Fatalf("pruned chain changed the answer\n got: %v\nwant: %v", pruned, full) + } + if probes != 1 || len(prunedStats) != 1 || len(fullStats) != 1 { + t.Fatalf("presence probes %d, stats queries %d (full %d)", probes, len(prunedStats), len(fullStats)) + } + if len(prunedStats[0])*2 > len(fullStats[0]) || !strings.Contains(prunedStats[0], `fields (level) result_prefix "__j_"`) { + t.Fatalf("the chain was not pruned (%d bytes against %d):\n%s", len(prunedStats[0]), len(fullStats[0]), prunedStats[0]) + } +} + +// A dashboard refreshing a relative range moves its window forward every few +// seconds: a refresh checks only the part younger than the settle horizon +// (plus what the clock moved), not the hour behind it. +func TestCoveredWindowProbeRefreshChecksOnlyTheRecentTail(t *testing.T) { + srv, _ := newJSONVolumeFakeVL(t, nil) + p := newSlidingTestProxy(t, srv.URL) + now := time.Now() + var checked []time.Duration + probe := func(shift time.Duration) { + t.Helper() + if _, _, err := p.coveredWindowProbe(context.Background(), "refresh-test", now.Add(-time.Hour+shift), now.Add(shift), func(from, end time.Time, acc string) (string, bool, error) { + checked = append(checked, end.Sub(from)) + return acc, false, nil + }); err != nil { + t.Fatal(err) + } + } + probe(0) + probe(5 * time.Second) + probe(10 * time.Second) + probe(15 * time.Second) + if len(checked) != 4 || checked[0] != time.Hour { + t.Fatalf("checked windows %v", checked) + } + for i, got := range checked[1:] { + // the settle horizon, the seconds the window moved, and the step's slack + if limit := statsPushdownRiskSettle + time.Duration(i+1)*5*time.Second + time.Second; got > limit { + t.Errorf("refresh %d checked %v, more than the %v the horizon allows", i+1, got, limit) + } + } +} + +// Two callers of one tenant who may see different data (forwarded +// credentials, routing) never share a remembered window: a mask or verdict +// measured for the narrower scope says nothing about the wider. +func TestCoveredWindowProbeKeepsScopesApart(t *testing.T) { + srv, _ := newJSONVolumeFakeVL(t, nil) + p := newSlidingTestProxy(t, srv.URL) + p.forwardHeaders = []string{"Authorization"} + scoped := func(token string) context.Context { + r := httptest.NewRequest(http.MethodGet, "/", nil) + r.Header.Set("X-Scope-OrgID", "0") + r.Header.Set("Authorization", token) + return withOrgID(r).Context() + } + s0 := time.Unix(1700000400, 0).UTC() + checks := 0 + probe := func(ctx context.Context) { + t.Helper() + if _, _, err := p.coveredWindowProbe(ctx, "scope-test", s0, s0.Add(30*time.Minute), func(_, _ time.Time, acc string) (string, bool, error) { + checks++ + return acc, false, nil + }); err != nil { + t.Fatal(err) + } + } + probe(scoped("Bearer narrow")) + probe(scoped("Bearer wide")) + probe(scoped("Bearer narrow")) + if checks != 2 { + t.Fatalf("%d checks, want 2: the second scope must not reuse the first's window, the third must reuse its own", checks) + } +} + +// A request holding the chain twice (two segments, perhaps on different +// bases) keeps both full: the mask of the first base says nothing about the +// second. +func TestPruneLevelChainLeavesTwoChainsAlone(t *testing.T) { + srv, fake := newJSONVolumeFakeVL(t, nil) + fake.presence = true + p := newSlidingTestProxy(t, srv.URL) + chain := fullLevelChain() + for _, tc := range []struct { + name, query string + pruned bool + }{ + {"one chain", `app:="a" ` + chain + ` | stats by (detected_level) count()`, true}, + {"two chains", `app:="a" ` + chain + ` | stats by (detected_level) count() ||| app:="b" ` + chain + ` | stats by (detected_level) count()`, false}, + } { + params := url.Values{"query": {tc.query}, "start": {"2023-11-14T22:00:00Z"}, "end": {"2023-11-14T22:30:00Z"}} + out, _ := p.pruneLevelChain(context.Background(), "/select/logsql/stats_query_range", params) + got := out.Get("query") + if (got != tc.query) != tc.pruned { + t.Errorf("%s: pruned=%v, want %v", tc.name, got != tc.query, tc.pruned) + } + } + // an instant query has no window to probe + params := url.Values{"query": {`app:="a" ` + chain + ` | stats by (detected_level) count()`}, "time": {"2023-11-14T22:30:00Z"}} + if got, _ := p.pruneLevelChain(context.Background(), "/select/logsql/stats_query", params); got.Get("query") != params.Get("query") { + t.Errorf("an instant query was changed") + } +} + +// A window whose rows all carry a stored level (a stream label, as Loki +// pipelines send it) is answered from stats grouped by the stored fields alone: +// one stats query, no chain, no level guard and no presence probe, with the +// level normalised as Loki's ingest does (a known spelling in any case becomes +// its level, any other word stays) and a filter on detected_level compared on +// that value. The stored level wins over a different body level. +// conformance: semantics/json-metric-detected-level-derivation +func TestStoredLevelDecidesDetectedLevel(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + // The detected_level Loki v3.7.7 gives a stream level (normalizeLogLevel: + // a case-insensitive match of its spellings, any other word as it is), + // written out by hand. + loki := map[string]string{"INFO": "info", "warn": "warn", "Warning": "warn", "ERR": "error", "notice": "notice", "dbg": "debug", "info": "info", "Information": "info", "unknown": "unknown"} + levels := []string{"INFO", "warn", "Warning", "ERR", "notice", "dbg", "info", "Information", "unknown"} + var lines []jsonVolumeLine + for i := 0; i < 108; i++ { + msg := `{"level":"error","n":` + strconv.Itoa(i) + `}` + lines = append(lines, jsonVolumeLine{ts: s0.Add(time.Duration(i) * 5 * time.Second), level: levels[i%len(levels)], msg: msg}) + } + start, end := s0.Add(time.Minute), s0.Add(8*time.Minute) + // reference counts the lines of (t-1m, t] that keep(detected_level) holds + // for, by the labels of groupBy, from the table above. + reference := func(groupBy []string, keep func(detected string) bool) map[string]map[int64]string { + out := map[string]map[int64]string{} + for at := start; !at.After(end); at = at.Add(time.Minute) { + counts := map[string]int{} + for _, line := range lines { + if !line.ts.After(at.Add(-time.Minute)) || line.ts.After(at) { + continue + } + detected := loki[line.level] + if !keep(detected) { + continue + } + labels := map[string]string{} + for _, name := range groupBy { + labels[name] = map[string]string{"level": line.level, "detected_level": detected}[name] + } + counts[canonicalLabelsKey(labels)]++ + } + for key, n := range counts { + if out[key] == nil { + out[key] = map[int64]string{} + } + out[key][at.Unix()] = strconv.Itoa(n) + } + } + return out + } + all := func(string) bool { return true } + for _, tc := range []struct { + name, query string + groupBy []string + keep func(string) bool + }{ + {"json", `sum by (level, detected_level) (count_over_time({app="api"} | json | drop __error__ [1m]))`, []string{"level", "detected_level"}, all}, + {"logfmt", `sum by (level, detected_level) (count_over_time({app="api"} | logfmt | drop __error__ [1m]))`, []string{"level", "detected_level"}, all}, + {"plain", `sum by (level, detected_level) (count_over_time({app="api"} | drop __error__ [1m]))`, []string{"level", "detected_level"}, all}, + {"json, detected_level filter", `sum by (level, detected_level) (count_over_time({app="api"} | json | drop __error__ | detected_level="info" [1m]))`, []string{"level", "detected_level"}, func(d string) bool { return d == "info" }}, + {"json, detected_level regexp", `sum by (detected_level) (count_over_time({app="api"} | json | drop __error__ | detected_level=~"warn|error|notice" [1m]))`, []string{"detected_level"}, func(d string) bool { return d == "warn" || d == "error" || d == "notice" }}, + } { + t.Run(tc.name, func(t *testing.T) { + srv, fake := newJSONVolumeFakeVL(t, lines) + fake.presence = true + p := newSlidingTestProxy(t, srv.URL) + got := runJSONVolumeQueryRange(t, p, tc.query, start, end, time.Minute) + want := reference(tc.groupBy, tc.keep) + if len(want) == 0 || !reflect.DeepEqual(got, want) { + t.Fatalf("result differs from Loki's answers\n got: %v\nwant: %v", got, want) + } + fake.mu.Lock() + defer fake.mu.Unlock() + if fake.rawCalls != 0 || len(fake.statsCalls) != 1 || len(fake.presenceCalls) != 0 || strings.Contains(fake.statsCalls[0], "__dl_s") || + !strings.Contains(fake.statsCalls[0], " | copy detected_level as __dl_d, level as __dl_l") { + t.Fatalf("expected one stored-level stats query: raw=%d stats=%q presence probes=%d", fake.rawCalls, fake.statsCalls, len(fake.presenceCalls)) + } + for _, guard := range fake.guardCalls { + if strings.Contains(guard, `_msg:"LEVEL"`) || strings.Contains(guard, "_stream:{detected_level") { + t.Fatalf("a level guard probe ran: %.120s", guard) + } + } + }) + } +} + +// One row without a stored level in the window sends the whole query to the +// chain (the line decides that row), and the verdict is remembered, so the next +// ask of the base goes there at once instead of repeating the attempt. +// conformance: semantics/json-metric-detected-level-derivation +func TestStoredLevelAttemptFallsBackToChain(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + var lines []jsonVolumeLine + for i := 0; i < 60; i++ { + lines = append(lines, jsonVolumeLine{ts: s0.Add(time.Duration(i) * 5 * time.Second), level: "info", msg: `{"n":` + strconv.Itoa(i) + `}`}) + } + lines[40].level, lines[40].msg = "", `{"severity":"ERROR"}` // the one row a stored level does not decide + srv, fake := newJSONVolumeFakeVL(t, lines) + fake.presence = true + p := newSlidingTestProxy(t, srv.URL) + query := `sum by (detected_level) (count_over_time({app="api"} | json | drop __error__ [5m]))` + end := s0.Add(5 * time.Minute) + for ask := 0; ask < 2; ask++ { + at := end.Add(time.Duration(ask) * time.Second) // a window the response cache has not seen + want := map[string]map[int64]string{canonicalLabelsKey(map[string]string{"detected_level": "info"}): {at.Unix(): "58"}, canonicalLabelsKey(map[string]string{"detected_level": "error"}): {at.Unix(): "1"}} + if got := runJSONVolumeQueryRange(t, p, query, at, at, time.Minute); !reflect.DeepEqual(got, want) { + t.Fatalf("ask %d: got %v, want %v", ask, got, want) + } + fake.mu.Lock() + stored, chain := len(fake.statsCalls)-len(chainStatsCalls(fake.statsCalls)), len(chainStatsCalls(fake.statsCalls)) + fake.mu.Unlock() + // the first ask tries the stored fields and then the chain; the second only the chain + if stored != 1 || chain != ask+1 { + t.Fatalf("ask %d: %d stored-level and %d chain stats queries so far", ask, stored, chain) + } + } +} + +// A mask is remembered for the part of its window older than the settle +// horizon, so a line that arrives late for a timestamp inside it can hold a +// key the pruned chain no longer reads (here severity, in a window whose +// remembered mask lists level only). The pruned query flags such a row +// (levelWitnessPipes); the request is then sent again with the full chain, so +// the answer is Loki's (the severity key, warn), not the keyword scan's +// (error), and the mask is probed again. +// conformance: semantics/json-metric-detected-level-derivation +func TestDetectedLevelWitnessCatchesALateLine(t *testing.T) { + for _, query := range []string{ + `sum by (detected_level) (count_over_time({env="prod"} | json | drop __error__ [5m]))`, // the pushdown + `sum by (detected_level) (count_over_time({env="prod"}[5m]))`, // the native route + } { + t.Run(query, func(t *testing.T) { detectedLevelWitnessLateLine(t, query) }) + } +} + +func detectedLevelWitnessLateLine(t *testing.T, query string) { + t.Helper() + s0 := time.Unix(1700000400, 0).UTC() + stream := map[string]string{"app": "api", "env": "prod"} + var rows []pushdownRow + for i := 0; i < 40; i++ { + rows = append(rows, pushdownRow{ts: s0.Add(6*time.Minute + time.Duration(i)*time.Second), stream: stream, msg: `{"level":"info","n":` + strconv.Itoa(i) + `}`}) + } + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.presence = true + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + key := func(level string) string { return canonicalLabelsKey(map[string]string{"detected_level": level}) } + ask := func(end time.Time) map[string]map[int64]string { + t.Helper() + code, got, body := detectedLevelQuery(t, p, query, end, end, time.Minute) + if code != http.StatusOK { + t.Fatalf("status %d %.200s", code, body) + } + return got + } + end := s0.Add(10 * time.Minute) + if got, want := ask(end), map[string]map[int64]string{key("info"): {end.Unix(): "40"}}; !reflect.DeepEqual(got, want) { + t.Fatalf("first request: got %v, want %v", got, want) + } + // the late line, stamped inside the window the mask covers + fake.mu.Lock() + fake.rows = append(fake.rows, pushdownRow{ts: s0.Add(8 * time.Minute), stream: stream, msg: `{"msg":"error retry","severity":"warn"}`}) + fake.mu.Unlock() + end = end.Add(time.Second) + if got, want := ask(end), map[string]map[int64]string{key("info"): {end.Unix(): "40"}, key("warn"): {end.Unix(): "1"}}; !reflect.DeepEqual(got, want) { + t.Fatalf("late line: got %v, want %v", got, want) + } + fake.mu.Lock() + chain := chainStatsCalls(fake.stats) + probes := len(fake.presenceCalls) + fake.mu.Unlock() + // request 1: pruned; request 2: pruned, flagged, then the full chain + if len(chain) != 3 || !strings.Contains(chain[0], `fields (level) result_prefix "__j_"`) || !strings.Contains(chain[1], "__lw") || !strings.Contains(chain[2], levelChain) { + t.Fatalf("expected a pruned query, the same flagged, and the full chain: %d queries", len(chain)) + } + // the mask was dropped: the next request probes again and keeps severity + end = end.Add(time.Second) + ask(end) + fake.mu.Lock() + chain, again := chainStatsCalls(fake.stats), len(fake.presenceCalls) + fake.mu.Unlock() + if again != probes+1 || len(chain) != 4 || !strings.Contains(chain[3], `fields (level, severity) result_prefix "__j_"`) { + t.Fatalf("expected a new probe (%d, had %d) and a chain unpacking level and severity: %d queries", again, probes, len(chain)) + } +} + +// The witness field is empty in every group of a clean answer and is taken out +// of the labels wherever it sits; a group with it set sends the request again. +func TestLevelWitnessResponse(t *testing.T) { + p := &Proxy{} + for _, tc := range []struct { + name, body, want string + flagged bool + }{ + {"last", `{"metric":{"__name__":"c","detected_level":"info","__lw":""},"values":[]}`, `{"metric":{"__name__":"c","detected_level":"info"},"values":[]}`, false}, + {"first", `{"metric":{"__lw":"","__name__":"c"},"values":[]}`, `{"metric":{"__name__":"c"},"values":[]}`, false}, + {"alone", `{"metric":{"__lw":""},"values":[]}`, `{"metric":{},"values":[]}`, false}, + {"flagged", `{"metric":{"__name__":"c","__lw":"1"},"values":[]}`, "", true}, + {"a value that mentions it", `{"metric":{"x":"\"__lw\":\"1\""},"values":[]}`, `{"metric":{"x":"\"__lw\":\"1\""},"values":[]}`, false}, + } { + t.Run(tc.name, func(t *testing.T) { + resp := &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(tc.body))} + flagged, err := p.levelWitnessResponse(resp) + if err != nil || flagged != tc.flagged { + t.Fatalf("flagged %v, err %v", flagged, err) + } + if !tc.flagged { + got, _ := io.ReadAll(resp.Body) + if string(got) != tc.want { + t.Fatalf("got %s, want %s", got, tc.want) + } + } + }) + } + // an answer too large to check is sent again with the full chain + old := maxStatsBucketResponseBytes + maxStatsBucketResponseBytes = 16 + t.Cleanup(func() { maxStatsBucketResponseBytes = old }) + resp := &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(strings.Repeat("x", 100)))} + if flagged, err := p.levelWitnessResponse(resp); err != nil || !flagged { + t.Fatalf("a body past the limit: flagged %v, err %v", flagged, err) + } +} + +// The presence probe is a backend request of its own (it may queue for +// admission), so it runs before the stats request takes a +// stats_query_range slot: a slot is never held while waiting for it. +// conformance: semantics/json-metric-detected-level-derivation +func TestLevelPresenceProbeRunsBeforeTheStatsSlot(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + end := s0.Add(5 * time.Minute) + row := pushdownRow{ts: s0.Add(100 * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: `{"msg":"x","severity":"error"}`} + srv, fake := newPushdownFakeVL(t, []pushdownRow{row}, nil) + fake.presence = true + fake.presenceGate = make(chan struct{}) + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + p.statsQueryRangeSem = makeStatsQueryRangeSem(1) + done := make(chan int, 1) + go func() { + code, _, _ := detectedLevelQuery(t, p, `sum by (detected_level) (count_over_time({env="prod"} | json | drop __error__ [5m]))`, end, end, time.Minute) + done <- code + }() + deadline := time.Now().Add(10 * time.Second) + for { + fake.mu.Lock() + probing := len(fake.presenceCalls) > 0 + fake.mu.Unlock() + if probing { + break + } + if time.Now().After(deadline) { + t.Fatal("the presence probe never ran") + } + time.Sleep(5 * time.Millisecond) + } + if free := len(p.statsQueryRangeSem); free != cap(p.statsQueryRangeSem) { + t.Errorf("a stats_query_range slot is held while the probe runs: %d of %d free", free, cap(p.statsQueryRangeSem)) + } + close(fake.presenceGate) + if code := <-done; code != http.StatusOK { + t.Fatalf("status %d", code) + } +} + +// VictoriaLogs refuses a query longer than 16 KiB (-search.maxQueryLen). The +// witness is the largest thing a pruned chain adds: with nothing in the window +// every position is left out, and a long selector (about 6.7 KB, seven long +// regexp names) still has to fit with the pruned chain after it. +func TestPrunedChainWithWitnessFitsTheQueryLimit(t *testing.T) { + srv, fake := newJSONVolumeFakeVL(t, nil) + fake.presence = true + p := newSlidingTestProxy(t, srv.URL) + var names []string + for i := 0; i < 7; i++ { + names = append(names, strings.Repeat("n", 120)+strconv.Itoa(i)) + } + base := `app:~"` + strings.Repeat(strings.Join(names, "|")+"|", 8) + `x"` + query := base + " " + fullLevelChain() + " | stats by (detected_level) count() as c" + params := url.Values{"query": {query}, "start": {"2023-11-14T22:00:00Z"}, "end": {"2023-11-14T22:30:00Z"}} + out, witness := p.pruneLevelChain(context.Background(), "/select/logsql/stats_query_range", params) + if witness == nil { + t.Fatal("nothing was pruned") + } + got := out.Get("query") + if len(base) < 6000 || len(got) > 16*1024 || len(got) >= len(query) { + t.Fatalf("base %d bytes; unpruned request %d, pruned with the witness %d (limit 16384)", len(base), len(query), len(got)) + } +} diff --git a/internal/proxy/detected_level_shared_unpack_test.go b/internal/proxy/detected_level_shared_unpack_test.go new file mode 100644 index 000000000..495ca6136 --- /dev/null +++ b/internal/proxy/detected_level_shared_unpack_test.go @@ -0,0 +1,173 @@ +package proxy + +import ( + "net/http" + "reflect" + "strings" + "testing" + "time" +) + +// sharedUnpackProbe is a stored line and the level and detected_level labels +// Loki gives it for `sum by (level, detected_level) (... | json | drop +// __error__ ...)`. The answers follow Loki v3.7.7: detected_level is set at +// ingest from the first level label of the stream, then of the structured +// metadata, then the OTLP severity number, then the line +// (pkg/distributor/field_detection.go), and the level label is the stream's or +// the metadata's `level`, else the parser's `level` key (a body key that +// collides with a stored label is renamed level_extracted). They are pinned +// against real Loki by the e2e test +// TestRangeMetricCompatibilityJSONDetectedLevelSharedUnpack. +type sharedUnpackProbe struct { + name string + stream map[string]string + vl map[string]string + msg string + level, detect string // the labels Loki gives the line ("" level: no label) +} + +var sharedUnpackProbes = []sharedUnpackProbe{ + {name: "stored severity, body level", vl: map[string]string{"severity": "warn"}, msg: `{"level":"error","n":1}`, level: "error", detect: "warn"}, + {name: "stored lvl, body level", vl: map[string]string{"lvl": "info"}, msg: `{"level":"error","n":1}`, level: "error", detect: "info"}, + {name: "stored severity number, body level", vl: map[string]string{"severity_number": "17"}, msg: `{"level":"debug","n":1}`, level: "debug", detect: "error"}, + {name: "stored Level, body level", vl: map[string]string{"Level": "info"}, msg: `{"level":"debug","n":1}`, level: "debug", detect: "info"}, + {name: "stored LEVEL, body level of another spelling", vl: map[string]string{"LEVEL": "WARN"}, msg: `{"level":"Information"}`, level: "Information", detect: "warn"}, + {name: "stream level, body level", stream: map[string]string{"level": "warn"}, msg: `{"level":"error","n":1}`, level: "warn", detect: "warn"}, + {name: "stored level, body level", vl: map[string]string{"level": "WARN"}, msg: `{"level":"info"}`, level: "WARN", detect: "warn"}, + {name: "stored severity, no body level", vl: map[string]string{"severity": "warn"}, msg: `{"n":1}`, level: "", detect: "warn"}, + {name: "body level only", msg: `{"level":"debug","n":1}`, level: "debug", detect: "debug"}, + {name: "body level of an unknown spelling, keyword in the line", msg: `{"level":"notice","msg":"error connecting"}`, level: "notice", detect: "error"}, +} + +// A plan that groups by `level` unpacks it from the body, and the chain that +// derives detected_level reads the same line: one unpack_json serves both. The +// label must be what Loki gives it wherever a level is stored (a stored field +// of another name decides the row's detected_level, and the parsed `level` +// still counts), also when the presence probe finds no body key the chain +// would read (the plan's label is no body key of the chain). +// conformance: semantics/json-metric-detected-level-derivation +func TestSharedUnpackKeepsThePlansLabelsLikeLoki(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(2*time.Minute) + for _, pruned := range []bool{false, true} { + for _, probe := range sharedUnpackProbes { + name := probe.name + if pruned { + name += " pruned" + } + t.Run(name, func(t *testing.T) { + // A second line no stored level decides keeps the chain route in play. + rows := []pushdownRow{ + probe.row(s0.Add(100 * time.Second)), + {ts: s0.Add(101 * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: `{"n":2}`}, + } + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.presence = pruned + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + labels := map[string]string{"detected_level": probe.detect} + if probe.level != "" { + labels["level"] = probe.level + } + want := map[string]map[int64]string{ + canonicalLabelsKey(labels): {end.Unix(): "1"}, + canonicalLabelsKey(map[string]string{"detected_level": "unknown"}): {end.Unix(): "1"}, + } + query := `sum by (level, detected_level) (count_over_time({env="prod"} | json | drop __error__ [5m]))` + code, got, body := detectedLevelQuery(t, p, query, start, end, time.Minute) + if code != http.StatusOK || !reflect.DeepEqual(got, want) { + t.Fatalf("status %d %.200s, got %v, want %v", code, body, got, want) + } + fake.mu.Lock() + defer fake.mu.Unlock() + for _, q := range fake.stats { + i := strings.Index(q, "unpack_json") + println("DBG", q[max(0, i-80):min(len(q), i+500)]) + j := strings.Index(q, "as detected_level") + if j >= 0 { + println("DBGEND", q[j:]) + } + } + chain := chainStatsCalls(fake.stats) + if len(chain) != 1 || fake.raw != 0 { + t.Fatalf("expected one chain stats query without raw rows: chain=%d raw=%d", len(chain), fake.raw) + } + // The query carries one parse of the line, from the chain, and the + // plan's own unpack is gone. + if n := strings.Count(chain[0], "unpack_json"); n != 1 { + t.Fatalf("%d unpack_json pipes, want 1: %s", n, chain[0]) + } + if !strings.Contains(chain[0], `result_prefix "__j_"`) || strings.Contains(chain[0], "fields (level) keep_original_fields") { + t.Fatalf("the plan keeps its own unpack: %s", chain[0]) + } + if pruned && len(chain[0]) >= len(levelChain) { + t.Fatalf("the query carries the full chain (%d bytes)", len(chain[0])) + } + // The chain that holds the plan's unpack is found and narrowed like any + // other: the witness rides on it. + if pruned && !strings.Contains(chain[0], levelWitnessField) { + t.Fatalf("the pruned chain carries no witness: %s", chain[0]) + } + }) + } + } +} + +func (c sharedUnpackProbe) row(ts time.Time) pushdownRow { + labels := map[string]string{"app": "api", "env": "prod"} + for k, v := range c.stream { + labels[k] = v + } + return pushdownRow{ts: ts, stream: labels, vl: c.vl, msg: c.msg} +} + +// Only a JSON plan that groups by plain labels hands its unpack to the chain: +// logfmt keeps its own (the chain does not read a line Loki reads as JSON as +// logfmt, the plan's parser does), a name_extracted label has a scratch unpack +// of its own, the stored-level plan has no chain, a key that is no plain name +// would be quoted or flattened, and a plan that groups by detected_level alone +// has no label to hand over. +// conformance: semantics/json-metric-detected-level-derivation +func TestSharedLevelUnpackEligibility(t *testing.T) { + for _, tc := range []struct { + name, query string + storedLevel bool + want []string + }{ + {"level and detected_level", `sum by (level, detected_level) (count_over_time({app="x"} | json | drop __error__ [1m]))`, false, []string{"level"}}, + {"two labels", `sum by (app_name, level, detected_level) (count_over_time({app="x"} | json | drop __error__ [1m]))`, false, []string{"app_name", "level"}}, + {"a filter on a label", `sum by (level, detected_level) (count_over_time({app="x"} | json | status="500" | drop __error__ [1m]))`, false, []string{"level", "status"}}, + {"detected_level alone", `sum by (detected_level) (count_over_time({app="x"} | json | drop __error__ [1m]))`, false, nil}, + {"the stored-level plan", `sum by (level, detected_level) (count_over_time({app="x"} | json | drop __error__ [1m]))`, true, nil}, + {"logfmt", `sum by (level, detected_level) (count_over_time({app="x"} | logfmt | drop __error__ [1m]))`, false, nil}, + {"a name_extracted label", `sum by (level, level_extracted, detected_level) (count_over_time({app="x"} | json | drop __error__ [1m]))`, false, nil}, + {"no level derived", `sum by (level) (count_over_time({app="x"} | json | drop __error__ [1m]))`, false, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + plan, ok := compileOrderedJSONMetric(tc.query) + if !ok { + t.Fatalf("no plan for %s", tc.query) + } + _, allKeys, extracted, _ := splitPushdownKeys(plan.unpackKeys()) + got := plan.sharedLevelUnpack(tc.storedLevel, allKeys, extracted, nil) + if tc.want == nil { + if got != nil { + t.Fatalf("shared %v, want none", got) + } + return + } + // the unpack parses a row only where a label is missing as a stored field + cond := orderedJSONUnpackCond(tc.want, nil) + if got == nil || !reflect.DeepEqual(got.Keys, tc.want) || got.Cond != cond { + t.Fatalf("shared %+v, want keys %v with condition %q", got, tc.want, cond) + } + }) + } + // A key that is no plain name is never handed over. + plan, _ := compileOrderedJSONMetric(`sum by (level, detected_level) (count_over_time({app="x"} | json | drop __error__ [1m]))`) + for _, key := range []string{"log.level", "a-b", "9a", "__dl_x", ""} { + if got := plan.sharedLevelUnpack(false, []string{"level", key}, nil, nil); got != nil { + t.Errorf("key %q was shared", key) + } + } +} diff --git a/internal/proxy/ordered_json_detected_level_chain_test.go b/internal/proxy/ordered_json_detected_level_chain_test.go new file mode 100644 index 000000000..bee1abb4b --- /dev/null +++ b/internal/proxy/ordered_json_detected_level_chain_test.go @@ -0,0 +1,336 @@ +package proxy + +import ( + "encoding/json" + "regexp" + "strconv" + "strings" + + "github.com/ReliablyObserve/Loki-VL-proxy/internal/translator" +) + +// emulateDetectedLevelChainFor evaluates translator.BuildDetectedLevelChain over +// one row's fields the way VictoriaLogs runs it, so the unit tests can follow +// the chain's own answer (and where it differs from Loki's) without a +// VictoriaLogs binary: the first non-empty stored level field decides (a +// severity text VictoriaLogs derived from the OTLP number is none) and a known +// spelling is normalized; otherwise, for a row no stored field decides, a line +// starting with a brace is unpacked as JSON for the level keys in the chain's +// fixed order and any other line as logfmt, and the first value found counts +// when it is a known spelling; else the line is scanned for a keyword, and a +// stored word the scan does not replace stays as it is; last the OTLP number. +// The chain's real answers are pinned against Loki by the e2e test +// TestRangeMetricCompatibilityJSONDetectedLevelProbes. +// A chain pruned to the fields and keys of present (nil: the full chain) reads +// no other: a field or key it does not list is not read, as if the row did not +// hold it. +func emulateDetectedLevelChainFor(values map[string]string, present *translator.DetectedLevelPresence) string { + names := translator.DetectedLevelFieldNames + keys := translator.DetectedLevelBodyKeys() + if present != nil { + kept := map[string]string{"_msg": values["_msg"]} + for _, name := range present.Stored { + kept[name] = values[name] + } + values, keys = kept, present.BodyKeys + } + number := values["severity_number"] + stored, has := "", number != "" + for _, name := range names[:len(names)-1] { + v := values[name] + if v == "" { + continue + } + has = true + if stored == "" && !synthesizedSeverity(name, v, number) { + stored = v + } + } + if has && unspecifiedOnly(values) { + has = false // translator.BuildHasLevelFilter excludes these rows + } + value := stored + if !has { + fields := []map[string]string{vlUnpackJSONFields(values["_msg"])} + if !bracedLine.MatchString(values["_msg"]) { + fields = append(fields, slidingLogfmtFields(values["_msg"])) + } + for _, fields := range fields { + for _, name := range keys { + if v := fields[name]; v != "" && value == "" { + value = v + } + } + } + } + if level := canonicalLevel([]byte(value)); level != "" { + return level + } + if !has { + if level := levelFromKeywords([]byte(values["_msg"])); level != levelUnknown { + return level + } + } + if stored != "" { + return stored // a stored word Loki does not know stays as it is + } + if number != "" { + return levelFromSeverityNumber([]byte(number)) + } + // the pipes that read a level from the severity names VictoriaLogs derived + // from the OTLP number (before v1.50) are part of the chain only when the + // window holds a severity field or a number + if m := severityDerivedName.FindStringSubmatch(values["severity"]); m != nil && (present == nil || !containsString(present.Stored, noSeverityNamePipes)) { + return strings.ToLower(m[1]) + } + return levelUnknown +} + +// chainStoredLevelDecides reports whether the chain built for present reads a +// stored level field of the row as its level (the first of +// emulateDetectedLevelChainFor): a non-empty level field that VictoriaLogs did +// not derive from the OTLP number, or a number other than the 0 that goes with +// a synthesised Unspecified. It is the rows the chain's __dl_h marker flags. +func chainStoredLevelDecides(values map[string]string, present *translator.DetectedLevelPresence) bool { + if present != nil { + kept := map[string]string{} + for _, name := range present.Stored { + kept[name] = values[name] + } + values = kept + } + number := values["severity_number"] + has := number != "" + for _, name := range translator.DetectedLevelFieldNames[:len(translator.DetectedLevelFieldNames)-1] { + if values[name] != "" { + has = true + } + } + return has && !unspecifiedOnly(values) +} + +var ( + bracedLine = regexp.MustCompile(translator.BracedLineRegexp) + severityDerivedName = regexp.MustCompile(`^(Trace|Debug|Info|Warn|Error|Fatal)[234]?$`) +) + +// synthesizedSeverity reports whether a stored level field holds a text +// VictoriaLogs derived from the OTLP severity number: severity_text naming the +// family of its number (Info, Info2..4 for 9..12) or Unspecified for a number +// outside 1..24, and, from before v1.50, a severity field naming a level +// without any number stored. +func synthesizedSeverity(name, value, number string) bool { + switch name { + case "severity": + return number == "" && (value == "Unspecified" || severityDerivedName.MatchString(value)) + case "severity_text": + n, err := strconv.Atoi(number) + if err != nil { + return false + } + if n < 1 || n > 24 { + return value == "Unspecified" + } + return strings.HasPrefix(value, vlSeverityTexts[(n-1)/4*4+1]) && severityDerivedName.MatchString(value) + } + return false +} + +// unspecifiedOnly reports whether the only level text of a row is the +// "Unspecified" VictoriaLogs writes for an OTLP record without severity: with +// severity_number 0 in severity_text, or alone in severity. +func unspecifiedOnly(values map[string]string) bool { + names := translator.DetectedLevelFieldNames + others := func(skip ...string) bool { + for _, name := range names[:len(names)-1] { + if values[name] != "" && !containsString(skip, name) { + return false + } + } + return true + } + return values["severity_text"] == "Unspecified" && values["severity_number"] == "0" && others("severity_text") || + values["severity"] == "Unspecified" && values["severity_number"] == "" && others("severity") +} + +// noSeverityNamePipes marks, in the presence read back from a chain, that it +// lacks the pipes mapping a derived severity name to a level. +const noSeverityNamePipes = "\x00no-severity-name-pipes" + +var ( + chainStoredPickRE = regexp.MustCompile(`"<([^>]+)>" as __dl_s keep_original_fields`) + chainBodyKeysRE = regexp.MustCompile(`unpack_json[^|]*? fields \(([^)]*)\) result_prefix "__j_"`) +) + +// chainPresence reads the stored fields and body keys a detected_level chain +// in query was built from. +func chainPresence(query string) *translator.DetectedLevelPresence { + present := &translator.DetectedLevelPresence{} + for _, m := range chainStoredPickRE.FindAllStringSubmatch(query, -1) { + present.Stored = append(present.Stored, m[1]) + } + if strings.Contains(query, `severity_number:*) "info" as __dl keep_original_fields`) { + present.Stored = append(present.Stored, "severity_number") + } + if !strings.Contains(query, "-severity_number:* severity:~") { + present.Stored = append(present.Stored, noSeverityNamePipes) + } + if m := chainBodyKeysRE.FindStringSubmatch(query); m != nil { + // A chain that holds the unpack of its plan lists the plan's labels too. + for _, key := range strings.Split(m[1], ", ") { + if containsString(translator.DetectedLevelBodyKeys(), key) { + present.BodyKeys = append(present.BodyKeys, key) + } + } + } + return present +} + +// chainSharedKeysRE finds the copies a chain built with translator.SharedUnpack +// ends with, one per label of the plan it unpacks for. +var chainSharedKeysRE = regexp.MustCompile(`format if \(-(\w+):\* __j_(\w+):\*\) "<__j_(\w+)>" as (\w+)`) + +// chainSharedCopy reports whether pipe is the copy of a label a chain with a +// shared unpack ends with. +func chainSharedCopy(pipe string) bool { return chainSharedUnpack(" | "+pipe) != "" } + +// chainSharedUnpack returns the plan's own unpack a chain stands in for: +// `unpack_json fields (labels) keep_original_fields`, or "" for a chain with +// none. The unit fakes evaluate that pipe in place of the chain's copies; the +// equivalence of the two is checked against VictoriaLogs by the e2e tests. +func chainSharedUnpack(chain string) string { + var keys []string + for _, m := range chainSharedKeysRE.FindAllStringSubmatch(chain, -1) { + if m[1] == m[2] && m[1] == m[3] && m[1] == m[4] { // a copy of the label onto itself, not a body-key pick + keys = append(keys, m[1]) + } + } + if len(keys) == 0 { + return "" + } + return " | unpack_json fields (" + strings.Join(keys, ", ") + ") keep_original_fields" +} + +// chainStart reports whether pipe is the first of a detected_level chain, +// whatever its pruning leaves first. +func chainStart(pipe string) bool { + if strings.HasPrefix(pipe, "copy ") || strings.HasPrefix(pipe, "filter (__dl_d:*") { + return false // the stored-level copies of the cheap plan and its filters + } + return strings.Contains(pipe, "__dl") || strings.Contains(pipe, `result_prefix "__j_"`) +} + +// withoutLevelChain cuts the detected_level chain out of query. +func withoutLevelChain(query string) string { + const chainEnd = "| delete __dl, __dl_*, __j_*, __l_*" + from := len(query) + for _, marker := range []string{"__dl", `result_prefix "__j_"`} { + if i := strings.Index(query, marker); i >= 0 && i < from { + from = i + } + } + from = strings.LastIndex(query[:from], " | ") + at := strings.Index(query, chainEnd) + end, unpack := at+len(chainEnd), "" + if at >= from { // the native route's chain has no closing delete + unpack = chainSharedUnpack(query[from:end]) + } + return query[:from] + unpack + query[end:] +} + +// isLevelPresenceQuery reports whether query is the presence probe. +func isLevelPresenceQuery(query string) bool { + return strings.Contains(query, ` | stats count() if (`) && strings.Contains(query, ` as p0, `) +} + +// presenceBase is the part of a presence probe the stats query shares. +func presenceBase(query string) string { + return query[:strings.Index(query, " | format if (")] +} + +// What the fake counts for the presence probe: its own copies of the stored +// fields, the body keys and the stored fields that do not decide a row by +// themselves, in the probe's order (16 stored fields, 14 body keys, the escape +// flag). They are not shared with the code under test, so a change to the +// probe's tables or word test there is a difference the tests see. +var ( + fakePresenceStored = []string{ + "detected_level", + "level", "LEVEL", "Level", "log.level", + "severity", "SEVERITY", "Severity", "SeverityText", + "lvl", "LVL", "Lvl", + "severity_text", "Severity_Text", "SEVERITY_TEXT", + "severity_number", + } + fakePresenceBody = []string{ + "level", "LEVEL", "Level", "log.level", + "severity", "SEVERITY", "Severity", "SeverityText", + "lvl", "LVL", "Lvl", + "severity_text", "Severity_Text", "SEVERITY_TEXT", + } + fakePresenceUndecided = []string{"severity", "severity_text", "severity_number"} +) + +// fakeNotWordRune is true for the runes that end a word of the line (ASCII +// letters, digits and the underscore continue it). +func fakeNotWordRune(r rune) bool { + return r != '_' && (r < '0' || r > '9') && (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') +} + +// levelPresenceResponse evaluates the presence probe over the values of the +// rows its base selects, the way VictoriaLogs counts them: a stored field is +// present when some row holds it non-empty, a body key when its words are +// tokens of the line of a row no definite level field decides, and an escape +// when such a line holds a backslash and a u. +func levelPresenceResponse(rows []map[string]string) string { + counts := make([]int, len(fakePresenceStored)+len(fakePresenceBody)+1) + for _, row := range rows { + decided := false + for i, name := range fakePresenceStored { + if row[name] == "" { + continue + } + counts[i]++ + decided = decided || !containsString(fakePresenceUndecided, name) + } + if decided { + continue + } + tokens := map[string]bool{} + for _, w := range strings.FieldsFunc(row["_msg"], fakeNotWordRune) { + tokens[w] = true + } + for i, key := range fakePresenceBody { + all := true + for _, w := range strings.FieldsFunc(key, fakeNotWordRune) { + all = all && tokens[w] + } + if all { + counts[len(fakePresenceStored)+i]++ + } + } + if strings.Contains(row["_msg"], `\u`) { + counts[len(counts)-1]++ + } + } + out := map[string]string{} + for i, n := range counts { + out["p"+strconv.Itoa(i)] = strconv.Itoa(n) + } + line, _ := json.Marshal(out) + return string(line) + "\n" +} + +// chainStatsCalls drops the stored-level attempts from the stats queries a +// plan reading detected_level sent: the buckets grouped by the stored fields, +// which end in errLevelNotStored when a row has no stored level and are asked +// again with the chain. +func chainStatsCalls(calls []string) []string { + var out []string + for _, call := range calls { + if !strings.Contains(call, "copy detected_level as __dl_d, level as __dl_l") { + out = append(out, call) + } + } + return out +} diff --git a/internal/proxy/ordered_json_detected_level_test.go b/internal/proxy/ordered_json_detected_level_test.go new file mode 100644 index 000000000..b6232152b --- /dev/null +++ b/internal/proxy/ordered_json_detected_level_test.go @@ -0,0 +1,440 @@ +package proxy + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "strconv" + "strings" + "testing" + "time" +) + +// detectedLevelProbe is one stored line and the detected_level Loki gives it. +// +// The body-only answers are Loki v3.7.7's own: the logic of +// pkg/distributor/field_detection.go (extractLogLevelFromLogLine: the first +// string value of an allowed level key, at depth 0 or 1 in document order, +// counts only as a known spelling, otherwise the whole line is keyword +// scanned) was run on each line. The answers for a level held by the stream +// or by a stored field follow the log-line derivation (deriveRowLevel) and +// are pinned against real Loki by the e2e test +// TestRangeMetricCompatibilityJSONDetectedLevelProbes. +type detectedLevelProbe struct { + name string + stream map[string]string // stream labels besides app and env + vl map[string]string // stored fields that are not stream labels + msg string + want string // the series' detected_level + // dropOnly marks lines with a parser error: Loki fails the plain `| json` + // query on them, so only the form that drops the error is valid. + dropOnly bool + // rowOnly marks a line the stats pushdown reads differently from Loki + // without a guard able to see it (semantics/detected-level-pipes-known- + // differences): only the raw-row evaluator's forms are asserted. + rowOnly bool + // rawRows marks the lines the stats pushdown must not answer: the pipes of + // translator.BuildDetectedLevelChain would read them differently, or + // might, so a guard probe sends the query to the raw-row evaluator. + rawRows bool +} + +var detectedLevelProbes = []detectedLevelProbe{ + {name: "pino numeric level", msg: `{"level":30,"msg":"hi"}`, want: "unknown"}, + {name: "pino numeric level with an error object", msg: `{"level":50,"msg":"boom","err":{"type":"Error"}}`, want: "error"}, + {name: "boolean level", msg: `{"level":true}`, want: "unknown"}, + {name: "null level", msg: `{"level":null}`, want: "unknown", rawRows: true}, + {name: "unknown spelling", msg: `{"level":"notice"}`, want: "unknown"}, + {name: "unknown spelling and an error word", msg: `{"level":"notice","msg":"error connecting"}`, want: "error"}, + {name: "padded level", msg: `{"level":" info"}`, want: "info"}, + {name: "severity key", msg: `{"severity":"ERROR"}`, want: "error"}, + {name: "Level key", msg: `{"Level":"Information"}`, want: "info"}, + {name: "LEVEL key", msg: `{"LEVEL":"warn"}`, want: "warn"}, + {name: "nested log.level", msg: `{"log":{"level":"warn"}}`, want: "warn"}, + {name: "no level key, error word", msg: `{"msg":"request failed with error"}`, want: "error"}, + {name: "object level", msg: `{"level":{"name":"info"}}`, want: "info", rawRows: true}, + {name: "array level", msg: `{"level":["info"]}`, want: "info"}, + {name: "body detected_level key", msg: `{"detected_level":"error","level":"info"}`, want: "info"}, + {name: "text after the closing brace", msg: `{"level":"info"} trailing`, want: "info", rawRows: true}, + {name: "no level at all", msg: `{"n":1}`, want: "unknown"}, + {name: "plain level key", msg: `{"level":"info"}`, want: "info"}, + {name: "severity before level", msg: `{"msg":"x","severity":"error","level":"info"}`, want: "error", rawRows: true}, + {name: "level before severity", msg: `{"level":"info","severity":"error"}`, want: "info", rawRows: true}, + {name: "stream level, unknown spelling", stream: map[string]string{"level": "notice"}, msg: `{"n":1}`, want: "notice"}, + {name: "stream severity", stream: map[string]string{"severity": "error"}, msg: `{"n":1}`, want: "error"}, + {name: "stream level wins over the body", stream: map[string]string{"level": "INFO"}, msg: `{"level":"error"}`, want: "info"}, + {name: "stream detected_level keeps its stored spelling", stream: map[string]string{"detected_level": "INFO"}, msg: `{"n":1}`, want: "INFO", rawRows: true}, + {name: "stream detected_level wins over a stream level", stream: map[string]string{"detected_level": "INFO", "level": "warn"}, msg: `{"n":1}`, want: "INFO", rawRows: true}, + {name: "stream detected_level wins over the body", stream: map[string]string{"detected_level": "Warning"}, msg: `{"level":"error"}`, want: "Warning", rawRows: true}, + {name: "stored level field", vl: map[string]string{"level": "WARN"}, msg: `{"level":"info"}`, want: "warn"}, + {name: "stored detected_level field", vl: map[string]string{"detected_level": "Error"}, msg: `{"level":"info"}`, want: "error"}, + {name: "body detected_level key alone", msg: `{"detected_level":"verbose"}`, want: "unknown"}, + // A mention of a level key inside a string is no second key. + {name: "level key mentioned in the message", msg: `{"level":"info","msg":"set log level=debug"}`, want: "info"}, + {name: "level name followed by a colon inside a string", msg: `{"msg":"level: high","level":"warn"}`, want: "warn"}, + {name: "stream severity before a stored level field", stream: map[string]string{"severity": "error"}, vl: map[string]string{"level": "info"}, msg: `{"n":1}`, want: "error", rowOnly: true}, + {name: "stored severity field", vl: map[string]string{"severity": "error"}, msg: `{"n":1}`, want: "error"}, + // OTLP records. VictoriaLogs from v1.50 stores severity_text (the text, or + // the name of the number when the record has none) with severity_number; + // before v1.50 one severity field and no number. A synthesised text is no + // level for Loki, which reads the number (and the line when it is 0). + {name: "OTLP without severity", vl: map[string]string{"severity_text": "Unspecified", "severity_number": "0"}, msg: `hello`, want: "unknown", dropOnly: true}, + {name: "OTLP without severity, error word", vl: map[string]string{"severity_text": "Unspecified", "severity_number": "0"}, msg: `request error`, want: "error", dropOnly: true}, + {name: "OTLP fine severity number", vl: map[string]string{"severity_text": "Info2", "severity_number": "10"}, msg: `hello`, want: "info", dropOnly: true}, + {name: "OTLP severity number", vl: map[string]string{"severity_text": "Warn", "severity_number": "13"}, msg: `hello`, want: "warn", dropOnly: true}, + {name: "OTLP severity number out of range", vl: map[string]string{"severity_text": "Unspecified", "severity_number": "30"}, msg: `request error`, want: "unknown", dropOnly: true}, + {name: "OTLP own text", vl: map[string]string{"severity_text": "notice", "severity_number": "0"}, msg: `hello`, want: "notice", dropOnly: true}, + {name: "OTLP own text over its number", vl: map[string]string{"severity_text": "Fine", "severity_number": "9"}, msg: `hello`, want: "Fine", dropOnly: true}, + {name: "OTLP without severity, before v1.50", vl: map[string]string{"severity": "Unspecified"}, msg: `hello`, want: "unknown", dropOnly: true}, + {name: "OTLP without severity, before v1.50, error word", vl: map[string]string{"severity": "Unspecified"}, msg: `request error`, want: "error", dropOnly: true}, + {name: "OTLP fine severity number, before v1.50", vl: map[string]string{"severity": "Info2"}, msg: `hello`, want: "info", dropOnly: true}, + {name: "OTLP severity number, before v1.50", vl: map[string]string{"severity": "Warn"}, msg: `hello`, want: "warn", dropOnly: true}, + {name: "OTLP own text, before v1.50", vl: map[string]string{"severity": "notice"}, msg: `hello`, want: "notice", dropOnly: true}, + // Malformed JSON is not JSON for Loki's ingest: the keyword scan reads it. + {name: "body detected_level key, syntax error", msg: `{"detected_level":"warn","x":{bad`, want: "warn", dropOnly: true}, + {name: "body detected_level and level keys, syntax error", msg: `{"detected_level":"warn","level":"info","x":{bad`, want: "warn", dropOnly: true, rawRows: true}, + {name: "level key, syntax error", msg: `{"level":"info","x":{bad`, want: "info", dropOnly: true, rawRows: true}, + // A line that is JSON for Loki (a brace first and last) never goes to + // the logfmt parser, whatever its text holds. + {name: "logfmt-looking text inside a JSON message", msg: `{"msg":"error: cannot set level=debug now"}`, want: "error"}, + {name: "logfmt-looking text inside a JSON message, level key", msg: `{"msg":"set level=debug","level":"info"}`, want: "info"}, + // One that only starts with a brace is logfmt for Loki. + {name: "logfmt after a brace", msg: `{ error level=info`, want: "info", dropOnly: true, rawRows: true}, + {name: "logfmt level keys of two spellings", msg: `Level=info LEVEL=error`, want: "info", dropOnly: true, rawRows: true}, + // Loki reads the raw bytes of a JSON value: an escaped spelling is no known + // level for it, where VictoriaLogs decodes it. Keys are decoded by both. + {name: "escaped level value", msg: `{"level":"\u0069nfo"}`, want: "unknown", rawRows: true}, + {name: "escaped level key", msg: `{"\u006cevel":"warn"}`, want: "warn"}, + // A spelling the keyword scan cannot find (information is no whole word + // of the scan): only the decoded key gives info, as in Loki v3.7.7. + {name: "escaped level key, long spelling", msg: `{"\u006cevel":"Information"}`, want: "info"}, + // The keyword scan of Loki v3.7.7 (detectLevelFromLogLine): the first word + // of trace, debug, fatal, critical, error, err, warning, warn, info by + // position, between its boundaries, case folded. A plain line is not JSON, + // so only the form that drops the parser error is valid for it. + {name: "keyword in capitals", msg: "request ERROR upstream", want: "error", dropOnly: true}, + {name: "keyword in title case", msg: "Error: connection refused", want: "error", dropOnly: true}, + {name: "keyword in mixed case", msg: "eRrOr in module", want: "error", dropOnly: true}, + {name: "plural is no keyword", msg: "3 errors found", want: "unknown", dropOnly: true}, + {name: "keyword inside a word", msg: "terror alert", want: "unknown", dropOnly: true}, + {name: "long spelling of warn", msg: "disk warning: low", want: "warn", dropOnly: true}, + {name: "plural of warn", msg: "warnings ignored", want: "unknown", dropOnly: true}, + {name: "two keywords, the first by position", msg: "info then error", want: "info", dropOnly: true}, + {name: "two keywords, the long spelling first", msg: "warning: error", want: "warn", dropOnly: true}, + {name: "critical before error", msg: "critical error", want: "critical", dropOnly: true}, + {name: "error before critical", msg: "error then critical", want: "error", dropOnly: true}, + {name: "hyphen before the keyword", msg: "foo-error", want: "unknown", dropOnly: true}, + {name: "underscore before the keyword", msg: "foo_error", want: "unknown", dropOnly: true}, + {name: "digit before the keyword", msg: "5error", want: "unknown", dropOnly: true}, + {name: "digits after the keyword", msg: "error404", want: "unknown", dropOnly: true}, + {name: "full stop after the keyword", msg: "error.", want: "unknown", dropOnly: true}, + {name: "short spelling of error", msg: "err: timeout", want: "error", dropOnly: true}, + {name: "keyword in brackets", msg: "(warn) slow", want: "warn", dropOnly: true}, + {name: "keyword in quotes", msg: "retry \"debug\" mode", want: "debug", dropOnly: true}, + {name: "keyword after an equals sign", msg: "x=trace on", want: "trace", dropOnly: true}, + {name: "fatal in capitals", msg: "FATAL: out of memory", want: "fatal", dropOnly: true}, + {name: "no-break space before the keyword", msg: "\u00a0error", want: "unknown", dropOnly: true}, + {name: "letter before the keyword", msg: "éerror", want: "unknown", dropOnly: true}, + {name: "letter after the keyword", msg: "errorж", want: "unknown", dropOnly: true}, + {name: "carriage return before the keyword", msg: "x\rerror", want: "unknown", dropOnly: true}, + {name: "plural in a JSON message", msg: `{"msg":"3 errors found"}`, want: "unknown"}, + {name: "two keywords in a JSON message", msg: `{"msg":"info then error"}`, want: "info"}, + {name: "long spelling first in a JSON message", msg: `{"msg":"warning: error"}`, want: "warn"}, + {name: "keyword alone in a JSON message", msg: `{"msg":"ERROR"}`, want: "error"}, + {name: "hyphen before the keyword in a JSON message", msg: `{"msg":"foo-error"}`, want: "unknown"}, +} + +// storedLevelDecides reports whether a stored level field, and no stored +// detected_level, decides the line's level (a level field of the stream or of +// the structured metadata): the stats grouped by the stored fields answer it. +func storedLevelDecides(c detectedLevelProbe) bool { + _, streamLevel := c.stream["level"] + _, metaLevel := c.vl["level"] + _, streamDL := c.stream["detected_level"] + _, metaDL := c.vl["detected_level"] + return (streamLevel || metaLevel) && !streamDL && !metaDL +} + +func (c detectedLevelProbe) row(ts time.Time) pushdownRow { + labels := map[string]string{"app": "api", "env": "prod"} + for k, v := range c.stream { + labels[k] = v + } + return pushdownRow{ts: ts, stream: labels, vl: c.vl, msg: c.msg} +} + +// The detected_level of `sum by (detected_level) (... | json ...)` is the +// level Loki sets at ingest (distributor/field_detection.go), whatever shape +// the level has in the line: a stream label keeps its stored value, the first +// string value of a level key in document order counts only as a known +// spelling, anything else falls to a keyword scan of the line, and a body key +// named detected_level never replaces it (Loki renames it +// detected_level_extracted). Every route answers the same: the raw-row +// evaluator (plain `| json`) and the stats pushdown with its guard +// (`| json | drop __error__`). +// conformance: semantics/json-metric-detected-level-derivation +func TestOrderedJSONDetectedLevelLikeLoki(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(2*time.Minute) + // Each line is asked twice: through the full chain, and through the chain + // pruned to the fields and keys the window holds (the presence probe + // answered from the rows), which must give the same answer. + for _, pruned := range []bool{false, true} { + for _, probe := range detectedLevelProbes { + want := map[string]map[int64]string{ + canonicalLabelsKey(map[string]string{"detected_level": probe.want}): {end.Unix(): "1"}, + } + for _, pipeline := range []string{`| json`, `| json | drop __error__`, `| json | __error__=""`} { + if probe.dropOnly && pipeline != `| json | drop __error__` || probe.rowOnly && pipeline == `| json | drop __error__` { + continue + } + name := probe.name + " " + pipeline + if pruned { + name += " pruned" + } + t.Run(name, func(t *testing.T) { + query := `sum by (detected_level) (count_over_time({env="prod"} ` + pipeline + ` [5m]))` + srv, fake := newPushdownFakeVL(t, []pushdownRow{probe.row(s0.Add(100 * time.Second))}, nil) + fake.presence = pruned + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + if code, got, body := detectedLevelQuery(t, p, query, start, end, time.Minute); code != http.StatusOK || !reflect.DeepEqual(got, want) { + t.Fatalf("status %d %.200s, got %v, want %v", code, body, got, want) + } + if pipeline != `| json | drop __error__` { + return + } + fake.mu.Lock() + defer fake.mu.Unlock() + if viaRaw := fake.raw > 0; viaRaw != probe.rawRows { + t.Fatalf("raw rows read: %v, want %v (stats queries %d, probes %d)", viaRaw, probe.rawRows, len(fake.stats), len(fake.guards)) + } + // A stored level (and no stored detected_level) decides in one stats + // query grouped by the stored fields, with no chain, no guard and + // no presence probe; any other line asks again with the chain. + chain := chainStatsCalls(fake.stats) + if storedLevelDecides(probe) { + if len(fake.stats) != 1 || len(chain) != 0 || len(fake.presenceCalls) != 0 { + t.Fatalf("a stored level should decide in one stats query: stats=%d chain=%d presence probes=%d", len(fake.stats), len(chain), len(fake.presenceCalls)) + } + return + } + // (a guard that finds a risky line cancels the stats request, and with + // it a presence probe not yet sent) + if pruned && !probe.rawRows && len(fake.presenceCalls) != 1 { + t.Fatalf("%d presence probes, want 1", len(fake.presenceCalls)) + } + if pruned && len(chain) == 1 && len(chain[0]) >= len(levelChain) { + t.Fatalf("the stats query carries the full chain (%d bytes)", len(chain[0])) + } + }) + } + } + } +} + +// Two lines the VictoriaLogs pipes cannot read like Loki and no guard probe can +// see (semantics/detected-level-pipes-known-differences): Loki's logfmt decoder +// compares keys without case and stops at a syntax error, where unpack_logfmt +// is case-sensitive and skips over it. The raw-row evaluator reads them like +// Loki; the stats pushdown keeps the pipes' value, which the entry records. +// conformance: semantics/detected-level-pipes-known-differences +func TestOrderedJSONDetectedLevelKnownDifferences(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(2*time.Minute) + for _, tc := range []struct{ name, msg, loki, pipes string }{ + {"logfmt level key of another case", `msg="error x" LvL=warn`, "warn", "error"}, + {"logfmt syntax error before the level", `msg="unterminated error level=info`, "error", "info"}, + } { + t.Run(tc.name, func(t *testing.T) { + row := pushdownRow{ts: s0.Add(100 * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: tc.msg} + srv, fake := newPushdownFakeVL(t, []pushdownRow{row}, nil) + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + series := func(level string) map[string]map[int64]string { + return map[string]map[int64]string{canonicalLabelsKey(map[string]string{"detected_level": level}): {end.Unix(): "1"}} + } + plan, ok := compileOrderedJSONMetric(`sum by (detected_level) (count_over_time({env="prod"} | logfmt | drop __error__ [5m]))`) + if !ok { + t.Fatal("no plan") + } + if got := detectedLevelRowMatrix(t, p, plan, start, end, time.Minute); !reflect.DeepEqual(got, series(tc.loki)) { + t.Errorf("row evaluator: got %v, want Loki's %s", got, tc.loki) + } + query := `sum by (detected_level) (count_over_time({env="prod"} | logfmt | drop __error__ [5m]))` + if code, got, body := detectedLevelQuery(t, p, query, start, end, time.Minute); code != http.StatusOK || !reflect.DeepEqual(got, series(tc.pipes)) { + t.Errorf("stats pushdown: status %d %.200s, got %v, want the pipes' %s", code, body, got, tc.pipes) + } + }) + } +} + +// The raw-row evaluator groups by the same value whether the rows come in +// one response or many, for count and bytes alike, and a window mixing every +// shape yields one series per level. +func TestOrderedJSONDetectedLevelMixedRows(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(5*time.Minute) + var rows []pushdownRow + want := map[string]map[string]int{} // level -> function -> sum at the last step + for i, probe := range detectedLevelProbes { + if probe.dropOnly { + continue // the plain `| json` query fails on them, as in Loki + } + rows = append(rows, probe.row(s0.Add(time.Duration(100+i)*time.Second))) + if want[probe.want] == nil { + want[probe.want] = map[string]int{} + } + want[probe.want]["count_over_time"]++ + want[probe.want]["bytes_over_time"] += len(probe.msg) + } + for _, function := range []string{"count_over_time", "bytes_over_time"} { + t.Run(function, func(t *testing.T) { + query := `sum by (detected_level) (` + function + `({env="prod"} | json [5m]))` + plan, ok := compileOrderedJSONMetric(query) + if !ok { + t.Fatal("no plan") + } + srv, fake := newPushdownFakeVL(t, rows, nil) + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + got := detectedLevelRowMatrix(t, p, plan, start, end, time.Minute) + if len(got) != len(want) { + t.Fatalf("%d series, want %d: %v", len(got), len(want), got) + } + for level, sums := range want { + key := canonicalLabelsKey(map[string]string{"detected_level": level}) + if have, ok := got[key][end.Unix()]; !ok || have != strconv.Itoa(sums[function]) { + t.Errorf("%s at %d: %q, want %d", level, end.Unix(), have, sums[function]) + } + } + }) + } +} + +// detectedLevelRowMatrix answers plan with the row evaluator over every +// selected row. +func detectedLevelRowMatrix(t *testing.T, p *Proxy, plan *orderedJSONMetricPlan, start, end time.Time, step time.Duration) map[string]map[int64]string { + t.Helper() + series, err := p.collectOrderedJSONMetric(context.Background(), plan, start, end, step) + if err != nil { + t.Fatalf("row evaluator: %v", err) + } + body, err := buildOrderedJSONMetric(context.Background(), plan, series, start, end, step, true, 1<<30) + if err != nil { + t.Fatalf("row evaluator: %v", err) + } + return decodeExtractedMatrix(t, body) +} + +// detectedLevelQuery runs query through the proxy's query_range handler. +func detectedLevelQuery(t *testing.T, p *Proxy, query string, start, end time.Time, step time.Duration) (int, map[string]map[int64]string, string) { + t.Helper() + params := url.Values{"query": {query}, "start": {strconv.FormatInt(start.UnixNano(), 10)}, "end": {strconv.FormatInt(end.UnixNano(), 10)}, "step": {strconv.FormatFloat(step.Seconds(), 'f', -1, 64)}} + rec := httptest.NewRecorder() + p.handleQueryRange(rec, httptest.NewRequest(http.MethodGet, "/loki/api/v1/query_range?"+params.Encode(), nil)) + if rec.Code != http.StatusOK { + return rec.Code, nil, rec.Body.String() + } + return rec.Code, decodeExtractedMatrix(t, rec.Body.Bytes()), "" +} + +// A filter on detected_level compares Loki's structured metadata, on every +// line whether or not its body parses, so it reads the derived value: the +// raw-row evaluator filters on it, and the stats pushdown (also for Logs +// Drilldown's detected_level breakdown, which is not the native histogram's) +// applies the filter after the pipes that compute it, `!= ""` being always +// true. +// conformance: semantics/json-metric-detected-level-derivation +func TestOrderedJSONDetectedLevelFilters(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(5*time.Minute), s0.Add(5*time.Minute) + pick := func(clean bool) (rows []pushdownRow, levels []string) { + for i, probe := range detectedLevelProbes { + if probe.dropOnly || clean && (probe.rawRows || probe.rowOnly) { + continue + } + rows = append(rows, probe.row(s0.Add(time.Duration(100+i)*time.Second))) + levels = append(levels, probe.want) + } + return rows, levels + } + for _, tc := range []struct { + name, pipeline string + keep func(level string) bool + pushdown bool // the stats pushdown serves lines the pipes read like Loki + filter string // the pipe the pushdown applies after the level pipes (a row the witness flags passes it) + }{ + {"not empty", `| json | drop __error__, __error_details__ | detected_level != ""`, func(string) bool { return true }, true, ""}, + {"equals", `| json | drop __error__ | detected_level="info"`, func(l string) bool { return l == "info" }, true, `| filter (detected_level:="info" OR __lw:="1")`}, + {"regexp", `| json | drop __error__ | detected_level=~"warn|error"`, func(l string) bool { return l == "warn" || l == "error" }, true, `| filter (detected_level:~"^(?:warn|error)$" OR __lw:="1")`}, + {"not equals", `| json | drop __error__ | detected_level!="unknown"`, func(l string) bool { return l != "unknown" }, true, `| filter (-detected_level:="unknown" OR __lw:="1")`}, + // No error drop: a parse error survives a filter on detected_level, which + // holds for every line, so the raw-row evaluator answers. + {"without the error drop", `| json | detected_level="info"`, func(l string) bool { return l == "info" }, false, ""}, + {"error label", `| json | __error__="" | detected_level!=""`, func(string) bool { return true }, false, ""}, + } { + for _, clean := range []bool{false, true} { + name := tc.name + " all probes" + if clean { + name = tc.name + " lines the pipes read like Loki" + } + t.Run(name, func(t *testing.T) { + rows, levels := pick(clean) + want := map[string]map[int64]string{} + counts := map[string]int{} + for _, level := range levels { + if tc.keep(level) { + counts[level]++ + } + } + for level, n := range counts { + want[canonicalLabelsKey(map[string]string{"detected_level": level})] = map[int64]string{end.Unix(): strconv.Itoa(n)} + } + if len(want) == 0 { + t.Fatal("fixture: the filter keeps nothing") + } + query := `sum by (detected_level) (count_over_time({env="prod"} ` + tc.pipeline + ` [5m]))` + for _, mode := range []struct{ drilldown, pruned bool }{{false, false}, {true, false}, {false, true}, {true, true}} { + drilldown := mode.drilldown + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.presence = mode.pruned + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + params := url.Values{"query": {query}, "start": {strconv.FormatInt(start.UnixNano(), 10)}, "end": {strconv.FormatInt(end.UnixNano(), 10)}, "step": {"300"}} + r := httptest.NewRequest(http.MethodGet, "/loki/api/v1/query_range?"+params.Encode(), nil) + if drilldown { + r.Header.Set("X-Query-Tags", "Source=grafana-lokiexplore-app") + } + rec := httptest.NewRecorder() + p.handleQueryRange(rec, r) + if rec.Code != http.StatusOK { + t.Fatalf("drilldown=%v pruned=%v: status %d %s", drilldown, mode.pruned, rec.Code, rec.Body) + } + if got := decodeExtractedMatrix(t, rec.Body.Bytes()); !reflect.DeepEqual(got, want) { + t.Fatalf("drilldown=%v pruned=%v: got %v, want %v", drilldown, mode.pruned, got, want) + } + fake.mu.Lock() + // Logs Drilldown's detected_level breakdown has the shape of its + // native histogram, but the label translator maps detected_level + // to level, so the native route is never exact for it + // (orderedJSONNativeGroupingIsExact): lines the pipes read like + // Loki are answered by the pushdown, behind its guard probes, + // with the filter applied after the pipes. + // The probe lines have no stored level, so the stored-level attempt is + // followed by the chain, whose answer is the guarded one. + chain := chainStatsCalls(fake.stats) + if clean && tc.pushdown && (fake.raw != 0 || len(chain) != 1 || len(fake.guards) == 0) { + t.Errorf("drilldown=%v: expected a guarded pushdown answer without raw rows: raw=%d stats=%d guards=%d", drilldown, fake.raw, len(chain), len(fake.guards)) + } + if clean && tc.pushdown && tc.filter != "" && len(chain) == 1 && !strings.Contains(chain[0], tc.filter+" | stats by (detected_level)") && !strings.Contains(chain[0], tc.filter+" | stats by (detected_level, __lw)") { + t.Errorf("drilldown=%v: filter not applied after the level pipes: %s", drilldown, chain[0]) + } + fake.mu.Unlock() + } + }) + } + } +} diff --git a/internal/proxy/ordered_json_filter_pushdown_test.go b/internal/proxy/ordered_json_filter_pushdown_test.go index 36842a32e..b55c5a02e 100644 --- a/internal/proxy/ordered_json_filter_pushdown_test.go +++ b/internal/proxy/ordered_json_filter_pushdown_test.go @@ -15,6 +15,8 @@ import ( "sync" "testing" "time" + + "github.com/ReliablyObserve/Loki-VL-proxy/internal/translator" ) // pushdownRow is one stored line: stream labels (spelled alike on both @@ -40,8 +42,19 @@ type pushdownFakeVL struct { stats []string guards []string raw int + // presence answers the level presence probe from the rows; otherwise the + // probe gets no counts (the proxy then keeps the full chain). presenceFail + // makes it a backend error, presenceLine replaces the honest answer. + presence bool + presenceFail bool + presenceLine func(honest string) string + presenceCalls []string + // presenceGate, when set, holds every presence probe until it is closed. + presenceGate chan struct{} } +var pushdownLiteralFormatRE = regexp.MustCompile("^format if \\((.+)\\) \"([^<\"]+)\" as (\\S+)$") + var pushdownFormatRE = regexp.MustCompile("^format if \\((.+)\\) \"<([^>]+)>\" as (\\S+)$") // vlUnpackJSONFields mirrors unpack_json: a line that is one JSON object after @@ -135,7 +148,7 @@ func splitLogsQLPipes(query string) []string { // fakeLogsQLFilter parses and evaluates the filter expressions the pushdown // and its probes emit with VictoriaLogs semantics over a row's values: -// `field:="v"`, `field:~"re"`, `field:*`, `field:"phrase"` and `field:word` +// `field:="v"`, `field:~"re"`, `field:*`, `field:*"text"*`, `field:"phrase"` and `field:word` // (a word delimited by non-word characters; letters, digits and underscores // are word characters), `-`/`!` negation, parentheses, implicit and, `or`. type fakeLogsQLFilter struct { @@ -235,8 +248,31 @@ func (f *fakeLogsQLFilter) fieldFilter() bool { return false } f.pos++ + if name == "_stream" && strings.HasPrefix(f.src[f.pos:], "{") { + // A stream filter with one regexp matcher: _stream:{label=~"re"}. + end := strings.IndexByte(f.src[f.pos:], '}') + label, pattern, _ := strings.Cut(f.src[f.pos+1:f.pos+end], "=~") + f.pos += end + 1 + pattern, _ = strconv.Unquote(pattern) + for _, m := range regexp.MustCompile(`([A-Za-z_][A-Za-z0-9_]*)="((?:[^"\\]|\\.)*)"`).FindAllStringSubmatch(f.vals["_stream"], -1) { + if m[1] == label && regexp.MustCompile("^(?:"+pattern+")$").MatchString(m[2]) { + return true + } + } + return false + } value := f.vals[name] switch { + case strings.HasPrefix(f.src[f.pos:], `*"`): + // A substring filter, field:*"text"*. + f.pos++ + sub := f.quoted() + if !strings.HasPrefix(f.src[f.pos:], "*") { + f.t.Errorf("fake VL: expected * after the substring at %d in %q", f.pos, f.src) + return false + } + f.pos++ + return strings.Contains(value, sub) case strings.HasPrefix(f.src[f.pos:], "*"): f.pos++ return value != "" @@ -296,7 +332,33 @@ func (f *pushdownFakeVL) applyPipes(t testing.TB, query string, row pushdownRow) if !fakeLogsQLMatch(t, pipes[0], values) { return values, false } + inChain := false + // A chain built with translator.SharedUnpack holds the unpack of its plan: + // the unpack and the copies after the level are run as VictoriaLogs runs + // them, the rest of the chain is emulated whole. + sharedChain := chainSharedUnpack(query) != "" for _, pipe := range pipes[1:] { + runPipe := true + switch { + case inChain: + // The rest of translator.BuildDetectedLevelChain: its result is + // already in detected_level. + inChain = pipe != "delete __dl, __dl_*, __j_*, __l_*" + runPipe = sharedChain && (!inChain || strings.HasPrefix(pipe, "unpack_json") || chainSharedCopy(pipe)) + case chainStart(pipe): + // The start of translator.BuildDetectedLevelChain; the chain is + // emulated whole by emulateDetectedLevelChainFor, for the fields and + // keys it was built from. + values["detected_level"] = emulateDetectedLevelChainFor(values, chainPresence(query)) + if chainStoredLevelDecides(values, chainPresence(query)) { + values["__dl_h"] = "1" // the rows a stored level decides, as the chain's marker pipes flag them + } + inChain = true + runPipe = sharedChain && (strings.HasPrefix(pipe, "unpack_json") || chainSharedCopy(pipe)) + } + if !runPipe { + continue + } switch { case strings.HasPrefix(pipe, "unpack_json") || strings.HasPrefix(pipe, "unpack_logfmt"): var unpacked map[string]string @@ -338,6 +400,13 @@ func (f *pushdownFakeVL) applyPipes(t testing.TB, query string, row pushdownRow) } } case strings.HasPrefix(pipe, "format if ("): + if lit := pushdownLiteralFormatRE.FindStringSubmatch(pipe); lit != nil { + // format if () "" as dst: dst is the text where the filter holds. + if fakeLogsQLMatch(t, lit[1], values) { + values[lit[3]] = lit[2] + } + continue + } m := pushdownFormatRE.FindStringSubmatch(pipe) if m == nil { t.Errorf("fake VL: unsupported format pipe %q", pipe) @@ -348,8 +417,10 @@ func (f *pushdownFakeVL) applyPipes(t testing.TB, query string, row pushdownRow) values[m[3]] = values[m[2]] } case strings.HasPrefix(pipe, "copy "): - src, dst, _ := strings.Cut(strings.TrimPrefix(pipe, "copy "), " as ") - values[dst] = values[strings.Trim(src, "`")] + for _, pair := range strings.Split(strings.TrimPrefix(pipe, "copy "), ", ") { + src, dst, _ := strings.Cut(pair, " as ") + values[dst] = values[strings.Trim(src, "`")] + } case strings.HasPrefix(pipe, "delete "): for _, field := range strings.Split(strings.TrimPrefix(pipe, "delete "), ", ") { if prefix, ok := strings.CutSuffix(field, "*"); ok { @@ -395,7 +466,9 @@ func newPushdownFakeVL(t testing.TB, rows []pushdownRow, stored func(string) str fake := &pushdownFakeVL{rows: rows, stored: stored} srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { - t.Errorf("parse form: %v", err) + if r.Context().Err() == nil { // a cancelled probe is not a failure + t.Errorf("parse form: %v", err) + } return } query := r.Form.Get("query") @@ -484,6 +557,35 @@ func newPushdownFakeVL(t testing.TB, rows []pushdownRow, stored func(string) str case "/select/logsql/query": start := parseFakeVLTime(t, r.Form.Get("start")) end := parseFakeVLTime(t, r.Form.Get("end")) + if isLevelPresenceQuery(query) { + fake.mu.Lock() + fake.presenceCalls = append(fake.presenceCalls, query) + fake.mu.Unlock() + if fake.presenceGate != nil { + <-fake.presenceGate + } + if fake.presenceFail { + http.Error(w, "backend failure", http.StatusInternalServerError) + return + } + line := "{}\n" + if fake.presence { + var rows []map[string]string + for _, row := range fake.rows { + if ts := row.ts.UnixNano(); ts < start || ts >= end { + continue + } + if values, kept := fake.applyPipes(t, presenceBase(query), row); kept { + rows = append(rows, values) + } + } + if line = levelPresenceResponse(rows); fake.presenceLine != nil { + line = fake.presenceLine(line) + } + } + _, _ = w.Write([]byte(line)) + return + } guard := strings.HasSuffix(query, " | limit 1") fake.mu.Lock() if guard { @@ -554,6 +656,9 @@ func lokiPushdownReferenceErr(plan *orderedJSONMetricPlan, rows []pushdownRow, s for k, v := range row.loki { base[k] = v } + if plan.derivesLevel { + referenceDerivedLevel(base, row) + } labels, ok, err := plan.processWithStreamLabels(row.msg, base, row.stream) if err != nil { return nil, fmt.Errorf("%q: %w", row.msg, err) @@ -622,6 +727,18 @@ func filterPushdownFixture(s0 time.Time) []pushdownRow { return rows } +// levelProbes is the number of guard probes a plan deriving detected_level +// adds to the pushdown's own, and levelChain the pipes that compute it. +const levelProbes = 3 + +var levelChain = translator.BuildDetectedLevelChain(translator.DetectedLevelFieldNames, translator.DetectedLevelChainGrouping, nil) + +// sharedLevelChain is the chain of a plan that hands its unpack to it: the +// labels, and the condition of the plan's own unpack ("" when it has none). +func sharedLevelChain(cond string, labels ...string) string { + return translator.BuildDetectedLevelChainShared(translator.DetectedLevelFieldNames, translator.DetectedLevelChainGrouping, nil, &translator.SharedUnpack{Keys: labels, Cond: cond}) +} + func newFilterPushdownProxy(t *testing.T, backendURL string) *Proxy { t.Helper() p := newSlidingTestProxy(t, backendURL) @@ -651,8 +768,8 @@ func TestOrderedJSONFilterPushdownUsesStatsBuckets(t *testing.T) { stats string }{ // Explore's logs volume for `{...} | json | service_version=`0.96.0` | pipeline=`logs/loki``. - {`sum by (level, detected_level) (count_over_time({app="api"} | json | service_version="0.96.0" | pipeline="logs/loki" | drop __error__ [1m]))`, 2, - ` | filter (service_version:* or ` + "`service.version`" + `:* or _msg:"service_version" or _msg:~"\\\\u") | filter (pipeline:* or _msg:"pipeline" or _msg:~"\\\\u") | unpack_json if ((-level:*) or (-service_version:* -` + "`service.version`" + `:*) or (-pipeline:*)) fields (level, service_version, pipeline) keep_original_fields` + stored + ` | filter service_version:="0.96.0" | filter pipeline:="logs/loki" | stats by (level, detected_level) count() as c`}, + {`sum by (level, detected_level) (count_over_time({app="api"} | json | service_version="0.96.0" | pipeline="logs/loki" | drop __error__ [1m]))`, 2 + levelProbes, + ` | filter (service_version:* or ` + "`service.version`" + `:* or _msg:"service_version" or _msg:~"\\\\u") | filter (pipeline:* or _msg:"pipeline" or _msg:~"\\\\u") ` + sharedLevelChain("(-level:*) or (-service_version:* -`service.version`:*) or (-pipeline:*)", "level", "service_version", "pipeline") + stored + ` | filter service_version:="0.96.0" | filter pipeline:="logs/loki" | stats by (level, detected_level) count() as c`}, // A user's grouped sum: the filter excludes unparsed lines, so errors need not be dropped. {`sum by (level) (count_over_time({app="api"} | json | pipeline="logs/loki" [1m]))`, 2, ` | filter (pipeline:* or _msg:"pipeline" or _msg:~"\\\\u") | unpack_json if ((-level:*) or (-pipeline:*)) fields (level, pipeline) keep_original_fields | filter pipeline:="logs/loki" | stats by (level) count() as c`}, @@ -667,8 +784,8 @@ func TestOrderedJSONFilterPushdownUsesStatsBuckets(t *testing.T) { // A field breakdown on a body key without dropping errors: `!=""` rejects unparsed lines. {`sum by (pipeline) (count_over_time({app="api"} | json | pipeline!="" [1m]))`, 2, ` | filter (pipeline:* or _msg:"pipeline" or _msg:~"\\\\u") | unpack_json if ((-pipeline:*)) fields (pipeline) keep_original_fields | filter -pipeline:="" | stats by (pipeline) count() as c`}, - {`sum by (level, detected_level) (bytes_over_time({app="api"} | json | service_version=~"0\\.9[0-9]\\.0" | drop __error__ [1m]))`, 2, - ` | filter (service_version:* or ` + "`service.version`" + `:* or _msg:"service_version" or _msg:~"\\\\u") | unpack_json if ((-level:*) or (-service_version:* -` + "`service.version`" + `:*)) fields (level, service_version) keep_original_fields` + stored + ` | filter service_version:~"^(?:0\\.9[0-9]\\.0)$" | stats by (level, detected_level) sum_len(_msg) as c, count() as __sample_count`}, + {`sum by (level, detected_level) (bytes_over_time({app="api"} | json | service_version=~"0\\.9[0-9]\\.0" | drop __error__ [1m]))`, 2 + levelProbes, + ` | filter (service_version:* or ` + "`service.version`" + `:* or _msg:"service_version" or _msg:~"\\\\u") ` + sharedLevelChain("(-level:*) or (-service_version:* -`service.version`:*)", "level", "service_version") + stored + ` | filter service_version:~"^(?:0\\.9[0-9]\\.0)$" | stats by (level, detected_level) sum_len(_msg) as c, count() as __sample_count`}, } { t.Run(tc.query, func(t *testing.T) { srv, fake := newPushdownFakeVL(t, rows, nil) @@ -688,11 +805,29 @@ func TestOrderedJSONFilterPushdownUsesStatsBuckets(t *testing.T) { } fake.mu.Lock() defer fake.mu.Unlock() - if fake.raw != 0 || len(fake.stats) != 1 || len(fake.guards) != tc.guards { - t.Fatalf("expected %d probes and one stats call without raw rows: raw=%d stats=%q guards=%q", tc.guards, fake.raw, fake.stats, fake.guards) + guards, stats := tc.guards, fake.stats + if strings.Contains(tc.query, "detected_level") { + if chain := chainStatsCalls(fake.stats); len(chain) == 0 { + // Every row the filters keep has a stream level: the stored + // level fields decide, in one stats query grouped by their + // copies and with none of the chain's guard probes. + guards -= levelProbes + if len(fake.stats) != 1 || !strings.Contains(fake.stats[0], " | copy detected_level as __dl_d, level as __dl_l | unpack_json") || !strings.Contains(fake.stats[0], " | stats by (level, __dl_d, __dl_l) ") { + t.Fatalf("expected one stored-level stats query: %q", fake.stats) + } + stats = nil + } else { + // Some do not: the stored-level attempt, with its label probes, + // then the chain with the level probes. + guards += tc.guards - levelProbes + stats = chain + } + } + if fake.raw != 0 || len(fake.guards) != guards || stats != nil && len(stats) != 1 { + t.Fatalf("expected %d probes and one stats call without raw rows: raw=%d stats=%d have %d guards", guards, fake.raw, len(stats), len(fake.guards)) } - if !strings.HasSuffix(fake.stats[0], tc.stats) { - t.Fatalf("stats query %q does not end with %q", fake.stats[0], tc.stats) + if stats != nil && !strings.HasSuffix(stats[0], tc.stats) { + t.Fatalf("stats query %q does not end with %q", stats[0], tc.stats) } }) } @@ -767,7 +902,7 @@ func TestOrderedJSONFilterPushdownProbesKeepRawEvaluator(t *testing.T) { // The probes run concurrently and the first one that finds a risky // line cancels the others, so a sibling probe may not reach // VictoriaLogs at all. - if fake.raw != 1 || len(fake.guards) == 0 || len(fake.guards) > tc.guards { + if fake.raw != 1 || len(fake.guards) == 0 || len(fake.guards) > tc.guards+levelProbes { t.Fatalf("expected the probe to keep the raw evaluator: raw=%d guards=%q stats=%q", fake.raw, fake.guards, fake.stats) } }) @@ -952,7 +1087,7 @@ func TestOrderedJSONFilterPushdownEligibility(t *testing.T) { {`sum by (level) (count_over_time({env="production"} | json | pipeline="" [1m]))`, false, nil, nil}, {`sum by (level) (count_over_time({env="production"} | json | pipeline="x" | drop __error__ [1m]))`, true, []string{"level"}, nil}, {`sum by (level) (count_over_time({env="production"} | json | pipeline="x" | trace_id!="" | drop __error_details__ [1m]))`, true, []string{"level"}, []string{"pipeline", "trace_id"}}, - {`sum by (service_version, detected_level) (count_over_time({env="production"} | json | drop __error__ [1m]))`, true, []string{"service_version", "level"}, nil}, + {`sum by (service_version, detected_level) (count_over_time({env="production"} | json | drop __error__ [1m]))`, true, []string{"service_version"}, nil}, {`sum by (k8s_pod_name) (count_over_time({env="production"} | json | drop __error__ | k8s_pod_name!="" [1m]))`, true, []string{"k8s_pod_name"}, nil}, {`sum(count_over_time({env="production"} | json | drop __error__ [1m]))`, false, nil, nil}, {`sum by (__error__) (count_over_time({env="production"} | json | pipeline="x" [1m]))`, false, nil, nil}, @@ -967,7 +1102,8 @@ func TestOrderedJSONFilterPushdownEligibility(t *testing.T) { {`sum by (level, detected_level) (count_over_time({env="production"} | logfmt | level_extracted!="" [1m]))`, true, []string{"level"}, []string{"level_extracted"}}, {`sum by (level_extracted, detected_level) (count_over_time({env="production"} | drop __error__ [1m]))`, false, nil, nil}, {`sum by (service_name) (count_over_time({env="production"} | json | drop __error__ [1m]))`, false, nil, nil}, - {`sum by (level) (count_over_time({env="production"} | json | detected_level="error" | drop __error__ [1m]))`, false, nil, nil}, + // A filter on detected_level compares the derived value after the pipes that compute it. + {`sum by (level) (count_over_time({env="production"} | json | detected_level="error" | drop __error__ [1m]))`, true, []string{"level"}, nil}, {`sum by (a_b_c_d_e_f_g_h) (count_over_time({env="production"} | json | drop __error__ [1m]))`, false, nil, nil}, {`sum by (level, detected_level) (count_over_time({env="production"} | logfmt | service_version="1" | drop __error__ [1m]))`, true, []string{"level"}, nil}, {`sum by (level, detected_level) (count_over_time({env="production"} | logfmt | service_version="1" [1m]))`, true, []string{"level"}, []string{"service_version"}}, @@ -1022,8 +1158,11 @@ func TestLogfmtFilterPushdownAliasKeepsRawEvaluator(t *testing.T) { } fake.mu.Lock() defer fake.mu.Unlock() - if fake.raw != tc.raw || len(fake.guards) != 1 { - t.Fatalf("raw=%d guards=%q, want raw=%d", fake.raw, fake.guards, tc.raw) + // A risky line cancels the probes still running; the stored level + // fields alone decide the level of a window of stream levels, without + // the level probes. + if fake.raw != tc.raw || len(fake.guards) == 0 || len(fake.guards) > 1+levelProbes || tc.raw == 0 && len(fake.guards) != 1 && len(fake.guards) != 1+levelProbes { + t.Fatalf("raw=%d guards=%d, want raw=%d", fake.raw, len(fake.guards), tc.raw) } }) } @@ -1187,3 +1326,17 @@ func TestOrderedJSONPipelineErrorMatchesLokiText(t *testing.T) { t.Fatalf("expected the stats pushdown and its two probes to answer the filtered query: raw=%d guards=%q", fake.raw-raw, fake.guards[guards:]) } } + +// referenceDerivedLevel gives a fixture row the detected_level the raw +// evaluator derives from what VictoriaLogs stores for it. +func referenceDerivedLevel(base map[string]string, row pushdownRow) { + entry := map[string]string{"_msg": row.msg} + for k, v := range row.stream { + entry[k] = v + } + for k, v := range row.vl { + entry[k] = v + } + levels := logRowLevels{bodyScan: true} + withDerivedLevel(base, levels.stringRow(entry, logRowStream{labels: row.stream, levels: levelFieldsFromLabels(row.stream)}), row.stream) +} diff --git a/internal/proxy/ordered_json_metric.go b/internal/proxy/ordered_json_metric.go index 3791c3fda..70c2ece9d 100644 --- a/internal/proxy/ordered_json_metric.go +++ b/internal/proxy/ordered_json_metric.go @@ -96,6 +96,11 @@ type orderedJSONMetricPlan struct { // parser is "json", "logfmt" or "" (no parser). Only JSON plans have a raw // evaluator; the others answer through the stats pushdown or fall through. parser string + // derivesLevel reports that the plan reads Loki's detected_level: it + // groups by it, keeps it under `without`, or filters on it. The raw + // evaluator then gives every row the value Loki's ingest set + // (withDerivedLevel) before any stage runs. + derivesLevel bool } // handleOrderedJSONMetric uses the existing metric accumulators, but executes @@ -302,6 +307,7 @@ func compileOrderedJSONMetric(query string) (*orderedJSONMetricPlan, bool) { if !ok || (hasJSON && hasLogfmt) { return nil, false } + plan.setDetectedLevel() if !hasJSON { return plan.levelVolumePlan(hasLogfmt, retained) } @@ -369,7 +375,9 @@ func (plan *orderedJSONMetricPlan) levelVolumePlan(logfmt bool, lineFilters stri } plan.fetchQuery = plan.selector + lineFilters plan.setStatsPushdown() - if plan.grouping == nil || !containsString(plan.grouping.Labels, "detected_level") || len(plan.unpackFields) == 0 { + if plan.grouping == nil || !containsString(plan.grouping.Labels, "detected_level") || !plan.pushdown || + plan.parser == "" && len(plan.unpackFields) == 0 { + // A plain plan grouped by detected_level alone is the native route's. return nil, false } return plan, true @@ -718,8 +726,11 @@ func logfmtKeyAliasPattern(labels []string) string { // Loki suffixes a parsed key that collides with a stream label or structured // metadata with _extracted, which unpack_json keep_original_fields reproduces // by keeping stored values. detected_level is structured metadata Loki sets -// at ingest, so JSON never yields it; it is derived from level like the other -// metric paths. An ungrouped sum unpacks only its filter keys; without any it +// at ingest, so JSON never yields it: a JSON plan grouped by it gets Loki's +// value from the same VictoriaLogs pipes as the native metric path +// (translator.BuildDetectedLevelChain), guarded by the detectedLevel*Risk +// probes for the lines those pipes read differently, which the raw evaluator +// then answers. An ungrouped sum unpacks only its filter keys; without any it // is served natively (withoutJSON). func (plan *orderedJSONMetricPlan) setStatsPushdown() { if !plan.aggregated || (plan.grouping != nil && plan.grouping.Without) { @@ -733,8 +744,11 @@ func (plan *orderedJSONMetricPlan) setStatsPushdown() { if plan.grouping != nil { for _, label := range plan.grouping.Labels { if label == "detected_level" { - label = "level" - } else if !orderedJSONPushdownLabelOK(label) || plan.parser == "" && strings.HasSuffix(label, "_extracted") { + // Computed by VictoriaLogs from the whole row + // (translator.BuildDetectedLevelChain), not from one key. + continue + } + if !orderedJSONPushdownLabelOK(label) || plan.parser == "" && strings.HasSuffix(label, "_extracted") { return } if !containsString(fields, label) { @@ -742,7 +756,7 @@ func (plan *orderedJSONMetricPlan) setStatsPushdown() { } } } - if len(fields) == 0 && len(filters) == 0 { + if len(fields) == 0 && len(filters) == 0 && !plan.derivesLevel { return } plan.pushdown = true @@ -775,9 +789,20 @@ func (plan *orderedJSONMetricPlan) pushdownLabelFilters() (filters []translator. case stage.line != nil: continue case stage.filter != nil: - if !parsed || !orderedJSONPushdownLabelOK(stage.filter.Field) || plan.parser == "" && strings.HasSuffix(stage.filter.Field, "_extracted") { + // Loki's detected_level is on every line, parsed or not: a filter + // on it compares the derived value (it never rejects an unparsed + // line, so it cannot stand for the error drop) and `!= ""` always + // holds. + level := stage.filter.Field == detectedLevelLabel && plan.derivesLevel + if !parsed || !level && (!orderedJSONPushdownLabelOK(stage.filter.Field) || plan.parser == "" && strings.HasSuffix(stage.filter.Field, "_extracted")) { return nil, nil, false } + if level { + if stage.filter.Op != "!=" || stage.filter.Value != "" { + filters = append(filters, *stage.filter) + } + continue + } filters = append(filters, *stage.filter) if !stage.filter.Matches("") && !containsString(errorFilters, stage.filter.Field) { errorFilters = append(errorFilters, stage.filter.Field) @@ -802,61 +827,101 @@ func (plan *orderedJSONMetricPlan) pushdownLabelFilters() (filters []translator. // logsQLLabelFilter renders a Loki string label filter as a LogsQL filter pipe // with the same semantics: exact comparison, and a fully anchored RE2 regexp. func logsQLLabelFilter(filter translator.DropCondition) string { + return " | filter " + logsQLLabelCondition(filter) +} + +// logsQLLabelCondition is the condition of logsQLLabelFilter. +func logsQLLabelCondition(filter translator.DropCondition) string { field := quoteLogsQLIdent(filter.Field) switch filter.Op { case "=": - return " | filter " + field + ":=" + logsql.QuoteValue(filter.Value) + return field + ":=" + logsql.QuoteValue(filter.Value) case "!=": - return " | filter -" + field + ":=" + logsql.QuoteValue(filter.Value) + return "-" + field + ":=" + logsql.QuoteValue(filter.Value) case "=~": // strconv.Quote, not QuotePattern: VictoriaLogs unquotes double-quoted // strings with Go rules, so a regexp escape such as \d must stay escaped // or the query is rejected. - return " | filter " + field + ":~" + strconv.Quote("^(?:"+filter.Value+")$") + return field + ":~" + strconv.Quote("^(?:"+filter.Value+")$") default: - return " | filter -" + field + ":~" + strconv.Quote("^(?:"+filter.Value+")$") - } -} - -// lokiNormalizedLevel mirrors the level normalization Loki applies when it -// sets detected_level at ingest; an empty level is unknown. -func lokiNormalizedLevel(level string) string { - switch strings.ToLower(level) { - case "": - return "unknown" - case "trace", "trc": - return "trace" - case "debug", "dbg": - return "debug" - case "info", "inf", "information": - return "info" - case "warn", "wrn", "warning": - return "warn" - case "error", "err": - return "error" - case "critical": - return "critical" - case "fatal": - return "fatal" - } - return level + return "-" + field + ":~" + strconv.Quote("^(?:"+filter.Value+")$") + } +} + +// levelChainFilter is the filter on detected_level after the chain. A row the +// witness flagged (levelWitnessPipes: the pruned chain may have misread it) +// passes it, so that the flag reaches the stats and the request is sent again +// with the full chain; without a witness nothing is flagged. +func levelChainFilter(filter translator.DropCondition) string { + return " | filter (" + logsQLLabelCondition(filter) + " OR " + levelWitnessField + `:="1")` +} + +// setDetectedLevel records whether the plan reads detected_level (see +// derivesLevel). +func (plan *orderedJSONMetricPlan) setDetectedLevel() { + if plan.aggregated && plan.grouping != nil && containsString(plan.grouping.Labels, detectedLevelLabel) != plan.grouping.Without { + plan.derivesLevel = true + } + for _, stage := range plan.stages { + if stage.filter != nil && stage.filter.Field == detectedLevelLabel { + plan.derivesLevel = true + } + } +} + +// withDerivedLevel gives a row the structured metadata Loki's ingest set +// before any parser runs: detected_level, which no parser yields. A body key +// of that name is therefore renamed detected_level_extracted, as Loki does, +// and a stream label detected_level keeps its stored value, the derived one +// then living under detected_level_extracted (detectedLevelName). The value is +// the one the log lines carry (deriveRowLevel): one derivation for every route. +func withDerivedLevel(base map[string]string, level detectedLevel, stream map[string]string) { + base[detectedLevelName(stream)] = level.String() } +// errLevelNotStored reports that a window holds rows whose detected_level no +// stored detected_level or level field decides, so the pipes of the chain must +// compute it. +var errLevelNotStored = errors.New("rows without a stored level") + // orderedJSONStatsBucketsWithReason serves an eligible plan (setStatsPushdown) // from stats_query_range buckets on the anchored sliding grid instead of raw // rows. served is false when the raw evaluator must answer: no bucket grid, a // possible series-limit overflow, or a line a probe found to be read // differently by the two parsers (parseDisagreement). +// +// A plan reading detected_level first asks for the stored fields alone: the +// buckets are grouped by the stored detected_level and level fields, and the +// level of each is Loki's value for every row they decide (the first +// non-empty of the two, a known spelling in its canonical form). That is +// main's plan, with no pipe and no guard probe per row. A bucket without a +// stored level, or with a stored detected_level (a stream label keeps its +// spelling, translator.BuildDetectedLevelChain), is a row only the chain and +// its guards read exactly: the query is asked again with them. func (p *Proxy) orderedJSONStatsBucketsWithReason(ctx context.Context, plan *orderedJSONMetricPlan, start, end time.Time, step time.Duration) (body []byte, served bool, parseDisagreement bool, err error) { if !plan.pushdown || p.labelTranslator == nil { return nil, false, false, nil } - unpack := append([]string(nil), plan.unpackFields...) - for _, filter := range plan.pushdownFilters { - if !containsString(unpack, filter.Field) { - unpack = append(unpack, filter.Field) + if plan.derivesLevel && !p.detectedLevelBodyScan { + // VictoriaLogs pipes always read the line; without the body scan only + // the stored fields decide, which the raw evaluator derives. + return nil, false, false, nil + } + if plan.derivesLevel { + body, served, parseDisagreement, err = p.orderedJSONStatsBuckets(ctx, plan, start, end, step, true) + if !errors.Is(err, errLevelNotStored) && !errors.Is(err, errStatsGroupGuard) { + return body, served, parseDisagreement, err } } + return p.orderedJSONStatsBuckets(ctx, plan, start, end, step, false) +} + +// orderedJSONStatsBuckets is one attempt of orderedJSONStatsBucketsWithReason: +// storedLevel reads detected_level from the stored fields (and returns +// errLevelNotStored when they do not decide every row), otherwise the chain +// computes it. +func (p *Proxy) orderedJSONStatsBuckets(ctx context.Context, plan *orderedJSONMetricPlan, start, end time.Time, step time.Duration, storedLevel bool) (body []byte, served bool, parseDisagreement bool, err error) { + unpack := plan.unpackKeys() // A name_extracted label is computed after the plain unpack from a scratch // unpack of the key name (translator.ExtractedLabelPipes); the plain unpack // and the parse-risk probes read both keys, the probes with the stored @@ -887,25 +952,9 @@ func (p *Proxy) orderedJSONStatsBucketsWithReason(ctx context.Context, plan *ord return nil, false, false, err } windowStart := start.Add(-plan.window) - // The parse-risk check and the bucket query run concurrently; the buckets - // are discarded when the check finds a line the parsers read differently. - riskCtx, cancelRisk := context.WithCancel(ctx) - defer cancelRisk() - type riskResult struct { - risky bool - err error - } - riskDone := make(chan riskResult, 1) - go func() { - risky, err := p.pushdownParseRisk(riskCtx, plan, base, plain, extracted, extractedKeys, stored, windowStart, end) - riskDone <- riskResult{risky, err} - }() statsCtx, cancelStats := context.WithCancel(ctx) defer cancelStats() - groupBy := plan.unpackFields - if plan.grouping != nil && containsString(plan.grouping.Labels, "detected_level") { - groupBy = append(append([]string(nil), groupBy...), "detected_level") - } + groupBy := plan.statsGroupBy(storedLevel) statsAggFunc := "count() as c" if plan.function == "bytes_rate" || plan.function == "bytes_over_time" { statsAggFunc = "sum_len(_msg) as c, count() as __sample_count" @@ -917,17 +966,55 @@ func (p *Proxy) orderedJSONStatsBucketsWithReason(ctx context.Context, plan *ord } } remember, compute := translator.ExtractedLabelPipes("unpack_"+plan.parser, extracted, stored) - query := base + remember + orderedJSONUnpackPipes(plan.parser, allKeys, stored, required) + compute + beforeUnpack := plan.levelPipes(storedLevel) + shared := plan.sharedLevelUnpack(storedLevel, allKeys, extracted, stored) + if shared != nil { + // One parse of every line: the chain's unpack reads the plan's labels too. + beforeUnpack = " " + translator.BuildDetectedLevelChainShared(translator.DetectedLevelFieldNames, translator.DetectedLevelChainGrouping, nil, shared) + statsCtx = withLevelSharedUnpack(statsCtx, shared) + } + query := base + remember + orderedJSONUnpackPipesShared(plan.parser, allKeys, stored, required, beforeUnpack, shared != nil) + compute + // A filter on detected_level compares the derived value: after the chain in + // VictoriaLogs, or on the stored spellings (levelPipes) and again on the + // normalised series below. for _, filter := range plan.pushdownFilters { + if storedLevel && filter.Field == detectedLevelLabel { + continue + } + if filter.Field == detectedLevelLabel { + query += levelChainFilter(filter) + continue + } query += logsQLLabelFilter(filter) } + if storedLevel && (p.levelNeedsChain(ctx, base) || p.levelNeedsChain(ctx, base+levelShapeSeparator+query)) { + return nil, false, false, errLevelNotStored + } + // The parse-risk check and the bucket query run concurrently; the buckets + // are discarded when the check finds a line the parsers read differently. + riskCtx, cancelRisk := context.WithCancel(ctx) + defer cancelRisk() + type riskResult struct { + risky bool + err error + } + riskDone := make(chan riskResult, 1) + go func() { + risky, err := p.pushdownParseRisk(riskCtx, plan, base, plain, extracted, extractedKeys, stored, storedLevel, windowStart, end) + riskDone <- riskResult{risky, err} + }() type statsResult struct { series map[string]manualSeriesSamples err error } statsDone := make(chan statsResult, 1) go func() { - series, err := p.collectRangeMetricHits(statsCtx, query, groupBy, groupBy, false, statsAggFunc, windowStart, end, bucket) + readCtx := statsCtx + if storedLevel { + // The spellings of one level are groups here, series in the answer. + readCtx = guardStatsGroups(statsCtx) + } + series, err := p.collectRangeMetricHits(readCtx, query, groupBy, groupBy, false, statsAggFunc, windowStart, end, bucket) statsDone <- statsResult{series, err} }() risk := <-riskDone @@ -937,13 +1024,39 @@ func (p *Proxy) orderedJSONStatsBucketsWithReason(ctx context.Context, plan *ord } stats := <-statsDone series, err := stats.series, stats.err + if errors.Is(err, errStatsGroupGuard) { + // More groups than the guard reads: the chain's groups are the answer's + // series, so it counts them and answers Loki's series limit itself. + p.markLevelNeedsChain(ctx, base+levelShapeSeparator+query) + } if err != nil { return nil, false, false, err } - // Only above the limit: a query with exactly the limit's series passes. - if series, err = capSeriesForRequest(ctx, series, p.resolvedMaxStatsQuerySeries(ctx)); err != nil { + if storedLevel { + if series, ok = storedLevelSeries(series, plan.levelFilters()); !ok { + p.markLevelNeedsChain(ctx, base) + return nil, false, false, errLevelNotStored + } + } else if series, err = capSeriesForRequest(ctx, series, p.resolvedMaxStatsQuerySeries(ctx)); err != nil { + // Only above the limit: a query with exactly the limit's series passes. return nil, false, false, err } + merged := plan.mergeGroups(series) + if storedLevel { + // The spellings of one level are one series, as in Loki: only the + // merged answer meets the limit. + if merged, err = capSeriesForRequest(ctx, merged, p.resolvedMaxStatsQuerySeries(ctx)); err != nil { + return nil, false, false, err + } + } + body, err = buildHitsRangeMetricMatrix(plan.function, merged, start, end, step, plan.window, p.limits().BufferedBackendBodyBytes) + return body, err == nil, false, err +} + +// mergeGroups folds the stats groups into the plan's series: groups that +// differ only in a field the plan does not group by (the copies of the stored +// level fields, the spellings of one level) are one series. +func (plan *orderedJSONMetricPlan) mergeGroups(series map[string]manualSeriesSamples) map[string]manualSeriesSamples { merged := make(map[string]manualSeriesSamples, len(series)) var grouped []string if plan.grouping != nil { @@ -953,12 +1066,6 @@ func (p *Proxy) orderedJSONStatsBucketsWithReason(ctx context.Context, plan *ord metric := make(map[string]string, len(grouped)) for _, label := range grouped { value := entry.Metric[label] - if label == "detected_level" { - if value == "" { - value = entry.Metric["level"] - } - value = lokiNormalizedLevel(value) - } if value != "" { metric[label] = value } @@ -974,8 +1081,67 @@ func (p *Proxy) orderedJSONStatsBucketsWithReason(ctx context.Context, plan *ord } merged[key] = target } - body, err = buildHitsRangeMetricMatrix(plan.function, merged, start, end, step, plan.window, p.limits().BufferedBackendBodyBytes) - return body, err == nil, false, err + return merged +} + +// statsGroupBy lists the fields the pushdown's stats group by: the grouped +// labels it unpacks, and the field(s) detected_level comes from. +func (plan *orderedJSONMetricPlan) statsGroupBy(storedLevel bool) []string { + groupBy := plan.unpackFields + switch { + case storedLevel: + groupBy = append(append([]string(nil), groupBy...), storedDetectedLevelField, storedLevelField) + case plan.grouping != nil && containsString(plan.grouping.Labels, "detected_level"): + groupBy = append(append([]string(nil), groupBy...), "detected_level") + } + return groupBy +} + +// levelPipes returns the pipes before the unpack that give a plan reading +// detected_level its field: the copies of the stored fields (taken before any +// unpack, so a body key never looks stored) or the chain. +func (plan *orderedJSONMetricPlan) levelPipes(storedLevel bool) string { + if storedLevel { + // The filters on detected_level run in VictoriaLogs on the stored + // spellings (translator.BuildStoredLevelCopyFilter), so the groups it + // returns are those of the rows the filters keep. + return " | copy detected_level as " + storedDetectedLevelField + ", level as " + storedLevelField + + translator.BuildStoredLevelCopyFilter(storedLevelField, storedDetectedLevelField, plan.levelFilters()) + } + return plan.levelChain() +} + +// levelFilters lists the filters on detected_level of a plan reading it. +func (plan *orderedJSONMetricPlan) levelFilters() []translator.DropCondition { + var filters []translator.DropCondition + for _, filter := range plan.pushdownFilters { + if filter.Field == detectedLevelLabel { + filters = append(filters, filter) + } + } + return filters +} + +// unpackKeys lists the keys the pushdown unpacks: the grouped labels and the +// filtered ones, except detected_level, which no key holds. +func (plan *orderedJSONMetricPlan) unpackKeys() []string { + unpack := append([]string(nil), plan.unpackFields...) + for _, filter := range plan.pushdownFilters { + if !containsString(unpack, filter.Field) && (!plan.derivesLevel || filter.Field != detectedLevelLabel) { + unpack = append(unpack, filter.Field) + } + } + return unpack +} + +// levelChain returns the pipes that compute detected_level for a plan reading +// it, placed before the unpack so that the stored level fields and the line +// decide, and a body key never looks stored. +func (plan *orderedJSONMetricPlan) levelChain() string { + if !plan.derivesLevel { + return "" + } + return " " + fullLevelChain() } // splitPushdownKeys splits the labels a pushdown reads: the plain ones, every @@ -1000,7 +1166,7 @@ func splitPushdownKeys(unpack []string) (plain, allKeys, extracted, extractedKey // lines), and for the name_extracted labels over both keys with the stored // value of name deleted, because the scratch unpack reads the body whatever // the stream holds. -func (p *Proxy) pushdownParseRisk(ctx context.Context, plan *orderedJSONMetricPlan, base string, plain, extracted, extractedKeys []string, stored map[string]string, start, end time.Time) (bool, error) { +func (p *Proxy) pushdownParseRisk(ctx context.Context, plan *orderedJSONMetricPlan, base string, plain, extracted, extractedKeys []string, stored map[string]string, storedLevel bool, start, end time.Time) (bool, error) { type result struct { risky bool err error @@ -1024,13 +1190,15 @@ func (p *Proxy) pushdownParseRisk(ctx context.Context, plan *orderedJSONMetricPl } extractedRisk = make(chan result, 1) go func() { - risky, err := p.cachedStatsPushdownRisk(ctx, plan.parser, base+" | delete "+strings.Join(names, ", "), extractedKeys, nil, nil, start, end) + risky, err := p.cachedStatsPushdownRisk(ctx, plan.parser, base+" | delete "+strings.Join(names, ", "), extractedKeys, nil, nil, false, start, end) extractedRisk <- result{risky, err} }() } var out result - if len(plain) > 0 { - out.risky, out.err = p.cachedStatsPushdownRisk(ctx, plan.parser, base, plain, stored, errorFilters, start, end) + if len(plain) > 0 || plan.derivesLevel && !storedLevel { + // A plan without a parser has no raw evaluator to fall back to. The + // level probes are for the lines only the chain reads. + out.risky, out.err = p.cachedStatsPushdownRisk(ctx, plan.parser, base, plain, stored, errorFilters, plan.derivesLevel && !storedLevel && plan.parser != "", start, end) } if extractedRisk != nil { if second := <-extractedRisk; out.err == nil && !out.risky { @@ -1128,7 +1296,7 @@ func (plan *orderedJSONMetricPlan) requiredFields(base string) []string { } var fields []string for _, filter := range plan.pushdownFilters { - if !filter.Matches("") && !containsString(fields, filter.Field) { + if !filter.Matches("") && !containsString(fields, filter.Field) && filter.Field != detectedLevelLabel { fields = append(fields, filter.Field) } } @@ -1144,6 +1312,22 @@ func (plan *orderedJSONMetricPlan) requiredFields(base string) []string { // lines are dropped before the parse (parsedFieldPrefilter), and the parser // runs only on lines that lack one of the labels as a stored field. func orderedJSONUnpackPipes(parser string, unpack []string, stored map[string]string, required []string) string { + return orderedJSONUnpackPipesAfter(parser, unpack, stored, required, "") +} + +// orderedJSONUnpackPipesAfter is orderedJSONUnpackPipes with the pipes of +// beforeUnpack placed after the prefilters and before the unpack, so they +// run on the lines that can count and see only stored fields. With no label +// to unpack, only beforeUnpack remains. +func orderedJSONUnpackPipesAfter(parser string, unpack []string, stored map[string]string, required []string, beforeUnpack string) string { + return orderedJSONUnpackPipesShared(parser, unpack, stored, required, beforeUnpack, false) +} + +// orderedJSONUnpackPipesShared is orderedJSONUnpackPipesAfter, optionally +// without the unpack itself: the pipes of beforeUnpack have unpacked the +// labels already (a detected_level chain built with translator.SharedUnpack), +// and only the stored aliases are left to apply. +func orderedJSONUnpackPipesShared(parser string, unpack []string, stored map[string]string, required []string, beforeUnpack string, unpacked bool) string { var sb strings.Builder if parser != "" { for _, field := range required { @@ -1153,13 +1337,12 @@ func orderedJSONUnpackPipes(parser string, unpack []string, stored map[string]st } sb.WriteString(parsedFieldPrefilter(parser, field, alias...)) } - conditions := make([]string, len(unpack)) - for i, field := range unpack { - conditions[i] = "(" + storedFieldsAbsent(field, stored) + ")" - } + } + sb.WriteString(beforeUnpack) + if parser != "" && !unpacked && (len(unpack) > 0 || beforeUnpack == "") { sb.WriteString(" | unpack_" + parser) if len(required) != 0 { - sb.WriteString(" if (" + strings.Join(conditions, " or ") + ")") + sb.WriteString(" if (" + orderedJSONUnpackCond(unpack, stored) + ")") } sb.WriteString(" fields (" + strings.Join(unpack, ", ") + ") keep_original_fields") } @@ -1171,6 +1354,43 @@ func orderedJSONUnpackPipes(parser string, unpack []string, stored map[string]st return sb.String() } +// orderedJSONUnpackCond is the condition of a plan's unpack when it has one: +// a row is unpacked when any label is missing as a stored field. +func orderedJSONUnpackCond(unpack []string, stored map[string]string) string { + conditions := make([]string, len(unpack)) + for i, field := range unpack { + conditions[i] = "(" + storedFieldsAbsent(field, stored) + ")" + } + return strings.Join(conditions, " or ") +} + +// sharedLevelUnpackKeyRE is the label names a chain can unpack for its plan: +// plain JSON keys, which no quoting, flattening of nested objects or renaming +// (log.level is the label log_level) can make differ from the plan's own +// unpack, and never a name of the chain's scratch fields. +var sharedLevelUnpackKeyRE = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_]*$`) + +// sharedLevelUnpack returns the unpack a plan hands to the detected_level chain +// so that the line is parsed once, or nil when the plan keeps its own: the +// chain exists only for a JSON plan that derives the level, and the labels +// must be plain keys with no name_extracted label among them (their scratch +// unpack, ExtractedLabelPipes, reads the body its own way). +func (plan *orderedJSONMetricPlan) sharedLevelUnpack(storedLevel bool, allKeys, extracted []string, stored map[string]string) *translator.SharedUnpack { + if storedLevel || !plan.derivesLevel || plan.parser != "json" || len(extracted) > 0 || len(allKeys) == 0 { + return nil + } + for _, key := range allKeys { + if !sharedLevelUnpackKeyRE.MatchString(key) || key == detectedLevelLabel { + return nil + } + } + // The unpack parses a row only where a label is missing as a stored field + // (the rest keep their stored values whatever the body says), or where the + // chain reads the level from the line; the plan's own unpack parses every + // row when no label is required, and the chain's before it again. + return &translator.SharedUnpack{Keys: allKeys, Cond: orderedJSONUnpackCond(allKeys, stored)} +} + // statsPushdownStoredFieldRisk reports whether a selected line carries one of // the labels the pushdown relies on to exclude unparsed lines as a stored // field while its body yields no value for it: such a line passes the filter @@ -1331,19 +1551,115 @@ func (p *Proxy) extractedBothKeysRisk(ctx context.Context, parser, base string, return p.statsPushdownRiskExists(ctx, query, start, end) } -// statsPushdownRiskCacheTTL bounds how long a risk-free window is remembered. -// Coverage never extends past statsPushdownRiskSettle before now, so lines that -// arrive late for recent timestamps are still checked on the next request. -const ( - statsPushdownRiskCacheTTL = 5 * time.Minute - statsPushdownRiskSettle = 5 * time.Minute +// detectedLevelStreamRisk reports whether a selected line belongs to a stream +// with a detected_level label. Loki keeps that label's stored value for +// detected_level (the derived level becomes detected_level_extracted), where +// the pipes of translator.BuildDetectedLevelChain overwrite the field with the +// derived one. +func (p *Proxy) detectedLevelStreamRisk(ctx context.Context, base string, start, end time.Time) (bool, error) { + query := base + ` | filter _stream:{detected_level=~".+"} | limit 1` + return p.statsPushdownRiskExists(ctx, query, start, end) +} + +// detectedLevelKeyNames lists the body keys Loki reads a level from. +var detectedLevelKeyNames = translator.DetectedLevelFieldNames[1 : len(translator.DetectedLevelFieldNames)-1] + +// detectedLevelNames is the alternation of the level key names, escaped. +var detectedLevelNames = func() string { + names := make([]string, len(detectedLevelKeyNames)) + for i, name := range detectedLevelKeyNames { + names[i] = regexp.QuoteMeta(name) + } + return strings.Join(names, "|") +}() + +// detectedLevelJSONKey matches a level key of a JSON line, quoted and followed +// by a colon (a mention inside a string, such as "log level: high", does not +// match); detectedLevelLogfmtKey matches one of a logfmt line. +var ( + detectedLevelJSONKey = `"(?:` + detectedLevelNames + `)"\s*:` + detectedLevelLogfmtKey = `(?:^|\s)(?:` + detectedLevelNames + `)=` ) +// detectedLevelWordFilter passes the lines that hold a level key as a word, +// which VictoriaLogs answers from its token index. +func detectedLevelWordFilter() string { + var words []string + for _, name := range detectedLevelKeyNames { + if !strings.Contains(name, ".") { + words = append(words, name) + } + } + return logsqlWordPrefilter(append(words, "level")) +} + +// detectedLevelStoredFilter is the filter excluding the lines a stored level +// field decides, which the pipes read first, as Loki does. +func detectedLevelStoredFilter() string { + if has := translator.BuildHasLevelFilter(translator.DetectedLevelFieldNames); has != "" { + return " | filter !(" + has + ")" + } + return "" +} + +// detectedLevelKeysRisk reports whether a selected line mentions a level key +// twice, holds a level value with an escape, or is read as logfmt although it +// starts with a brace. Loki takes the first level key in document order (and, +// for a value that is not a known spelling, scans the line), while the pipes +// of translator.BuildDetectedLevelChain read the keys in a fixed order: a line +// with two of them can be read differently. Loki reads a JSON value's raw +// bytes, so an escaped spelling (\u0069nfo) is no known level for it, where +// VictoriaLogs decodes it. A line is JSON for Loki when it starts with a brace +// and ends with a closing one: one that starts with a brace only is logfmt, +// which the chain does not unpack. A line starting with a brace is read for +// quoted keys, any other for logfmt keys. The patterns also match a mention +// inside a longer string, a superset that only costs the raw evaluator. +func (p *Proxy) detectedLevelKeysRisk(ctx context.Context, base string, start, end time.Time) (bool, error) { + const closed = `\}[\s\v\x{85}\p{Z}]*$` + braced := strconv.Quote(translator.BracedLineRegexp) + twice := func(key string) string { return key + `(?s:.*)` + key } + query := base + detectedLevelStoredFilter() + " | filter " + detectedLevelWordFilter() + + " ((_msg:~" + braced + " _msg:~" + strconv.Quote(twice(detectedLevelJSONKey)) + ")" + + " or (_msg:~" + braced + " _msg:~" + strconv.Quote(detectedLevelJSONKey+`\s*"[^"\\]*\\`) + ")" + + " or (_msg:~" + braced + " -_msg:~" + strconv.Quote(closed) + " _msg:~" + strconv.Quote(detectedLevelLogfmtKey) + ")" + + " or (-_msg:~" + braced + " _msg:~" + strconv.Quote(twice(detectedLevelLogfmtKey)) + ")) | limit 1" + return p.statsPushdownRiskExists(ctx, query, start, end) +} + +// detectedLevelUnpackRisk reports whether a selected JSON line mentions a +// level key that unpack_json yields no value for: a key nested other than +// under log (Loki reads level keys at depth 0 and 1), a line VictoriaLogs +// does not parse as a whole (leading whitespace, text after the closing brace, +// a syntax error: Loki keeps the keys read before it), an array or a +// mention inside text. Loki reads such a key, the pipes of +// translator.BuildDetectedLevelChain do not. +func (p *Proxy) detectedLevelUnpackRisk(ctx context.Context, base string, start, end time.Time) (bool, error) { + names := make([]string, len(detectedLevelKeyNames)) + empty := make([]string, len(detectedLevelKeyNames)) + for i, name := range detectedLevelKeyNames { + names[i] = quoteLogsQLIdent(name) + empty[i] = quoteLogsQLIdent("__lr_"+name) + `:=""` + } + query := base + detectedLevelStoredFilter() + " | filter " + detectedLevelWordFilter() + + " _msg:~" + strconv.Quote(`^\s*\{(?s:.*)`+detectedLevelJSONKey) + + " | unpack_json from _msg fields (" + strings.Join(names, ", ") + `) result_prefix "__lr_"` + + " | filter " + strings.Join(empty, " ") + " | limit 1" + return p.statsPushdownRiskExists(ctx, query, start, end) +} + +// statsPushdownRiskCacheTTL bounds how long a risk-free window is remembered, +// counted from the check of its oldest part (coveredWindowProbe). Coverage +// never extends past statsPushdownRiskSettle before now, so lines that arrive +// late for recent timestamps are still checked on the next request. +var statsPushdownRiskCacheTTL = 5 * time.Minute + +const statsPushdownRiskSettle = 5 * time.Minute + // cachedStatsPushdownRisk answers the parse-risk check for [start, end], // remembering the window already found free of risky lines per tenant, query -// and field set, so a refresh or a wider range checks only the uncovered part. -// A window with a risky line is never cached. -func (p *Proxy) cachedStatsPushdownRisk(ctx context.Context, parser, base string, fields []string, stored map[string]string, errorFilters []string, start, end time.Time) (bool, error) { +// and field set, so a refresh or a wider range checks only the uncovered part +// (coveredWindowProbe). A window with a risky line is never cached. +func (p *Proxy) cachedStatsPushdownRisk(ctx context.Context, parser, base string, fields []string, stored map[string]string, errorFilters []string, levelRisk bool, start, end time.Time) (bool, error) { check := func(from, to time.Time) (bool, error) { // The probes run beside each other; one round trip. type result struct { @@ -1358,12 +1674,23 @@ func (p *Proxy) cachedStatsPushdownRisk(ctx context.Context, parser, base string return p.statsPushdownStoredFieldRisk(ctx, parser, base, errorFilters, stored, from, to) }, } - if parser != "" { + if levelRisk { + probes = append(probes, + func(ctx context.Context) (bool, error) { return p.detectedLevelStreamRisk(ctx, base, from, to) }, + func(ctx context.Context) (bool, error) { return p.detectedLevelKeysRisk(ctx, base, from, to) }, + func(ctx context.Context) (bool, error) { return p.detectedLevelUnpackRisk(ctx, base, from, to) }) + } + // A plan that only derives detected_level has no label to probe. + fieldParser := parser + if len(fields) == 0 { + fieldParser = "" + } + if fieldParser != "" { probes = append(probes, func(ctx context.Context) (bool, error) { - return p.extractedBothKeysRisk(ctx, parser, base, fields, from, to) + return p.extractedBothKeysRisk(ctx, fieldParser, base, fields, from, to) }) } - switch parser { + switch fieldParser { case "json": probes = append(probes, func(ctx context.Context) (bool, error) { @@ -1396,9 +1723,6 @@ func (p *Proxy) cachedStatsPushdownRisk(ctx context.Context, parser, base string } return false, nil } - if p.cache == nil { - return check(start, end) - } spelled := make([]string, len(fields)) for i, field := range fields { spelled[i] = field @@ -1406,45 +1730,12 @@ func (p *Proxy) cachedStatsPushdownRisk(ctx context.Context, parser, base string spelled[i] += "=" + vl } } - key := "stats-pushdown-risk:v2:" + getOrgID(ctx) + ":" + parser + ":" + strings.Join(spelled, ",") + ":" + strings.Join(errorFilters, ",") + ":" + base - type window struct{ from, to time.Time } - todo := []window{{start, end}} - coveredFrom, coveredTo := start, end - if raw, _, ok := p.cache.GetWithTTL(key); ok { - var fromNs, toNs int64 - if _, err := fmt.Sscanf(string(raw), "%d %d", &fromNs, &toNs); err == nil { - cachedFrom, cachedTo := time.Unix(0, fromNs), time.Unix(0, toNs) - if !start.After(cachedTo) && !end.Before(cachedFrom) { - todo = todo[:0] - if start.Before(cachedFrom) { - todo = append(todo, window{start, cachedFrom}) - } else { - coveredFrom = cachedFrom - } - if end.After(cachedTo) { - todo = append(todo, window{cachedTo, end}) - } else { - coveredTo = cachedTo - } - } - } - } - for _, w := range todo { - risky, err := check(w.from, w.to) - if err != nil || risky { - return risky, err - } - } - if settled := time.Now().Add(-statsPushdownRiskSettle); coveredTo.After(settled) { - coveredTo = settled - } - if len(todo) > 0 && coveredTo.After(coveredFrom) { - // Only a window that was actually checked refreshes the entry; sliding the - // TTL on a pure cache hit would freeze the interior verdict forever, and a - // line arriving late for a timestamp inside it would never be seen. - p.cache.SetLocalOnlyWithTTL(key, []byte(fmt.Sprintf("%d %d", coveredFrom.UnixNano(), coveredTo.UnixNano())), statsPushdownRiskCacheTTL) - } - return false, nil + key := "stats-pushdown-risk:v4:" + getOrgID(ctx) + ":" + parser + ":" + strings.Join(spelled, ",") + ":" + strings.Join(errorFilters, ",") + ":" + strconv.FormatBool(levelRisk) + ":" + base + _, risky, err := p.coveredWindowProbe(ctx, key, start, end, func(from, to time.Time, _ string) (string, bool, error) { + risky, err := check(from, to) + return "", risky, err + }) + return risky, err } // statsPushdownRiskExists runs a `| limit 1` risk query over [start, end] and @@ -1970,12 +2261,6 @@ func (plan *orderedJSONMetricPlan) groupLabels(labels map[string]string) map[str group[name] = value } } - // detected_level is structured metadata Loki sets at ingest, so no parser - // yields it; derive it from the level this row carries, normalised the way - // Loki normalises it, exactly as the stats pushdown does. - if group["detected_level"] == "" && containsString(plan.grouping.Labels, "detected_level") != plan.grouping.Without { - group["detected_level"] = lokiNormalizedLevel(labels["level"]) - } return group } @@ -2024,6 +2309,7 @@ func (p *Proxy) collectOrderedJSONMetric(ctx context.Context, plan *orderedJSONM streamLabels := make(map[string]map[string]string) descriptors := make(map[string]cachedLogQueryStreamDescriptor) exposures := make(map[string][]metadataFieldExposure) + levels := p.newLogRowLevels() rows := 0 for scanner.Scan() { if err := ctx.Err(); err != nil { @@ -2052,6 +2338,9 @@ func (p *Proxy) collectOrderedJSONMetric(ctx context.Context, plan *orderedJSONM if err != nil { return nil, err } + if plan.derivesLevel { + withDerivedLevel(base, levels.stringRow(row, levels.streamLabels(row["_stream"], desc.streamLabels)), desc.streamLabels) + } labels, keep, err := plan.processWithContext(ctx, row["_msg"], base, desc.translatedLabels) if err != nil { return nil, err diff --git a/internal/proxy/ordered_json_stats_pushdown_test.go b/internal/proxy/ordered_json_stats_pushdown_test.go index b9a627a3e..711de4502 100644 --- a/internal/proxy/ordered_json_stats_pushdown_test.go +++ b/internal/proxy/ordered_json_stats_pushdown_test.go @@ -33,6 +33,12 @@ type jsonVolumeFakeVL struct { statsCalls []string guardCalls []string rawCalls int + // presence answers the level presence probe from the lines (see + // pushdownFakeVL); presenceCalls records the probes. + presence bool + presenceFail bool + presenceLine func(honest string) string + presenceCalls []string } // vlUnpackJSONLevel mirrors VictoriaLogs unpack_json for the level key: only a @@ -60,12 +66,26 @@ var jsonVolumeMsgFilterRE = regexp.MustCompile(`_msg:~("(?:[^"\\]|\\.)*")`) // renders for a Loki level=/!= label filter. var fakeVLLevelFilterRE = regexp.MustCompile(`\| filter (-?)level:="([^"]*)"`) +// fakeStoredLevelFilters applies the filter pipes on the copies of the stored +// level fields (translator.BuildStoredLevelCopyFilter) to a stream level. +func fakeStoredLevelFilters(t testing.TB, query, level string) bool { + values := map[string]string{"__dl_l": level} + for _, pipe := range splitLogsQLPipes(query) { + if strings.HasPrefix(pipe, "filter (__dl_d:*") && !fakeLogsQLMatch(t, strings.TrimPrefix(pipe, "filter "), values) { + return false + } + } + return true +} + func newJSONVolumeFakeVL(t testing.TB, lines []jsonVolumeLine) (*httptest.Server, *jsonVolumeFakeVL) { t.Helper() fake := &jsonVolumeFakeVL{lines: lines} srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { - t.Errorf("parse form: %v", err) + if r.Context().Err() == nil { // a cancelled probe is not a failure + t.Errorf("parse form: %v", err) + } return } query := r.Form.Get("query") @@ -78,6 +98,15 @@ func newJSONVolumeFakeVL(t testing.TB, lines []jsonVolumeLine) (*httptest.Server if !ok { return } + // The pipes of translator.BuildDetectedLevelChain are emulated whole + // below; the unpack they hold is not the plan's own. + // The stored-level plan copies the stored fields, the chain computes the level. + storedLevel := strings.Contains(query, "copy detected_level as __dl_d, level as __dl_l") + chain := !storedLevel && strings.Contains(query, "__dl") + chainQuery := query + if chain { + query = withoutLevelChain(query) + } keep := strings.Contains(query, "fields (level) keep_original_fields") groupStart := strings.Index(query, "stats by (") if groupStart < 0 { @@ -87,7 +116,7 @@ func newJSONVolumeFakeVL(t testing.TB, lines []jsonVolumeLine) (*httptest.Server rest := query[groupStart+len("stats by ("):] groupBy := strings.Split(rest[:strings.Index(rest, ")")], ", ") bytesMetric := strings.Contains(query, "sum_len(_msg) as c") - type key struct{ name, level string } + type key struct{ name, level, derived string } points := map[key]map[int64]float64{} add := func(k key, ts int64, v float64) { if points[k] == nil { @@ -112,22 +141,38 @@ func newJSONVolumeFakeVL(t testing.TB, lines []jsonVolumeLine) (*httptest.Server // Without keep_original_fields unpacking overwrites stored fields. level = unpacked } + if storedLevel && !fakeStoredLevelFilters(t, query, line.level) { + continue + } if m := fakeVLLevelFilterRE.FindStringSubmatch(query); m != nil && (level == m[2]) == (m[1] == "-") { continue } + derived := "" + if chain { + stored := map[string]string{"_msg": line.msg} + if line.level != "" { + stored["level"] = line.level + } + derived = emulateDetectedLevelChainFor(stored, chainPresence(chainQuery)) + } + if storedLevel { + derived = line.level // __dl_l: the stream's level, before any unpack + } bucket := vlTruncate(ts, step, offset) value := 1.0 if bytesMetric { value = float64(len(line.msg)) - add(key{"__sample_count", level}, bucket, 1) + add(key{"__sample_count", level, derived}, bucket, 1) } - add(key{"c", level}, bucket, value) + add(key{"c", level, derived}, bucket, value) } keys := make([]key, 0, len(points)) for k := range points { keys = append(keys, k) } - sort.Slice(keys, func(i, j int) bool { return keys[i].name+keys[i].level < keys[j].name+keys[j].level }) + sort.Slice(keys, func(i, j int) bool { + return keys[i].name+keys[i].level+"\x00"+keys[i].derived < keys[j].name+keys[j].level+"\x00"+keys[j].derived + }) var sb strings.Builder sb.WriteString(`{"status":"success","data":{"resultType":"matrix","result":[`) for i, k := range keys { @@ -137,8 +182,11 @@ func newJSONVolumeFakeVL(t testing.TB, lines []jsonVolumeLine) (*httptest.Server fmt.Fprintf(&sb, `{"metric":{"__name__":%q`, k.name) for _, field := range groupBy { value := "" - if field == "level" { + switch field { + case "level": value = k.level + case "detected_level", "__dl_l": + value = k.derived } // VictoriaLogs reports every grouped field, empty or not. fmt.Fprintf(&sb, `,%q:%q`, field, value) @@ -163,11 +211,42 @@ func newJSONVolumeFakeVL(t testing.TB, lines []jsonVolumeLine) (*httptest.Server case "/select/logsql/query": start := parseFakeVLTime(t, r.Form.Get("start")) end := parseFakeVLTime(t, r.Form.Get("end")) + if isLevelPresenceQuery(query) { + fake.mu.Lock() + fake.presenceCalls = append(fake.presenceCalls, query) + fake.mu.Unlock() + if fake.presenceFail { + http.Error(w, "backend failure", http.StatusInternalServerError) + return + } + line := "{}\n" + if fake.presence { + var rows []map[string]string + for _, l := range fake.lines { + if ts := l.ts.UnixNano(); ts < start || ts >= end { + continue + } + values := map[string]string{"_msg": l.msg} + if l.level != "" { + values["level"] = l.level + } + rows = append(rows, values) + } + if line = levelPresenceResponse(rows); fake.presenceLine != nil { + line = fake.presenceLine(line) + } + } + _, _ = w.Write([]byte(line)) + return + } guard := strings.HasSuffix(query, " | limit 1") - logfmtGuard := guard && !strings.Contains(query, "unpack_json") + // The guard probes of the detected_level pipes, which the generic + // fake evaluates over the same lines. + levelGuard := guard && (strings.Contains(query, `_msg:"LEVEL"`) || strings.Contains(query, "filter _stream:{detected_level")) + logfmtGuard := guard && !levelGuard && !strings.Contains(query, "unpack_json") var msgRE *regexp.Regexp fake.mu.Lock() - if logfmtGuard { + if levelGuard || logfmtGuard { fake.guardCalls = append(fake.guardCalls, query) } else if guard { fake.guardCalls = append(fake.guardCalls, query) @@ -187,10 +266,19 @@ func newJSONVolumeFakeVL(t testing.TB, lines []jsonVolumeLine) (*httptest.Server if ts < start || ts >= end { continue } + if levelGuard { + labels := map[string]string{"app": "api"} + if line.level != "" { + labels["level"] = line.level + } + if !new(pushdownFakeVL).guardMatches(t, query, pushdownRow{ts: line.ts, stream: labels, msg: line.msg}) { + continue + } + } if logfmtGuard && (line.level != "" || !logfmtLineIsRisk(line.msg, "level")) { continue } - if guard && !logfmtGuard && (line.level != "" || !msgRE.MatchString(line.msg) || vlUnpackJSONLevel(line.msg) != "") { + if guard && !levelGuard && !logfmtGuard && (line.level != "" || !msgRE.MatchString(line.msg) || vlUnpackJSONLevel(line.msg) != "") { continue } row := map[string]string{"_time": line.ts.UTC().Format(time.RFC3339Nano), "_msg": line.msg, "_stream": `{app="api"}`, "app": "api"} @@ -221,7 +309,11 @@ func lokiJSONVolumeReference(lines []jsonVolumeLine, fn string, groupBy []string } // lokiLevelVolumeReference is lokiJSONVolumeReference for parser "json", -// "logfmt" (fixture lines are simple key=value tokens) or "" (no parser). +// "logfmt" (fixture lines are simple key=value tokens) or "" (no parser). It is +// a hand-built model of Loki's answer for these simple fixtures, not Loki's +// own: detected_level follows Loki's ingest (a stream level, else the level +// key of the body, normalized to a known spelling, else a keyword scan of the +// line). The real-Loki comparison lives in the e2e tests. func lokiLevelVolumeReference(lines []jsonVolumeLine, parser, fn string, groupBy []string, start, end time.Time, step, window time.Duration) map[string]map[int64]string { out := map[string]map[int64]string{} for t := start; !t.After(end); t = t.Add(step) { @@ -238,13 +330,27 @@ func lokiLevelVolumeReference(lines []jsonVolumeLine, parser, fn string, groupBy case parser == "logfmt": level = slidingLogfmtFields(line.msg)["level"] } + // Loki's ingest reads the body of a stream without a level for + // detected_level whatever the query parses (a logfmt or JSON level + // key here). + detected := line.level + if detected == "" { + if detected = slidingLogfmtFields(line.msg)["level"]; detected == "" { + detected = vlUnpackJSONLevel(line.msg) + } + } + if known := canonicalLevel([]byte(detected)); known != "" { + detected = known + } else if line.level == "" { + detected = levelFromKeywords([]byte(line.msg)) + } labels := map[string]string{} for _, name := range groupBy { switch { case name == "level" && level != "": labels[name] = level - case name == "detected_level" && level != "": - labels[name] = level + case name == "detected_level" && detected != "": + labels[name] = detected case name == "detected_level": labels[name] = "unknown" } @@ -345,9 +451,9 @@ func TestOrderedJSONLogsVolumeUsesStatsBuckets(t *testing.T) { window time.Duration stats string }{ - {`sum by (level, detected_level) (count_over_time({app="api"} | json | drop __error__[1m]))`, "count_over_time", []string{"level", "detected_level"}, time.Minute, time.Minute, "unpack_json fields (level) keep_original_fields | stats by (level, detected_level) count() as c"}, + {`sum by (level, detected_level) (count_over_time({app="api"} | json | drop __error__[1m]))`, "count_over_time", []string{"level", "detected_level"}, time.Minute, time.Minute, "| delete __dl, __dl_*, __j_*, __l_* | stats by (level, detected_level) count() as c"}, {`sum by (level) (rate({app="api"} | json | drop __error__, __error_details__ [2m]))`, "rate", []string{"level"}, 30 * time.Second, 2 * time.Minute, "unpack_json fields (level) keep_original_fields | stats by (level) count() as c"}, - {`sum by (detected_level) (bytes_over_time({app="api"} | json | drop __error__ [90s]))`, "bytes_over_time", []string{"detected_level"}, time.Minute, 90 * time.Second, "unpack_json fields (level) keep_original_fields | stats by (level, detected_level) sum_len(_msg) as c, count() as __sample_count"}, + {`sum by (detected_level) (bytes_over_time({app="api"} | json | drop __error__ [90s]))`, "bytes_over_time", []string{"detected_level"}, time.Minute, 90 * time.Second, "| delete __dl, __dl_*, __j_*, __l_* | stats by (detected_level) sum_len(_msg) as c, count() as __sample_count"}, } { t.Run(tc.query, func(t *testing.T) { srv, fake := newJSONVolumeFakeVL(t, lines) @@ -359,11 +465,22 @@ func TestOrderedJSONLogsVolumeUsesStatsBuckets(t *testing.T) { } fake.mu.Lock() defer fake.mu.Unlock() - if fake.rawCalls != 0 || len(fake.statsCalls) != 1 || len(fake.guardCalls) != 1 { - t.Fatalf("expected one guard and one stats call without raw rows: raw=%d stats=%q guard=%q", fake.rawCalls, fake.statsCalls, fake.guardCalls) + // The detected_level pipes add the guard probes of levelProbes. + guards := 1 + if strings.Contains(tc.query, "detected_level") { + guards += levelProbes + } + if !strings.Contains(tc.query, "by (level") { + guards-- // no label to probe: only the detected_level probes run + } else if strings.Contains(tc.query, "detected_level") { + guards++ // the stored-level attempt (a row has no stored level) probes the label on its own + } + chain := chainStatsCalls(fake.statsCalls) + if fake.rawCalls != 0 || len(chain) != 1 || len(fake.guardCalls) != guards { + t.Fatalf("expected %d guards and one chain stats call without raw rows: raw=%d stats=%q guard=%q", guards, fake.rawCalls, fake.statsCalls, fake.guardCalls) } - if !strings.HasSuffix(fake.statsCalls[0], tc.stats) { - t.Fatalf("stats query %q does not end with %q", fake.statsCalls[0], tc.stats) + if !strings.HasSuffix(chain[0], tc.stats) { + t.Fatalf("stats query %q does not end with %q", chain[0], tc.stats) } }) } @@ -422,7 +539,7 @@ func TestOrderedJSONStatsPushdownEligibility(t *testing.T) { fields []string }{ {`sum by (level, detected_level) (count_over_time({env="production"} | json | drop __error__[1m]))`, []string{"level"}}, - {`sum by (detected_level) (count_over_time({env="production"} |= "x" | json | drop __error_details__, __error__[1m]))`, []string{"level"}}, + {`sum by (detected_level) (count_over_time({env="production"} |= "x" | json | drop __error_details__, __error__[1m]))`, nil}, {`sum by (app, level) (bytes_rate({env="production"} | json | drop __error__ [1m]))`, []string{"app", "level"}}, {`sum by (level) (rate({env="production"} | json | drop __error__ | json[1m]))`, nil}, {`sum by (level) (count_over_time({env="production"} | json [1m]))`, nil}, @@ -443,8 +560,8 @@ func TestOrderedJSONStatsPushdownEligibility(t *testing.T) { {`sum by (level) (count_over_time({env="production"} | json | drop __error__, level [1m]))`, nil}, {`count_over_time({env="production"} | json | drop __error__ [1m])`, nil}, {`sum by (level, detected_level) (count_over_time({env="production"} | drop __error__[1m]))`, []string{"level"}}, - {`sum by (detected_level) (rate({env="production"} |= "x" | logfmt | drop __error__, __error_details__[1m]))`, []string{"level"}}, - {`sum by (detected_level) (count_over_time({env="production"} | logfmt | level="error" | drop __error__[1m]))`, []string{"level"}}, + {`sum by (detected_level) (rate({env="production"} |= "x" | logfmt | drop __error__, __error_details__[1m]))`, nil}, + {`sum by (detected_level) (count_over_time({env="production"} | logfmt | level="error" | drop __error__[1m]))`, nil}, } { t.Run(tc.query, func(t *testing.T) { plan, ok := compileOrderedJSONMetric(tc.query) @@ -504,7 +621,8 @@ func TestLogfmtParseRiskPatterns(t *testing.T) { // Grafana's logs volume for a plain selector and for `| logfmt` groups by the // detected_level alias. VictoriaLogs stats buckets answer both: level keeps -// the stored value, detected_level mirrors it and falls back to unknown. +// the stored value, detected_level is Loki's value for the line (the stream's +// level, else the body's level key or keyword) and unknown without any. func TestLevelVolumePlainAndLogfmtUseStatsBuckets(t *testing.T) { s0 := time.Unix(1700000400, 0).UTC() var lines []jsonVolumeLine @@ -527,21 +645,10 @@ func TestLevelVolumePlainAndLogfmtUseStatsBuckets(t *testing.T) { guards int stats string }{ - {`sum by (level, detected_level) (count_over_time({app="api"} | drop __error__[1m]))`, "", 0, `app:="api" | stats by (level, detected_level) count() as c`}, - {`sum by (level, detected_level) (count_over_time({app="api"} | logfmt | drop __error__[1m]))`, "logfmt", 1, "unpack_logfmt fields (level) keep_original_fields | stats by (level, detected_level) count() as c"}, + {`sum by (level, detected_level) (count_over_time({app="api"} | drop __error__[1m]))`, "", 0, "| delete __dl, __dl_*, __j_*, __l_* | stats by (level, detected_level) count() as c"}, + {`sum by (level, detected_level) (count_over_time({app="api"} | logfmt | drop __error__[1m]))`, "logfmt", 2 + levelProbes, "unpack_logfmt fields (level) keep_original_fields | stats by (level, detected_level) count() as c"}, } { t.Run(tc.query, func(t *testing.T) { - lines := lines - if tc.parser == "" { - // Loki detects a body level at ingest for streams without one; - // the plain fixture keeps level-less bodies free of levels. - lines = append([]jsonVolumeLine(nil), lines...) - for i := range lines { - if lines[i].level == "" { - lines[i].msg = strings.ReplaceAll(lines[i].msg, "level=", "stage=") - } - } - } srv, fake := newJSONVolumeFakeVL(t, lines) p := newSlidingTestProxy(t, srv.URL) want := lokiLevelVolumeReference(lines, tc.parser, "count_over_time", []string{"level", "detected_level"}, start, end, time.Minute, time.Minute) @@ -551,11 +658,12 @@ func TestLevelVolumePlainAndLogfmtUseStatsBuckets(t *testing.T) { } fake.mu.Lock() defer fake.mu.Unlock() - if fake.rawCalls != 0 || len(fake.statsCalls) != 1 || len(fake.guardCalls) != tc.guards { - t.Fatalf("expected %d guard and one stats call without raw rows: raw=%d stats=%q guard=%q", tc.guards, fake.rawCalls, fake.statsCalls, fake.guardCalls) + chain := chainStatsCalls(fake.statsCalls) + if fake.rawCalls != 0 || len(chain) != 1 || len(fake.guardCalls) != tc.guards { + t.Fatalf("expected %d guard and one chain stats call without raw rows: raw=%d stats=%q guard=%q", tc.guards, fake.rawCalls, fake.statsCalls, fake.guardCalls) } - if !strings.HasSuffix(fake.statsCalls[0], tc.stats) { - t.Fatalf("stats query %q does not end with %q", fake.statsCalls[0], tc.stats) + if !strings.HasSuffix(chain[0], tc.stats) { + t.Fatalf("stats query %q does not end with %q", chain[0], tc.stats) } }) } @@ -596,8 +704,8 @@ func TestOrderedJSONLogsVolumeWithLabelFilterUsesStatsBuckets(t *testing.T) { for _, tc := range []struct { query, op, stats string }{ - {`sum by (level, detected_level) (count_over_time({app="api"} | json | level="warn" | drop __error__[1m]))`, "=", ` | filter (level:* or _msg:"level" or _msg:~"\\\\u") | unpack_json if ((-level:*)) fields (level) keep_original_fields | filter level:="warn" | stats by (level, detected_level) count() as c`}, - {`sum by (level, detected_level) (count_over_time({app="api"} | json | drop __error__ | level!="warn" [1m]))`, "!=", `unpack_json fields (level) keep_original_fields | filter -level:="warn" | stats by (level, detected_level) count() as c`}, + {`sum by (level, detected_level) (count_over_time({app="api"} | json | level="warn" | drop __error__[1m]))`, "=", ` | filter (level:* or _msg:"level" or _msg:~"\\\\u") ` + sharedLevelChain("(-level:*)", "level") + ` | filter level:="warn" | stats by (level, detected_level) count() as c`}, + {`sum by (level, detected_level) (count_over_time({app="api"} | json | drop __error__ | level!="warn" [1m]))`, "!=", `| delete __dl, __dl_*, __j_*, __l_* | filter -level:="warn" | stats by (level, detected_level) count() as c`}, } { t.Run(tc.query, func(t *testing.T) { srv, fake := newJSONVolumeFakeVL(t, lines) @@ -619,8 +727,9 @@ func TestOrderedJSONLogsVolumeWithLabelFilterUsesStatsBuckets(t *testing.T) { } fake.mu.Lock() defer fake.mu.Unlock() - if fake.rawCalls != 0 || len(fake.statsCalls) != 1 || !strings.HasSuffix(fake.statsCalls[0], tc.stats) { - t.Fatalf("expected one stats call ending %q without raw rows: raw=%d stats=%q", tc.stats, fake.rawCalls, fake.statsCalls) + chain := chainStatsCalls(fake.statsCalls) + if fake.rawCalls != 0 || len(chain) != 1 || !strings.HasSuffix(chain[0], tc.stats) { + t.Fatalf("expected one chain stats call ending %q without raw rows: raw=%d stats=%q", tc.stats, fake.rawCalls, fake.statsCalls) } }) } @@ -667,7 +776,7 @@ func TestCachedStatsPushdownRiskChecksOnlyUncoveredWindow(t *testing.T) { {0, 40 * time.Minute, 2}, {-10 * time.Minute, 40 * time.Minute, 3}, } { - if risky, err := p.cachedStatsPushdownRisk(ctx, "json", `app:="api"`, []string{"level"}, nil, nil, s0.Add(tc.from), s0.Add(tc.to)); err != nil || risky { + if risky, err := p.cachedStatsPushdownRisk(ctx, "json", `app:="api"`, []string{"level"}, nil, nil, false, s0.Add(tc.from), s0.Add(tc.to)); err != nil || risky { t.Fatalf("step %d: risky=%v err=%v", i, risky, err) } if got := calls(); got != tc.calls { diff --git a/internal/proxy/range_metric_compat.go b/internal/proxy/range_metric_compat.go index 726c026b4..43054d328 100644 --- a/internal/proxy/range_metric_compat.go +++ b/internal/proxy/range_metric_compat.go @@ -1183,16 +1183,30 @@ func snapSlidingBucketNanos(ts int64, start time.Time, hitStep time.Duration) in // series limit error once it holds more result entries than the limit allows // (two per series when byte presence counts ride along). A Drilldown request // reads it all and keeps the busiest series instead. +// +// With the groups of the response guarded (guardStatsGroups) a VictoriaLogs +// group is no series of the answer: groups merge into fewer series after the +// read (the spellings of one level), and Loki's limit applies to the merged +// answer only. The read then stops at splitGroupSlack times the limit as a +// memory guard, with errStatsGroupGuard instead of the series limit error, and +// the caller asks again in a form whose groups are its series. func (p *Proxy) readStatsBucketBody(ctx context.Context, body io.Reader, limit int, withPresence bool) ([]byte, error) { counter := &seriesCountingReader{r: body} + guarded := statsGroupsGuarded(ctx) if !requestKeepsPartialSeries(ctx) { counter.limit = limit if withPresence { counter.limit = 2 * limit } + if guarded { + counter.limit *= splitGroupSlack + } } out, err := readBodyLimited(counter, maxStatsBucketResponseBytes) if errors.Is(err, errSeriesCountExceeded) { + if guarded { + return nil, errStatsGroupGuard + } return nil, &seriesLimitError{limit: limit} } if err != nil { @@ -1201,6 +1215,27 @@ func (p *Proxy) readStatsBucketBody(ctx context.Context, body io.Reader, limit i return out, nil } +// splitGroupSlack scales the series limit into the number of groups a guarded +// stats read holds in memory before the caller asks in another form. +const splitGroupSlack = 2 + +// errStatsGroupGuard reports a guarded stats response with more than +// splitGroupSlack times the series limit of groups: no verdict on the series +// the answer holds, the memory guard of the read. +var errStatsGroupGuard = errors.New("stats response holds more groups than the guarded read keeps") + +type statsGroupGuardKey struct{} + +// guardStatsGroups marks the stats reads of ctx as guarded (see readStatsBucketBody). +func guardStatsGroups(ctx context.Context) context.Context { + return context.WithValue(ctx, statsGroupGuardKey{}, true) +} + +func statsGroupsGuarded(ctx context.Context) bool { + guarded, _ := ctx.Value(statsGroupGuardKey{}).(bool) + return guarded +} + // collectRangeMetricHits calls VL's /select/logsql/stats_query_range endpoint and // returns pre-bucketed samples (ts=bucket left edge ns on the start+k*hitStep // grid, value=bucket_value). This avoids reading all raw log entries for @@ -1237,6 +1272,10 @@ func (p *Proxy) collectRangeMetricHits( params.Set("query", statsQuery) p.setSlidingStatsRangeParams(params, start, end, hitStep) + // The detected_level chain is narrowed (a probe request) before a slot is + // taken, not while holding one. + ctx = p.pruneBeforeSlot(ctx, params) + // Acquire concurrency slot. The Drilldown Fields page fires ~30 of these // in parallel; without a cap all 30 hit VL simultaneously, causing a CPU storm. // Block until a slot is free or the request is cancelled. @@ -1356,6 +1395,11 @@ func (p *Proxy) collectRangeMetricHits( seriesMap[seriesKey] = manualSeriesSamples{Metric: metric, Samples: samples} } } + if statsGroupsGuarded(ctx) { + // The groups are not the answer's series: the caller applies the limit + // to the merged answer. + return seriesMap, nil + } // More series than -max-stats-query-series: Loki's error, or for Drilldown the // busiest series (complete, with byte values and presence) and a warning. return capSeriesForRequest(ctx, seriesMap, limit) diff --git a/internal/proxy/stored_level_series_limit_test.go b/internal/proxy/stored_level_series_limit_test.go new file mode 100644 index 000000000..c30d9c6d8 --- /dev/null +++ b/internal/proxy/stored_level_series_limit_test.go @@ -0,0 +1,221 @@ +package proxy + +import ( + "fmt" + "net/http" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "github.com/ReliablyObserve/Loki-VL-proxy/internal/translator" +) + +// Loki counts the series of the answer: the spellings of one level are one +// series (the stream level INFO, info and Information all give +// detected_level=info), and a query is refused only when the answer holds more +// than the limit. The stored-level plan reads those spellings as separate +// VictoriaLogs groups, so the limit applies to the merged answer, and a +// filter on detected_level runs in VictoriaLogs, keeping the groups of the +// rows that can match. +// +// conformance: semantics/json-metric-detected-level-derivation +func TestStoredLevelPlanCountsTheSeriesOfTheAnswer(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + // Loki v3.7.7's detected_level for each stream level (normalizeLogLevel). + loki := map[string]string{"INFO": "info", "warn": "warn", "Warning": "warn", "ERR": "error", "notice": "notice", "dbg": "debug", "info": "info", "Information": "info", "unknown": "unknown"} + levels := []string{"INFO", "warn", "Warning", "ERR", "notice", "dbg", "info", "Information", "unknown"} + var lines []jsonVolumeLine + for i := 0; i < 108; i++ { + lines = append(lines, jsonVolumeLine{ts: s0.Add(time.Duration(i) * 5 * time.Second), level: levels[i%len(levels)], msg: `{"n":` + strconv.Itoa(i) + `}`}) + } + start, end := s0.Add(time.Minute), s0.Add(8*time.Minute) + // matrix counts the lines of (t-1m, t] by the labels of groupBy. + matrix := func(groupBy string, keep func(detected string) bool) map[string]map[int64]string { + out := map[string]map[int64]string{} + for at := start; !at.After(end); at = at.Add(time.Minute) { + counts := map[string]int{} + for _, line := range lines { + if !line.ts.After(at.Add(-time.Minute)) || line.ts.After(at) || !keep(loki[line.level]) { + continue + } + label := map[string]string{"level": line.level, "detected_level": loki[line.level]}[groupBy] + counts[canonicalLabelsKey(map[string]string{groupBy: label})]++ + } + for key, n := range counts { + if out[key] == nil { + out[key] = map[int64]string{} + } + out[key][at.Unix()] = strconv.Itoa(n) + } + } + return out + } + any := func(string) bool { return true } + for _, tc := range []struct { + name, query, groupBy string + limit int + keep func(string) bool + wantSeries int // 0: Loki's series limit error + wantChain bool + }{ + // three spellings of info: three series at the limit of three + {"filter on a level keeps its spellings only", `sum by (level) (count_over_time({app="api"} | json | drop __error__ | detected_level="info" [1m]))`, "level", 3, + func(d string) bool { return d == "info" }, 3, false}, + {"regexp filter", `sum by (level) (count_over_time({app="api"} | json | drop __error__ | detected_level=~"warn|notice" [1m]))`, "level", 3, + func(d string) bool { return d == "warn" || d == "notice" }, 3, false}, + {"negated filter", `sum by (level) (count_over_time({app="api"} | json | drop __error__ | detected_level!="info" [1m]))`, "level", 6, + func(d string) bool { return d != "info" }, 6, false}, + {"a word Loki does not know compares as stored", `sum by (level) (count_over_time({app="api"} | json | drop __error__ | detected_level="notice" [1m]))`, "level", 1, + func(d string) bool { return d == "notice" }, 1, false}, + // nine spellings, six levels: exactly the limit passes, one more is refused + {"merged answer at the limit", `sum by (detected_level) (count_over_time({app="api"} | json | drop __error__ [1m]))`, "detected_level", 6, any, 6, false}, + {"merged answer over the limit", `sum by (detected_level) (count_over_time({app="api"} | json | drop __error__ [1m]))`, "detected_level", 5, any, 0, false}, + // 9 groups are past twice the limit of 4: the chain's groups are the series + {"past the group guard the chain counts", `sum by (detected_level) (count_over_time({app="api"} | json | drop __error__ [1m]))`, "detected_level", 4, any, 0, true}, + } { + t.Run(tc.name, func(t *testing.T) { + srv, fake := newJSONVolumeFakeVL(t, lines) + fake.presence = true + p := newSlidingTestProxy(t, srv.URL) + p.maxStatsQuerySeries = tc.limit + code, got, body := detectedLevelQuery(t, p, tc.query, start, end, time.Minute) + fake.mu.Lock() + calls := append([]string(nil), fake.statsCalls...) + raw := fake.rawCalls + fake.mu.Unlock() + if tc.wantSeries == 0 { + if code != http.StatusBadRequest || !strings.Contains(body, fmt.Sprintf("maximum number of series (%d) reached", tc.limit)) { + t.Fatalf("expected Loki's series limit error, got %d %.200s", code, body) + } + } else { + want := matrix(tc.groupBy, tc.keep) + if code != http.StatusOK || len(got) != tc.wantSeries || !reflect.DeepEqual(got, want) { + t.Fatalf("status %d %.200s\n got: %v\nwant: %v\nstats: %q", code, body, got, want, calls) + } + } + if chain := len(chainStatsCalls(calls)) > 0; chain != tc.wantChain || raw != 0 { + t.Fatalf("chain stats query ran: %v (want %v), raw reads %d, stats %q", chain, tc.wantChain, raw, calls) + } + }) + } +} + +// 30 pods with a line of each of three levels: the filter keeps a third of the +// groups in VictoriaLogs, so a limit that only the answer's series (30) fit +// under is not tripped by the 90 groups an unfiltered stored-level read holds. +// Without a filter the 90 groups pass the guard (twice the limit) of 40 and +// the chain counts the 30 series instead; the verdict is remembered for that +// query, so the next request goes to the chain at once. +// conformance: semantics/json-metric-detected-level-derivation +func TestStoredLevelPlanManyPodsUnderTheSeriesLimit(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + end := s0.Add(5 * time.Minute) + var rows []pushdownRow + for pod := 0; pod < 30; pod++ { + for l, level := range []string{"error", "info", "warn"} { + rows = append(rows, pushdownRow{ts: s0.Add(time.Duration(10*pod+l+1) * time.Second), stream: map[string]string{"app": "api", "env": "prod", "level": level}, msg: `{"pod":"p` + strconv.Itoa(pod) + `"}`}) + } + } + want := func(count string) map[string]map[int64]string { + out := map[string]map[int64]string{} + for pod := 0; pod < 30; pod++ { + out[canonicalLabelsKey(map[string]string{"pod": "p" + strconv.Itoa(pod)})] = map[int64]string{end.Unix(): count} + } + return out + } + run := func(t *testing.T, query string, limit int) (*pushdownFakeVL, *Proxy) { + t.Helper() + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.presence = true + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + p.maxStatsQuerySeries = limit + code, got, body := detectedLevelQuery(t, p, query, end, end, time.Minute) + if code != http.StatusOK || !reflect.DeepEqual(got, want("1")) { + t.Fatalf("status %d %.200s\n got: %v\n stats: %q", code, body, got, fake.stats) + } + return fake, p + } + t.Run("filtered", func(t *testing.T) { + fake, _ := run(t, `sum by (pod) (count_over_time({env="prod"} | json | drop __error__ | detected_level="error" [5m]))`, 50) + fake.mu.Lock() + defer fake.mu.Unlock() + if len(fake.stats) != 1 || fake.raw != 0 || !strings.Contains(fake.stats[0], "copy detected_level as __dl_d, level as __dl_l") { + t.Fatalf("expected one stored-level query: %q (raw %d)", fake.stats, fake.raw) + } + }) + t.Run("past the guard", func(t *testing.T) { + // three spellings of one level per pod: 90 groups, 30 series + rows = rows[:0] + for pod := 0; pod < 30; pod++ { + for l, level := range []string{"error", "ERR", "Err"} { + rows = append(rows, pushdownRow{ts: s0.Add(time.Duration(10*pod+l+1) * time.Second), stream: map[string]string{"app": "api", "env": "prod", "level": level}, msg: `{"pod":"p` + strconv.Itoa(pod) + `"}`}) + } + } + query := `sum by (pod, detected_level) (count_over_time({env="prod"} | json | drop __error__ [10m]))` + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.presence = true + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + p.maxStatsQuerySeries = 40 // the guard keeps 80 groups: the stored-level read has 90 + for ask := 0; ask < 2; ask++ { + at := end.Add(time.Duration(ask) * time.Second) + code, got, body := detectedLevelQuery(t, p, query, at, at, time.Minute) + if code != http.StatusOK || len(got) != 30 { + t.Fatalf("ask %d: status %d %.200s, %d series", ask, code, body, len(got)) + } + for key, points := range got { + if !strings.Contains(key, `detected_level="error"`) || points[at.Unix()] != "3" { + t.Fatalf("ask %d: %s: %v", ask, key, points) + } + } + fake.mu.Lock() + stored, chain := len(fake.stats)-len(chainStatsCalls(fake.stats)), len(chainStatsCalls(fake.stats)) + fake.mu.Unlock() + // the first ask tries the stored fields and then the chain; the verdict is + // remembered, so the second goes to the chain alone + if stored != 1 || chain != ask+1 { + t.Fatalf("ask %d: %d stored-level and %d chain stats queries so far", ask, stored, chain) + } + } + }) +} + +// The filter on detected_level that the stored-level plan sends to +// VictoriaLogs keeps exactly the rows whose stored level normalises to a value +// the filter accepts, and every row the stored fields do not decide (no stored +// level, or a stored detected_level), which the plan sends to the chain. +// conformance: semantics/json-metric-detected-level-derivation +func TestStoredLevelCopyFilterIsExact(t *testing.T) { + words := []string{"info", "INFO", "Information", "inf", "warn", "WARNING", "wrn", "error", "ERR", "Err", "trace", "TRC", "debug", "dbg", "critical", "CRITICAL", "fatal", + "notice", "unknown", "30", " info", "info ", "infoo", "errors", "Zulu"} + for _, tc := range []struct{ op, value string }{ + {"=", "info"}, {"=", "error"}, {"=", "unknown"}, {"=", "notice"}, {"=", "INFO"}, {"=", " info"}, {"=", "30"}, + {"!=", "info"}, {"!=", "warn"}, {"!=", "notice"}, {"!=", "unknown"}, + {"=~", "warn|error"}, {"=~", "err.*"}, {"=~", "info|notice"}, {"=~", ".+"}, {"=~", "(?i)zulu"}, {"=~", "unk.*"}, + {"!~", "warn|error"}, {"!~", "info"}, {"!~", "n.*"}, + } { + filter, err := translator.NewDropCondition("detected_level", tc.op, tc.value) + if err != nil { + t.Fatal(err) + } + pipes := splitLogsQLPipes("x" + translator.BuildStoredLevelCopyFilter("__dl_l", "__dl_d", []translator.DropCondition{filter})) + if len(pipes) != 2 || !strings.HasPrefix(pipes[1], "filter ") { + t.Fatalf("%s %q: %q", tc.op, tc.value, pipes) + } + expr := strings.TrimPrefix(pipes[1], "filter ") + for _, word := range words { + want := filter.Matches(translator.NormalizeStoredLevel(word)) + if got := fakeLogsQLMatch(t, expr, map[string]string{"__dl_l": word}); got != want { + t.Errorf("detected_level%s%q on a stored level %q: VictoriaLogs filter keeps it: %v, Loki's answer %v\n%s", tc.op, tc.value, word, got, want, expr) + } + } + for name, values := range map[string]map[string]string{"no stored level": {}, "stored detected_level": {"__dl_d": "x", "__dl_l": "info"}} { + if !fakeLogsQLMatch(t, expr, values) { + t.Errorf("detected_level%s%q drops a row with %s, which only the chain can read", tc.op, tc.value, name) + } + } + } +} diff --git a/internal/proxy/stream_processing.go b/internal/proxy/stream_processing.go index 446314a0a..a795b05a3 100644 --- a/internal/proxy/stream_processing.go +++ b/internal/proxy/stream_processing.go @@ -946,6 +946,32 @@ func (l *logRowLevels) mapRow(entry map[string]interface{}, msg string, s logRow return l.unpackedRow(s, &fields, func() string { return storedLogLineFromEntry(msg, entry, s.labels, nil, l.defaultMsg) }) } +// stringRow is mapRow for a row held as strings (the raw-row metric +// evaluator); the stream's labels are not repeated among its fields. +func (l *logRowLevels) stringRow(entry map[string]string, s logRowStream) detectedLevel { + var fields levelFields + for k, v := range entry { + if v == "" || !mayBeLevelField([]byte(k)) { + continue + } + if _, isStream := s.labels[k]; isStream { + continue + } + fields.observe([]byte(k), []byte(v)) + } + msg := entry["_msg"] + if !isVLMissingMsg(msg, l.defaultMsg) { + return deriveRowLevel(s.levels, &fields, []byte(msg), false, l.bodyScan) + } + return l.unpackedRow(s, &fields, func() string { + decoded := make(map[string]interface{}, len(entry)) + for k, v := range entry { + decoded[k] = v + } + return storedLogLineFromEntry(msg, decoded, s.labels, nil, l.defaultMsg) + }) +} + // unpackedRow derives detected_level for a row whose _msg is VictoriaLogs' // missing-message value (built-in, -backend-default-msg-value, or empty): its // fields came from a JSON body. The keyword scan reads the diff --git a/internal/translator/detected_level_keyword_test.go b/internal/translator/detected_level_keyword_test.go new file mode 100644 index 000000000..595f8f38e --- /dev/null +++ b/internal/translator/detected_level_keyword_test.go @@ -0,0 +1,352 @@ +package translator + +import ( + "math/rand" + "regexp" + "strings" + "testing" + "unicode" + "unicode/utf8" +) + +// vlAnyCasePhrase is VictoriaLogs' i(phrase) over one value +// (lib/logstorage/filter_any_case_phrase.go matchAnyCasePhrase): the value is +// lowercased rune by rune (unicode.ToLower) and the phrase must stand between +// non-token runes, a token rune being a Unicode letter or digit or '_'. +func vlAnyCasePhrase(s, phrase string) bool { + var lower strings.Builder + for _, r := range s { + lower.WriteRune(unicode.ToLower(r)) + } + v := lower.String() + isToken := func(r rune) bool { return unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' } + for pos := 0; ; { + n := strings.Index(v[pos:], phrase) + if n < 0 { + return false + } + pos += n + before, after := true, true + if pos > 0 { + r, _ := utf8.DecodeLastRuneInString(v[:pos]) + before = !isToken(r) + } + if end := pos + len(phrase); end < len(v) { + r, _ := utf8.DecodeRuneInString(v[end:]) + after = !isToken(r) + } + if before && after { + return true + } + pos++ + } +} + +// gateWordRE reads the phrases out of the text of the gate. +var gateWordRE = regexp.MustCompile(`i\(([^)]*)\)`) + +// levelKeywordGateMatches evaluates the text LevelKeywordGate() returns over +// one line: any of the phrases it lists, case-insensitively. The phrases come +// from the text, not from the keyword list, so a gate that lists fewer words +// than the regexp is seen. +func levelKeywordGateMatches(line string) bool { + for _, m := range gateWordRE.FindAllStringSubmatch(LevelKeywordGate(), -1) { + if vlAnyCasePhrase(line, m[1]) { + return true + } + } + return false +} + +// chainKeywordLevel is what the chain's keyword scan gives a line: the gate, +// then the regexp, then the finals (error|err is error, warning|warn is warn, +// any other word lowercased); unknown when the gate or the regexp finds none. +func chainKeywordLevel(re *regexp.Regexp, line string) string { + if !levelKeywordGateMatches(line) { + return "unknown" + } + m := re.FindStringSubmatch(line) + if m == nil { + return "unknown" + } + switch w := strings.ToLower(m[1]); w { + case "err": + return "error" + case "warning": + return "warn" + default: + return w + } +} + +// The answers of Loki v3.7.7's detectLevelFromLogLine +// (pkg/distributor/field_detection.go), written out by hand from its source: +// the line is lowercased, each of trace, debug, fatal, critical, error, err, +// warning, warn, info is searched for its first occurrence with a left +// boundary of ' ', '\t', '\n', '[', '(', '{', '"', '=' (or the line start) and a +// right boundary of ' ', '\t', '\n', '[', ']', '(', ')', '{', '}', ':', ',', '!', +// '"', '=' (or the line end), and the occurrence with the smallest position +// wins, so the order of the words only matters at one position. The table +// does not call the chain to build its expectations. +// +// conformance: severity-detected-level-derivation +var lokiKeywordAnswers = []struct{ line, want string }{ + // case + {"ERROR something", "error"}, {"Error: x", "error"}, {"eRrOr", "error"}, {"WARNING", "warn"}, + {"INFO", "info"}, {"Debug", "debug"}, {"TRACE", "trace"}, {"FaTaL", "fatal"}, {"Critical", "critical"}, + // substrings are no words + {"errors found", "unknown"}, {"terror", "unknown"}, {"warnings", "unknown"}, {"information", "unknown"}, + {"informational", "unknown"}, {"debugger", "unknown"}, {"fatalities", "unknown"}, {"errno 5", "unknown"}, + {"preinfo", "unknown"}, {"tracer", "unknown"}, {"critically", "unknown"}, {"errx", "unknown"}, + // the long and short spellings + {"warning: disk", "warn"}, {"this is a warning", "warn"}, {"warn", "warn"}, {"err:", "error"}, + {"failed err!", "error"}, {"an err", "error"}, + // several keywords: the smallest position wins + {"info then error", "info"}, {"error then info", "error"}, {"warning: error", "warn"}, + {"error warn", "error"}, {"warn error", "warn"}, {"critical error", "critical"}, {"error critical", "error"}, + {"fatal critical", "fatal"}, {"critical fatal", "critical"}, {"x info debug trace", "info"}, + {"debug info", "debug"}, {"x errors then warn", "warn"}, {"terror then info", "info"}, + // left boundaries + {"x=error", "error"}, {`"error"`, "error"}, {"[error]", "error"}, {"(err)", "error"}, {"{debug}", "debug"}, + {"a\terror", "error"}, {"a\nerror", "error"}, + {"foo-error", "unknown"}, {"foo_error", "unknown"}, {"5error", "unknown"}, {":error", "unknown"}, + {",error", "unknown"}, {"!error", "unknown"}, {"]error", "unknown"}, {")error", "unknown"}, + {"}error", "unknown"}, {"/error", "unknown"}, {".error", "unknown"}, {"'error", "unknown"}, + {"\rerror", "unknown"}, {" error", "unknown"}, {"éerror", "unknown"}, {"€error", "unknown"}, + {"\xffeerror", "unknown"}, {"\xffinfo", "unknown"}, + // right boundaries + {"error404", "unknown"}, {"error.", "unknown"}, {"error;", "unknown"}, {"error/x", "unknown"}, + {"error-x", "unknown"}, {"error_x", "unknown"}, {"error\r", "unknown"}, {"error x", "unknown"}, + {"errorж", "unknown"}, {"error\xff", "unknown"}, {"error'", "unknown"}, {"error>", "unknown"}, + {"error[", "error"}, {"error]", "error"}, {"error(", "error"}, {"error)", "error"}, {"error{", "error"}, + {"error}", "error"}, {"error,", "error"}, {"error!", "error"}, {`error"`, "error"}, {"error=1", "error"}, + {"error\tx", "error"}, {"error\nx", "error"}, + // a rejected occurrence does not hide a later one + {"terror error", "error"}, {"errors, then err", "error"}, {"x-info y=info", "info"}, + // nothing + {"", "unknown"}, {"hello world", "unknown"}, {"GET /api/v1/users 200 12ms", "unknown"}, +} + +// conformance: severity-detected-level-derivation +func TestKeywordScanLikeLoki(t *testing.T) { + re := regexp.MustCompile(levelKeywordRegexp) + for _, c := range lokiKeywordAnswers { + if got := chainKeywordLevel(re, c.line); got != c.want { + t.Errorf("%q: chain %q, Loki %q", c.line, got, c.want) + } + // the regexp alone, without the gate, gives the same answer: the gate removes no match + m := re.FindStringSubmatch(c.line) + alone := "unknown" + if m != nil { + alone = map[string]string{"err": "error", "warning": "warn"}[strings.ToLower(m[1])] + if alone == "" { + alone = strings.ToLower(m[1]) + } + } + if alone != c.want { + t.Errorf("%q: the regexp alone gives %q, Loki %q", c.line, alone, c.want) + } + } +} + +// Known difference, as before the gate: Loki lowercases with strings.ToLower +// (Unicode), the regexp folds ASCII only, so a line that spells a keyword with +// U+0130 reads info in Loki and unknown here. The gate lowercases like Loki, +// lets the line through, and the regexp finds nothing. +// conformance: severity-detected-level-derivation +func TestKeywordScanUnicodeFoldIsUnchangedByTheGate(t *testing.T) { + re := regexp.MustCompile(levelKeywordRegexp) + if got := chainKeywordLevel(re, "İNFO ready"); got != "unknown" { + t.Fatalf("İNFO: %q", got) + } + if re.MatchString("İNFO ready") { + t.Fatal("the regexp folds U+0130") + } +} + +// The gate is a superset of the regexp: any line the regexp matches holds a +// keyword as a whole word of the tokenizer. The lines are made of random +// pieces around the keywords: every ASCII byte, bytes of multi-byte runes, +// invalid UTF-8, the other case of the letters. +// conformance: severity-detected-level-derivation +func TestKeywordGateIsASupersetOfTheRegexp(t *testing.T) { + re := regexp.MustCompile(levelKeywordRegexp) + rng := rand.New(rand.NewSource(7)) + var seps []string + for b := 0; b < 128; b++ { + seps = append(seps, string(rune(b))) + } + for _, c := range " \t\n[](){}:,!\"=" { // Loki's boundaries, drawn often so that most lines match + for k := 0; k < 12; k++ { + seps = append(seps, string(c)) + } + } + seps = append(seps, "\u00a0", "é", "ж", "€", "\xff", "\xc2", "\u0130", "\u017f", "\u212a", "\u0663", "\u2028") + words := append(append([]string(nil), levelKeywords...), "errors", "terror", "warnings", "inf", "wrn", "ERROR", "Warning", "iNfO") + matched := 0 + for i := 0; i < 400000; i++ { + var sb strings.Builder + for j, n := 0, 1+rng.Intn(4); j < n; j++ { + sb.WriteString(seps[rng.Intn(len(seps))]) + if rng.Intn(3) == 0 { + sb.WriteString(seps[rng.Intn(len(seps))]) + } + w := words[rng.Intn(len(words))] + if rng.Intn(2) == 0 { + w = strings.ToUpper(w) + } + sb.WriteString(w) + } + sb.WriteString(seps[rng.Intn(len(seps))]) + line := sb.String() + if re.MatchString(line) { + matched++ + if !levelKeywordGateMatches(line) { + t.Fatalf("the regexp matches %q and the gate does not", line) + } + } + } + if matched < 40000 { + t.Fatalf("only %d of the lines matched; the generator is too tame", matched) + } +} + +// conformance: severity-detected-level-derivation +func TestLevelKeywordGateText(t *testing.T) { + want := "_msg:(i(trace) OR i(debug) OR i(fatal) OR i(critical) OR i(error) OR i(err) OR i(warning) OR i(warn) OR i(info))" + if got := LevelKeywordGate(); got != want { + t.Fatalf("gate = %s", got) + } + // every chain that scans the line gates the scan on it: the full chain, a chain with no key or field + for _, present := range []*DetectedLevelPresence{nil, {}, {Stored: []string{"level"}, BodyKeys: []string{"level"}}} { + chain := BuildDetectedLevelChain(nil, DetectedLevelChainGrouping, present) + if !strings.Contains(chain, "| extract_regexp if (") || !strings.Contains(chain, `__dl:"" `+want+") `(?i)") { + t.Fatalf("keyword scan of %v is not gated: %s", present, chain) + } + } +} + +// A chain that also unpacks the labels of its plan parses each line once: the +// level keys it reads and the labels share one unpack_json, the level keys are +// read only for rows no stored level decides although the unpack parses more +// rows, and every label ends up with the value the plan's own `unpack_json +// fields (...) keep_original_fields` would give it. +// conformance: severity-detected-level-derivation +func TestBuildDetectedLevelChainShared(t *testing.T) { + shared := &SharedUnpack{Keys: []string{"level", "app_name"}} + chain := BuildDetectedLevelChainShared(DetectedLevelFieldNames, DetectedLevelChainGrouping, nil, shared) + if n := strings.Count(chain, "unpack_json"); n != 1 { + t.Fatalf("%d unpack_json pipes: %s", n, chain) + } + for _, want := range []string{ + // every row is parsed (the plan unpacks every row), the level keys and the labels together + `| unpack_json from _msg fields (level, LEVEL, Level, log.level, severity, SEVERITY, Severity, SeverityText, lvl, LVL, Lvl, severity_text, Severity_Text, SEVERITY_TEXT, app_name) result_prefix "__j_"`, + // the level keys are read only where no stored level decides, whichever rows were parsed + `| format if (-__dl_h:* __j_level:*) "<__j_level>" as __dl_v keep_original_fields`, + `| format if (-__dl_h:* __j_SEVERITY_TEXT:*) "<__j_SEVERITY_TEXT>" as __dl_v keep_original_fields`, + // the labels: the body's value where the field is not stored, before the closing delete + `| format "<__dl>" as detected_level | format if (-level:* __j_level:*) "<__j_level>" as level | format if (-app_name:* __j_app_name:*) "<__j_app_name>" as app_name | delete __dl, __dl_*, __j_*, __l_*`, + } { + if !strings.Contains(chain, want) { + t.Errorf("shared chain lacks %q:\n%s", want, chain) + } + } + // The logfmt reads are the chain's own and unchanged. + if !strings.Contains(chain, "| unpack_logfmt if (-__dl_h:* -_msg:~") { + t.Errorf("the logfmt unpack changed: %s", chain) + } + + // Without a shared unpack, or in a mode that has no labels, nothing changes. + for _, mode := range []DetectedLevelChainMode{DetectedLevelChainGroup, DetectedLevelChainLogs} { + if got, want := BuildDetectedLevelChainShared(DetectedLevelFieldNames, mode, nil, shared), BuildDetectedLevelChain(DetectedLevelFieldNames, mode, nil); got != want { + t.Errorf("mode %d changed by a shared unpack", mode) + } + } + if got, want := BuildDetectedLevelChainShared(DetectedLevelFieldNames, DetectedLevelChainGrouping, nil, &SharedUnpack{}), BuildDetectedLevelChain(DetectedLevelFieldNames, DetectedLevelChainGrouping, nil); got != want { + t.Error("an empty shared unpack changed the chain") + } + + // The plan's own condition: a row it parses is parsed whether or not a stored + // level decides it, and the chain's rows are parsed too. + conditional := BuildDetectedLevelChainShared([]string{"level", "app"}, DetectedLevelChainGrouping, nil, &SharedUnpack{Keys: []string{"app_name"}, Cond: "(-app_name:*)"}) + if !strings.Contains(conditional, "| unpack_json if (-__dl_h:* or (-app_name:*)) from _msg fields (") { + t.Errorf("conditional shared unpack: %s", conditional) + } + + // Pruned: a label is no body key of the chain, so the mask never drops it, + // and a window with no body key still unpacks the labels (and reads no + // level key from them). + none := BuildDetectedLevelChainShared(nil, DetectedLevelChainGrouping, &DetectedLevelPresence{Stored: []string{"Level"}}, &SharedUnpack{Keys: []string{"level"}}) + for _, want := range []string{ + `| unpack_json from _msg fields (level) result_prefix "__j_"`, + `| format if (-level:* __j_level:*) "<__j_level>" as level | delete`, + } { + if !strings.Contains(none, want) { + t.Errorf("chain without body keys lacks %q:\n%s", want, none) + } + } + if strings.Contains(none, "as __dl_v keep_original_fields | format if (-__dl_h:* __j_") || strings.Contains(none, "unpack_logfmt") { + t.Errorf("a window without body keys reads a level key: %s", none) + } + // With a plan condition and no body key the chain asks nothing of its own. + only := BuildDetectedLevelChainShared(nil, DetectedLevelChainGrouping, &DetectedLevelPresence{Stored: []string{"level"}}, &SharedUnpack{Keys: []string{"level"}, Cond: "(-level:*)"}) + if !strings.Contains(only, "| unpack_json if ((-level:*)) from _msg fields (level)") { + t.Errorf("plan condition alone: %s", only) + } +} + +// FuzzKeywordGateIsASupersetOfTheRegexp lets the fuzzer look for a line the +// keyword regexp matches and the gate (VictoriaLogs' i() over the nine words) +// does not, and for a line where the chain's scan disagrees with a plain +// search for Loki's first bounded keyword. +// conformance: severity-detected-level-derivation +func FuzzKeywordGateIsASupersetOfTheRegexp(f *testing.F) { + for _, c := range lokiKeywordAnswers { + f.Add(c.line) + } + f.Add("İNFO ready") + f.Add("\xff\xfeerror\xc2") + re := regexp.MustCompile(levelKeywordRegexp) + f.Fuzz(func(t *testing.T, line string) { + if re.MatchString(line) && !levelKeywordGateMatches(line) { + t.Fatalf("the regexp matches %q and the gate does not", line) + } + // Loki's own search, written out: the smallest position of a keyword + // between its boundaries in the lowercased line (ASCII fold, as the regexp). + lower := strings.Map(func(r rune) rune { + if r >= 'A' && r <= 'Z' { + return r + 'a' - 'A' + } + return r + }, line) + best, bestAt := "unknown", len(lower)+1 + for _, w := range levelKeywords { + for from := 0; ; { + at := strings.Index(lower[from:], w) + if at < 0 { + break + } + at += from + left := at == 0 || strings.ContainsRune(" \t\n[({\"=", rune(lower[at-1])) + end := at + len(w) + right := end == len(lower) || strings.ContainsRune(" \t\n[](){}:,!\"=", rune(lower[end])) + if left && right { + if at < bestAt { + best, bestAt = w, at + } + break + } + from = at + 1 + } + } + switch best { + case "err": + best = "error" + case "warning": + best = "warn" + } + if got := chainKeywordLevel(re, line); got != best { + t.Fatalf("%q: the chain's scan says %q, a search for the first bounded keyword says %q", line, got, best) + } + }) +} diff --git a/internal/translator/detected_level_logsql.go b/internal/translator/detected_level_logsql.go index f0c98c53a..03235f4a7 100644 --- a/internal/translator/detected_level_logsql.go +++ b/internal/translator/detected_level_logsql.go @@ -3,6 +3,8 @@ package translator import ( "fmt" "regexp" + "slices" + "strconv" "strings" "github.com/ReliablyObserve/Loki-VL-proxy/internal/logsql" @@ -22,10 +24,17 @@ import ( // order rather than document order, nested objects are not searched by key // (the keyword scan still sees their words), unpack_logfmt keys are // case-sensitive and do not stop at syntax errors, and the keyword regexp -// folds ASCII only. The stored-field filters do not tell stream fields from +// folds ASCII only. unpack_logfmt skips the lines Loki reads as JSON, which +// start with a brace (the guard probes check the ones that do not also end +// with one). The stored-field filters do not tell stream fields from // other fields: a detected_level stream field is an index label for Loki and // does not decide the value, so callers leave it out of existingFields. +// BracedLineRegexp matches the start of a line Loki's ingest reads as JSON +// (field_detection.go isJSON: the first non-space rune is a brace and the last +// one a closing brace), where it never reads logfmt keys. +const BracedLineRegexp = `^[\s\v\x{85}\p{Z}]*\{` + // DetectedLevelFieldNames lists the stored fields that decide detected_level, // in priority order: detected_level, Loki's log_level_fields, severity_number. var DetectedLevelFieldNames = []string{ @@ -68,18 +77,43 @@ var detectedLevelSeverityNumberRegexp = map[string]string{ // detectedLevelCanonical lists every value the body chain can produce. var detectedLevelCanonical = []string{"trace", "debug", "info", "warn", "error", "critical", "fatal", "unknown"} -// vlSynthesizedSeverityTexts is VictoriaLogs' severity_text for OTel severity -// numbers 0..24 when an OTLP record has no text. -var vlSynthesizedSeverityTexts = []string{ - "Unspecified", - "Trace", "Trace2", "Trace3", "Trace4", - "Debug", "Debug2", "Debug3", "Debug4", - "Info", "Info2", "Info3", "Info4", - "Warn", "Warn2", "Warn3", "Warn4", - "Error", "Error2", "Error3", "Error4", - "Fatal", "Fatal2", "Fatal3", "Fatal4", +// levelKeywords are the words of Loki's keyword scan (field_detection.go +// levelPatterns), in its order. +var levelKeywords = []string{"trace", "debug", "fatal", "critical", "error", "err", "warning", "warn", "info"} + +// levelKeywordRegexp finds the first keyword of a line that stands between +// Loki's word boundaries (isLeftWordBoundary, isRightWordBoundary), ASCII +// case folded, which is what extract_regexp runs over the rows that need it. +var levelKeywordRegexp = `(?i)(?:^|[ \t\n\[({"=])(?P<__dl_e>` + strings.Join(levelKeywords, "|") + `)(?:$|[ \t\n\[\](){}:,!"=])` + +// LevelKeywordGate returns the filter in front of the keyword regexp: a line +// that holds one of the keywords as a whole word of VictoriaLogs' tokenizer, +// case-insensitively (i(), which finds the words in the block's token index +// and lowercases only what is left). It is a superset of the lines the regexp +// matches, never a second decision: every byte Loki accepts around a keyword +// (space, tab, newline, brackets, braces, quote, '=', ':', ',', '!') is +// a token separator, and the folding of (?i) maps no non-ASCII rune onto the +// letters of the nine words, so the matched text is an ASCII word of the line +// delimited by separators. The regexp, not the gate, decides which rows are +// matched, so a line the gate lets through and Loki's boundaries reject +// (`foo-error`, `error.`) is still no match. +func LevelKeywordGate() string { + words := make([]string, len(levelKeywords)) + for i, w := range levelKeywords { + words[i] = "i(" + w + ")" + } + return "_msg:(" + strings.Join(words, " OR ") + ")" } +// vlSeverityFamilies are the severity texts VictoriaLogs derives from OTel +// severity numbers 1..24 when an OTLP record has no text: the stem for the +// first number of a family of four, the stem with 2, 3 and 4 for the rest +// (Info for 9, Info2 for 10), and Unspecified for 0 and every other number. +var vlSeverityFamilies = []struct { + stem string + lo, hi int +}{{"Trace", 1, 4}, {"Debug", 5, 8}, {"Info", 9, 12}, {"Warn", 13, 16}, {"Error", 17, 20}, {"Fatal", 21, 24}} + // DetectedLevelStoredFields returns the level-deciding fields present in // existingFields, in priority order. func DetectedLevelStoredFields(existingFields []string) []string { @@ -110,13 +144,77 @@ const ( DetectedLevelChainGrouping ) +// DetectedLevelPresence names the level fields and body keys a window holds, +// so the chain carries only the pipes that can match a row of it. Each pipe +// of the chain passes every block of the window whether or not it changes a +// row, which makes the full chain's cost grow with the 16 stored fields and 14 +// body keys it knows. Stored lists the DetectedLevelFieldNames present as +// fields; BodyKeys lists the DetectedLevelBodyKeys that may occur in a line. +// Both must over-report: a name left out is never read. +type DetectedLevelPresence struct { + Stored, BodyKeys []string +} + +// DetectedLevelBodyKeys lists the body keys Loki reads a level from, in the +// order the chain reads them. +func DetectedLevelBodyKeys() []string { return detectedLevelLogLevelFieldNames() } + // BuildDetectedLevelChain returns the body chain (starting with "| "). When // existingFields hold stored level fields, the unpack and regexp pipes skip -// rows that have one. -func BuildDetectedLevelChain(existingFields []string, mode DetectedLevelChainMode) string { +// the rows that have one (BuildHasLevelFilter), which a scratch field +// __dl_h marks once instead of repeating the filter in every pipe. A non-nil +// present limits the chain to the fields and keys it lists (and replaces +// existingFields); nil keeps every pick. +func BuildDetectedLevelChain(existingFields []string, mode DetectedLevelChainMode, present *DetectedLevelPresence) string { + return BuildDetectedLevelChainShared(existingFields, mode, present, nil) +} + +// SharedUnpack names the JSON keys a stats plan unpacks itself, so that the +// chain's own unpack_json reads them too and the plan runs none: one parse of +// every line instead of two. Keys are plain label names (letters, digits and +// underscores), unpacked with the plan's keep_original_fields rule: a field +// that is already stored and non-empty keeps its value, the body's fills only +// the others. Cond is the plan's own condition on its unpack ("" when it +// unpacks every row); a row it excludes holds every key as a stored field. +type SharedUnpack struct { + Keys []string + Cond string +} + +// BuildDetectedLevelChainShared is BuildDetectedLevelChain for a plan that +// unpacks Keys itself (DetectedLevelChainGrouping only; the other modes ignore +// shared). The chain's unpack_json reads the level keys it needs and Keys with +// result_prefix "__j_", so a key never mistakes a stored field for a body +// value, and before the closing delete every key gets the value the plan's own +// `unpack_json fields (...) keep_original_fields` would have given it: the +// body's, where the field is not stored. The rows the level keys are read +// from stay those without a stored level (-__dl_h:*), whatever rows the unpack +// parsed for the plan. +func BuildDetectedLevelChainShared(existingFields []string, mode DetectedLevelChainMode, present *DetectedLevelPresence, shared *SharedUnpack) string { + var sharedKeys []string + if shared != nil && mode == DetectedLevelChainGrouping { + sharedKeys = shared.Keys + } + pruned := present != nil + bodyKeys := DetectedLevelBodyKeys() + if pruned { + existingFields, bodyKeys = present.Stored, present.BodyKeys + } + stored := DetectedLevelStoredFields(existingFields) + // A level text (any stored field but the number) sets __dl_s; the number + // decides after the text and the line, and VictoriaLogs before v1.50 holds + // the OTLP severity in severity alone (severityNameClause), so the number + // pipes stay for either of them. + texts := levelTextFields(DetectedLevelFieldNames) + if pruned { + texts = levelTextFields(stored) + } + hasText := len(texts) > 0 + hasNumber := !pruned || hasField(stored, "severity_number") + hasBody := len(bodyKeys) > 0 cond := "" - if has := BuildHasLevelFilter(existingFields); has != "" { - cond = "!(" + has + ")" + if len(stored) > 0 { + cond = "-__dl_h:*" } withCond := func(extra string) string { switch { @@ -130,81 +228,231 @@ func BuildDetectedLevelChain(existingFields []string, mode DetectedLevelChainMod return " if (" + cond + " " + extra + ")" } } - names := detectedLevelLogLevelFieldNames() - quoted := make([]string, len(names)) - for i, n := range names { + quoted := make([]string, len(bodyKeys)) + for i, n := range bodyKeys { quoted[i] = logsqlFieldName(n) } fieldList := strings.Join(quoted, ", ") - var b strings.Builder - // Stored fields first, in Loki's priority order: the first non-empty one - // decides, and keep_original_fields keeps that first write. - first := true - writeStored := func(field string) { - quotedField := logsqlFieldName(field) - if first { - fmt.Fprintf(&b, `| format if (%s:*) "<%s>" as __dl_s keep_original_fields`, quotedField, field) - first = false - return + var pipes []string + add := func(format string, args ...any) { pipes = append(pipes, fmt.Sprintf(format, args...)) } + pipes = append(pipes, storedLevelPicks(texts, stored)...) + if hasText { + add(`format if (__dl_s:*) "<__dl_s>" as __dl_v keep_original_fields`) + } + if hasBody || len(sharedKeys) > 0 { + unpackFields, unpackCond := fieldList, withCond("") + if len(sharedKeys) > 0 { + unpackFields = unpackFieldList(quoted, sharedKeys) + unpackCond = sharedUnpackCond(cond, hasBody, shared.Cond) } - fmt.Fprintf(&b, ` | format if (%s:*) "<%s>" as __dl_s keep_original_fields`, quotedField, field) - } - for _, field := range DetectedLevelFieldNames { - if field == "severity_number" { - continue + add(`unpack_json%s from _msg fields (%s) result_prefix "__j_"`, unpackCond, unpackFields) + } + if hasBody { + add(`unpack_logfmt%s from _msg fields (%s) result_prefix "__l_"`, withCond("-_msg:~"+quoteLevelRegexp(BracedLineRegexp)), fieldList) + for _, prefix := range []string{"__j_", "__l_"} { + for _, n := range bodyKeys { + f := logsqlFieldName(prefix + n) + if prefix == "__j_" && len(sharedKeys) > 0 { + // The shared unpack parsed rows a stored level decides too. + add(`format%s "<%s>" as __dl_v keep_original_fields`, withCond(f+":*"), prefix+n) + continue + } + add(`format if (%s:*) "<%s>" as __dl_v keep_original_fields`, f, prefix+n) + } } - writeStored(field) } - fmt.Fprintf(&b, ` | format if (__dl_s:*) "<__dl_s>" as __dl_v keep_original_fields`) - fmt.Fprintf(&b, ` | unpack_json%s from _msg fields (%s) result_prefix "__j_"`, withCond(""), fieldList) - fmt.Fprintf(&b, ` | unpack_logfmt%s from _msg fields (%s) result_prefix "__l_"`, withCond(""), fieldList) - for _, prefix := range []string{"__j_", "__l_"} { - for _, n := range names { - f := logsqlFieldName(prefix + n) - fmt.Fprintf(&b, ` | format if (%s:*) "<%s>" as __dl_v keep_original_fields`, f, prefix+n) + if hasText || hasBody { + for _, s := range detectedLevelSpellings { + add("format if (__dl_v:~%s) %q as __dl keep_original_fields", quoteLevelRegexp("(?i)^("+strings.Join(s.spellings, "|")+")$"), s.level) } } - for _, s := range detectedLevelSpellings { - fmt.Fprintf(&b, " | format if (__dl_v:~%s) %q as __dl keep_original_fields", quoteLevelRegexp("(?i)^("+strings.Join(s.spellings, "|")+")$"), s.level) - } - b.WriteString(" | extract_regexp") - b.WriteString(withCond(`__dl:""`)) - b.WriteString(" ") - b.WriteString(quoteLevelRegexp(`(?i)(?:^|[ \t\n\[({"=])(?P<__dl_e>trace|debug|fatal|critical|error|err|warning|warn|info)(?:$|[ \t\n\[\](){}:,!"=])`)) - b.WriteString(" from _msg") - b.WriteString(` | format if (__dl:"" __dl_e:~"(?i)^(error|err)$") "error" as __dl keep_original_fields`) - b.WriteString(` | format if (__dl:"" __dl_e:~"(?i)^(warning|warn)$") "warn" as __dl keep_original_fields`) - b.WriteString(` | format if (__dl:"" __dl_e:*) "" as __dl keep_original_fields`) + add("extract_regexp%s %s from _msg", withCond(`__dl:"" `+LevelKeywordGate()), quoteLevelRegexp(levelKeywordRegexp)) + add(`format if (__dl:"" __dl_e:~"(?i)^(error|err)$") "error" as __dl keep_original_fields`) + add(`format if (__dl:"" __dl_e:~"(?i)^(warning|warn)$") "warn" as __dl keep_original_fields`) + add(`format if (__dl:"" __dl_e:*) "" as __dl keep_original_fields`) // A stored word Loki does not know stays as it is (a level field holding // "notice" is "notice"), unlike a word read from the line. - b.WriteString(` | format if (__dl:"" __dl_s:*) "<__dl_s>" as __dl keep_original_fields`) + if hasText { + add(`format if (__dl:"" __dl_s:*) "<__dl_s>" as __dl keep_original_fields`) + } // severity_number decides only when no level field did, as in Loki. - for _, level := range detectedLevelCanonical { - pattern, ok := detectedLevelSeverityNumberRegexp[level] - if !ok { - continue + if hasNumber || hasField(stored, "severity") { + pipes = append(pipes, severityNumberPipes()...) + } + if hasNumber { + add(`format if (__dl:"" -__dl_s:* severity_number:*) "info" as __dl keep_original_fields`) + } + add(`format "unknown" as __dl keep_original_fields`) + pipes = append(pipes, chainEnding(mode, cond, sharedKeys)...) + return "| " + strings.Join(pipes, " | ") +} + +// storedLevelPicks returns the pipes that read the stored level fields, in +// Loki's priority order (the first non-empty one decides, and +// keep_original_fields keeps that first write), then the ones that mark the +// rows a stored level decides (__dl_h). +func storedLevelPicks(texts, stored []string) []string { + var pipes []string + add := func(format string, args ...any) { pipes = append(pipes, fmt.Sprintf(format, args...)) } + for _, field := range texts { + quotedField := logsqlFieldName(field) + cond := quotedField + ":*" + // A severity text VictoriaLogs derived from the OTLP number is no level + // field of Loki's: the number (or a later field) decides. + switch field { + case "severity_text": + cond += " !" + synthesizedSeverityText() + ` !(severity_text:="Unspecified" severity_number:="0")` + case "severity": + cond += " !" + synthesizedSeverityName() + } + add(`format if (%s) "<%s>" as __dl_s keep_original_fields`, cond, field) + } + if len(stored) > 0 { + // A row has a stored level (BuildHasLevelFilter) when __dl_s holds a + // level text, or the number is one other than the 0 that goes with a + // synthesised Unspecified, or the severity field names a finer level. + add(`format if (__dl_s:*) "1" as __dl_h`) + if hasField(stored, "severity_number") { + add(`format if (-__dl_h:* severity_number:* !(severity_number:="0" severity_text:="Unspecified")) "1" as __dl_h`) + } + if hasField(stored, "severity") { + add(`format if (-__dl_h:* %s -severity:="Unspecified") "1" as __dl_h`, synthesizedSeverityName()) } - fmt.Fprintf(&b, ` | format if (__dl:"" -__dl_s:* severity_number:~%s) %q as __dl keep_original_fields`, - quoteLevelRegexp(pattern), level) } - b.WriteString(` | format if (__dl:"" -__dl_s:* severity_number:*) "info" as __dl keep_original_fields`) - b.WriteString(` | format "unknown" as __dl keep_original_fields`) + return pipes +} + +// chainEnding returns the closing pipes of the chain: the value written to +// detected_level (for every row, or where no stored level decided), the labels +// of a shared unpack, and the delete of the scratch fields. +func chainEnding(mode DetectedLevelChainMode, cond string, sharedKeys []string) []string { switch mode { case DetectedLevelChainGrouping: - b.WriteString(` | format "<__dl>" as detected_level`) - b.WriteString(" | delete __dl, __dl_*, __j_*, __l_*") + out := []string{`format "<__dl>" as detected_level`} + for _, key := range sharedKeys { + out = append(out, fmt.Sprintf(`format if (-%s:* %s:*) "<__j_%s>" as %s`, logsqlFieldName(key), logsqlFieldName("__j_"+key), key, logsqlFieldName(key))) + } + return append(out, "delete __dl, __dl_*, __j_*, __l_*") case DetectedLevelChainLogs: if cond == "" { - b.WriteString(" | rename __dl as detected_level") - } else { - fmt.Fprintf(&b, ` | format if (%s) "<__dl>" as detected_level`, cond) + return []string{"rename __dl as detected_level", "delete __dl, __dl_*, __j_*, __l_*"} + } + return []string{fmt.Sprintf(`format if (%s) "<__dl>" as detected_level`, cond), "delete __dl, __dl_*, __j_*, __l_*"} + default: + return []string{"delete __dl_*, __j_*, __l_*"} + } +} + +// unpackFieldList joins the level keys of the chain (already quoted) and the +// keys a plan shares, each once. +func unpackFieldList(levelKeys, shared []string) string { + fields := append([]string(nil), levelKeys...) + for _, key := range shared { + if q := logsqlFieldName(key); !slices.Contains(fields, q) { + fields = append(fields, q) } - b.WriteString(" | delete __dl, __dl_*, __j_*, __l_*") + } + return strings.Join(fields, ", ") +} + +// sharedUnpackCond is the condition of the unpack both the chain and the plan +// read: the rows the chain reads level keys from (chainCond, none meaning +// every row, when needChain) and the rows the plan unpacks (planCond, none +// meaning every row). +func sharedUnpackCond(chainCond string, needChain bool, planCond string) string { + switch { + case planCond == "", needChain && chainCond == "": + return "" + case !needChain: + return " if (" + planCond + ")" default: - b.WriteString(" | delete __dl_*, __j_*, __l_*") + return " if (" + chainCond + " or " + planCond + ")" + } +} + +// levelTextFields returns the level fields of fields that hold a level text +// (all but severity_number), in the order given. +func levelTextFields(fields []string) []string { + out := make([]string, 0, len(fields)) + for _, f := range fields { + if f != "severity_number" { + out = append(out, f) + } + } + return out +} + +// severityNumberPipes maps severity_number (and, before VictoriaLogs v1.50, +// the severity names derived from it) to a level for rows no level text decided. +func severityNumberPipes() []string { + var out []string + for _, level := range detectedLevelCanonical { + pattern, ok := detectedLevelSeverityNumberRegexp[level] + if !ok { + continue + } + match := "severity_number:~" + quoteLevelRegexp(pattern) + if names := severityNameClause([]string{level}); names != "" { + match = "(" + match + " OR " + names + ")" + } + out = append(out, fmt.Sprintf(`format if (__dl:"" -__dl_s:* %s) %q as __dl keep_original_fields`, match, level)) + } + return out +} + +// NormalizeStoredLevel is the detected_level of a row whose first stored level +// field holds value, as the chain computes it: a known spelling (any case) +// becomes its canonical level, any other word stays as stored. +func NormalizeStoredLevel(value string) string { + for _, s := range detectedLevelSpellings { + for _, spelling := range s.spellings { + if strings.EqualFold(value, spelling) { + return s.level + } + } } - return b.String() + return value +} + +// BuildStoredLevelCopyFilter returns the pipes that keep, of a stats query reading +// the stored level (stored holds the copy of the level field, meta that of +// detected_level), the rows a filter on detected_level can match: a row whose +// stored level normalises (NormalizeStoredLevel) to a value the filter +// accepts, and every row the stored fields do not decide exactly (a stored +// detected_level, or no stored level at all), which the caller sees and sends +// to the full chain. A known spelling, in any case, stands for its canonical +// level; any other word compares as stored. The caller applies the filters +// again to the normalised value, so this one only has to keep every row that +// can match. +func BuildStoredLevelCopyFilter(stored, meta string, filters []DropCondition) string { + if len(filters) == 0 { + return "" + } + known := stored + ":~" + quoteLevelRegexp("(?i)^(?:"+strings.Join(allDetectedLevelSpellings(), "|")+")$") + var sb strings.Builder + for _, f := range filters { + var alts []string + for _, s := range detectedLevelSpellings { + if f.Matches(s.level) { + alts = append(alts, stored+":~"+quoteLevelRegexp("(?i)^(?:"+strings.Join(s.spellings, "|")+")$")) + } + } + var word string + switch f.Op { + case "=": + word = stored + ":=" + logsql.QuoteValue(f.Value) + case "!=": + word = "-" + stored + ":=" + logsql.QuoteValue(f.Value) + case "=~": + word = stored + ":~" + strconv.Quote("^(?:"+f.Value+")$") + default: + word = "-" + stored + ":~" + strconv.Quote("^(?:"+f.Value+")$") + } + alts = append(alts, "(-"+known+" "+word+")") + fmt.Fprintf(&sb, " | filter (%s:* OR -%s:* OR %s)", meta, stored, strings.Join(alts, " OR ")) + } + return sb.String() } func detectedLevelLogLevelFieldNames() []string { @@ -239,13 +487,39 @@ func BuildHasLevelFilter(existingFields []string) string { if len(parts) > 1 { has = "(" + has + ")" } - if hasField(stored, "severity_text") && hasField(stored, "severity_number") { - zero := append(emptyPrefix(stored, "severity_text"), `severity_text:="Unspecified"`, `severity_number:="0"`) - has += " !(" + strings.Join(zero, " ") + ")" + for _, noLevel := range noLevelRows(stored) { + has += " !(" + noLevel + ")" } return has } +// noLevelRows returns the filters of the rows whose only level text is the +// "Unspecified" VictoriaLogs synthesises for an OTLP record without severity: +// severity_text with severity_number 0 (v1.50 and newer), or severity without +// any number (older). Loki stores no level for them, and the other stored +// level fields, when set, still decide. +func noLevelRows(stored []string) []string { + var out []string + if hasField(stored, "severity_text") && hasField(stored, "severity_number") { + out = append(out, strings.Join(append(emptyExcept(stored, "severity_text", "severity_number"), `severity_text:="Unspecified"`, `severity_number:="0"`), " ")) + } + if hasField(stored, "severity") { + out = append(out, strings.Join(append(emptyExcept(stored, "severity", "severity_number"), `severity:="Unspecified"`, "-severity_number:*"), " ")) + } + return out +} + +// emptyExcept returns `f:""` filters for the stored fields other than skip. +func emptyExcept(stored []string, skip ...string) []string { + out := make([]string, 0, len(stored)) + for _, f := range stored { + if !hasField(skip, f) { + out = append(out, logsqlFieldName(f)+`:""`) + } + } + return out +} + func hasField(fields []string, name string) bool { for _, f := range fields { if f == name { @@ -362,6 +636,24 @@ func (m detectedLevelMatcher) storedTextClause(field string, targets []string) s } } +// joinOr joins the non-empty filters with OR. +func joinOr(parts ...string) string { + var alts []string + for _, p := range parts { + if p != "" { + alts = append(alts, p) + } + } + switch len(alts) { + case 0: + return "" + case 1: + return alts[0] + default: + return "(" + strings.Join(alts, " OR ") + ")" + } +} + func severityNumberClause(targets []string) string { var alts []string for _, t := range targets { @@ -383,16 +675,53 @@ func severityNumberClause(targets []string) string { } // synthesizedSeverityText matches OTLP rows whose severity_text VictoriaLogs -// derived from a non-zero severity_number. +// derived from a non-zero severity_number: a name of the number's family (the +// pairing inside a family is not checked: a record sending Info2 with number +// 12 is read as synthesised, and still gets info), or Unspecified for a number +// outside 0..24. func synthesizedSeverityText() string { - alts := make([]string, 0, len(vlSynthesizedSeverityTexts)) - for n := 1; n < len(vlSynthesizedSeverityTexts); n++ { - alts = append(alts, fmt.Sprintf(`(severity_text:=%q severity_number:="%d")`, vlSynthesizedSeverityTexts[n], n)) + alts := make([]string, 0, len(vlSeverityFamilies)+1) + for _, f := range vlSeverityFamilies { + nums := make([]string, 0, 4) + for n := f.lo; n <= f.hi; n++ { + nums = append(nums, strconv.Itoa(n)) + } + alts = append(alts, "(severity_text:~"+quoteLevelRegexp("^"+f.stem+"[234]?$")+" severity_number:~"+quoteLevelRegexp("^("+strings.Join(nums, "|")+")$")+")") } - alts = append(alts, `(severity_text:="Unspecified" severity_number:~"^(-[1-9][0-9]*|2[5-9]|[3-9][0-9]|[1-9][0-9]{2,})$")`) + alts = append(alts, `(severity_text:="Unspecified" severity_number:~`+quoteLevelRegexp(`^(-[1-9][0-9]*|2[5-9]|[3-9][0-9]|[1-9][0-9]{2,})$`)+")") return "(" + strings.Join(alts, " OR ") + ")" } +// severityStems returns the stems of vlSeverityFamilies for the canonical +// levels given (all of them when none is). +func severityStems(levels ...string) []string { + var stems []string + for _, f := range vlSeverityFamilies { + if len(levels) == 0 || slices.Contains(levels, strings.ToLower(f.stem)) { + stems = append(stems, f.stem) + } + } + return stems +} + +// synthesizedSeverityName matches a severity field holding a name +// VictoriaLogs before v1.50 derived from the OTLP number (it stores no +// severity_number then): "Unspecified", which says nothing, and the finer +// names, which Loki reads from the number as the coarse level. +func synthesizedSeverityName() string { + return "(severity:~" + quoteLevelRegexp("^(Unspecified|("+strings.Join(severityStems(), "|")+")[234]?)$") + " -severity_number:*)" +} + +// severityNameClause matches a severity field holding a derived name of one +// of the targets (see synthesizedSeverityName), or "" when none has one. +func severityNameClause(targets []string) string { + stems := severityStems(targets...) + if len(stems) == 0 || len(targets) == 0 { + return "" + } + return "(-severity_number:* severity:~" + quoteLevelRegexp("^("+strings.Join(stems, "|")+")[234]?$") + ")" +} + // BuildStoredLevelFilter returns a filter matching rows whose stored fields // decide a detected_level accepted by the matcher, "" when no stored row can // match. Negated matchers return the positive filter: callers combine it with @@ -429,6 +758,14 @@ func (m detectedLevelMatcher) storedFilter(existingFields []string) string { if match != "" && f == "severity_text" && otel { match += ` !` + synthesizedSeverityText() + ` !(severity_text:="Unspecified" severity_number:="0")` } + if f == "severity" { + if match != "" { + match = "(" + match + " !" + synthesizedSeverityName() + ")" + } + if names := severityNameClause(targets); names != "" { + match = joinOr(match, names) + } + } } if match == "" { continue @@ -508,7 +845,7 @@ func BuildDetectedLevelFilter(op, value string, existingFields []string) (Detect } else { plan.Prefilter = detectedLevelPrefilter(targets, has, stored != "") } - plan.Pipes = BuildDetectedLevelChain(existingFields, DetectedLevelChainGroup) + " | filter " + final + plan.Pipes = BuildDetectedLevelChain(existingFields, DetectedLevelChainGroup, nil) + " | filter " + final return plan, nil } diff --git a/internal/translator/detected_level_logsql_test.go b/internal/translator/detected_level_logsql_test.go index 23fe18b3a..0e1ddc785 100644 --- a/internal/translator/detected_level_logsql_test.go +++ b/internal/translator/detected_level_logsql_test.go @@ -1,6 +1,7 @@ package translator import ( + "regexp" "strings" "testing" ) @@ -25,6 +26,11 @@ func TestBuildHasLevelFilter(t *testing.T) { {[]string{"app", "msg"}, ""}, {[]string{"x", "level"}, "level:*"}, {[]string{"severity_number", "severity_text", "log.level"}, `(log.level:* OR severity_text:* OR severity_number:*) !(log.level:"" severity_text:="Unspecified" severity_number:="0")`}, + // VictoriaLogs before v1.50 stores the OTLP severity as one field, with + // no number: "Unspecified" carries no level. Every other level field, + // not only the ones ranked before it, keeps a row in. + {[]string{"severity", "level", "lvl"}, `(level:* OR severity:* OR lvl:*) !(level:"" lvl:"" severity:="Unspecified" -severity_number:*)`}, + {[]string{"severity_number", "severity_text", "Severity_Text"}, `(severity_text:* OR Severity_Text:* OR severity_number:*) !(Severity_Text:"" severity_text:="Unspecified" severity_number:="0")`}, } for _, tc := range cases { if got := BuildHasLevelFilter(tc.fields); got != tc.want { @@ -68,11 +74,13 @@ func TestBuildStoredLevelFilterOTelSynthesizedText(t *testing.T) { } for _, want := range []string{ "severity_text:~`(?i)^(warn|wrn|warning)$`", - `!((severity_text:="Trace" severity_number:="1") OR `, - `(severity_text:="Warn" severity_number:="13")`, + // The names of a number's family, whatever the number's place in it. + "(severity_text:~`^Trace[234]?$` severity_number:~`^(1|2|3|4)$`)", + "(severity_text:~`^Warn[234]?$` severity_number:~`^(13|14|15|16)$`)", + "(severity_text:=\"Unspecified\" severity_number:~`^(-[1-9][0-9]*|2[5-9]|[3-9][0-9]|[1-9][0-9]{2,})$`)", `!(severity_text:="Unspecified" severity_number:="0")`, "severity_number:~`^1[3-6]$`", - `(severity_text:"" OR (((severity_text:="Trace" severity_number:="1")`, + `(severity_text:"" OR (((severity_text:~`, } { if !strings.Contains(got, want) { t.Fatalf("filter lacks %q:\n%s", want, got) @@ -80,6 +88,27 @@ func TestBuildStoredLevelFilterOTelSynthesizedText(t *testing.T) { } } +// VictoriaLogs before v1.50 stores an OTLP record's severity as one field named +// severity and no number: its finer names (Info2) mean the coarse level, and +// Unspecified means none, so neither is a raw level word. +func TestBuildStoredLevelFilterSynthesizedSeverityName(t *testing.T) { + got, err := BuildStoredLevelFilter("=", "info", []string{"severity"}) + if err != nil { + t.Fatal(err) + } + want := "severity:~`(?i)^(info|inf|information)$`" + if !strings.Contains(got, want) || !strings.Contains(got, "(-severity_number:* severity:~`^(Info)[234]?$`)") { + t.Fatalf("filter lacks the derived names:\n%s", got) + } + got, err = BuildStoredLevelFilter("=~", "Unspecified|Info2", []string{"severity"}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(got, "!(severity:~`^(Unspecified|(Trace|Debug|Info|Warn|Error|Fatal)[234]?)$` -severity_number:*)") { + t.Fatalf("a derived name matched as a raw word:\n%s", got) + } +} + func TestBuildStoredLevelFilterRejectsBadMatchers(t *testing.T) { if _, err := BuildStoredLevelFilter("=~", "(", nil); err == nil { t.Fatal("invalid regexp accepted") @@ -136,7 +165,7 @@ func TestBuildDetectedLevelFilterPlans(t *testing.T) { } return } - chain := BuildDetectedLevelChain(tc.fields, DetectedLevelChainGroup) + chain := BuildDetectedLevelChain(tc.fields, DetectedLevelChainGroup, nil) if plan.Pipes != chain+" "+tc.filter { t.Fatalf("pipes tail = %q, want chain + %q", strings.TrimPrefix(plan.Pipes, chain), tc.filter) } @@ -146,7 +175,7 @@ func TestBuildDetectedLevelFilterPlans(t *testing.T) { // conformance: severity-detected-level-derivation func TestBuildDetectedLevelChain(t *testing.T) { - chain := BuildDetectedLevelChain(nil, DetectedLevelChainGroup) + chain := BuildDetectedLevelChain(nil, DetectedLevelChainGroup, nil) pipes := strings.Split(strings.TrimPrefix(chain, "| "), " | ") // 16 stored picks + 1 stored copy + 2 unpack + 28 body picks + 7 // normalisers + 1 keyword regexp + 3 keyword finals + 1 stored word + @@ -160,12 +189,17 @@ func TestBuildDetectedLevelChain(t *testing.T) { 14: `format if (SEVERITY_TEXT:*) "" as __dl_s keep_original_fields`, 15: `format if (__dl_s:*) "<__dl_s>" as __dl_v keep_original_fields`, 16: `unpack_json from _msg fields (level, LEVEL, Level, log.level, severity, SEVERITY, Severity, SeverityText, lvl, LVL, Lvl, severity_text, Severity_Text, SEVERITY_TEXT) result_prefix "__j_"`, - 17: `unpack_logfmt from _msg fields (level, LEVEL, Level, log.level, severity, SEVERITY, Severity, SeverityText, lvl, LVL, Lvl, severity_text, Severity_Text, SEVERITY_TEXT) result_prefix "__l_"`, + // Lines Loki reads as JSON (a leading brace) never go to logfmt. + 17: "unpack_logfmt if (-_msg:~`^[\\s\\v\\x{85}\\p{Z}]*\\{`) from _msg fields (level, LEVEL, Level, log.level, severity, SEVERITY, Severity, SeverityText, lvl, LVL, Lvl, severity_text, Severity_Text, SEVERITY_TEXT) result_prefix \"__l_\"", + // A severity text VictoriaLogs derived from the OTLP number is no level + // field: Unspecified and the finer names (Info2) of v1.47 (severity, + // no number) and the text matching its number of v1.50+ (severity_text). + 5: "format if (severity:* !(severity:~`^(Unspecified|(Trace|Debug|Info|Warn|Error|Fatal)[234]?)$` -severity_number:*)) \"\" as __dl_s keep_original_fields", 18: `format if (__j_level:*) "<__j_level>" as __dl_v keep_original_fields`, 45: `format if (__l_SEVERITY_TEXT:*) "<__l_SEVERITY_TEXT>" as __dl_v keep_original_fields`, 46: "format if (__dl_v:~`(?i)^(trace|trc)$`) \"trace\" as __dl keep_original_fields", 52: "format if (__dl_v:~`(?i)^(fatal)$`) \"fatal\" as __dl keep_original_fields", - 53: "extract_regexp if (__dl:\"\") `(?i)(?:^|[ \\t\\n\\[({\"=])(?P<__dl_e>trace|debug|fatal|critical|error|err|warning|warn|info)(?:$|[ \\t\\n\\[\\](){}:,!\"=])` from _msg", + 53: "extract_regexp if (__dl:\"\" _msg:(i(trace) OR i(debug) OR i(fatal) OR i(critical) OR i(error) OR i(err) OR i(warning) OR i(warn) OR i(info))) `(?i)(?:^|[ \\t\\n\\[({\"=])(?P<__dl_e>trace|debug|fatal|critical|error|err|warning|warn|info)(?:$|[ \\t\\n\\[\\](){}:,!\"=])` from _msg", 57: `format if (__dl:"" __dl_s:*) "<__dl_s>" as __dl keep_original_fields`, 66: `format "unknown" as __dl keep_original_fields`, 67: `delete __dl_*, __j_*, __l_*`, @@ -175,22 +209,38 @@ func TestBuildDetectedLevelChain(t *testing.T) { } } - logs := BuildDetectedLevelChain(nil, DetectedLevelChainLogs) + logs := BuildDetectedLevelChain(nil, DetectedLevelChainLogs, nil) if !strings.HasSuffix(logs, `| format "unknown" as __dl keep_original_fields | rename __dl as detected_level | delete __dl, __dl_*, __j_*, __l_*`) { t.Fatalf("logs chain ending: %s", logs[len(logs)-160:]) } - stored := BuildDetectedLevelChain([]string{"level", "app"}, DetectedLevelChainLogs) + stored := BuildDetectedLevelChain([]string{"level", "app"}, DetectedLevelChainLogs, nil) for _, want := range []string{ - `| unpack_json if (!(level:*)) from _msg fields (`, - `| unpack_logfmt if (!(level:*)) from _msg fields (`, - "| extract_regexp if (!(level:*) __dl:\"\") `", - `| format if (!(level:*)) "<__dl>" as detected_level | delete __dl, __dl_*, __j_*, __l_*`, + // A row with a level field is marked once, and skipped by the pipes + // that read the line. + `| format if (__dl_s:*) "1" as __dl_h | format if (__dl_s:*) "<__dl_s>" as __dl_v`, + `| unpack_json if (-__dl_h:*) from _msg fields (`, + "| unpack_logfmt if (-__dl_h:* -_msg:~`^[\\s\\v\\x{85}\\p{Z}]*\\{`) from _msg fields (", + "| extract_regexp if (-__dl_h:* __dl:\"\" _msg:(i(trace) OR ", + `| format if (-__dl_h:*) "<__dl>" as detected_level | delete __dl, __dl_*, __j_*, __l_*`, } { if !strings.Contains(stored, want) { t.Fatalf("chain with stored level lacks %q", want) } } + + // The OTLP severity of VictoriaLogs v1.50+ (severity_text, severity_number) + // and before (severity alone): a row decides by a level text that is not + // synthesised, a number other than 0, or a finer severity name. + otlp := BuildDetectedLevelChain([]string{"severity_text", "severity_number", "severity"}, DetectedLevelChainGroup, nil) + for _, want := range []string{ + `| format if (-__dl_h:* severity_number:* !(severity_number:="0" severity_text:="Unspecified")) "1" as __dl_h`, + "| format if (-__dl_h:* (severity:~`^(Unspecified|(Trace|Debug|Info|Warn|Error|Fatal)[234]?)$` -severity_number:*) -severity:=\"Unspecified\") \"1\" as __dl_h", + } { + if !strings.Contains(otlp, want) { + t.Fatalf("OTLP chain lacks %q", want) + } + } } // Loki groups metric queries by the detected_level it derived at ingest, not @@ -219,3 +269,183 @@ func TestMetricGroupingDerivesDetectedLevel(t *testing.T) { t.Fatalf("unrelated grouping changed: %q", plain) } } + +// A chain built from the fields and keys a window holds carries only their +// pipes, keeps the pipes the OTLP severity needs, and with every field and key +// present is the full chain. +// conformance: severity-detected-level-derivation +func TestBuildDetectedLevelChainPruned(t *testing.T) { + all := &DetectedLevelPresence{Stored: DetectedLevelFieldNames, BodyKeys: DetectedLevelBodyKeys()} + for _, mode := range []DetectedLevelChainMode{DetectedLevelChainGroup, DetectedLevelChainLogs, DetectedLevelChainGrouping} { + if got, want := BuildDetectedLevelChain(nil, mode, all), BuildDetectedLevelChain(DetectedLevelFieldNames, mode, nil); got != want { + t.Fatalf("mode %d: every field and key present is not the full chain:\n%s\n%s", mode, got, want) + } + } + pipes := func(present *DetectedLevelPresence) []string { + chain := BuildDetectedLevelChain(nil, DetectedLevelChainGrouping, present) + if !strings.HasPrefix(chain, "| ") { + t.Fatalf("chain does not start with a pipe: %.80s", chain) + } + return strings.Split(strings.TrimPrefix(chain, "| "), " | ") + } + contains := func(list []string, prefix string) bool { + for _, p := range list { + if strings.HasPrefix(p, prefix) { + return true + } + } + return false + } + + // One stored field and one body key: 1 pick, the marker, the stored copy, 2 + // unpacks, 2 key picks, 7 normalisers, the keyword regexp and its 3 finals, + // the stored word, unknown, the final copy and delete. + level := pipes(&DetectedLevelPresence{Stored: []string{"level"}, BodyKeys: []string{"level"}}) + if len(level) != 22 { + t.Fatalf("level chain has %d pipes, want 22: %v", len(level), level) + } + for i, want := range map[int]string{ + 0: `format if (level:*) "" as __dl_s keep_original_fields`, + 1: `format if (__dl_s:*) "1" as __dl_h`, + 3: `unpack_json if (-__dl_h:*) from _msg fields (level) result_prefix "__j_"`, + } { + if level[i] != want { + t.Fatalf("pipe %d = %q, want %q", i, level[i], want) + } + } + if contains(level, "format if (__dl:\"\" -__dl_s:* severity_number") { + t.Fatalf("a window without severity_number or severity keeps the number pipes: %v", level) + } + + // Nothing stored and no key: the keyword scan decides, or unknown. + none := pipes(&DetectedLevelPresence{}) + if !strings.HasPrefix(none[0], "extract_regexp ") || len(none) != 7 { + t.Fatalf("empty chain: %d pipes, first %.60s: %v", len(none), none[0], none) + } + for _, gone := range []string{"unpack_", "format if (__dl_s", "format if (__dl_v", "format if (__dl:\"\" -__dl_s", "format if (__dl:\"\" __dl_s"} { + if contains(none, gone) { + t.Fatalf("empty chain still has a %q pipe: %v", gone, none) + } + } + + // The OTLP severity: v1.50+ stores severity_text with severity_number, and + // the synthesised text defers to the number; before it, severity alone + // carries the derived name (severityNameClause), which only the number + // pipes read. + otlp := pipes(&DetectedLevelPresence{Stored: []string{"severity_text", "severity_number"}}) + for _, want := range []string{ + `format if (severity_text:* !`, + `format if (-__dl_h:* severity_number:* !(severity_number:="0" severity_text:="Unspecified")) "1" as __dl_h`, + `format if (__dl:"" -__dl_s:* severity_number:*) "info" as __dl keep_original_fields`, + } { + if !contains(otlp, want) { + t.Fatalf("OTLP v1.50 chain lacks %q: %v", want, otlp) + } + } + old := pipes(&DetectedLevelPresence{Stored: []string{"severity"}}) + if !contains(old, `format if (severity:* !(severity:~`) || !contains(old, `format if (-__dl_h:* (severity:~`) || !contains(old, `format if (__dl:"" -__dl_s:* (severity_number:~`) { + t.Fatalf("pre-v1.50 OTLP chain lacks a severity pick, marker or number pipe: %v", old) + } + if contains(old, `format if (__dl:"" -__dl_s:* severity_number:*) "info"`) { + t.Fatalf("a window without severity_number keeps the number fallback: %v", old) + } + number := pipes(&DetectedLevelPresence{Stored: []string{"severity_number"}}) + if !strings.HasPrefix(number[0], `format if (__dl_s:*) "1" as __dl_h`) { + t.Fatalf("number-only chain starts with %.60s", number[0]) + } + for _, want := range []string{`format if (-__dl_h:* severity_number:*`, `format if (__dl:"" -__dl_s:* severity_number:*) "info"`} { + if !contains(number, want) { + t.Fatalf("number-only chain lacks %q: %v", want, number) + } + } +} + +// The answers of Loki v3.7.7's normalizeLogLevel (pkg/distributor/field_detection.go), +// written out by hand from its source: a case-insensitive match of exactly +// these spellings (bytes.EqualFold, no trimming), any other value as it is. +// The table does not call NormalizeStoredLevel to build its expectations. +// conformance: severity-detected-level-derivation +func TestNormalizeStoredLevelIsLokisNormalizeLogLevel(t *testing.T) { + for value, want := range map[string]string{ + "trace": "trace", "TRACE": "trace", "trc": "trace", "Trc": "trace", + "debug": "debug", "Debug": "debug", "dbg": "debug", "DBG": "debug", + "info": "info", "INFO": "info", "inf": "info", "inf ": "inf ", "information": "info", "Information": "info", "INFORMATION": "info", + "warn": "warn", "WARN": "warn", "wrn": "warn", "WRN": "warn", "warning": "warn", "WARNING": "warn", + "error": "error", "Error": "error", "err": "error", "Err": "error", "ERR": "error", + "critical": "critical", "CRITICAL": "critical", + "fatal": "fatal", "Fatal": "fatal", + // not Loki's spellings: the value stays as it is + "notice": "notice", "30": "30", " info": " info", "info ": "info ", "unknown": "unknown", "": "", + "emerg": "emerg", "alert": "alert", "verbose": "verbose", "warnings": "warnings", "errors": "errors", "crit": "crit", + } { + if got := NormalizeStoredLevel(value); got != want { + t.Errorf("NormalizeStoredLevel(%q) = %q, Loki's normalizeLogLevel gives %q", value, got, want) + } + } +} + +// The level of a row a stored level field decides is computed in Go from the +// grouped stored value; it must be what the chain's spelling pipes give: the +// anchored, case-insensitive regexps of detectedLevelSpellings, and any other +// word as stored. +// conformance: severity-detected-level-derivation +func TestNormalizeStoredLevelMatchesTheChain(t *testing.T) { + corpus := []string{"info", "INFO", "Info", "inf", "Information", "INFORMATION", "warn", "WRN", "Warning", "WARNING", "error", "ERR", "Err", + "critical", "CRITICAL", "fatal", "Fatal", "trace", "TRC", "debug", "DBG", "unknown", "notice", "NOTICE", "verbose", "30", "true", "", + " info", "info ", "info\n", "infoo", "in fo", "warning2", "ünfo", "ınfo", "ſtrace"} + for _, value := range corpus { + want := value + for _, s := range detectedLevelSpellings { + if regexp.MustCompile("(?i)^(" + strings.Join(s.spellings, "|") + ")$").MatchString(value) { + want = s.level + break + } + } + if got := NormalizeStoredLevel(value); got != want { + t.Errorf("NormalizeStoredLevel(%q) = %q, the chain's pipes give %q", value, got, want) + } + } +} + +// Whatever a Loki label filter holds, the filter pipes sent to VictoriaLogs for +// the stored level are well formed (balanced parentheses outside quoted text, +// one pipe per filter) and never panic. +func FuzzStoredLevelCopyFilter(f *testing.F) { + for _, seed := range [][2]string{{"=", "info"}, {"!=", "error"}, {"=~", "warn|error"}, {"!~", "(?i)inf.*"}, {"=", ""}, {"=~", `a"b`}, {"=", "`"}, {"=~", `\d+`}} { + f.Add(seed[0], seed[1]) + } + f.Fuzz(func(t *testing.T, op, value string) { + filter, err := NewDropCondition("detected_level", op, value) + if err != nil || (op != "=" && op != "!=" && op != "=~" && op != "!~") { + return + } + got := BuildStoredLevelCopyFilter("__dl_l", "__dl_d", []DropCondition{filter}) + if !strings.HasPrefix(got, " | filter (__dl_d:* OR -__dl_l:* OR ") || strings.Count(got, " | filter ") != 1 { + t.Fatalf("%q %q: %s", op, value, got) + } + depth, quote := 0, byte(0) + for i := 0; i < len(got); i++ { + c := got[i] + switch { + case quote != 0: + if c == '\\' && quote == '"' { + i++ + } else if c == quote { + quote = 0 + } + case c == '"' || c == '`': + quote = c + case c == '(': + depth++ + case c == ')': + depth-- + } + if depth < 0 { + t.Fatalf("%q %q: unbalanced: %s", op, value, got) + } + } + if depth != 0 || quote != 0 { + t.Fatalf("%q %q: unbalanced or unterminated: %s", op, value, got) + } + }) +} diff --git a/internal/translator/translator.go b/internal/translator/translator.go index b69ac9f2c..e84b50e0c 100644 --- a/internal/translator/translator.go +++ b/internal/translator/translator.go @@ -2891,7 +2891,7 @@ func applyDetectedLevelGrouping(query string, existingFields []string) string { if !strings.Contains(query, detectedLevelLabelName) { return query } - chain := BuildDetectedLevelChain(existingFields, DetectedLevelChainGrouping) + chain := BuildDetectedLevelChain(existingFields, DetectedLevelChainGrouping, nil) return applyDerivedGrouping(query, detectedLevelLabelName, func(q string) string { pipeAt := firstTopLevelPipe(q) if pipeAt < 0 { diff --git a/test/e2e-compat/detected_level_variants_test.go b/test/e2e-compat/detected_level_variants_test.go new file mode 100644 index 000000000..0ea60f800 --- /dev/null +++ b/test/e2e-compat/detected_level_variants_test.go @@ -0,0 +1,238 @@ +//go:build e2e + +package e2e_compat + +import ( + "encoding/json" + "fmt" + "math/rand" + "net/http" + "net/url" + "regexp" + "sort" + "strconv" + "strings" + "testing" + "time" + + "github.com/ReliablyObserve/Loki-VL-proxy/internal/translator" +) + +// variantLine is a generated row: the body and the stored fields besides it. +type variantLine struct { + tag string + msg string + stored map[string]string +} + +var variantBodies = []string{ + // the keyword scan: case, whole words, boundaries, several keywords + "request ERROR upstream", "Error: connection refused", "eRrOr in module", "3 errors found", "terror alert", + "disk warning: low", "warnings ignored", "info then error", "warning: error", "critical error", "error then critical", + "foo-error", "foo_error", "5error", "error404", "error.", "err: timeout", "(warn) slow", `retry "debug" mode`, + "x=trace on", "FATAL: out of memory", " error", "éerror", "errorж", "x\rerror", "İNFO ready", "hello world", + "GET /api/v1/users 200 12ms", "Trace enter f", "[info] started", "{debug}", "a\terror", "a\nwarn", "x info y", + // JSON bodies: a level key of every kind of value and name, the keyword scan behind them + `{"level":"info"}`, `{"level":"INFO","event":"a"}`, `{"level":"Information"}`, `{"level":"warning"}`, `{"level":"ERR","event":"b"}`, + `{"level":"notice"}`, `{"level":"notice","msg":"error connecting"}`, `{"level":30,"msg":"hi"}`, `{"level":50,"err":{"type":"Error"}}`, + `{"level":true}`, `{"level":null}`, `{"level":" info"}`, `{"level":""}`, `{"level":{"name":"info"}}`, `{"level":["info"]}`, + `{"severity":"ERROR"}`, `{"Level":"Information"}`, `{"LEVEL":"warn"}`, `{"log":{"level":"warn"}}`, `{"a":{"level":"error"}}`, + `{"lvl":"dbg"}`, `{"SeverityText":"fatal"}`, `{"severity_text":"trc","event":"c"}`, `{"msg":"x","severity":"error","level":"info"}`, + `{"level":"info","severity":"error"}`, `{"msg":"request failed with error"}`, `{"msg":"3 errors found"}`, `{"msg":"info then error"}`, + `{"msg":"warning: error"}`, `{"msg":"ERROR"}`, `{"msg":"foo-error"}`, `{"n":1}`, `{"event":"d","n":2}`, `{"detected_level":"error","level":"info"}`, + `{"detected_level":"verbose"}`, `{"level":"info"} trailing`, `{"level":"warn","x":{bad`, `{"level":"info","x":{bad`, + `{"level":"warn"}`, `{"level":"Information"}`, `{"level":"info"}`, `{"msg":"error: cannot set level=debug now"}`, + `{"msg":"set level=debug","level":"info"}`, `{"msg": "level: high", "level": "warn"}`, ` {"level":"error"}`, + // logfmt bodies + `level=warn msg=x`, `Level=info LEVEL=error`, `msg="error x" LvL=warn`, `msg="unterminated error level=info`, `{ error level=info`, + `level=info event=e1`, `lvl=err event=e2`, `ts=1 level=debug`, `msg=hello`, `x=1 y=2`, +} + +var variantStored = []map[string]string{ + nil, nil, nil, nil, nil, nil, nil, nil, + {"level": "info"}, {"level": "WARN"}, {"level": "notice"}, {"Level": "info"}, {"LEVEL": "error"}, {"lvl": "debug"}, + {"severity": "warn"}, {"severity": "Info2"}, {"severity": "Unspecified"}, {"severity_text": "Unspecified", "severity_number": "0"}, + {"severity_text": "Info2", "severity_number": "10"}, {"severity_text": "Warn", "severity_number": "13"}, {"severity_number": "17"}, + {"severity_number": "30"}, {"severity_text": "notice", "severity_number": "0"}, {"detected_level": "INFO"}, + {"level": "info", "severity": "error"}, {"Level": "info", "level": "warn"}, +} + +// variantQuery runs query over the fixture's window on VictoriaLogs and returns +// the rows by tag as "level|event|detected_level". +func variantQuery(t *testing.T, query string, start, end time.Time) map[string]string { + t.Helper() + form := url.Values{"query": {query}, "start": {start.UTC().Format(time.RFC3339Nano)}, "end": {end.UTC().Format(time.RFC3339Nano)}} + status, body := hardeningRequest(t, http.MethodPost, vlURL+"/select/logsql/query", form.Encode(), map[string]string{"Content-Type": "application/x-www-form-urlencoded"}) + if status != http.StatusOK { + t.Fatalf("VictoriaLogs answered %d for %.300s: %.300s", status, query, body) + } + out := map[string]string{} + for _, line := range strings.Split(strings.TrimSpace(string(body)), "\n") { + var row map[string]string + if err := json.Unmarshal([]byte(line), &row); err != nil { + t.Fatalf("row %q: %v", line, err) + } + out[row["tag"]] = row["level"] + "|" + row["event"] + "|" + row["detected_level"] + } + return out +} + +// The pipes that derive detected_level in VictoriaLogs were made cheaper in +// three ways, each of which must give the answer of the pipes it replaced on +// every row, whatever the row holds: the keyword regexp runs behind a filter +// on the keyword words, the plan's own unpack_json is the chain's (the labels +// are copied from its prefixed fields), and the presence probe and witness find +// an escape with a substring filter. This test ingests generated rows into +// VictoriaLogs alone (no Loki: the answers against Loki are the probe tests') +// and runs the pipes as they were and as they are, per row. +// conformance: semantics/json-metric-detected-level-derivation +func TestRangeMetricCompatibilityDetectedLevelPipeVariants(t *testing.T) { + now := time.Now() + s0 := now.Add(-3 * time.Hour).Truncate(time.Minute) + app := fmt.Sprintf("level-variants-%d", now.UnixNano()) + rng := rand.New(rand.NewSource(42)) + var lines []variantLine + // every body with no stored field, then every body with random stored fields + for i, body := range variantBodies { + lines = append(lines, variantLine{tag: "a" + strconv.Itoa(i), msg: body}) + } + for i := 0; i < 1400; i++ { + lines = append(lines, variantLine{tag: "b" + strconv.Itoa(i), msg: variantBodies[rng.Intn(len(variantBodies))], stored: variantStored[rng.Intn(len(variantStored))]}) + } + var rows strings.Builder + stored, bodyKeys := map[string]bool{}, map[string]bool{} + for i, l := range lines { + fields := map[string]string{"_time": s0.Add(time.Duration(i) * time.Second).UTC().Format(time.RFC3339Nano), "_msg": l.msg, "app": app, "tag": l.tag} + for k, v := range l.stored { + fields[k] = v + stored[k] = true + } + row, _ := json.Marshal(fields) + rows.Write(row) + rows.WriteByte('\n') + } + for _, key := range translator.DetectedLevelBodyKeys() { + bodyKeys[key] = true // the chain pruned to what the fixture uses: every key (some line holds each of the words) + } + status, body := hardeningRequest(t, http.MethodPost, vlURL+"/insert/jsonline?_stream_fields=app", rows.String(), map[string]string{"Content-Type": "application/stream+json"}) + if status != http.StatusOK { + t.Fatalf("VL ingest: %d %s", status, body) + } + forceVLFlush(t) + start, end := s0.Add(-time.Minute), s0.Add(time.Duration(len(lines)+60)*time.Second) + base := fmt.Sprintf("app:=%q", app) + if got := variantQuery(t, base+` | stats by (tag) count() c`, start, end); len(got) != len(lines) { + t.Fatalf("VictoriaLogs holds %d of %d rows", len(got), len(lines)) + } + + labels := []string{"level", "event"} + const stats = " | stats by (tag, level, event, detected_level) count() c" + // The pipes as they were: the plan's own unpack follows the chain, and the + // keyword regexp is not gated. + gateRE := regexp.MustCompile(`(extract_regexp if \([^)]*__dl:"") _msg:\((?:i\(\w+\)(?: OR )?)+\)\)`) + ungated := func(chain string) string { + out := gateRE.ReplaceAllString(chain, "$1)") + if out == chain { + t.Fatal("the chain has no keyword gate to take out") + } + return out + } + was := func(chain, planUnpack string) string { + return base + " " + ungated(chain) + planUnpack + stats + } + var present *translator.DetectedLevelPresence + { + var st []string + for _, name := range translator.DetectedLevelFieldNames { + if stored[name] { + st = append(st, name) + } + } + present = &translator.DetectedLevelPresence{Stored: st, BodyKeys: translator.DetectedLevelBodyKeys()} + } + for _, tc := range []struct { + name string + present *translator.DetectedLevelPresence + cond string // the condition of the plan's own unpack + }{ + {"full chain", nil, ""}, + {"chain pruned to the stored fields of the fixture", present, ""}, + {"full chain, plan unpack on rows lacking a label", nil, "(-level:*) or (-event:*)"}, + {"pruned chain, plan unpack on rows lacking a label", present, "(-level:*) or (-event:*)"}, + } { + t.Run(tc.name, func(t *testing.T) { + existing := translator.DetectedLevelFieldNames + planUnpack := " | unpack_json fields (level, event) keep_original_fields" + if tc.cond != "" { + planUnpack = " | unpack_json if (" + tc.cond + ") fields (level, event) keep_original_fields" + } + chain := translator.BuildDetectedLevelChain(existing, translator.DetectedLevelChainGrouping, tc.present) + old := variantQuery(t, was(chain, planUnpack), start, end) + gated := variantQuery(t, base+" "+chain+planUnpack+stats, start, end) + shared := &translator.SharedUnpack{Keys: labels, Cond: tc.cond} + sharedRows := variantQuery(t, base+" "+translator.BuildDetectedLevelChainShared(existing, translator.DetectedLevelChainGrouping, tc.present, shared)+stats, start, end) + if len(old) != len(lines) || len(sharedRows) != len(lines) { + t.Fatalf("rows: was %d, now %d, fixture %d", len(old), len(sharedRows), len(lines)) + } + bad := 0 + for _, l := range lines { + if old[l.tag] != sharedRows[l.tag] || old[l.tag] != gated[l.tag] { + if bad++; bad <= 15 { + t.Errorf("row %s %q stored %v: was %q, gated %q, shared %q", l.tag, l.msg, l.stored, old[l.tag], gated[l.tag], sharedRows[l.tag]) + } + } + } + if bad > 0 { + t.Fatalf("%d of %d rows differ", bad, len(lines)) + } + }) + } +} + +// The probe and the witness look for the two bytes backslash and u with a +// substring filter; it selects the lines the regular expression did. +// conformance: semantics/json-metric-detected-level-derivation +func TestRangeMetricCompatibilityEscapeSubstringFilter(t *testing.T) { + now := time.Now() + app := fmt.Sprintf("level-escape-%d", now.UnixNano()) + msgs := []string{`{"x":"Abc"}`, "no escape here", `u only \ then u`, `a\\u double`, `tail\u`, `U upper \U0041`, `x\ u`, `\u`, `u\`, `\\u`, "a/u", "é\\u00e9", `\u0000`, `p\\\u1`} + var rows strings.Builder + for i, m := range msgs { + row, _ := json.Marshal(map[string]string{"_time": now.Add(-time.Duration(i+1) * time.Second).UTC().Format(time.RFC3339Nano), "_msg": m, "app": app, "tag": strconv.Itoa(i)}) + rows.Write(row) + rows.WriteByte('\n') + } + if status, body := hardeningRequest(t, http.MethodPost, vlURL+"/insert/jsonline?_stream_fields=app", rows.String(), map[string]string{"Content-Type": "application/stream+json"}); status != http.StatusOK { + t.Fatalf("VL ingest: %d %s", status, body) + } + forceVLFlush(t) + tags := func(filter string) string { + form := url.Values{"query": {fmt.Sprintf("app:=%q %s | fields tag", app, filter)}} + status, body := hardeningRequest(t, http.MethodPost, vlURL+"/select/logsql/query", form.Encode(), map[string]string{"Content-Type": "application/x-www-form-urlencoded"}) + if status != http.StatusOK { + t.Fatalf("%d %s", status, body) + } + var got []string + for _, line := range strings.Split(strings.TrimSpace(string(body)), "\n") { + if line == "" { + continue + } + var row map[string]string + _ = json.Unmarshal([]byte(line), &row) + got = append(got, row["tag"]) + } + sort.Strings(got) + return strings.Join(got, ",") + } + var want []string + for i, m := range msgs { + if strings.Contains(m, `\u`) { + want = append(want, strconv.Itoa(i)) + } + } + sort.Strings(want) + byRegexp, substring := tags(`_msg:~"\\\\u"`), tags(`_msg:*"\\u"*`) + if byRegexp != strings.Join(want, ",") || substring != byRegexp { + t.Fatalf("regexp selects %q, substring %q, the lines holding a backslash and u are %q", byRegexp, substring, strings.Join(want, ",")) + } +} diff --git a/test/e2e-compat/extracted_metric_pushdown_test.go b/test/e2e-compat/extracted_metric_pushdown_test.go index d8220629d..739e180bd 100644 --- a/test/e2e-compat/extracted_metric_pushdown_test.go +++ b/test/e2e-compat/extracted_metric_pushdown_test.go @@ -96,7 +96,7 @@ func TestCompat_ExtractedLabelMetricPushdown(t *testing.T) { after := parserMetricEvaluations(t, route) raw := 0 for _, call := range recorder.reset() { - if strings.HasPrefix(call, "/select/logsql/query ") && !strings.HasSuffix(call, " | limit 1") { + if strings.HasPrefix(call, "/select/logsql/query ") && !strings.HasSuffix(call, " | limit 1") && !isLevelPresenceCall(call) { raw++ } } diff --git a/test/e2e-compat/json_detected_level_series_limit_test.go b/test/e2e-compat/json_detected_level_series_limit_test.go new file mode 100644 index 000000000..886e17644 --- /dev/null +++ b/test/e2e-compat/json_detected_level_series_limit_test.go @@ -0,0 +1,78 @@ +//go:build e2e + +package e2e_compat + +import ( + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/ReliablyObserve/Loki-VL-proxy/internal/proxy" +) + +// Loki counts the series of the answer: the spellings of one level are one +// series (the stream levels `error`, `ERR` and `Err` all give +// detected_level=error), and a filter on detected_level narrows the series +// before they are counted. The stats pushdown reads those spellings as +// separate VictoriaLogs groups, so the series limit applies to the merged +// answer: a query whose answer fits the limit gets the 200 and the series Loki +// gives, even when its groups do not, and one more series than the limit is +// refused with Loki's message. The stack's Loki has a limit far above this +// fixture, so the limit under test is the in-process proxy's +// (-max-stats-query-series) and the series are compared with Loki's. +// conformance: semantics/json-metric-detected-level-derivation +// conformance: limits/series-limit-error +func TestRangeMetricCompatibilityJSONDetectedLevelSeriesLimit(t *testing.T) { + now := time.Now() + s0 := now.Add(-9 * time.Hour).Truncate(time.Hour) + app := fmt.Sprintf("json-level-limit-%d", now.UnixNano()) + const pods = 12 + spellings := []string{"error", "ERR", "Err"} // three spellings of one level + var lines []jsonVolumeStreamLine + for p := 0; p < pods; p++ { + pod := fmt.Sprintf("p%02d", p) + for i, level := range append(spellings, "info") { + lines = append(lines, jsonVolumeStreamLine{ + ts: s0.Add(time.Duration(p*4+i) * time.Second), level: level, + labels: map[string]string{"pod": pod}, msg: fmt.Sprintf(`{"n":%d}`, p*4+i), + }) + } + } + ingestJSONVolumeFixture(t, map[string][]jsonVolumeStreamLine{app: lines}) + start, end := s0.Add(-time.Minute), s0.Add(2*time.Minute) + + // every pod x (error, info) is a series of its own: 24 answered by + // sum by (pod, detected_level); the error filter keeps the 12 pods. + byPodLevel := `sum by (pod, detected_level) (count_over_time({app="` + app + `"} | json | drop __error__ [1m]))` + filtered := `sum by (pod) (count_over_time({app="` + app + `"} | json | drop __error__ | detected_level="error" [1m]))` + for _, tc := range []struct { + name, query string + series int + }{{"all levels", byPodLevel, 2 * pods}, {"filtered to error", filtered, pods}} { + loki := slidingRangeSeries(t, lokiURL, tc.query, start, end, time.Minute, nil) + if len(loki) != tc.series { + t.Fatalf("%s: Loki returned %d series, want %d", tc.name, len(loki), tc.series) + } + // at the limit, one above it and far above it: Loki's answer, not a 400. + // VictoriaLogs holds 48 groups for the first query (three spellings of + // error and info per pod) and 12 for the second (its filter runs there): + // more than the limit in the first, so a plan counting groups fails. + for _, limit := range []int{tc.series, tc.series + 1, 10 * tc.series} { + t.Run(fmt.Sprintf("%s/limit %d", tc.name, limit), func(t *testing.T) { + route, _ := newJSONVolumeRouteProxyWithConfig(t, func(cfg *proxy.Config) { cfg.MaxStatsQuerySeries = limit }) + assertSlidingParity(t, tc.query+fmt.Sprintf(" [limit %d]", limit), loki, slidingRangeSeries(t, route, tc.query, start, end, time.Minute, nil)) + }) + } + // one series past the limit: Loki's error + t.Run(tc.name+"/over the limit", func(t *testing.T) { + limit := tc.series - 1 + route, _ := newJSONVolumeRouteProxyWithConfig(t, func(cfg *proxy.Config) { cfg.MaxStatsQuerySeries = limit }) + status, body, _ := alignLimitRangeQuery(t, route, tc.query, start, end, nil) + if status != http.StatusBadRequest || !strings.Contains(body, fmt.Sprintf("maximum number of series (%d) reached for a single query", limit)) { + t.Fatalf("expected Loki's series limit error for %d series over a limit of %d, got %d %s", tc.series, limit, status, body) + } + }) + } +} diff --git a/test/e2e-compat/json_detected_level_test.go b/test/e2e-compat/json_detected_level_test.go new file mode 100644 index 000000000..98419bcf0 --- /dev/null +++ b/test/e2e-compat/json_detected_level_test.go @@ -0,0 +1,516 @@ +//go:build e2e + +package e2e_compat + +import ( + "cmp" + "fmt" + "net/http" + "strings" + "testing" + "time" +) + +// detected_level is the level Loki sets at ingest, normalized +// (distributor/field_detection.go): a level stream label, else a level key of +// the JSON body, whatever the spelling (info, INFO, inf, information). A +// `sum by (detected_level)` over `| json` is answered by the row evaluator, or +// by the stats pushdown with the `drop __error__` form, and both must group +// the spellings of one level into one series like Loki, with the level in the +// body, as a stream label, or both (the stream's wins). The pushdown computes +// the value in VictoriaLogs, the row evaluator in the proxy. +// conformance: semantics/json-metric-detected-level-derivation +func TestRangeMetricCompatibilityJSONDetectedLevel(t *testing.T) { + now := time.Now() + s0 := now.Add(-9 * time.Hour).Truncate(time.Hour) + body := fmt.Sprintf("json-level-body-%d", now.UnixNano()) + stream := fmt.Sprintf("json-level-stream-%d", now.UnixNano()) + both := fmt.Sprintf("json-level-both-%d", now.UnixNano()) + spellings := []string{"info", "INFO", "Info", "inf", "information", "warn", "WARN", "warning", "error", "ERR", "debug"} + var bodyLines, streamLines, bothLines []jsonVolumeStreamLine + for i := 0; i < 66; i++ { + ts := s0.Add(time.Duration(i) * 5 * time.Second) + level := spellings[i%len(spellings)] + other := spellings[(i+3)%len(spellings)] + bodyLines = append(bodyLines, jsonVolumeStreamLine{ts: ts, msg: fmt.Sprintf(`{"level":%q,"n":%d}`, level, i)}) + streamLines = append(streamLines, jsonVolumeStreamLine{ts: ts, level: level, msg: fmt.Sprintf(`{"n":%d}`, i)}) + bothLines = append(bothLines, jsonVolumeStreamLine{ts: ts, level: level, msg: fmt.Sprintf(`{"level":%q,"n":%d}`, other, i)}) + } + // A line without any level, the last of each fixture. + last := s0.Add(331 * time.Second) + bodyLines = append(bodyLines, jsonVolumeStreamLine{ts: last, msg: `{"n":-1}`}) + streamLines = append(streamLines, jsonVolumeStreamLine{ts: last, msg: `{"n":-1}`}) + bothLines = append(bothLines, jsonVolumeStreamLine{ts: last, msg: `{"n":-1}`}) + ingestJSONVolumeFixture(t, map[string][]jsonVolumeStreamLine{body: bodyLines, stream: streamLines, both: bothLines}) + + route, _ := newJSONVolumeRouteProxy(t) + targets := map[string]string{"proxy": proxyURL, "in-process": route} + start, end := s0.Add(-2*time.Minute), s0.Add(9*time.Minute) + for _, tc := range []struct{ name, app string }{ + {"level in the body", body}, + {"level as a stream label with mixed spellings", stream}, + {"level in both, the stream's wins", both}, + } { + for _, pipeline := range []string{`| json`, `| json | drop __error__`} { + for _, function := range []string{"count_over_time", "bytes_over_time"} { + query := `sum by (detected_level) (` + function + `({app="` + tc.app + `"} ` + pipeline + ` [1m]))` + t.Run(tc.name+"/"+function+" "+pipeline, func(t *testing.T) { + loki := slidingRangeSeries(t, lokiURL, query, start, end, time.Minute, nil) + if len(loki) < 4 { + t.Fatalf("Loki sanity: expected several detected_level series, got %d: %v", len(loki), loki) + } + for name, base := range targets { + assertSlidingParity(t, query+" ["+name+"]", loki, slidingRangeSeries(t, base, query, start, end, time.Minute, nil)) + } + }) + } + } + } +} + +// detectedLevelProbe is one stored line and the detected_level Loki gives it +// (want, checked against Loki itself by the test). labels are further stream +// labels and meta structured metadata; dropOnly lines have a parser error, so +// only the `drop __error__` pipeline is valid for them. route, when set, is +// the evaluator the `drop __error__` query must use: "stats" for a line the +// VictoriaLogs pipes read like Loki, "raw" for one a guard probe hands to the +// raw-row evaluator. +type detectedLevelProbe struct { + name string + labels map[string]string + meta map[string]string + msg string + want string + dropOnly bool + route string + // rowOnly marks a line the stats pushdown reads differently from Loki + // without a guard able to see it (semantics/detected-level-pipes-known- + // differences): only the raw-row evaluator's form, `| json`, is compared. + rowOnly bool + // otlp lines are pushed as OTLP records (the severity VictoriaLogs stores + // depends on its version: severity_text and severity_number from v1.50, one + // severity field before) under the service name; severityNumber and + // severityText are the record's. + otlp bool + severityNumber int + severityText string +} + +// selector returns the stream selector of the probe's app (an OTLP record +// carries the app as its service name). +func (c detectedLevelProbe) selector(app string) string { + if c.otlp { + return `{service_name="` + app + `"}` + } + return `{app="` + app + `"}` +} + +var detectedLevelProbes = []detectedLevelProbe{ + {name: "pino numeric level", msg: `{"level":30,"msg":"hi"}`, want: "unknown", route: "stats"}, + {name: "pino numeric level with an error object", msg: `{"level":50,"msg":"boom","err":{"type":"Error"}}`, want: "error", route: "stats"}, + {name: "boolean level", msg: `{"level":true}`, want: "unknown"}, + {name: "null level", msg: `{"level":null}`, want: "unknown"}, + {name: "unknown spelling", msg: `{"level":"notice"}`, want: "unknown", route: "stats"}, + {name: "unknown spelling and an error word", msg: `{"level":"notice","msg":"error connecting"}`, want: "error", route: "stats"}, + {name: "padded level", msg: `{"level":" info"}`, want: "info", route: "stats"}, + {name: "severity key", msg: `{"severity":"ERROR"}`, want: "error", route: "stats"}, + {name: "Level key", msg: `{"Level":"Information"}`, want: "info", route: "stats"}, + {name: "LEVEL key", msg: `{"LEVEL":"warn"}`, want: "warn", route: "stats"}, + {name: "nested log.level", msg: `{"log":{"level":"warn"}}`, want: "warn", route: "stats"}, + {name: "no level key, error word", msg: `{"msg":"request failed with error"}`, want: "error", route: "stats"}, + {name: "object level", msg: `{"level":{"name":"info"}}`, want: "info"}, + {name: "array level", msg: `{"level":["info"]}`, want: "info"}, + {name: "body detected_level key", msg: `{"detected_level":"error","level":"info"}`, want: "info", route: "stats"}, + {name: "text after the closing brace", msg: `{"level":"info"} trailing`, want: "info", route: "raw"}, + {name: "no level at all", msg: `{"n":1}`, want: "unknown", route: "stats"}, + {name: "plain level key", msg: `{"level":"info"}`, want: "info", route: "stats"}, + {name: "severity before level", msg: `{"msg":"x","severity":"error","level":"info"}`, want: "error", route: "raw"}, + {name: "level before severity", msg: `{"level":"info","severity":"error"}`, want: "info", route: "raw"}, + {name: "body detected_level key alone", msg: `{"detected_level":"verbose"}`, want: "unknown", route: "stats"}, + {name: "level key mentioned in the message", msg: `{"level":"info","msg":"set log level=debug"}`, want: "info", route: "stats"}, + {name: "level name followed by a colon inside a string", msg: `{"msg":"level: high","level":"warn"}`, want: "warn", route: "stats"}, + {name: "syntax error after the level", msg: `{"level":"warn","x":{bad`, want: "warn", dropOnly: true, route: "raw"}, + {name: "body detected_level key, syntax error", msg: `{"detected_level":"warn","x":{bad`, want: "warn", dropOnly: true}, + {name: "body detected_level and level keys, syntax error", msg: `{"detected_level":"warn","level":"info","x":{bad`, want: "warn", dropOnly: true, route: "raw"}, + {name: "level key, syntax error", msg: `{"level":"info","x":{bad`, want: "info", dropOnly: true, route: "raw"}, + {name: "stream severity before stored level metadata", labels: map[string]string{"severity": "error"}, meta: map[string]string{"level": "info"}, msg: `{"n":1}`, want: "error", rowOnly: true}, + {name: "stored severity metadata", meta: map[string]string{"severity": "error"}, msg: `{"n":1}`, want: "error", route: "stats"}, + {name: "stream level, unknown spelling", labels: map[string]string{"level": "notice"}, msg: `{"n":1}`, want: "notice", route: "stats"}, + {name: "stream severity", labels: map[string]string{"severity": "error"}, msg: `{"n":1}`, want: "error", route: "stats"}, + {name: "stream level wins over the body", labels: map[string]string{"level": "INFO"}, msg: `{"level":"error"}`, want: "info", route: "stats"}, + {name: "stream detected_level keeps its stored spelling", labels: map[string]string{"detected_level": "INFO"}, msg: `{"n":1}`, want: "INFO", route: "raw"}, + {name: "stream detected_level wins over a stream level", labels: map[string]string{"detected_level": "INFO", "level": "warn"}, msg: `{"n":1}`, want: "INFO", route: "raw"}, + {name: "stream detected_level wins over the body", labels: map[string]string{"detected_level": "Warning"}, msg: `{"level":"error"}`, want: "Warning", route: "raw"}, + {name: "stored level metadata", meta: map[string]string{"level": "WARN"}, msg: `{"level":"info"}`, want: "warn", route: "stats"}, + {name: "stored detected_level metadata", meta: map[string]string{"detected_level": "Error"}, msg: `{"level":"info"}`, want: "error", route: "stats"}, + // A line that is JSON for Loki (a brace first and last) never goes to the + // logfmt parser; one that only starts with a brace is logfmt for it. + {name: "logfmt-looking text inside a JSON message", msg: `{"msg":"error: cannot set level=debug now"}`, want: "error", route: "stats"}, + {name: "logfmt after a brace", msg: `{ error level=info`, want: "info", dropOnly: true, route: "raw"}, + {name: "logfmt level keys of two spellings", msg: `Level=info LEVEL=error`, want: "info", dropOnly: true, route: "raw"}, + // Loki reads the raw bytes of a JSON value, so an escaped spelling is no + // known level for it; keys it decodes, as VictoriaLogs does. + {name: "escaped level value", msg: `{"level":"\u0069nfo"}`, want: "unknown", route: "raw"}, + {name: "escaped level key", msg: `{"\u006cevel":"warn"}`, want: "warn", route: "stats"}, + // A spelling the keyword scan cannot find (information is no whole word + // of the scan): only the decoded key gives info, as in Loki v3.7.7. + {name: "escaped level key, long spelling", msg: `{"\u006cevel":"Information"}`, want: "info", route: "stats"}, + // OTLP records. A severity text VictoriaLogs synthesised from the number + // (Unspecified, Info2) is no level for Loki, which reads the number, or the + // line when the record has none. + {name: "OTLP without severity", otlp: true, msg: "hello", want: "unknown", dropOnly: true, route: "stats"}, + {name: "OTLP without severity, error word", otlp: true, msg: "request error", want: "error", dropOnly: true, route: "stats"}, + {name: "OTLP fine severity number", otlp: true, severityNumber: 10, msg: "hello", want: "info", dropOnly: true, route: "stats"}, + {name: "OTLP severity number", otlp: true, severityNumber: 13, msg: "hello", want: "warn", dropOnly: true, route: "stats"}, + {name: "OTLP severity text and number", otlp: true, severityNumber: 17, severityText: "Error", msg: "hello", want: "error", dropOnly: true, route: "stats"}, + {name: "OTLP own text", otlp: true, severityText: "notice", msg: "hello", want: "notice", dropOnly: true, route: "stats"}, + {name: "OTLP own text over its number", otlp: true, severityNumber: 9, severityText: "Fine", msg: "hello", want: "Fine", dropOnly: true, route: "stats"}, + // The keyword scan of Loki v3.7.7 (detectLevelFromLogLine): case, whole + // words only, the first keyword by position, Loki's boundary bytes. A plain + // line is no JSON, so only `| json | drop __error__` is valid for it. + {name: "keyword in capitals", msg: "request ERROR upstream", want: "error", dropOnly: true, route: "stats"}, + {name: "keyword in title case", msg: "Error: connection refused", want: "error", dropOnly: true, route: "stats"}, + {name: "keyword in mixed case", msg: "eRrOr in module", want: "error", dropOnly: true, route: "stats"}, + {name: "plural is no keyword", msg: "3 errors found", want: "unknown", dropOnly: true, route: "stats"}, + {name: "keyword inside a word", msg: "terror alert", want: "unknown", dropOnly: true, route: "stats"}, + {name: "long spelling of warn", msg: "disk warning: low", want: "warn", dropOnly: true, route: "stats"}, + {name: "plural of warn", msg: "warnings ignored", want: "unknown", dropOnly: true, route: "stats"}, + {name: "two keywords, the first by position", msg: "info then error", want: "info", dropOnly: true, route: "stats"}, + {name: "two keywords, the long spelling first", msg: "warning: error", want: "warn", dropOnly: true, route: "stats"}, + {name: "critical before error", msg: "critical error", want: "critical", dropOnly: true, route: "stats"}, + {name: "error before critical", msg: "error then critical", want: "error", dropOnly: true, route: "stats"}, + {name: "hyphen before the keyword", msg: "foo-error", want: "unknown", dropOnly: true, route: "stats"}, + {name: "underscore before the keyword", msg: "foo_error", want: "unknown", dropOnly: true, route: "stats"}, + {name: "digit before the keyword", msg: "5error", want: "unknown", dropOnly: true, route: "stats"}, + {name: "digits after the keyword", msg: "error404", want: "unknown", dropOnly: true, route: "stats"}, + {name: "full stop after the keyword", msg: "error.", want: "unknown", dropOnly: true, route: "stats"}, + {name: "short spelling of error", msg: "err: timeout", want: "error", dropOnly: true, route: "stats"}, + {name: "keyword in brackets", msg: "(warn) slow", want: "warn", dropOnly: true, route: "stats"}, + {name: "keyword in quotes", msg: "retry \"debug\" mode", want: "debug", dropOnly: true, route: "stats"}, + {name: "keyword after an equals sign", msg: "x=trace on", want: "trace", dropOnly: true, route: "stats"}, + {name: "fatal in capitals", msg: "FATAL: out of memory", want: "fatal", dropOnly: true, route: "stats"}, + {name: "no-break space before the keyword", msg: "\u00a0error", want: "unknown", dropOnly: true, route: "stats"}, + {name: "letter before the keyword", msg: "éerror", want: "unknown", dropOnly: true, route: "stats"}, + {name: "letter after the keyword", msg: "errorж", want: "unknown", dropOnly: true, route: "stats"}, + {name: "carriage return before the keyword", msg: "x\rerror", want: "unknown", dropOnly: true, route: "stats"}, + {name: "plural in a JSON message", msg: `{"msg":"3 errors found"}`, want: "unknown", route: "stats"}, + {name: "two keywords in a JSON message", msg: `{"msg":"info then error"}`, want: "info", route: "stats"}, + {name: "long spelling first in a JSON message", msg: `{"msg":"warning: error"}`, want: "warn", route: "stats"}, + {name: "keyword alone in a JSON message", msg: `{"msg":"ERROR"}`, want: "error", route: "stats"}, + {name: "hyphen before the keyword in a JSON message", msg: `{"msg":"foo-error"}`, want: "unknown", route: "stats"}, +} + +// Every shape of level a line can hold gets the detected_level Loki's ingest +// gave it, on both routes of `sum by (detected_level) (... | json ...)`: a +// level key that is not a known string spelling falls to a keyword scan of the +// whole line (pino's numeric levels, `notice`), the first level key in +// document order wins whatever its name (severity, Level, LEVEL, log.level), +// a stream label detected_level keeps its stored spelling, and a body key +// named detected_level never replaces the value. The proxy's two routes are +// compared with Loki's own answer, which must also be the one recorded in the +// table; the pushdown must serve the lines the VictoriaLogs pipes read like +// Loki and hand the others to the raw-row evaluator. +// conformance: semantics/json-metric-detected-level-derivation +func TestRangeMetricCompatibilityJSONDetectedLevelProbes(t *testing.T) { + now := time.Now() + s0 := now.Add(-9 * time.Hour).Truncate(time.Hour) + apps := make([]string, len(detectedLevelProbes)) + fixture := map[string][]jsonVolumeStreamLine{} + var otlpApps []string + for i, probe := range detectedLevelProbes { + apps[i] = fmt.Sprintf("json-level-probe-%02d-%d", i, now.UnixNano()) + if probe.otlp { + otlpApps = append(otlpApps, apps[i]) + var records [][]byte + for k := 0; k < 2; k++ { + records = append(records, otlpLogRecord(s0.Add(time.Duration(10+k)*time.Second), probe.severityNumber, probe.severityText, probe.msg, nil)) + } + data := otlpLogsData(map[string]string{"service.name": apps[i]}, records) + dlPost(t, lokiURL+"/otlp/v1/logs", "application/x-protobuf", data) + dlPost(t, vlURL+"/insert/opentelemetry/v1/logs", "application/x-protobuf", data) + continue + } + for k := 0; k < 2; k++ { + line := jsonVolumeStreamLine{ts: s0.Add(time.Duration(10+k) * time.Second), msg: probe.msg, labels: map[string]string{}, meta: probe.meta, vlMeta: probe.meta} + for name, value := range probe.labels { + if name == "level" { + line.level = value + } else { + line.labels[name] = value + } + } + fixture[apps[i]] = append(fixture[apps[i]], line) + } + } + ingestJSONVolumeFixture(t, fixture) + start, end := s0, s0.Add(2*time.Minute) + if len(otlpApps) > 0 { + forceVLFlush(t) + if status, body := hardeningRequest(t, http.MethodPost, lokiURL+"/flush", "", nil); status >= 300 { + t.Fatalf("Loki flush: %d %s", status, body) + } + for _, app := range otlpApps { + dlWaitBothBackends(t, dlFixture{start: start, end: end}, `{service_name="`+app+`"}`, fmt.Sprintf(`{service.name=%q}`, app), 2) + } + } + + route, rec := newJSONVolumeRouteProxy(t) + for i, probe := range detectedLevelProbes { + for _, pipeline := range []string{`| json`, `| json | drop __error__`} { + if probe.dropOnly && pipeline == `| json` || probe.rowOnly && pipeline != `| json` { + continue + } + query := `sum by (detected_level) (count_over_time(` + probe.selector(apps[i]) + ` ` + pipeline + ` [1m]))` + t.Run(probe.name+" "+pipeline, func(t *testing.T) { + loki := slidingRangeSeries(t, lokiURL, query, start, end, time.Minute, nil) + if len(loki) != 1 { + t.Fatalf("Loki sanity: expected one detected_level series, got %v", loki) + } + for key := range loki { + if key != "detected_level="+probe.want { + t.Errorf("Loki answers {%s}, the table says detected_level=%s", key, probe.want) + } + } + rec.reset() + assertSlidingParity(t, query+" [in-process]", loki, slidingRangeSeries(t, route, query, start, end, time.Minute, nil)) + calls := rec.reset() + assertSlidingParity(t, query+" [proxy]", loki, slidingRangeSeries(t, proxyURL, query, start, end, time.Minute, nil)) + if pipeline != `| json | drop __error__` || probe.route == "" { + return + } + stats, raw := false, false + for _, call := range calls { + stats = stats || strings.HasPrefix(call, "/select/logsql/stats_query_range ") + raw = raw || strings.HasPrefix(call, "/select/logsql/query ") && !strings.HasSuffix(call, " | limit 1") && !isLevelPresenceCall(call) + } + if probe.route == "stats" && (raw || !stats) || probe.route == "raw" && !raw { + t.Errorf("route %q expected: stats call=%v raw rows read=%v; VictoriaLogs calls %q", probe.route, stats, raw, calls) + } + }) + } + } + + // A filter on detected_level compares Loki's structured metadata of every + // line, and Logs Drilldown's detected_level breakdown (not the native + // histogram: the label translator maps detected_level to level) keeps its + // guard. One selector holds the lines the VictoriaLogs pipes read like + // Loki and one the lines they do not. + var clean, risky, valid, otlp []string + for i, probe := range detectedLevelProbes { + if probe.otlp { + otlp = append(otlp, apps[i]) + continue + } + switch { + case probe.route == "stats": + clean = append(clean, apps[i]) + case probe.route == "raw": + risky = append(risky, apps[i]) + } + if !probe.dropOnly { + valid = append(valid, apps[i]) + } + } + drilldown := map[string]string{"X-Query-Tags": "Source=grafana-lokiexplore-app"} + for _, tc := range []struct { + name string + selector []string + pipeline string + headers map[string]string + guarded bool // the selector holds only lines the pipes read like Loki + label string // the selector's label (default app) + }{ + {"Drilldown breakdown, clean lines", clean, `| json | drop __error__, __error_details__ | detected_level != ""`, drilldown, true, ""}, + {"Drilldown breakdown, guarded lines", risky, `| json | drop __error__, __error_details__ | detected_level != ""`, drilldown, false, ""}, + {"equals, clean lines", clean, `| json | drop __error__ | detected_level="info"`, nil, true, ""}, + {"regexp, guarded lines", risky, `| json | drop __error__ | detected_level=~"warn|error|INFO"`, nil, false, ""}, + {"not equals, all lines", append(append([]string{}, clean...), risky...), `| json | drop __error__ | detected_level!="unknown"`, nil, false, ""}, + {"without the error drop, valid lines", valid, `| json | detected_level="info"`, nil, false, ""}, + {"error label, valid lines", valid, `| json | __error__="" | detected_level!=""`, nil, false, ""}, + {"equals, OTLP lines", otlp, `| json | drop __error__ | detected_level="info"`, nil, true, "service_name"}, + {"regexp, OTLP lines", otlp, `| json | drop __error__ | detected_level=~"warn|error|notice"`, nil, true, "service_name"}, + {"not equals, OTLP lines", otlp, `| json | drop __error__ | detected_level!="unknown"`, nil, true, "service_name"}, + } { + t.Run(tc.name, func(t *testing.T) { + label := cmp.Or(tc.label, "app") + query := `sum by (detected_level) (count_over_time({` + label + `=~"` + strings.Join(tc.selector, "|") + `"} ` + tc.pipeline + ` [1m]))` + loki := slidingRangeSeries(t, lokiURL, query, start, end, time.Minute, tc.headers) + rec.reset() + assertSlidingParity(t, query+" [in-process]", loki, slidingRangeSeries(t, route, query, start, end, time.Minute, tc.headers)) + calls := rec.reset() + assertSlidingParity(t, query+" [proxy]", loki, slidingRangeSeries(t, proxyURL, query, start, end, time.Minute, tc.headers)) + if !tc.guarded { + return + } + stats, attempts, guards, raw := 0, 0, 0, 0 + for _, call := range calls { + switch { + case isStoredLevelAttempt(call): + attempts++ + case strings.HasPrefix(call, "/select/logsql/stats_query_range "): + stats++ + case strings.HasPrefix(call, "/select/logsql/query ") && strings.HasSuffix(call, " | limit 1"): + guards++ + case isLevelPresenceCall(call): + case strings.HasPrefix(call, "/select/logsql/query "): + raw++ + } + } + // The stored-level attempt answers a window of stored levels (its + // guards are the labels' only, none here), the chain the rest, guarded. + if (attempts+stats != 1 && !(attempts == 1 && stats == 1)) || raw != 0 || (stats == 1 && guards == 0) { + t.Errorf("expected one guarded stats request without raw rows: stored-level attempts=%d chain stats=%d guards=%d raw=%d", attempts, stats, guards, raw) + } + }) + } +} + +// isLevelPresenceCall reports whether a recorded VictoriaLogs call is the +// presence probe that prunes the detected_level pipes: one stats query, no rows. +func isLevelPresenceCall(call string) bool { + return strings.HasPrefix(call, "/select/logsql/query ") && strings.Contains(call, " | stats count() if (") && strings.Contains(call, " as p0, ") +} + +// isStoredLevelAttempt reports whether a recorded call is the stats query a +// plan reading detected_level sends first: the buckets grouped by copies of the +// stored detected_level and level fields. It answers a window whose rows all +// carry a stored level; any other window asks again with the chain. +func isStoredLevelAttempt(call string) bool { + return strings.HasPrefix(call, "/select/logsql/stats_query_range ") && strings.Contains(call, "copy detected_level as __dl_d, level as __dl_l") +} + +// levelStatsOutcome returns the guard probes a plan reading detected_level may +// send, given the probes of its parsed labels (plain), the three of the chain +// (level), the stored-level attempts and the chain stats requests it made: the +// attempt alone answered (the labels' probes only), the attempt then the chain +// (both sets), or the chain alone (a remembered verdict). +func levelStatsOutcome(plain, level, attempts, stats int) (lo, hi int, ok bool) { + switch { + case attempts == 1 && stats == 0: + return plain, plain, true + case attempts == 1 && stats == 1: + return plain + level, 2*plain + level, true + case attempts == 0 && stats == 1: + return plain + level, plain + level, true + } + return 0, 0, false +} + +// A plan that groups by `level` and detected_level unpacks `level` from the +// body, and the pipes that derive detected_level read the same line: they +// share one parse of it. A level stored under another name decides the row's +// detected_level, and the parsed `level` still counts as the label; a stored +// `level` keeps the label and the body's moves to level_extracted. Each line +// sits in a window with a second line no stored level decides, so that the +// derivation runs in VictoriaLogs, and the labels are Loki's. +// conformance: semantics/json-metric-detected-level-derivation +func TestRangeMetricCompatibilityJSONDetectedLevelSharedUnpack(t *testing.T) { + type probe struct { + name string + labels, meta map[string]string + msg string + level, detected string // "" level: the line has no level label + otlp bool + severityNumber int + severityText string + } + probes := []probe{ + {name: "stored severity, body level", meta: map[string]string{"severity": "warn"}, msg: `{"level":"error","n":1}`, level: "error", detected: "warn"}, + {name: "stored lvl, body level", meta: map[string]string{"lvl": "info"}, msg: `{"level":"error","n":1}`, level: "error", detected: "info"}, + {name: "stored Level, body level", meta: map[string]string{"Level": "info"}, msg: `{"level":"debug","n":1}`, level: "debug", detected: "info"}, + {name: "stored LEVEL, body level of another spelling", meta: map[string]string{"LEVEL": "WARN"}, msg: `{"level":"Information"}`, level: "Information", detected: "warn"}, + {name: "stream level, body level", labels: map[string]string{"level": "warn"}, msg: `{"level":"error","n":1}`, level: "warn", detected: "warn"}, + {name: "stored level, body level", meta: map[string]string{"level": "WARN"}, msg: `{"level":"info"}`, level: "WARN", detected: "warn"}, + {name: "stored severity, no body level", meta: map[string]string{"severity": "warn"}, msg: `{"n":1}`, detected: "warn"}, + {name: "body level only", msg: `{"level":"debug","n":1}`, level: "debug", detected: "debug"}, + {name: "body level of an unknown spelling, keyword in the line", msg: `{"level":"notice","msg":"error connecting"}`, level: "notice", detected: "error"}, + {name: "OTLP severity number, body level", otlp: true, severityNumber: 17, msg: `{"level":"debug","n":1}`, level: "debug", detected: "error"}, + {name: "OTLP severity number and text, body level", otlp: true, severityNumber: 13, severityText: "Warn", msg: `{"level":"info"}`, level: "info", detected: "warn"}, + } + now := time.Now() + s0 := now.Add(-9 * time.Hour).Truncate(time.Hour) + apps := make([]string, len(probes)) + fixture := map[string][]jsonVolumeStreamLine{} + var otlpApps []string + for i, p := range probes { + apps[i] = fmt.Sprintf("json-level-shared-%02d-%d", i, now.UnixNano()) + if p.otlp { + otlpApps = append(otlpApps, apps[i]) + records := [][]byte{ + otlpLogRecord(s0.Add(10*time.Second), p.severityNumber, p.severityText, p.msg, nil), + otlpLogRecord(s0.Add(11*time.Second), 0, "", `{"n":2}`, nil), + } + data := otlpLogsData(map[string]string{"service.name": apps[i]}, records) + dlPost(t, lokiURL+"/otlp/v1/logs", "application/x-protobuf", data) + dlPost(t, vlURL+"/insert/opentelemetry/v1/logs", "application/x-protobuf", data) + continue + } + line := func(at time.Duration, msg string, withLabels bool) jsonVolumeStreamLine { + l := jsonVolumeStreamLine{ts: s0.Add(at), msg: msg, labels: map[string]string{}} + if withLabels { + l.meta, l.vlMeta = p.meta, p.meta + for name, value := range p.labels { + if name == "level" { + l.level = value + } else { + l.labels[name] = value + } + } + } + return l + } + // the probe's line and one no stored level decides, in separate streams + fixture[apps[i]] = append(fixture[apps[i]], line(10*time.Second, p.msg, true), line(11*time.Second, `{"n":2}`, false)) + } + ingestJSONVolumeFixture(t, fixture) + start, end := s0, s0.Add(2*time.Minute) + if len(otlpApps) > 0 { + forceVLFlush(t) + if status, body := hardeningRequest(t, http.MethodPost, lokiURL+"/flush", "", nil); status >= 300 { + t.Fatalf("Loki flush: %d %s", status, body) + } + for _, app := range otlpApps { + dlWaitBothBackends(t, dlFixture{start: start, end: end}, `{service_name="`+app+`"}`, fmt.Sprintf(`{service.name=%q}`, app), 2) + } + } + + route, rec := newJSONVolumeRouteProxy(t) + for i, p := range probes { + selector := `{app="` + apps[i] + `"}` + if p.otlp { + selector = `{service_name="` + apps[i] + `"}` + } + query := `sum by (level, detected_level) (count_over_time(` + selector + ` | json | drop __error__ [1m]))` + t.Run(p.name, func(t *testing.T) { + loki := slidingRangeSeries(t, lokiURL, query, start, end, time.Minute, nil) + wantKey := "detected_level=" + p.detected + if p.level != "" { + wantKey = wantKey + ",level=" + p.level + } + if _, ok := loki[wantKey]; !ok || len(loki) != 2 { + t.Fatalf("Loki answers %v, the table says {%s} and detected_level=unknown", loki, wantKey) + } + rec.reset() + assertSlidingParity(t, query+" [in-process]", loki, slidingRangeSeries(t, route, query, start, end, time.Minute, nil)) + calls := rec.reset() + assertSlidingParity(t, query+" [proxy]", loki, slidingRangeSeries(t, proxyURL, query, start, end, time.Minute, nil)) + // One stats request carries the derivation and one parse of the line; + // no raw rows are read. + chain, raw := 0, 0 + for _, call := range calls { + switch { + case strings.HasPrefix(call, "/select/logsql/stats_query_range ") && !isStoredLevelAttempt(call): + chain++ + if n := strings.Count(call, "unpack_json"); n != 1 { + t.Errorf("%d unpack_json pipes in the chain request, want 1: %.300s", n, call) + } + case strings.HasPrefix(call, "/select/logsql/query ") && !strings.HasSuffix(call, " | limit 1") && !isLevelPresenceCall(call): + raw++ + } + } + if chain != 1 || raw != 0 { + t.Errorf("expected one chain stats request without raw rows: chain=%d raw=%d calls %q", chain, raw, calls) + } + }) + } +} diff --git a/test/e2e-compat/json_filter_pushdown_test.go b/test/e2e-compat/json_filter_pushdown_test.go index 5aa81aaeb..3cfdb8a1e 100644 --- a/test/e2e-compat/json_filter_pushdown_test.go +++ b/test/e2e-compat/json_filter_pushdown_test.go @@ -129,26 +129,43 @@ func TestRangeMetricCompatibilityJSONFilterPushdown(t *testing.T) { continue } after := parserMetricEvaluations(t, route) - stats, guards, raw := 0, 0, 0 + stats, attempts, guards, raw := 0, 0, 0, 0 for _, call := range recorder.reset() { switch { + case isStoredLevelAttempt(call): + attempts++ case strings.HasPrefix(call, "/select/logsql/stats_query_range "): stats++ case strings.HasPrefix(call, "/select/logsql/query ") && strings.HasSuffix(call, " | limit 1"): guards++ + case isLevelPresenceCall(call): case strings.HasPrefix(call, "/select/logsql/query "): raw++ } } rawDelta := after["raw_rows/probe"] - before["raw_rows/probe"] statsDelta := after["vl_stats_buckets/pushdown"] - before["vl_stats_buckets/pushdown"] - if tc.pushed && (stats != 1 || guards != tc.guards || raw != 0 || rawDelta != 0 || statsDelta != 1) { - t.Fatalf("expected %d probes and one stats_query_range request without raw rows: stats=%d guards=%d raw=%d counter raw_rows+%d vl_stats_buckets+%d", tc.guards, stats, guards, raw, rawDelta, statsDelta) + // A grouping by detected_level adds the guard probes of its pipes + // unless the stored level fields answered (levelStatsOutcome). + wantGuards, wantMost := tc.guards, tc.guards + if strings.Contains(tc.query, "detected_level") { + lo, hi, ok := levelStatsOutcome(tc.guards, 3, attempts, stats) + if !ok { + lo, hi = -1, -1 + } + wantGuards, wantMost = lo, hi + stats += attempts + if attempts == 1 && stats == 2 { + stats = 1 // the attempt and the chain are one answer + } + } + if tc.pushed && (stats != 1 || guards < wantGuards || guards > wantMost || raw != 0 || rawDelta != 0 || statsDelta != 1) { + t.Fatalf("expected %d to %d probes and one stats answer without raw rows: stats=%d (stored-level attempts %d) guards=%d raw=%d counter raw_rows+%d vl_stats_buckets+%d", wantGuards, wantMost, stats, attempts, guards, raw, rawDelta, statsDelta) } // The bucket query runs beside the probe and is cancelled when the // probe finds a line the parsers read differently; the answer must // come from the raw evaluator. - if !tc.pushed && (guards != tc.guards || raw != 1 || rawDelta != 1 || statsDelta != 0) { + if !tc.pushed && (guards == 0 || guards > tc.guards+3 || raw != 1 || rawDelta != 1 || statsDelta != 0) { t.Fatalf("expected the probe to keep the raw evaluator: stats=%d guards=%d raw=%d counter raw_rows+%d vl_stats_buckets+%d", stats, guards, raw, rawDelta, statsDelta) } } @@ -218,6 +235,7 @@ func TestRangeMetricCompatibilityJSONFilterPushdown(t *testing.T) { switch { case strings.HasPrefix(call, "/select/logsql/query ") && strings.HasSuffix(call, " | limit 1"): guards++ + case isLevelPresenceCall(call): case strings.HasPrefix(call, "/select/logsql/query "): raw++ } diff --git a/test/e2e-compat/json_logs_volume_test.go b/test/e2e-compat/json_logs_volume_test.go index 5a96f0f21..b72c41546 100644 --- a/test/e2e-compat/json_logs_volume_test.go +++ b/test/e2e-compat/json_logs_volume_test.go @@ -12,6 +12,7 @@ import ( "net/http/httptest" "net/http/httputil" "net/url" + "sort" "strconv" "strings" "sync" @@ -32,6 +33,8 @@ type jsonVolumeStreamLine struct { msg string meta map[string]string vlMeta map[string]string + // labels are further stream labels (severity, detected_level, ...). + labels map[string]string } // vlQueryRecorder forwards to VictoriaLogs and records each request path with @@ -60,6 +63,13 @@ func newJSONVolumeRouteProxy(t *testing.T) (string, *vlQueryRecorder) { // -manual-range-metric-row-limit set (0: the default), so a query that falls // back to the raw-row evaluator over more rows fails with the limit error. func newJSONVolumeRouteProxyWithRowLimit(t *testing.T, rowLimit int) (string, *vlQueryRecorder) { + t.Helper() + return newJSONVolumeRouteProxyWithConfig(t, func(cfg *proxy.Config) { cfg.RangeMetricRowLimit = rowLimit }) +} + +// newJSONVolumeRouteProxyWithConfig is newJSONVolumeRouteProxy with the proxy's +// configuration adjusted by set (a series limit, a row limit). +func newJSONVolumeRouteProxyWithConfig(t *testing.T, set func(*proxy.Config)) (string, *vlQueryRecorder) { t.Helper() target, err := url.Parse(vlURL) if err != nil { @@ -82,11 +92,12 @@ func newJSONVolumeRouteProxyWithRowLimit(t *testing.T, rowLimit int) (string, *v })) t.Cleanup(backend.Close) - p, err := proxy.New(proxy.Config{ + cfg := proxy.Config{ BackendURL: backend.URL, Cache: cache.NewDisabled(), LogLevel: "error", LabelStyle: proxy.LabelStyleUnderscores, MetadataFieldMode: proxy.MetadataFieldModeHybrid, EmitStructuredMetadata: true, - RangeMetricRowLimit: rowLimit, - }) + } + set(&cfg) + p, err := proxy.New(cfg) if err != nil { t.Fatal(err) } @@ -109,14 +120,27 @@ func newJSONVolumeRouteProxyWithRowLimit(t *testing.T, rowLimit int) (string, *v // proves both backends hold every line of each app. func ingestJSONVolumeFixture(t *testing.T, apps map[string][]jsonVolumeStreamLine) { t.Helper() - var vlRows strings.Builder var streams []any for app, lines := range apps { - byLevel := map[string][]any{} + var vlRows strings.Builder + // Structured metadata is no stream field: each app names its own. + streamFields := map[string]bool{"app": true} + type lokiStream struct { + labels map[string]string + values []any + } + byLabels := map[string]*lokiStream{} for _, line := range lines { fields := map[string]string{"_time": line.ts.UTC().Format(time.RFC3339Nano), "_msg": line.msg, "app": app} + labels := map[string]string{"app": app} if line.level != "" { fields["level"] = line.level + labels["level"] = line.level + streamFields["level"] = true + } + for k, v := range line.labels { + fields[k], labels[k] = v, v + streamFields[k] = true } for k, v := range line.vlMeta { fields[k] = v @@ -128,22 +152,27 @@ func ingestJSONVolumeFixture(t *testing.T, apps map[string][]jsonVolumeStreamLin if len(line.meta) > 0 { entry = append(entry, line.meta) } - byLevel[line.level] = append(byLevel[line.level], entry) - } - for level, values := range byLevel { - labels := map[string]string{"app": app} - if level != "" { - labels["level"] = level + key := fmt.Sprint(labels) + if byLabels[key] == nil { + byLabels[key] = &lokiStream{labels: labels} } - streams = append(streams, map[string]any{"stream": labels, "values": values}) + byLabels[key].values = append(byLabels[key].values, entry) + } + for _, stream := range byLabels { + streams = append(streams, map[string]any{"stream": stream.labels, "values": stream.values}) + } + names := make([]string, 0, len(streamFields)) + for name := range streamFields { + names = append(names, name) + } + sort.Strings(names) + status, body := hardeningRequest(t, http.MethodPost, vlURL+"/insert/jsonline?_stream_fields="+strings.Join(names, ","), vlRows.String(), map[string]string{"Content-Type": "application/stream+json"}) + if status != http.StatusOK { + t.Fatalf("VL ingest: %d %s", status, body) } - } - status, body := hardeningRequest(t, http.MethodPost, vlURL+"/insert/jsonline?_stream_fields=app,level", vlRows.String(), map[string]string{"Content-Type": "application/stream+json"}) - if status != http.StatusOK { - t.Fatalf("VL ingest: %d %s", status, body) } payload, _ := json.Marshal(map[string]any{"streams": streams}) - status, body = hardeningRequest(t, http.MethodPost, lokiURL+"/loki/api/v1/push", string(payload), map[string]string{"Content-Type": "application/json", "X-Scope-OrgID": "0"}) + status, body := hardeningRequest(t, http.MethodPost, lokiURL+"/loki/api/v1/push", string(payload), map[string]string{"Content-Type": "application/json", "X-Scope-OrgID": "0"}) if status != http.StatusNoContent { t.Fatalf("Loki ingest: %d %s", status, body) } @@ -210,11 +239,13 @@ func TestRangeMetricCompatibilityJSONLogsVolume(t *testing.T) { jsonVolumeStreamLine{ts: at(4), level: "", msg: fmt.Sprintf(`{"level":["error"],"n":%d}`, i)}, ) // Level-less lines carry no level in the body, so Loki's ingest-time - // detected_level is unknown for them. + // detected_level is unknown for them; the last line's body holds one. fixture[plain] = append(fixture[plain], jsonVolumeStreamLine{ts: at(0), level: "info", msg: fmt.Sprintf("request served %d", i)}, jsonVolumeStreamLine{ts: at(1), level: "debug", msg: fmt.Sprintf("cache probe %d", i)}, jsonVolumeStreamLine{ts: at(2), level: "", msg: fmt.Sprintf("op=select n=%d", i)}, + // Loki's ingest reads a level key of the body whatever the query parses. + jsonVolumeStreamLine{ts: at(3), level: "", msg: fmt.Sprintf("level=warn op=update n=%d", i)}, ) fixture[logfmt] = append(fixture[logfmt], jsonVolumeStreamLine{ts: at(0), level: "info", msg: fmt.Sprintf("level=error msg=collision n=%d", i)}, @@ -236,16 +267,17 @@ func TestRangeMetricCompatibilityJSONLogsVolume(t *testing.T) { step time.Duration pushed bool minimum int // Loki series expected for the fixture + guards int // probes of a pushed query: the label parse risk, and three for detected_level }{ // Grafana's exact supplementary logs volume query. - {`sum by (level, detected_level) (count_over_time({app="` + app + `"} | json | drop __error__[1m]))`, time.Minute, true, 4}, - {`sum by (level) (count_over_time({app="` + app + `"} | json | drop __error__, __error_details__ [2m]))`, time.Minute, true, 4}, - {`sum by (app) (count_over_time({app=~"` + app + `"} | json | drop __error__[1m]))`, time.Minute, true, 1}, - {`sum by (detected_level) (bytes_over_time({app="` + app + `"} | json | drop __error__[90s]))`, 30 * time.Second, true, 4}, - {`sum by (level) (count_over_time({app="` + partial + `"} | json | drop __error__[1m]))`, time.Minute, false, 3}, + {`sum by (level, detected_level) (count_over_time({app="` + app + `"} | json | drop __error__[1m]))`, time.Minute, true, 4, 4}, + {`sum by (level) (count_over_time({app="` + app + `"} | json | drop __error__, __error_details__ [2m]))`, time.Minute, true, 4, 1}, + {`sum by (app) (count_over_time({app=~"` + app + `"} | json | drop __error__[1m]))`, time.Minute, true, 1, 1}, + {`sum by (detected_level) (bytes_over_time({app="` + app + `"} | json | drop __error__[90s]))`, 30 * time.Second, true, 4, 3}, + {`sum by (level) (count_over_time({app="` + partial + `"} | json | drop __error__[1m]))`, time.Minute, false, 3, 1}, // Grafana's logs volume for plain and logfmt selectors. - {`sum by (level, detected_level) (count_over_time({app="` + plain + `"} | drop __error__[1m]))`, time.Minute, true, 3}, - {`sum by (level, detected_level) (count_over_time({app="` + logfmt + `"} | logfmt | drop __error__[1m]))`, time.Minute, true, 3}, + {`sum by (level, detected_level) (count_over_time({app="` + plain + `"} | drop __error__[1m]))`, time.Minute, true, 3, 0}, + {`sum by (level, detected_level) (count_over_time({app="` + logfmt + `"} | logfmt | drop __error__[1m]))`, time.Minute, true, 3, 4}, } { t.Run(tc.query, func(t *testing.T) { loki := slidingRangeSeries(t, lokiURL, tc.query, start, end, tc.step, nil) @@ -258,23 +290,31 @@ func TestRangeMetricCompatibilityJSONLogsVolume(t *testing.T) { if name != "in-process" { continue } - stats, guards, raw := 0, 0, 0 + stats, attempts, guards, raw := 0, 0, 0, 0 for _, call := range recorder.reset() { switch { + case isStoredLevelAttempt(call): + attempts++ case strings.HasPrefix(call, "/select/logsql/stats_query_range "): stats++ case strings.HasPrefix(call, "/select/logsql/query ") && strings.HasSuffix(call, " | limit 1"): guards++ + case isLevelPresenceCall(call): case strings.HasPrefix(call, "/select/logsql/query "): raw++ } } - wantGuards := 1 - if !strings.Contains(tc.query, "json") && !strings.Contains(tc.query, "logfmt") { - wantGuards = 0 // no parser: stored fields only - } - if tc.pushed && (stats != 1 || guards != wantGuards || raw != 0) { - t.Fatalf("expected %d guard and one stats_query_range request without raw rows: stats=%d guards=%d raw=%d", wantGuards, stats, guards, raw) + if tc.pushed { + // Every shape here reads detected_level: tc.guards counts the + // labels' probes and the chain's three. + plain, level := tc.guards-3, 3 + if tc.guards == 0 { // the plain plan has no guard to run + plain, level = 0, 0 + } + lo, hi, ok := levelStatsOutcome(plain, level, attempts, stats) + if !ok || guards < lo || guards > hi || raw != 0 { + t.Fatalf("expected %d to %d guards and one stats_query_range answer without raw rows: stored-level attempts=%d chain stats=%d guards=%d raw=%d", lo, hi, attempts, stats, guards, raw) + } } // The bucket query runs beside the guard and is cancelled when the // guard finds a line the parsers read differently, so a stats @@ -304,12 +344,14 @@ func TestRangeMetricCompatibilityJSONLogsVolume(t *testing.T) { switch { case strings.HasSuffix(call, " | limit 1"): guards++ + case isLevelPresenceCall(call): case strings.HasPrefix(call, "/select/logsql/query "): rows++ } } - if guards != 1 || rows != 1 { - t.Fatalf("expected one risk check and one row fetch: guards=%d rows=%d", guards, rows) + // The detected_level probes run beside the label's; a risky line cancels the rest. + if guards == 0 || guards > 4 || rows != 1 { + t.Fatalf("expected the risk checks and one row fetch: guards=%d rows=%d", guards, rows) } }) } diff --git a/test/e2e-compat/log-generator.py b/test/e2e-compat/log-generator.py index 4ab9c0f6d..846a349f8 100644 --- a/test/e2e-compat/log-generator.py +++ b/test/e2e-compat/log-generator.py @@ -49,6 +49,11 @@ BACKFILL_ONLY = os.getenv("LOG_BACKFILL_ONLY", "0") == "1" BACKFILL_SEED = os.getenv("LOG_BACKFILL_SEED", "") BACKFILL_END = int(os.getenv("LOG_BACKFILL_END", "0")) # unix seconds; 0 = now +# LOG_NO_LEVEL_LABEL=1 leaves the level out of the stream labels: the level then +# lives only in the log line (JSON and logfmt services), or nowhere (plain +# text), as for a collector that does not promote it. bench/ab uses it to +# measure the detected_level routes on rows with no stored level. +NO_LEVEL_LABEL = os.getenv("LOG_NO_LEVEL_LABEL", "0") == "1" # Simulated clock for the backfill, in nanoseconds. None in live mode. While # set, every line gets its own timestamp 1 µs after the previous one, so no two @@ -669,6 +674,8 @@ def make_labels(app: str, namespace: str, cluster: str, level: str, "pod": rand_pod(app, pod_sfx), "container": app.split("-")[0], } + if NO_LEVEL_LABEL: + del labels["level"] if extra: labels.update(extra) return labels