diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3e8874544..c186d0b7a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -162,6 +162,9 @@ jobs: - name: Fuzz smoke (extract log patterns from window entries) run: go test ./internal/proxy -run '^$' -fuzz='^FuzzExtractLogPatternsFromWindowEntries$' -fuzztime=12s + - name: Fuzz smoke (json error filter split predicate) + run: go test ./internal/proxy -run '^$' -fuzz='^FuzzJSONValidSplitPredicate$' -fuzztime=12s + # --- internal/translator --- - name: Fuzz smoke (translate LogQL) run: go test ./internal/translator -run '^$' -fuzz=FuzzTranslateLogQL -fuzztime=12s diff --git a/CHANGELOG.md b/CHANGELOG.md index 1db72f783..340c40384 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,91 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Performance + +- **`| json | __error__=""` metrics are counted by VictoriaLogs stats instead of + reading every line (#758).** `sum by (app) (count_over_time({namespace="prod"} + | json | __error__="" [5m]))` and the same shape with rate, bytes, a label + filter or a grouping by a parsed key, by a stream label or by service_name + read every selected line as a raw row into the proxy, because VictoriaLogs + attaches no parser error and Loki's `JSONParserErr` depends on the query + (`pkg/logql/log/parser.go` `JSONParser.Process` walks a nested object only + when a required label starts with its key path). The stats pushdown now splits + the lines: a line not opening with `{` after space, tab, line feed or carriage + return fails Loki's parser and is dropped; a line VictoriaLogs parses as one + JSON object (a sentinel member spliced after the brace and read back with + `unpack_json`) and that holds none of the constructs VictoriaLogs' parser + accepts and Loki's rejects (a backslash in a key, a value starting with `.`, + `e`, `E` or `+`, NaN or Inf) is accepted by Loki under every hint set and + counted by VictoriaLogs; every other brace-opened line is counted per bucket + in the same scan and read as a raw row only from those buckets, then evaluated + by the row evaluator. A cached `| limit 1` probe looks for the rejected + constructs and only when it finds one is the scan repeated (the first scan is + cancelled) with such lines moved to the raw rows. Label filters run before the + stats pipe, and the lines read as raw rows lose the grouped fields there + (marked once with `format if (...) "1" as __vlr`), so they form one group + however many labels they hold. Loki's series limit applies to the merged + answer only, never to VictoriaLogs groups, which are not series: the + spellings of one level are separate groups and one detected_level series, + filtered-out or unparseable lines make groups that are no series at all, and + a line between two evaluation windows (a step wider than the range) makes a + group Loki's answer has no point for. The stats read therefore keeps at most + twice `-max-stats-query-series` in groups as a memory guard (a response of + 300,000 groups is no longer parsed whole), and a read past it, or a response + too large to read, only means the split does not answer: the row evaluator + does, as before this change, with its own row and byte limits and Loki's + series limit error where its series exceed the limit. Grafana Logs Drilldown + requests, which keep the busiest series of an over-limit answer, and + groupings by a field starting with `_` take the row evaluator as before. + A grouping by `detected_level` is split only from the stored level fields + (the stored `detected_level` and `level`, copied before anything else + touches them): when every counted line has a stored level the spellings of one + level merge into one series after the read; a level only the body holds, a + line without one, a stored `detected_level` label, another stored level + field (`severity`, `lvl`, OTLP severity), a read past the group guard or a + filter on `detected_level` leave the query to the row evaluator, + which derives the level of each line like Loki's ingest. The derivation of + `sum by (detected_level)` itself is #760's. Window sums add the bucket counts and the raw lines at each evaluation point + before rate divides once. service_name is taken from the first non-empty + source field that is a stream label of the row, as Loki assigns it at + ingest, never from structured metadata. Measured on VictoriaLogs v1.52.0 with + generator data against main: 1 h 0.56-2.44 s -> 0.03-0.07 s (Loki warm + 0.01-0.05 s); 6 h 3.3 s or a 502 -> 0.10-0.45 s. A fallback costs one + VictoriaLogs stats scan on top of the row evaluator: 0.1 s more at 1 h, and + where the row evaluator ends at once with the series limit error (a grouping + by a field with thousands of values) 1.5-4.8 s at 24 h, because VictoriaLogs + aggregates every group before it answers. The raw row evaluator also reads + only the fields it needs (`| fields _time, _stream, _msg, level` plus every + stored field that can expose a required label), which lightens every raw-row read + of a `| json` metric. An instant query of a `__error__=""` plan keeps the row + evaluator, as it did before the split. + +### Fixed + +- **`| json | __error__=""` metrics over 6 h and 24 h answer instead of a 502 + (#758).** These shapes read every selected line as a raw row and failed with + `manual range metric row limit exceeded` or `ordered JSON metric response + exceeds ... bytes` (502) once a range held about a million lines; Loki answers + them. With the stats pushdown above they answer in 0.4-1.4 s at 24 h on the + generator data. The row evaluator's guards (nesting depth 64, 1024 fields, 1 + MiB line, 64 KiB value) now apply only to the lines read raw: a line + VictoriaLogs vouches for is counted as Loki counts it, however wide and up to + 300 levels deep (VictoriaLogs' JSON parser limit); a deeper line is read raw + and keeps the row evaluator's depth guard. The row and byte limits + (`-manual-range-metric-row-limit`, `-ordered-json-metric-max-bytes`) bound the + raw rows with their existing errors. + `semantics/json-error-filter-metric-split-pushdown`; the open case + `semantics/json-empty-value-label-series` records that a parsed key with an + empty value is its own series in Loki and still merges into the label-less + series here, as before. +- **A `| json` metric line that opens with a form feed fails like Loki + (#758).** The one-row lookup that answers Loki's `pipeline error` before any + row is scanned matched lines opening with `{` after RE2's `\s`, which + includes the form feed; Loki's parser skips only space, tab, line feed and + carriage return (grafana/jsonparser `nextToken`), so such a line is now found + by the lookup instead of the row scan + (`semantics/pipeline-error-text-parity`). + ## [2.27.0] - 2026-10-22 ### Security diff --git a/bench/ab/results/2026-10-22-json-error-split-1h.json b/bench/ab/results/2026-10-22-json-error-split-1h.json new file mode 100644 index 000000000..05cc8ea96 --- /dev/null +++ b/bench/ab/results/2026-10-22-json-error-split-1h.json @@ -0,0 +1,560 @@ +{ + "label": "json-error-split-1h", + "date": "2026-10-22", + "set": "json-error-split", + "description": "`| json | __error__=\"\"` metrics: counted by one VictoriaLogs stats scan over the lines VictoriaLogs parses, with only the other brace-opened lines read as raw rows, instead of reading every selected line raw (502 at 24 h on the row and byte limits). Run with the control set.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 5, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "refresh_s": null, + "refresh_runs": null, + "timing": "warm", + "verdicts": { + "faster": 5 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "J1 json no-error count by service_name, one service", + "range": "1h", + "p50": { + "base": 0.696, + "pr": 0.032, + "loki": 0.024 + }, + "p95": { + "base": 0.807, + "pr": 0.035, + "loki": 0.044 + }, + "cold": { + "base": 0.711, + "pr": 0.032, + "loki": 0.029 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 38.57, + "pr": 38.57, + "loki": 38.57 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 2, + "entriesRequested": 2, + "bytesReceived": 14752 + }, + "statsResult": { + "entriesFound": 1, + "entriesRequested": 1, + "bytesReceived": 186 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 12, + "entriesRequested": 12, + "bytesReceived": 93130 + }, + "statsResult": { + "entriesFound": 6, + "entriesRequested": 6, + "bytesReceived": 1116 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "cold_cache_served": [ + "loki" + ], + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "faster", + "speedup": 21.75, + "confirmed": true + }, + { + "shape": "J2 json no-error count by app, namespace", + "range": "1h", + "p50": { + "base": 1.754, + "pr": 0.067, + "loki": 0.042 + }, + "p95": { + "base": 1.813, + "pr": 0.088, + "loki": 0.057 + }, + "cold": { + "base": 1.763, + "pr": 0.064, + "loki": 0.043 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 53.56, + "pr": 48.97, + "loki": 48.97 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 2, + "entriesRequested": 2, + "bytesReceived": 98051 + }, + "statsResult": { + "entriesFound": 1, + "entriesRequested": 1, + "bytesReceived": 177 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 12, + "entriesRequested": 12, + "bytesReceived": 620137 + }, + "statsResult": { + "entriesFound": 6, + "entriesRequested": 6, + "bytesReceived": 1062 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "cold_cache_served": [ + "loki" + ], + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "faster", + "speedup": 26.18, + "confirmed": true + }, + { + "shape": "J3 json no-error count by parsed method", + "range": "1h", + "p50": { + "base": 0.565, + "pr": 0.032, + "loki": 0.016 + }, + "p95": { + "base": 0.631, + "pr": 0.035, + "loki": 0.017 + }, + "cold": { + "base": 0.557, + "pr": 0.029, + "loki": 0.016 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 65.38, + "pr": 65.38, + "loki": 65.38 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 2, + "entriesRequested": 2, + "bytesReceived": 70221 + }, + "statsResult": { + "entriesFound": 1, + "entriesRequested": 1, + "bytesReceived": 186 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 12, + "entriesRequested": 12, + "bytesReceived": 444104 + }, + "statsResult": { + "entriesFound": 6, + "entriesRequested": 6, + "bytesReceived": 1116 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "cold_cache_served": [ + "loki" + ], + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "faster", + "speedup": 17.66, + "confirmed": true + }, + { + "shape": "J4 json no-error count by service_name, all services", + "range": "1h", + "p50": { + "base": 2.615, + "pr": 0.072, + "loki": 0.053 + }, + "p95": { + "base": 2.771, + "pr": 0.089, + "loki": 0.056 + }, + "cold": { + "base": 2.457, + "pr": 0.075, + "loki": 0.059 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 70.5, + "pr": 70.5, + "loki": 70.5 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 2, + "entriesRequested": 2, + "bytesReceived": 139905 + }, + "statsResult": { + "entriesFound": 1, + "entriesRequested": 1, + "bytesReceived": 177 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 12, + "entriesRequested": 12, + "bytesReceived": 884878 + }, + "statsResult": { + "entriesFound": 6, + "entriesRequested": 6, + "bytesReceived": 1062 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "cold_cache_served": [ + "loki" + ], + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "faster", + "speedup": 36.32, + "confirmed": true + }, + { + "shape": "J5 json no-error bytes_rate by app", + "range": "1h", + "p50": { + "base": 2.231, + "pr": 0.07, + "loki": 0.044 + }, + "p95": { + "base": 2.521, + "pr": 0.08, + "loki": 0.048 + }, + "cold": { + "base": 2.133, + "pr": 0.091, + "loki": 0.078 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 69.38, + "pr": 40.61, + "loki": 40.61 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 2, + "entriesRequested": 2, + "bytesReceived": 98033 + }, + "statsResult": { + "entriesFound": 1, + "entriesRequested": 1, + "bytesReceived": 177 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 12, + "entriesRequested": 12, + "bytesReceived": 620024 + }, + "statsResult": { + "entriesFound": 6, + "entriesRequested": 6, + "bytesReceived": 1062 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "cold_cache_served": [ + "loki" + ], + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "faster", + "speedup": 31.87, + "confirmed": true + } + ], + "isolated_windows": true +} diff --git a/bench/ab/results/2026-10-22-json-error-split-control-1h.json b/bench/ab/results/2026-10-22-json-error-split-control-1h.json new file mode 100644 index 000000000..6c927b125 --- /dev/null +++ b/bench/ab/results/2026-10-22-json-error-split-control-1h.json @@ -0,0 +1,2002 @@ +{ + "label": "json-error-split-control-1h", + "date": "2026-10-22", + "set": "control", + "description": "Shapes a change must not slow down or alter. Run on every performance-relevant PR.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 5, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "refresh_s": null, + "refresh_runs": null, + "timing": "warm", + "verdicts": { + "same": 20 + }, + "result_differs_from_reference": 1, + "result_differs_preexisting": 1, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "logs plain", + "range": "1h", + "p50": { + "base": 0.028, + "pr": 0.03, + "loki": 0.025 + }, + "p95": { + "base": 0.036, + "pr": 0.046, + "loki": 0.027 + }, + "cold": { + "base": 0.041, + "pr": 0.028, + "loki": 0.04 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 19.67, + "pr": 19.67, + "loki": 19.67 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 1, + "entriesRequested": 1, + "bytesReceived": 177 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 4, + "entriesRequested": 4, + "bytesReceived": 712 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.93 + }, + { + "shape": "logs | json", + "range": "1h", + "p50": { + "base": 0.056, + "pr": 0.052, + "loki": 0.081 + }, + "p95": { + "base": 0.071, + "pr": 0.079, + "loki": 0.108 + }, + "cold": { + "base": 0.048, + "pr": 0.052, + "loki": 0.05 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 34.07, + "pr": 34.07, + "loki": 34.07 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 1, + "entriesRequested": 1, + "bytesReceived": 178 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 4, + "entriesRequested": 4, + "bytesReceived": 712 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.08 + }, + { + "shape": "logs | logfmt", + "range": "1h", + "p50": { + "base": 0.073, + "pr": 0.074, + "loki": 0.049 + }, + "p95": { + "base": 0.083, + "pr": 0.089, + "loki": 0.054 + }, + "cold": { + "base": 0.098, + "pr": 0.078, + "loki": 0.047 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 76.29, + "pr": 76.29, + "loki": 76.29 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 1, + "entriesRequested": 1, + "bytesReceived": 178 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 4, + "entriesRequested": 4, + "bytesReceived": 712 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.99 + }, + { + "shape": "json volume no filters", + "range": "1h", + "p50": { + "base": 0.047, + "pr": 0.038, + "loki": 0.032 + }, + "p95": { + "base": 0.054, + "pr": 0.07, + "loki": 0.038 + }, + "cold": { + "base": 0.042, + "pr": 0.041, + "loki": 0.423 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 65.21, + "pr": 65.21, + "loki": 65.21 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 0, + "entriesRequested": 2, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 2, + "entriesRequested": 2, + "bytesReceived": 356 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 8, + "entriesRequested": 8, + "bytesReceived": 217720 + }, + "statsResult": { + "entriesFound": 4, + "entriesRequested": 4, + "bytesReceived": 712 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.24 + }, + { + "shape": "logfmt volume", + "range": "1h", + "p50": { + "base": 0.075, + "pr": 0.069, + "loki": 0.043 + }, + "p95": { + "base": 0.085, + "pr": 0.083, + "loki": 0.044 + }, + "cold": { + "base": 0.053, + "pr": 0.062, + "loki": 0.597 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 76.3, + "pr": 76.3, + "loki": 76.3 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 0, + "entriesRequested": 2, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 2, + "entriesRequested": 2, + "bytesReceived": 356 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 8, + "entriesRequested": 8, + "bytesReceived": 217735 + }, + "statsResult": { + "entriesFound": 4, + "entriesRequested": 4, + "bytesReceived": 712 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.09 + }, + { + "shape": "plain volume", + "range": "1h", + "p50": { + "base": 0.01, + "pr": 0.009, + "loki": 0.028 + }, + "p95": { + "base": 0.019, + "pr": 0.011, + "loki": 0.042 + }, + "cold": { + "base": 0.008, + "pr": 0.007, + "loki": 0.448 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 60.66, + "pr": 60.66, + "loki": 60.66 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 0, + "entriesRequested": 2, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 2, + "entriesRequested": 2, + "bytesReceived": 356 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 8, + "entriesRequested": 8, + "bytesReceived": 217656 + }, + "statsResult": { + "entriesFound": 4, + "entriesRequested": 4, + "bytesReceived": 712 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.11 + }, + { + "shape": "stream-label count by app", + "range": "1h", + "p50": { + "base": 0.013, + "pr": 0.013, + "loki": 0.027 + }, + "p95": { + "base": 0.017, + "pr": 0.015, + "loki": 0.036 + }, + "cold": { + "base": 0.014, + "pr": 0.014, + "loki": 0.399 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 10.85, + "pr": 10.85, + "loki": 10.85 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 0, + "entriesRequested": 2, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 2, + "entriesRequested": 2, + "bytesReceived": 356 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 8, + "entriesRequested": 8, + "bytesReceived": 804194 + }, + "statsResult": { + "entriesFound": 4, + "entriesRequested": 4, + "bytesReceived": 712 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.0 + }, + { + "shape": "ungrouped rate", + "range": "1h", + "p50": { + "base": 0.01, + "pr": 0.009, + "loki": 0.059 + }, + "p95": { + "base": 0.015, + "pr": 0.012, + "loki": 0.098 + }, + "cold": { + "base": 0.007, + "pr": 0.012, + "loki": 0.469 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 9.55, + "pr": 9.55, + "loki": 9.55 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 0, + "entriesRequested": 2, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 2, + "entriesRequested": 2, + "bytesReceived": 356 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 8, + "entriesRequested": 8, + "bytesReceived": 56188 + }, + "statsResult": { + "entriesFound": 4, + "entriesRequested": 4, + "bytesReceived": 712 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.11 + }, + { + "shape": "bytes_over_time by app", + "range": "1h", + "p50": { + "base": 0.024, + "pr": 0.025, + "loki": 0.05 + }, + "p95": { + "base": 0.025, + "pr": 0.029, + "loki": 0.058 + }, + "cold": { + "base": 0.025, + "pr": 0.025, + "loki": 0.327 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 22.21, + "pr": 22.21, + "loki": 22.21 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 0, + "entriesRequested": 2, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 2, + "entriesRequested": 2, + "bytesReceived": 356 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 8, + "entriesRequested": 8, + "bytesReceived": 804194 + }, + "statsResult": { + "entriesFound": 4, + "entriesRequested": 4, + "bytesReceived": 712 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.96 + }, + { + "shape": "topk rate by app", + "range": "1h", + "p50": { + "base": 0.024, + "pr": 0.028, + "loki": 0.053 + }, + "p95": { + "base": 0.026, + "pr": 0.031, + "loki": 0.066 + }, + "cold": { + "base": 0.024, + "pr": 0.028, + "loki": 0.456 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 20.22, + "pr": 17.42, + "loki": 17.42 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 0, + "entriesRequested": 2, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 2, + "entriesRequested": 2, + "bytesReceived": 356 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 8, + "entriesRequested": 8, + "bytesReceived": 270846 + }, + "statsResult": { + "entriesFound": 4, + "entriesRequested": 4, + "bytesReceived": 712 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "parity_vs_reference": "differs", + "baseline_parity_vs_reference": "differs", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.86 + }, + { + "shape": "detected_level volume", + "range": "1h", + "p50": { + "base": 0.024, + "pr": 0.025, + "loki": 0.024 + }, + "p95": { + "base": 0.029, + "pr": 0.032, + "loki": 0.027 + }, + "cold": { + "base": 0.036, + "pr": 0.033, + "loki": 0.43 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 35.42, + "pr": 35.42, + "loki": 35.42 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 0, + "entriesRequested": 2, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 2, + "entriesRequested": 2, + "bytesReceived": 356 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 8, + "entriesRequested": 8, + "bytesReceived": 217103 + }, + "statsResult": { + "entriesFound": 4, + "entriesRequested": 4, + "bytesReceived": 712 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.96 + }, + { + "shape": "by service_name", + "range": "1h", + "p50": { + "base": 0.014, + "pr": 0.015, + "loki": 0.021 + }, + "p95": { + "base": 0.016, + "pr": 0.015, + "loki": 0.023 + }, + "cold": { + "base": 0.013, + "pr": 0.013, + "loki": 0.307 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 23.68, + "pr": 23.68, + "loki": 23.68 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 0, + "entriesRequested": 2, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 2, + "entriesRequested": 2, + "bytesReceived": 356 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 8, + "entriesRequested": 8, + "bytesReceived": 805346 + }, + "statsResult": { + "entriesFound": 4, + "entriesRequested": 4, + "bytesReceived": 712 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.93 + }, + { + "shape": "drilldown level volume", + "range": "1h", + "p50": { + "base": 0.009, + "pr": 0.01, + "loki": 0.021 + }, + "p95": { + "base": 0.011, + "pr": 0.013, + "loki": 0.025 + }, + "cold": { + "base": 0.006, + "pr": 0.01, + "loki": 0.409 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 9.12, + "pr": 7.73, + "loki": 7.73 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 0, + "entriesRequested": 2, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 2, + "entriesRequested": 2, + "bytesReceived": 356 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 8, + "entriesRequested": 8, + "bytesReceived": 110024 + }, + "statsResult": { + "entriesFound": 4, + "entriesRequested": 4, + "bytesReceived": 712 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.9 + }, + { + "shape": "drilldown service landing", + "range": "1h", + "p50": { + "base": 0.013, + "pr": 0.013, + "loki": 0.016 + }, + "p95": { + "base": 0.015, + "pr": 0.017, + "loki": 0.026 + }, + "cold": { + "base": 0.016, + "pr": 0.013, + "loki": 0.398 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 11.3, + "pr": 11.3, + "loki": 11.3 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 0, + "entriesRequested": 2, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 2, + "entriesRequested": 2, + "bytesReceived": 356 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 8, + "entriesRequested": 8, + "bytesReceived": 405747 + }, + "statsResult": { + "entriesFound": 4, + "entriesRequested": 4, + "bytesReceived": 712 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.0 + }, + { + "shape": "drilldown detected_level", + "range": "1h", + "p50": { + "base": 0.019, + "pr": 0.019, + "loki": 0.02 + }, + "p95": { + "base": 0.021, + "pr": 0.023, + "loki": 0.021 + }, + "cold": { + "base": 0.02, + "pr": 0.017, + "loki": 0.417 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 24.84, + "pr": 24.84, + "loki": 24.84 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 0, + "entriesRequested": 2, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 2, + "entriesRequested": 2, + "bytesReceived": 356 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 8, + "entriesRequested": 8, + "bytesReceived": 110399 + }, + "statsResult": { + "entriesFound": 4, + "entriesRequested": 4, + "bytesReceived": 712 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.0 + }, + { + "shape": "json rate by level (no filter, no drop)", + "range": "1h", + "p50": { + "base": 0.003, + "pr": 0.003, + "loki": 0.07 + }, + "p95": { + "base": 0.003, + "pr": 0.004, + "loki": 0.089 + }, + "cold": { + "base": 0.005, + "pr": 0.004, + "loki": 0.061 + }, + "status": { + "base": "400", + "pr": "400", + "loki": "400" + }, + "vl_cpu_max": { + "base": 24.36, + "pr": 24.36, + "loki": 24.36 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "n/a", + "verdict": "same", + "speedup": null + }, + { + "shape": "logfmt count by level, filter, drop", + "range": "1h", + "p50": { + "base": 0.084, + "pr": 0.079, + "loki": 0.028 + }, + "p95": { + "base": 0.101, + "pr": 0.101, + "loki": 0.035 + }, + "cold": { + "base": 0.125, + "pr": 0.113, + "loki": 0.732 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 129.7, + "pr": 129.7, + "loki": 129.7 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 0, + "entriesRequested": 2, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 2, + "entriesRequested": 2, + "bytesReceived": 356 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 8, + "entriesRequested": 8, + "bytesReceived": 56832 + }, + "statsResult": { + "entriesFound": 4, + "entriesRequested": 4, + "bytesReceived": 712 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.06 + }, + { + "shape": "instant ungrouped rate", + "range": "instant", + "p50": { + "base": 0.005, + "pr": 0.005, + "loki": 0.037 + }, + "p95": { + "base": 0.006, + "pr": 0.006, + "loki": 0.043 + }, + "cold": { + "base": 0.005, + "pr": 0.006, + "loki": 0.033 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 72.86, + "pr": 72.86, + "loki": 72.86 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 1, + "entriesRequested": 1, + "bytesReceived": 178 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 4, + "entriesRequested": 4, + "bytesReceived": 712 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.0 + }, + { + "shape": "instant json volume", + "range": "instant", + "p50": { + "base": 0.018, + "pr": 0.011, + "loki": 0.076 + }, + "p95": { + "base": 0.042, + "pr": 0.012, + "loki": 0.087 + }, + "cold": { + "base": 0.01, + "pr": 0.01, + "loki": 0.042 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 13.89, + "pr": 13.89, + "loki": 13.89 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 1, + "entriesRequested": 1, + "bytesReceived": 178 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 4, + "entriesRequested": 4, + "bytesReceived": 712 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.64 + }, + { + "shape": "instant json filtered", + "range": "instant", + "p50": { + "base": 0.021, + "pr": 0.022, + "loki": 0.036 + }, + "p95": { + "base": 0.026, + "pr": 0.024, + "loki": 0.038 + }, + "cold": { + "base": 0.027, + "pr": 0.024, + "loki": 0.033 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 11.44, + "pr": 11.44, + "loki": 11.44 + }, + "cache": { + "loki": { + "cold": { + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 1, + "entriesRequested": 1, + "bytesReceived": 178 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + }, + "warm": { + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "statsResult": { + "entriesFound": 4, + "entriesRequested": 4, + "bytesReceived": 712 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "bytesReceived": 0 + } + } + } + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.95 + } + ], + "isolated_windows": true +} diff --git a/bench/ab/results/2026-10-22-json-error-split-long.json b/bench/ab/results/2026-10-22-json-error-split-long.json new file mode 100644 index 000000000..555113f37 --- /dev/null +++ b/bench/ab/results/2026-10-22-json-error-split-long.json @@ -0,0 +1,323 @@ +{ + "label": "json-error-split-long", + "date": "2026-10-22", + "set": "json-error-split", + "description": "`| json | __error__=\"\"` metrics: counted by one VictoriaLogs stats scan over the lines VictoriaLogs parses, with only the other brace-opened lines read as raw rows, instead of reading every selected line raw (502 at 24 h on the row and byte limits). Run with the control set.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 5, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "refresh_s": null, + "refresh_runs": null, + "timing": "warm", + "verdicts": { + "faster": 2, + "fixed": 8 + }, + "result_differs_from_reference": 0, + "result_differs_preexisting": 0, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "J1 json no-error count by service_name, one service", + "range": "6h", + "p50": { + "base": 4.648, + "pr": 0.142 + }, + "p95": { + "base": 4.793, + "pr": 0.155 + }, + "cold": { + "base": 4.698, + "pr": 0.259 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 209.94, + "pr": 45.77 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "faster", + "speedup": 32.73, + "confirmed": true + }, + { + "shape": "J1 json no-error count by service_name, one service", + "range": "24h", + "p50": { + "base": 14.677, + "pr": 0.625 + }, + "p95": { + "base": 17.408, + "pr": 0.697 + }, + "cold": { + "base": 15.353, + "pr": 1.914 + }, + "status": { + "base": "502", + "pr": "200" + }, + "vl_cpu_max": { + "base": 724.81, + "pr": 724.81 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "fixed", + "speedup": null + }, + { + "shape": "J2 json no-error count by app, namespace", + "range": "6h", + "p50": { + "base": 13.463, + "pr": 0.476 + }, + "p95": { + "base": 13.857, + "pr": 0.504 + }, + "cold": { + "base": 12.969, + "pr": 0.62 + }, + "status": { + "base": "502", + "pr": "200" + }, + "vl_cpu_max": { + "base": 351.88, + "pr": 221.37 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "fixed", + "speedup": null + }, + { + "shape": "J2 json no-error count by app, namespace", + "range": "24h", + "p50": { + "base": 11.952, + "pr": 1.46 + }, + "p95": { + "base": 14.227, + "pr": 1.779 + }, + "cold": { + "base": 11.791, + "pr": 2.237 + }, + "status": { + "base": "502", + "pr": "200" + }, + "vl_cpu_max": { + "base": 739.03, + "pr": 739.03 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "fixed", + "speedup": null + }, + { + "shape": "J3 json no-error count by parsed method", + "range": "6h", + "p50": { + "base": 4.512, + "pr": 0.159 + }, + "p95": { + "base": 5.012, + "pr": 0.19 + }, + "cold": { + "base": 4.613, + "pr": 0.209 + }, + "status": { + "base": "200", + "pr": "200" + }, + "vl_cpu_max": { + "base": 167.55, + "pr": 102.44 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "same", + "verdict": "faster", + "speedup": 28.38, + "confirmed": true + }, + { + "shape": "J3 json no-error count by parsed method", + "range": "24h", + "p50": { + "base": 12.18, + "pr": 0.611 + }, + "p95": { + "base": 14.019, + "pr": 0.753 + }, + "cold": { + "base": 12.457, + "pr": 0.584 + }, + "status": { + "base": "502", + "pr": "200" + }, + "vl_cpu_max": { + "base": 562.38, + "pr": 562.38 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "fixed", + "speedup": null + }, + { + "shape": "J4 json no-error count by service_name, all services", + "range": "6h", + "p50": { + "base": 13.093, + "pr": 0.502 + }, + "p95": { + "base": 14.871, + "pr": 0.749 + }, + "cold": { + "base": 14.255, + "pr": 0.991 + }, + "status": { + "base": "502", + "pr": "200" + }, + "vl_cpu_max": { + "base": 277.61, + "pr": 277.61 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "fixed", + "speedup": null + }, + { + "shape": "J4 json no-error count by service_name, all services", + "range": "24h", + "p50": { + "base": 12.016, + "pr": 2.01 + }, + "p95": { + "base": 12.453, + "pr": 2.262 + }, + "cold": { + "base": 14.243, + "pr": 3.142 + }, + "status": { + "base": "502", + "pr": "200" + }, + "vl_cpu_max": { + "base": 764.3, + "pr": 768.51 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "fixed", + "speedup": null + }, + { + "shape": "J5 json no-error bytes_rate by app", + "range": "6h", + "p50": { + "base": 11.027, + "pr": 0.346 + }, + "p95": { + "base": 11.833, + "pr": 0.396 + }, + "cold": { + "base": 11.657, + "pr": 0.503 + }, + "status": { + "base": "502", + "pr": "200" + }, + "vl_cpu_max": { + "base": 768.51, + "pr": 231.97 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "fixed", + "speedup": null + }, + { + "shape": "J5 json no-error bytes_rate by app", + "range": "24h", + "p50": { + "base": 12.305, + "pr": 1.518 + }, + "p95": { + "base": 14.803, + "pr": 2.04 + }, + "cold": { + "base": 12.587, + "pr": 3.001 + }, + "status": { + "base": "502", + "pr": "200" + }, + "vl_cpu_max": { + "base": 694.84, + "pr": 759.84 + }, + "parity_vs_reference": "n/a", + "baseline_parity_vs_reference": "n/a", + "parity_vs_baseline": "n/a", + "verdict": "fixed", + "speedup": null + } + ], + "isolated_windows": true +} diff --git a/bench/ab/results/oneoff/2026-10-22-json-error-split-fallback.json b/bench/ab/results/oneoff/2026-10-22-json-error-split-fallback.json new file mode 100644 index 000000000..f60590618 --- /dev/null +++ b/bench/ab/results/oneoff/2026-10-22-json-error-split-fallback.json @@ -0,0 +1,138 @@ +{ + "label": "json-error-split-fallback", + "date": "2026-10-22", + "description": "Shapes where the JSON error-filter split falls back to the row evaluator (more than twice -max-stats-query-series groups), and two within the guard: main vs branch warm p50 (3 runs at 24 h, 5 at 1 h, distinct windows), host proxies against one VictoriaLogs v1.52.0 stack seeded with VictoriaLogs only (8.5 days available, 25 h of window needed), restarts 0->0. Main is origin/main at run time (2410a427), branch is the PR head built before the rebase (7db060e8; the rebase changed no proxy code). Selector {env=\"production\"}, `| json | __error__=\"\"`, [5m], step 5 s at 1 h and 60 s at 24 h. The fallback costs one VictoriaLogs stats scan on top of main's row evaluator; where main's row evaluator fails fast with the series limit error (F1, F2) the branch is slower by that scan.", + "command": "python3 fallback.py --existing-end (host proxies with the shown -max-stats-query-series, a client timeout of 120 s)", + "rows": [ + { + "shape": "F1 sum by (session_id), groups past twice the limit", + "max_stats_query_series": 500, + "range": "1h", + "main": { + "status": "400", + "p50_s": 0.03 + }, + "branch": { + "status": "400", + "p50_s": 0.16 + } + }, + { + "shape": "F1 sum by (session_id), groups past twice the limit", + "max_stats_query_series": 500, + "range": "24h", + "main": { + "status": "400", + "p50_s": 0.03 + }, + "branch": { + "status": "400", + "p50_s": 5.29 + } + }, + { + "shape": "F3 sum by (user_id), groups within twice the limit (no fallback)", + "max_stats_query_series": 500, + "range": "1h", + "main": { + "status": "200", + "p50_s": 2.29 + }, + "branch": { + "status": "200", + "p50_s": 0.07 + } + }, + { + "shape": "F3 sum by (user_id), groups within twice the limit (no fallback)", + "max_stats_query_series": 500, + "range": "24h", + "main": { + "status": "502", + "p50_s": 8.5 + }, + "branch": { + "status": "200", + "p50_s": 1.76 + } + }, + { + "shape": "F4 sum by (detected_level), 3 series within twice the limit (no fallback on this data)", + "max_stats_query_series": 5, + "range": "1h", + "main": { + "status": "200", + "p50_s": 2.46 + }, + "branch": { + "status": "200", + "p50_s": 0.06 + } + }, + { + "shape": "F4 sum by (detected_level), 3 series within twice the limit (no fallback on this data)", + "max_stats_query_series": 5, + "range": "24h", + "main": { + "status": "502", + "p50_s": 8.0 + }, + "branch": { + "status": "200", + "p50_s": 1.05 + } + }, + { + "shape": "F2 sum by (service_name), 7 groups past twice the limit", + "max_stats_query_series": 3, + "range": "1h", + "main": { + "status": "400", + "p50_s": 0.0 + }, + "branch": { + "status": "400", + "p50_s": 0.07 + } + }, + { + "shape": "F2 sum by (service_name), 7 groups past twice the limit", + "max_stats_query_series": 3, + "range": "24h", + "main": { + "status": "400", + "p50_s": 0.0 + }, + "branch": { + "status": "400", + "p50_s": 1.94 + } + }, + { + "shape": "F4b sum by (detected_level), 3 series past twice the limit, fallback", + "max_stats_query_series": 1, + "range": "1h", + "main": { + "status": "400", + "p50_s": 0.0 + }, + "branch": { + "status": "400", + "p50_s": 0.0 + } + }, + { + "shape": "F4b sum by (detected_level), 3 series past twice the limit, fallback", + "max_stats_query_series": 1, + "range": "24h", + "main": { + "status": "400", + "p50_s": 0.0 + }, + "branch": { + "status": "400", + "p50_s": 0.0 + } + } + ] +} diff --git a/bench/ab/shapes.json b/bench/ab/shapes.json index d6d36133c..94e6a2796 100644 --- a/bench/ab/shapes.json +++ b/bench/ab/shapes.json @@ -1888,6 +1888,61 @@ ] } ] + }, + "json-error-split": { + "description": "`| json | __error__=\"\"` metrics: counted by one VictoriaLogs stats scan over the lines VictoriaLogs parses, with only the other brace-opened lines read as raw rows, instead of reading every selected line raw (502 at 24 h on the row and byte limits). Run with the control set.", + "ranges": [ + "1h", + "6h", + "24h" + ], + "shapes": [ + { + "name": "J1 json no-error count by service_name, one service", + "query": "sum by (service_name) (count_over_time({service_name=\"api-gateway\"} | json | __error__=\"\" [5m]))", + "covers": [ + "semantics/json-error-filter-metric-split-pushdown", + "parser-error-and-label-collision", + "loki_api_v1_query_range" + ] + }, + { + "name": "J2 json no-error count by app, namespace", + "query": "sum by (app) (count_over_time({namespace=\"prod\"} | json | __error__=\"\" [5m]))", + "covers": [ + "semantics/json-error-filter-metric-split-pushdown", + "parser-error-and-label-collision", + "loki_api_v1_query_range" + ] + }, + { + "name": "J3 json no-error count by parsed method", + "query": "sum by (method) (count_over_time({service_name=\"api-gateway\"} | json | __error__=\"\" [5m]))", + "covers": [ + "semantics/json-error-filter-metric-split-pushdown", + "parser-error-and-label-collision", + "loki_api_v1_query_range" + ] + }, + { + "name": "J4 json no-error count by service_name, all services", + "query": "sum by (service_name) (count_over_time({env=\"production\"} | json | __error__=\"\" [5m]))", + "covers": [ + "semantics/json-error-filter-metric-split-pushdown", + "parser-error-and-label-collision", + "loki_api_v1_query_range" + ] + }, + { + "name": "J5 json no-error bytes_rate by app", + "query": "sum by (app) (bytes_rate({namespace=\"prod\"} | json | __error__=\"\" [5m]))", + "covers": [ + "semantics/json-error-filter-metric-split-pushdown", + "parser-error-and-label-collision", + "loki_api_v1_query_range" + ] + } + ] } } } diff --git a/bench/visual/spec.json b/bench/visual/spec.json index 42120839f..816da5498 100644 --- a/bench/visual/spec.json +++ b/bench/visual/spec.json @@ -402,6 +402,19 @@ "semantics/unwrap-quantile-response-limit" ] }, + { + "id": "explore-json-error-filter-metric", + "kind": "explore", + "query": "sum by (app) (count_over_time({namespace=\"prod\"} | json | __error__=\"\" [$__auto]))", + "fixes": [ + "semantics/json-error-filter-metric-split-pushdown" + ], + "covers": [ + "semantics/json-error-filter-metric-split-pushdown", + "parser-error-and-label-collision", + "loki_api_v1_query_range" + ] + }, { "id": "explore-label-browser", "kind": "label-browser", diff --git a/conformance/registry/cases/profiles/json-line-stored-without-line.yaml b/conformance/registry/cases/profiles/json-line-stored-without-line.yaml index dd6fa8646..556804b92 100644 --- a/conformance/registry/cases/profiles/json-line-stored-without-line.yaml +++ b/conformance/registry/cases/profiles/json-line-stored-without-line.yaml @@ -11,7 +11,9 @@ why: | dotted names, numbers as strings) and sets _msg to "missing _msg field; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field". Key order, value types and nesting are lost, and the fields of the line - cannot be told from the push's structured metadata. The proxy rebuilds the + cannot be told from the push's structured metadata. Every `| json` metric + path (the stats pushdown, its parse-risk probes and the row evaluator) + reads the line from _msg, so such rows count as lines that are not JSON. The proxy rebuilds the line from every stored field (structured metadata included), reports them as structured metadata without a parser and as parsed labels after `| json` or `| logfmt`. Loki returns the original line and no labels from diff --git a/conformance/registry/cases/semantics/json-empty-value-label-series.yaml b/conformance/registry/cases/semantics/json-empty-value-label-series.yaml new file mode 100644 index 000000000..09b3136eb --- /dev/null +++ b/conformance/registry/cases/semantics/json-empty-value-label-series.yaml @@ -0,0 +1,44 @@ +id: semantics/json-empty-value-label-series +title: A parsed label with an empty value is its own series in Loki, merged into the label-less series by the proxy (open) +proves: + - parser-error-and-label-collision +consumer: api +why: | + Grouped by a key of the JSON body, Loki keeps a line whose key holds an + empty value (`{"a":""}`, or a value its unescape rejects such as a lone + surrogate `"\ud800"`, which pkg/logql/log/parser.go readValue turns into + "") in a series `{a=""}`, apart from the lines without the key. The + proxy's row evaluator drops empty values when it builds the series + (orderedJSONMetricPlan.groupLabels), and VictoriaLogs groups a missing and + an empty field alike, so both land in the label-less series. + + Measured on VictoriaLogs v1.52.0 against Loki v3.7.7 with a crafted + corpus: `sum by (a) (count_over_time({...} | json | __error__="" [1m]))` + answered 23 series on Loki and 22 on the proxy (main and this change + alike); the 6 lines of `{a=""}` were added to `{}`. +request: + method: GET + path: /loki/api/v1/query_range + params: + query: 'sum by (a) (count_over_time({app="json-empty"} | json | __error__="" [1m]))' + start: now-10m + end: now + step: "60" + headers: + X-Scope-OrgID: "0" + precondition: | + The stream holds lines `{"a":""}`, `{"a":"x"}` and `{"b":1}`. +compare: + mode: exact + fields: [series_labels, values] + expect: | + Three series: {a=""}, {a="x"} and {} (not met: {a=""} is merged into {}). +gap: + status: open + impact: explore-visible + effort: M + area: 'internal/proxy ordered_json_metric.go groupLabels (row evaluator); stats pushdown grouping (VictoriaLogs cannot tell an empty field from a missing one)' + loki: 'pkg/logql/log/parser.go:137-211 parseLabelValue/readValue; the vector aggregation keeps the empty label' + proxy: 'empty-valued parsed labels dropped from the series' + planned_test: 'a row-evaluator unit case and an e2e on a corpus with empty values' + clusters: [] diff --git a/conformance/registry/cases/semantics/json-error-filter-metric-split-pushdown.yaml b/conformance/registry/cases/semantics/json-error-filter-metric-split-pushdown.yaml new file mode 100644 index 000000000..f85c93201 --- /dev/null +++ b/conformance/registry/cases/semantics/json-error-filter-metric-split-pushdown.yaml @@ -0,0 +1,134 @@ +id: semantics/json-error-filter-metric-split-pushdown +title: A `| json | __error__=""` metric counts Loki's accepted lines from VictoriaLogs stats, reading only the doubtful lines raw (fixed) +proves: + - parser-error-and-label-collision + - loki_api_v1_query_range +consumer: api +why: | + `sum by (app) (count_over_time({namespace="prod"} | json | __error__="" [5m]))` + counts only the lines Loki's JSON parser accepts. VictoriaLogs attaches no + parser error, so the stats pushdown refused any `__error__` filter and every + such query read all selected lines as raw rows into the row evaluator: + 1.6 s at 1 h (Loki 0.04 s) and a 502 on the row or byte limit at 6-24 h. + + Loki (pkg/logql/log/parser.go:74-135 JSONParser.Process) runs + grafana/jsonparser ObjectEach over the line and walks a nested object only + when a required label starts with its key path (parser_hints.go + ShouldExtractPrefix); with `__error__` required parsing never stops early. + So whether Loki accepts a line depends on the query, and no single LogsQL + filter can match its JSONParserErr. The pushdown splits the selected lines + instead: + + - a line that does not open with `{` after space, tab, line feed or + carriage return fails ObjectEach (jsonparser nextToken), so Loki drops it; + - a line VictoriaLogs parses as one JSON object (a sentinel member spliced + after the brace and read back with unpack_json; the splice cannot repair a + broken line, while wrapping the line in a new object could be closed by + trailing text) and that holds none of the constructs fastjson accepts and + jsonparser rejects (a backslash in a key, a member value starting with + `. e E +`, NaN or Inf: parseRawNumber/parseValue vs getType) is accepted + by Loki under every hint set, so VictoriaLogs counts it; + - every other brace-opened line is counted per bucket in the same scan and + read as a raw row only from those buckets, projected to the fields the + row evaluator reads, and evaluated by it. + + The hazard constructs are looked for with a cached `| limit 1` probe; only + when it finds one does the scan run again with such lines moved to the raw + rows. The parse-risk probes for parsed labels check only lines + VictoriaLogs parses. Window sums add the bucket counts and the raw lines at + each evaluation point before rate is divided once. service_name is the + stream label Loki assigns from the other stream labels: VictoriaLogs takes + the first source field that is a stream label of the row (a stream filter + in `format if`), never structured metadata, as the row evaluator does. + + Series limit: VictoriaLogs groups are not Loki series. It makes a group for + every row reaching stats (label filters therefore run before the stats + pipe), groups the spellings of one level apart (info, INFO, inf and + information are four groups and one detected_level series) and the empty + service name apart from unknown_service, so no count of groups says how many + series the answer holds. The split therefore never ends with the series limit + error from a group count. The lines read as raw rows have the grouped fields + blanked once `format if () "1" as __vlr` has marked them + (`format if (__vlr:*) "" as `, pipes VictoriaLogs has had since + v1.40.0): they are counted per bucket whatever their labels, because the row + evaluator derives those itself, and so they form one group. Only the merged + answer, after the groups are keyed by their Loki labels and the raw rows are + added, is checked against `-max-stats-query-series`, with Loki's error. The + stats read keeps at most twice the limit in groups as a memory guard; a read + past it, or a response too large to read (`maxStatsBucketResponseBytes`: + many series over a long range at a fine step), only means the split does + not answer: the row evaluator does, as it did before the split, with its own + row and byte limits (`-manual-range-metric-row-limit`, + `-ordered-json-metric-max-bytes`) and Loki's series limit error where its + series exceed the limit. The fallback costs the stats scan on top of the row + evaluator (0.1 s at 1 h; at 24 h 1.5-4.8 s where the row evaluator would + have ended at once with the series limit error), because VictoriaLogs + aggregates every group before it answers. Grafana Logs Drilldown requests + (which keep the busiest series of an over-limit answer) and groupings by a + field starting with `_` take the row evaluator as before, and so does an + instant query (one evaluation time, no bucket grid to add the raw lines to). + + detected_level is Loki's value at ingest + (semantics/json-metric-detected-level-derivation). The split serves a + grouping by it only from the stored level fields: the pipes copy the stored + detected_level and level fields (`copy detected_level as __dl_d, level as + __dl_l`) before the sentinel splice, and nothing of the split touches the + copies except the blanking of the lines read as raw rows. It is served when + every counted group has a stored level and no stored detected_level (the + spellings of one level are separate groups and one series, merged after the + read, so the series limit counts series). The derivation chain, its + presence probe and its witness are never part of a split plan. A window + holding a counted line the stored fields do not decide (a level only the + body holds, none at all, a stored detected_level label), a line with another + stored level field (severity, lvl, the OTLP severity fields: the copy of + `level` cannot tell a stream label from metadata, and Loki reads the + stream's before the metadata's, found by a `| limit 1` probe), a read past the + group guard and a response too large to read leave the query to the row + evaluator, which derives the level of every line itself, as it does for the + lines read as raw rows. A filter on detected_level is not split: the filter + would need the derived level for the raw rows and the stored one for the + counted lines, so that plan keeps the row evaluator. Speed never outranks + Loki's answer: every doubtful window takes the row evaluator. + + A step wider than the range leaves gaps between the evaluation windows; the + scan covers them, but a series none of whose counted buckets falls in a + window has no point in Loki's answer, so it is dropped before the limit is + applied. + + Layer: VictoriaLogs native stats (`count() if (-__vlr:* ...)`, + `count() if (__vlr:*)`); bounded raw rows for the third set. +request: + method: GET + path: /loki/api/v1/query_range + params: + query: 'sum by (app) (count_over_time({namespace="prod"} | json | __error__="" [5m]))' + start: now-24h + end: now + step: "60" + headers: + X-Scope-OrgID: "0" +compare: + mode: exact + fields: [status, series_labels, values] + expect: | + HTTP 200 with Loki's series and values from one stats_query_range scan + (two when the hazard probe finds a line) plus raw reads limited to the + buckets holding brace-opened lines VictoriaLogs cannot vouch for. + Measured on VictoriaLogs v1.52.0 with generator data: 1.68 s -> 0.06 s at + 1 h (Loki 0.04 s); 502 -> 1.16 s at 24 h. +gap: + status: fixed + fixed_by: "#758" + impact: explore-visible + effort: M + area: 'internal/proxy ordered_json_valid_split.go, ordered_json_metric.go stats pushdown' + loki: 'pkg/logql/log/parser.go:74-135 JSONParser.Process; pkg/logql/log/parser_hints.go:73-188; grafana/jsonparser parser.go ObjectEach/getType' + proxy: 'every selected line read as a raw row (502 at the row and byte limits at 6-24 h)' + planned_test: 'TestRangeMetricCompatibilityJSONErrorFilterSplit, ...SplitSeriesLimit and ...SplitDetectedLevel (test/e2e-compat, group semantics: the stored-level, body-level, gap-line and deep-nested variants against Loki); proxy TestJSONValidSplitPredicateDifferential, FuzzJSONValidSplitPredicate, TestJSONLokiRejectHazardTable, TestJSONValidSplitEligibility, TestJSONValidSplitMatchesRawEvaluator, TestJSONValidSplitReadsOnlyRawRowBuckets, TestJSONValidSplitCountsDeepLinesLikeLoki, TestJSONValidSplitFilteredOutGroupsStayUnderSeriesLimit, TestJSONValidSplitRawOnlyGroupIsNotASeries, TestJSONValidSplitServiceNameFromStreamLabelWithNewline, TestJSONValidSplitHazardProbeUsesTheSplitBraceFilter, TestJSONValidSplitHazardProbeCancelsTheFirstScan, TestJSONValidSplitMergedAnswerOverTheLimitReturnsLokiError, TestJSONValidSplitOverTwiceTheLimitGroupsFallBackToRowEvaluator, TestJSONValidSplitLevelSpellingsMergeUnderTheLimit, TestJSONValidSplitLimitPlusOneSeriesWithoutRawRows, TestJSONValidSplitOversizedResponseOnlyFallsBackForSplitPlans, TestJSONValidSplitDrilldownRequestsKeepTheRowEvaluator, TestJSONValidSplitInstantQueriesKeepTheRowEvaluator, TestJSONValidSplitMarksRawCandidatesOnce, TestJSONValidSplitGapGroupsAreNoSeries, TestJSONValidSplitHugeResponseFallsBackWithBoundedMemory, TestJSONValidSplitFilteredOutGroupsUnderSizeCap, TestJSONValidSplitSeriesLimitFailsBeforeRawRows, TestJSONValidSplitReadStopsAtTwiceTheSeriesLimit, TestJSONValidSplitRawCandidatesFormOneGroup, TestJSONValidSplitRawCandidatesOverManyStreamsFormOneGroup, TestJSONValidSplitResponseTooLargeFallsBackToRawEvaluator, TestJSONValidSplitWithoutLabelsSendsNoEmptyUnpack, TestJSONValidSplitDetectedLevelWindowsMatchTheRowEvaluator, TestOrderedJSONRowProjectionKeepsTheStoredLevelFields' + note: | + The e2e runs on VictoriaLogs v1.49.0 (the floor) and v1.52.0 (the pin). The row evaluator's guards (nesting depth 64, + 1024 fields, 1 MiB line, 64 KiB value) now apply only to the raw rows: a line VictoriaLogs vouches + for is counted as Loki counts it, however wide and up to 300 levels deep (VictoriaLogs' JSON parser + limit); a deeper line is read raw and keeps the row evaluator's depth guard. A JSON key with an empty value still merges into the + label-less series (semantics/json-empty-value-label-series), on this path as on the row evaluator. + clusters: [] diff --git a/conformance/registry/cases/semantics/pipeline-error-text-parity.yaml b/conformance/registry/cases/semantics/pipeline-error-text-parity.yaml index bf087d79f..93c060abf 100644 --- a/conformance/registry/cases/semantics/pipeline-error-text-parity.yaml +++ b/conformance/registry/cases/semantics/pipeline-error-text-parity.yaml @@ -27,8 +27,10 @@ why: | Now, for a `| json` metric whose pipeline neither drops `__error__` nor carries a label filter an unparsed line fails (and that requires labels, so the parser runs), one `| limit 1` lookup of a selected line that is - not a JSON object (`-_msg:~"^\s*\{"`, Loki's `jsonparser.ObjectEach` - fails every such line) answers the error before any row is scanned, with + not a JSON object (`-_msg:~"^[ \t\n\r]*\{"`: Loki's `jsonparser.ObjectEach` + skips only space, tab, line feed and carriage return before the brace and + fails every other line, one opening with a form feed included) answers the + error before any row is scanned, with Loki's text and that line's series; the evaluator's own check keeps the same text for a malformed object it meets while scanning. Only lines Loki evaluates count: the lookup starts one nanosecond after `start - range` diff --git a/conformance/registry/generated/perf-evidence.json b/conformance/registry/generated/perf-evidence.json index b1c68a241..45e0271a9 100644 --- a/conformance/registry/generated/perf-evidence.json +++ b/conformance/registry/generated/perf-evidence.json @@ -30620,17 +30620,15 @@ "valid": true, "verdict_vs_baseline": "same", "warm_p50": 8.62 - } - ], - "metric-series-identity": [ + }, { - "baseline": "release", + "baseline": "base", "baseline_status": "200", - "cold": 0.014, + "cold": 0.032, "date": "2026-10-22", - "label": "daily-control", - "loki_cache_served": false, - "loki_cold": 0.337, + "label": "json-error-split-1h", + "loki_cache_served": true, + "loki_cold": 0.029, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -30643,8 +30641,8 @@ "entriesRequested": 0 }, "result": { - "bytesReceived": 0, - "entriesFound": 0, + "bytesReceived": 14752, + "entriesFound": 2, "entriesRequested": 2 }, "seriesResult": { @@ -30653,9 +30651,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 354, - "entriesFound": 2, - "entriesRequested": 2 + "bytesReceived": 186, + "entriesFound": 1, + "entriesRequested": 1 }, "volumeResult": { "bytesReceived": 0, @@ -30663,25 +30661,66 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.066, + "loki_warm_p50": 0.024, "parity_vs_loki": "same", "range": "1h", - "set": "control", - "shape": "stream-label count by app", + "set": "json-error-split", + "shape": "J1 json no-error count by service_name, one service", "slower_than_loki_cold": false, + "slower_than_loki_warm": false, "status": "200", "valid": true, - "verdict_vs_baseline": "same", - "warm_p50": 0.014 + "verdict_vs_baseline": "faster", + "warm_p50": 0.032 }, { - "baseline": "release", + "baseline": "base", + "baseline_status": "502", + "cold": 1.914, + "date": "2026-10-22", + "label": "json-error-split-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-error-split", + "shape": "J1 json no-error count by service_name, one service", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 0.625 + }, + { + "baseline": "base", "baseline_status": "200", - "cold": 0.132, + "cold": 0.259, "date": "2026-10-22", - "label": "daily-control", + "label": "json-error-split-long", "loki_cache_served": false, - "loki_cold": 5.058, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-error-split", + "shape": "J1 json no-error count by service_name, one service", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 0.142 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.064, + "date": "2026-10-22", + "label": "json-error-split-1h", + "loki_cache_served": true, + "loki_cold": 0.043, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -30694,9 +30733,9 @@ "entriesRequested": 0 }, "result": { - "bytesReceived": 0, - "entriesFound": 0, - "entriesRequested": 25 + "bytesReceived": 98051, + "entriesFound": 2, + "entriesRequested": 2 }, "seriesResult": { "bytesReceived": 0, @@ -30704,9 +30743,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 12434, - "entriesFound": 26, - "entriesRequested": 26 + "bytesReceived": 177, + "entriesFound": 1, + "entriesRequested": 1 }, "volumeResult": { "bytesReceived": 0, @@ -30714,63 +30753,66 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.068, + "loki_warm_p50": 0.042, "parity_vs_loki": "same", - "range": "24h", - "set": "control", - "shape": "stream-label count by app", + "range": "1h", + "set": "json-error-split", + "shape": "J2 json no-error count by app, namespace", "slower_than_loki_cold": false, + "slower_than_loki_warm": false, "status": "200", "valid": true, - "verdict_vs_baseline": "same", - "warm_p50": 0.117 + "verdict_vs_baseline": "faster", + "warm_p50": 0.067 }, { - "baseline": "main", - "baseline_status": "200", - "cold": 0.019, - "date": "2026-10-06", - "label": "plain-name-control", + "baseline": "base", + "baseline_status": "502", + "cold": 2.237, + "date": "2026-10-22", + "label": "json-error-split-long", "loki_cache_served": false, - "loki_cold": 0.185, - "loki_warm_p50": 0.005, - "parity_vs_loki": "same", - "range": "3h", - "set": "control", - "shape": "stream-label count by app", + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-error-split", + "shape": "J2 json no-error count by app, namespace", "slower_than_loki_cold": false, + "slower_than_loki_warm": false, "status": "200", "valid": true, - "verdict_vs_baseline": "same", - "warm_p50": 0.017 + "verdict_vs_baseline": "fixed", + "warm_p50": 1.46 }, { "baseline": "base", - "baseline_status": "200", - "cold": 0.172, - "date": "2026-10-17", - "label": "unwrapwithout-control-long", + "baseline_status": "502", + "cold": 0.62, + "date": "2026-10-22", + "label": "json-error-split-long", "loki_cache_served": false, "loki_cold": null, "loki_warm_p50": null, "parity_vs_loki": "n/a", "range": "6h", - "set": "control", - "shape": "stream-label count by app", + "set": "json-error-split", + "shape": "J2 json no-error count by app, namespace", "slower_than_loki_cold": false, + "slower_than_loki_warm": false, "status": "200", "valid": true, - "verdict_vs_baseline": "same", - "warm_p50": 0.078 + "verdict_vs_baseline": "fixed", + "warm_p50": 0.476 }, { - "baseline": "release", + "baseline": "base", "baseline_status": "200", - "cold": 0.015, + "cold": 0.029, "date": "2026-10-22", - "label": "daily-unwrap-ungrouped-sum", - "loki_cache_served": false, - "loki_cold": 0.022, + "label": "json-error-split-1h", + "loki_cache_served": true, + "loki_cold": 0.016, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -30783,8 +30825,8 @@ "entriesRequested": 0 }, "result": { - "bytesReceived": 0, - "entriesFound": 0, + "bytesReceived": 70221, + "entriesFound": 2, "entriesRequested": 2 }, "seriesResult": { @@ -30793,9 +30835,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 0, - "entriesFound": 0, - "entriesRequested": 2 + "bytesReceived": 186, + "entriesFound": 1, + "entriesRequested": 1 }, "volumeResult": { "bytesReceived": 0, @@ -30803,25 +30845,66 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.009, + "loki_warm_p50": 0.016, "parity_vs_loki": "same", "range": "1h", - "set": "unwrap-ungrouped-sum", - "shape": "U1 logfmt amount rate", + "set": "json-error-split", + "shape": "J3 json no-error count by parsed method", "slower_than_loki_cold": false, + "slower_than_loki_warm": false, "status": "200", "valid": true, - "verdict_vs_baseline": "same", - "warm_p50": 0.015 + "verdict_vs_baseline": "faster", + "warm_p50": 0.032 }, { - "baseline": "release", + "baseline": "base", + "baseline_status": "502", + "cold": 0.584, + "date": "2026-10-22", + "label": "json-error-split-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-error-split", + "shape": "J3 json no-error count by parsed method", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 0.611 + }, + { + "baseline": "base", "baseline_status": "200", - "cold": 0.228, + "cold": 0.209, "date": "2026-10-22", - "label": "daily-unwrap-ungrouped-sum", + "label": "json-error-split-long", "loki_cache_served": false, - "loki_cold": 0.294, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-error-split", + "shape": "J3 json no-error count by parsed method", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 0.159 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.075, + "date": "2026-10-22", + "label": "json-error-split-1h", + "loki_cache_served": true, + "loki_cold": 0.059, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -30834,9 +30917,9 @@ "entriesRequested": 0 }, "result": { - "bytesReceived": 0, - "entriesFound": 0, - "entriesRequested": 25 + "bytesReceived": 139905, + "entriesFound": 2, + "entriesRequested": 2 }, "seriesResult": { "bytesReceived": 0, @@ -30844,9 +30927,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 8342, - "entriesFound": 26, - "entriesRequested": 26 + "bytesReceived": 177, + "entriesFound": 1, + "entriesRequested": 1 }, "volumeResult": { "bytesReceived": 0, @@ -30854,25 +30937,66 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.014, + "loki_warm_p50": 0.053, "parity_vs_loki": "same", + "range": "1h", + "set": "json-error-split", + "shape": "J4 json no-error count by service_name, all services", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 0.072 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 3.142, + "date": "2026-10-22", + "label": "json-error-split-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", "range": "24h", - "set": "unwrap-ungrouped-sum", - "shape": "U1 logfmt amount rate", + "set": "json-error-split", + "shape": "J4 json no-error count by service_name, all services", "slower_than_loki_cold": false, + "slower_than_loki_warm": false, "status": "200", "valid": true, - "verdict_vs_baseline": "same", - "warm_p50": 0.227 + "verdict_vs_baseline": "fixed", + "warm_p50": 2.01 }, { - "baseline": "release", - "baseline_status": "200", - "cold": 0.072, + "baseline": "base", + "baseline_status": "502", + "cold": 0.991, "date": "2026-10-22", - "label": "daily-unwrap-ungrouped-sum", + "label": "json-error-split-long", "loki_cache_served": false, - "loki_cold": 0.107, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-error-split", + "shape": "J4 json no-error count by service_name, all services", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 0.502 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.091, + "date": "2026-10-22", + "label": "json-error-split-1h", + "loki_cache_served": true, + "loki_cold": 0.078, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -30885,9 +31009,9 @@ "entriesRequested": 0 }, "result": { - "bytesReceived": 0, - "entriesFound": 0, - "entriesRequested": 7 + "bytesReceived": 98033, + "entriesFound": 2, + "entriesRequested": 2 }, "seriesResult": { "bytesReceived": 0, @@ -30895,9 +31019,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 1212, - "entriesFound": 4, - "entriesRequested": 8 + "bytesReceived": 177, + "entriesFound": 1, + "entriesRequested": 1 }, "volumeResult": { "bytesReceived": 0, @@ -30905,44 +31029,68 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.014, + "loki_warm_p50": 0.044, "parity_vs_loki": "same", - "range": "6h", - "set": "unwrap-ungrouped-sum", - "shape": "U1 logfmt amount rate", + "range": "1h", + "set": "json-error-split", + "shape": "J5 json no-error bytes_rate by app", "slower_than_loki_cold": false, + "slower_than_loki_warm": false, "status": "200", "valid": true, - "verdict_vs_baseline": "same", - "warm_p50": 0.071 + "verdict_vs_baseline": "faster", + "warm_p50": 0.07 }, { "baseline": "base", - "baseline_status": "200", - "cold": 0.135, - "date": "2026-10-11", - "label": "unwrapsum-long", + "baseline_status": "502", + "cold": 3.001, + "date": "2026-10-22", + "label": "json-error-split-long", "loki_cache_served": false, "loki_cold": null, "loki_warm_p50": null, "parity_vs_loki": "n/a", - "range": "7d", - "set": "unwrap-ungrouped-sum", - "shape": "U1 logfmt amount rate", + "range": "24h", + "set": "json-error-split", + "shape": "J5 json no-error bytes_rate by app", "slower_than_loki_cold": false, + "slower_than_loki_warm": false, "status": "200", "valid": true, - "verdict_vs_baseline": "faster", - "warm_p50": 0.12 + "verdict_vs_baseline": "fixed", + "warm_p50": 1.518 }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 0.503, + "date": "2026-10-22", + "label": "json-error-split-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-error-split", + "shape": "J5 json no-error bytes_rate by app", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 0.346 + } + ], + "metric-series-identity": [ { "baseline": "release", "baseline_status": "200", - "cold": 0.033, + "cold": 0.014, "date": "2026-10-22", - "label": "daily-unwrap-ungrouped-sum", + "label": "daily-control", "loki_cache_served": false, - "loki_cold": 0.054, + "loki_cold": 0.337, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -30965,7 +31113,7 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 374, + "bytesReceived": 354, "entriesFound": 2, "entriesRequested": 2 }, @@ -30975,25 +31123,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.013, + "loki_warm_p50": 0.066, "parity_vs_loki": "same", "range": "1h", - "set": "unwrap-ungrouped-sum", - "shape": "U2 json duration_ms rate", + "set": "control", + "shape": "stream-label count by app", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.033 + "warm_p50": 0.014 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.596, + "cold": 0.132, "date": "2026-10-22", - "label": "daily-unwrap-ungrouped-sum", + "label": "daily-control", "loki_cache_served": false, - "loki_cold": 0.94, + "loki_cold": 5.058, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -31016,7 +31164,7 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 14982, + "bytesReceived": 12434, "entriesFound": 26, "entriesRequested": 26 }, @@ -31026,95 +31174,63 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.023, + "loki_warm_p50": 0.068, "parity_vs_loki": "same", "range": "24h", - "set": "unwrap-ungrouped-sum", - "shape": "U2 json duration_ms rate", + "set": "control", + "shape": "stream-label count by app", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.605 + "warm_p50": 0.117 }, { - "baseline": "release", + "baseline": "main", "baseline_status": "200", - "cold": 0.162, - "date": "2026-10-22", - "label": "daily-unwrap-ungrouped-sum", + "cold": 0.019, + "date": "2026-10-06", + "label": "plain-name-control", "loki_cache_served": false, - "loki_cold": 0.196, - "loki_cold_cache": { - "instantMetricResult": { - "bytesReceived": 0, - "entriesFound": 0, - "entriesRequested": 0 - }, - "labelResult": { - "bytesReceived": 0, - "entriesFound": 0, - "entriesRequested": 0 - }, - "result": { - "bytesReceived": 0, - "entriesFound": 0, - "entriesRequested": 7 - }, - "seriesResult": { - "bytesReceived": 0, - "entriesFound": 0, - "entriesRequested": 0 - }, - "statsResult": { - "bytesReceived": 3336, - "entriesFound": 8, - "entriesRequested": 8 - }, - "volumeResult": { - "bytesReceived": 0, - "entriesFound": 0, - "entriesRequested": 0 - } - }, - "loki_warm_p50": 0.022, + "loki_cold": 0.185, + "loki_warm_p50": 0.005, "parity_vs_loki": "same", - "range": "6h", - "set": "unwrap-ungrouped-sum", - "shape": "U2 json duration_ms rate", + "range": "3h", + "set": "control", + "shape": "stream-label count by app", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.162 + "warm_p50": 0.017 }, { "baseline": "base", - "baseline_status": "502", - "cold": 0.259, - "date": "2026-10-11", - "label": "unwrapsum-long", + "baseline_status": "200", + "cold": 0.172, + "date": "2026-10-17", + "label": "unwrapwithout-control-long", "loki_cache_served": false, "loki_cold": null, "loki_warm_p50": null, "parity_vs_loki": "n/a", - "range": "7d", - "set": "unwrap-ungrouped-sum", - "shape": "U2 json duration_ms rate", + "range": "6h", + "set": "control", + "shape": "stream-label count by app", "slower_than_loki_cold": false, "status": "200", "valid": true, - "verdict_vs_baseline": "fixed", - "warm_p50": 0.268 + "verdict_vs_baseline": "same", + "warm_p50": 0.078 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.032, + "cold": 0.015, "date": "2026-10-22", "label": "daily-unwrap-ungrouped-sum", "loki_cache_served": false, - "loki_cold": 0.063, + "loki_cold": 0.022, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -31137,8 +31253,8 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 374, - "entriesFound": 2, + "bytesReceived": 0, + "entriesFound": 0, "entriesRequested": 2 }, "volumeResult": { @@ -31147,25 +31263,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.013, + "loki_warm_p50": 0.009, "parity_vs_loki": "same", "range": "1h", "set": "unwrap-ungrouped-sum", - "shape": "U3 json duration_ms rate, error filter", + "shape": "U1 logfmt amount rate", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.035 + "warm_p50": 0.015 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.6, + "cold": 0.228, "date": "2026-10-22", "label": "daily-unwrap-ungrouped-sum", "loki_cache_served": false, - "loki_cold": 1.15, + "loki_cold": 0.294, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -31188,7 +31304,7 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 15070, + "bytesReceived": 8342, "entriesFound": 26, "entriesRequested": 26 }, @@ -31198,25 +31314,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.017, + "loki_warm_p50": 0.014, "parity_vs_loki": "same", "range": "24h", "set": "unwrap-ungrouped-sum", - "shape": "U3 json duration_ms rate, error filter", + "shape": "U1 logfmt amount rate", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.602 + "warm_p50": 0.227 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.167, + "cold": 0.072, "date": "2026-10-22", "label": "daily-unwrap-ungrouped-sum", "loki_cache_served": false, - "loki_cold": 0.241, + "loki_cold": 0.107, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -31239,8 +31355,8 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 3808, - "entriesFound": 8, + "bytesReceived": 1212, + "entriesFound": 4, "entriesRequested": 8 }, "volumeResult": { @@ -31249,21 +31365,21 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.017, + "loki_warm_p50": 0.014, "parity_vs_loki": "same", "range": "6h", "set": "unwrap-ungrouped-sum", - "shape": "U3 json duration_ms rate, error filter", + "shape": "U1 logfmt amount rate", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.169 + "warm_p50": 0.071 }, { "baseline": "base", - "baseline_status": "502", - "cold": 0.297, + "baseline_status": "200", + "cold": 0.135, "date": "2026-10-11", "label": "unwrapsum-long", "loki_cache_served": false, @@ -31272,21 +31388,21 @@ "parity_vs_loki": "n/a", "range": "7d", "set": "unwrap-ungrouped-sum", - "shape": "U3 json duration_ms rate, error filter", + "shape": "U1 logfmt amount rate", "slower_than_loki_cold": false, "status": "200", "valid": true, - "verdict_vs_baseline": "fixed", - "warm_p50": 0.27 + "verdict_vs_baseline": "faster", + "warm_p50": 0.12 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.124, + "cold": 0.033, "date": "2026-10-22", "label": "daily-unwrap-ungrouped-sum", "loki_cache_served": false, - "loki_cold": 0.313, + "loki_cold": 0.054, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -31309,8 +31425,8 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 0, - "entriesFound": 0, + "bytesReceived": 374, + "entriesFound": 2, "entriesRequested": 2 }, "volumeResult": { @@ -31319,25 +31435,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.055, - "parity_vs_loki": "differs", + "loki_warm_p50": 0.013, + "parity_vs_loki": "same", "range": "1h", "set": "unwrap-ungrouped-sum", - "shape": "U4 logfmt latency_ms rate, production", + "shape": "U2 json duration_ms rate", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.121 + "warm_p50": 0.033 }, { "baseline": "release", "baseline_status": "200", - "cold": 2.569, + "cold": 0.596, "date": "2026-10-22", "label": "daily-unwrap-ungrouped-sum", "loki_cache_served": false, - "loki_cold": 6.922, + "loki_cold": 0.94, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -31360,7 +31476,7 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 5162, + "bytesReceived": 14982, "entriesFound": 26, "entriesRequested": 26 }, @@ -31370,25 +31486,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.069, - "parity_vs_loki": "differs", + "loki_warm_p50": 0.023, + "parity_vs_loki": "same", "range": "24h", "set": "unwrap-ungrouped-sum", - "shape": "U4 logfmt latency_ms rate, production", + "shape": "U2 json duration_ms rate", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 2.566 + "warm_p50": 0.605 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.656, + "cold": 0.162, "date": "2026-10-22", "label": "daily-unwrap-ungrouped-sum", "loki_cache_served": false, - "loki_cold": 1.608, + "loki_cold": 0.196, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -31411,7 +31527,7 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 1432, + "bytesReceived": 3336, "entriesFound": 8, "entriesRequested": 8 }, @@ -31421,21 +31537,21 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.066, - "parity_vs_loki": "differs", + "loki_warm_p50": 0.022, + "parity_vs_loki": "same", "range": "6h", "set": "unwrap-ungrouped-sum", - "shape": "U4 logfmt latency_ms rate, production", + "shape": "U2 json duration_ms rate", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.659 + "warm_p50": 0.162 }, { "baseline": "base", - "baseline_status": "200", - "cold": 1.179, + "baseline_status": "502", + "cold": 0.259, "date": "2026-10-11", "label": "unwrapsum-long", "loki_cache_served": false, @@ -31444,21 +31560,21 @@ "parity_vs_loki": "n/a", "range": "7d", "set": "unwrap-ungrouped-sum", - "shape": "U4 logfmt latency_ms rate, production", + "shape": "U2 json duration_ms rate", "slower_than_loki_cold": false, "status": "200", "valid": true, - "verdict_vs_baseline": "faster", - "warm_p50": 1.058 + "verdict_vs_baseline": "fixed", + "warm_p50": 0.268 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.007, + "cold": 0.032, "date": "2026-10-22", "label": "daily-unwrap-ungrouped-sum", "loki_cache_served": false, - "loki_cold": 0.007, + "loki_cold": 0.063, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -31473,7 +31589,7 @@ "result": { "bytesReceived": 0, "entriesFound": 0, - "entriesRequested": 0 + "entriesRequested": 2 }, "seriesResult": { "bytesReceived": 0, @@ -31481,9 +31597,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 191, - "entriesFound": 1, - "entriesRequested": 1 + "bytesReceived": 374, + "entriesFound": 2, + "entriesRequested": 2 }, "volumeResult": { "bytesReceived": 0, @@ -31491,25 +31607,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.007, + "loki_warm_p50": 0.013, "parity_vs_loki": "same", - "range": "instant", + "range": "1h", "set": "unwrap-ungrouped-sum", - "shape": "U5 logfmt amount rate, instant", + "shape": "U3 json duration_ms rate, error filter", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.007 + "warm_p50": 0.035 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.009, + "cold": 0.6, "date": "2026-10-22", "label": "daily-unwrap-ungrouped-sum", "loki_cache_served": false, - "loki_cold": 0.011, + "loki_cold": 1.15, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -31524,7 +31640,7 @@ "result": { "bytesReceived": 0, "entriesFound": 0, - "entriesRequested": 0 + "entriesRequested": 25 }, "seriesResult": { "bytesReceived": 0, @@ -31532,9 +31648,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 187, - "entriesFound": 1, - "entriesRequested": 1 + "bytesReceived": 15070, + "entriesFound": 26, + "entriesRequested": 26 }, "volumeResult": { "bytesReceived": 0, @@ -31542,25 +31658,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.011, + "loki_warm_p50": 0.017, "parity_vs_loki": "same", - "range": "instant", + "range": "24h", "set": "unwrap-ungrouped-sum", - "shape": "U6 json duration_ms rate, instant", + "shape": "U3 json duration_ms rate, error filter", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.009 + "warm_p50": 0.602 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.008, + "cold": 0.167, "date": "2026-10-22", "label": "daily-unwrap-ungrouped-sum", "loki_cache_served": false, - "loki_cold": 0.011, + "loki_cold": 0.241, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -31575,7 +31691,7 @@ "result": { "bytesReceived": 0, "entriesFound": 0, - "entriesRequested": 0 + "entriesRequested": 7 }, "seriesResult": { "bytesReceived": 0, @@ -31583,9 +31699,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 187, - "entriesFound": 1, - "entriesRequested": 1 + "bytesReceived": 3808, + "entriesFound": 8, + "entriesRequested": 8 }, "volumeResult": { "bytesReceived": 0, @@ -31593,25 +31709,44 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.011, + "loki_warm_p50": 0.017, "parity_vs_loki": "same", - "range": "instant", + "range": "6h", "set": "unwrap-ungrouped-sum", - "shape": "U7 json duration_ms sum_over_time, instant", + "shape": "U3 json duration_ms rate, error filter", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.009 + "warm_p50": 0.169 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 0.297, + "date": "2026-10-11", + "label": "unwrapsum-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "7d", + "set": "unwrap-ungrouped-sum", + "shape": "U3 json duration_ms rate, error filter", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 0.27 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.019, + "cold": 0.124, "date": "2026-10-22", "label": "daily-unwrap-ungrouped-sum", "loki_cache_served": false, - "loki_cold": 0.027, + "loki_cold": 0.313, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -31634,8 +31769,8 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 382, - "entriesFound": 2, + "bytesReceived": 0, + "entriesFound": 0, "entriesRequested": 2 }, "volumeResult": { @@ -31644,25 +31779,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.009, - "parity_vs_loki": "same", + "loki_warm_p50": 0.055, + "parity_vs_loki": "differs", "range": "1h", "set": "unwrap-ungrouped-sum", - "shape": "U8 logfmt amount rate by service (grouped)", + "shape": "U4 logfmt latency_ms rate, production", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.018 + "warm_p50": 0.121 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.267, + "cold": 2.569, "date": "2026-10-22", "label": "daily-unwrap-ungrouped-sum", "loki_cache_served": false, - "loki_cold": 0.345, + "loki_cold": 6.922, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -31685,7 +31820,7 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 9982, + "bytesReceived": 5162, "entriesFound": 26, "entriesRequested": 26 }, @@ -31695,25 +31830,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.01, - "parity_vs_loki": "same", + "loki_warm_p50": 0.069, + "parity_vs_loki": "differs", "range": "24h", "set": "unwrap-ungrouped-sum", - "shape": "U8 logfmt amount rate by service (grouped)", + "shape": "U4 logfmt latency_ms rate, production", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.263 + "warm_p50": 2.566 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.081, + "cold": 0.656, "date": "2026-10-22", "label": "daily-unwrap-ungrouped-sum", "loki_cache_served": false, - "loki_cold": 0.088, + "loki_cold": 1.608, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -31736,7 +31871,7 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 2896, + "bytesReceived": 1432, "entriesFound": 8, "entriesRequested": 8 }, @@ -31746,21 +31881,21 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.01, - "parity_vs_loki": "same", + "loki_warm_p50": 0.066, + "parity_vs_loki": "differs", "range": "6h", "set": "unwrap-ungrouped-sum", - "shape": "U8 logfmt amount rate by service (grouped)", + "shape": "U4 logfmt latency_ms rate, production", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.08 + "warm_p50": 0.659 }, { "baseline": "base", "baseline_status": "200", - "cold": 0.143, + "cold": 1.179, "date": "2026-10-11", "label": "unwrapsum-long", "loki_cache_served": false, @@ -31769,21 +31904,21 @@ "parity_vs_loki": "n/a", "range": "7d", "set": "unwrap-ungrouped-sum", - "shape": "U8 logfmt amount rate by service (grouped)", + "shape": "U4 logfmt latency_ms rate, production", "slower_than_loki_cold": false, "status": "200", "valid": true, - "verdict_vs_baseline": "same", - "warm_p50": 0.145 + "verdict_vs_baseline": "faster", + "warm_p50": 1.058 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.102, + "cold": 0.007, "date": "2026-10-22", "label": "daily-unwrap-ungrouped-sum", - "loki_cache_served": true, - "loki_cold": 0.103, + "loki_cache_served": false, + "loki_cold": 0.007, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -31798,7 +31933,7 @@ "result": { "bytesReceived": 0, "entriesFound": 0, - "entriesRequested": 2 + "entriesRequested": 0 }, "seriesResult": { "bytesReceived": 0, @@ -31806,9 +31941,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 0, - "entriesFound": 0, - "entriesRequested": 0 + "bytesReceived": 191, + "entriesFound": 1, + "entriesRequested": 1 }, "volumeResult": { "bytesReceived": 0, @@ -31816,25 +31951,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.018, - "parity_vs_loki": "differs", - "range": "1h", + "loki_warm_p50": 0.007, + "parity_vs_loki": "same", + "range": "instant", "set": "unwrap-ungrouped-sum", - "shape": "U9 logfmt amount rate_counter (folded per series)", + "shape": "U5 logfmt amount rate, instant", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.1 + "warm_p50": 0.007 }, { "baseline": "release", - "baseline_status": "502", - "cold": 0.472, + "baseline_status": "200", + "cold": 0.009, "date": "2026-10-22", "label": "daily-unwrap-ungrouped-sum", - "loki_cache_served": true, - "loki_cold": 1.656, + "loki_cache_served": false, + "loki_cold": 0.011, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -31849,7 +31984,7 @@ "result": { "bytesReceived": 0, "entriesFound": 0, - "entriesRequested": 25 + "entriesRequested": 0 }, "seriesResult": { "bytesReceived": 0, @@ -31857,9 +31992,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 0, - "entriesFound": 0, - "entriesRequested": 0 + "bytesReceived": 187, + "entriesFound": 1, + "entriesRequested": 1 }, "volumeResult": { "bytesReceived": 0, @@ -31867,25 +32002,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.019, - "parity_vs_loki": "differs", - "range": "24h", + "loki_warm_p50": 0.011, + "parity_vs_loki": "same", + "range": "instant", "set": "unwrap-ungrouped-sum", - "shape": "U9 logfmt amount rate_counter (folded per series)", + "shape": "U6 json duration_ms rate, instant", "slower_than_loki_cold": false, - "status": "502", + "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.469 + "warm_p50": 0.009 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.761, + "cold": 0.008, "date": "2026-10-22", "label": "daily-unwrap-ungrouped-sum", - "loki_cache_served": true, - "loki_cold": 0.709, + "loki_cache_served": false, + "loki_cold": 0.011, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -31900,7 +32035,7 @@ "result": { "bytesReceived": 0, "entriesFound": 0, - "entriesRequested": 7 + "entriesRequested": 0 }, "seriesResult": { "bytesReceived": 0, @@ -31908,9 +32043,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 0, - "entriesFound": 0, - "entriesRequested": 0 + "bytesReceived": 187, + "entriesFound": 1, + "entriesRequested": 1 }, "volumeResult": { "bytesReceived": 0, @@ -31918,44 +32053,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.018, - "parity_vs_loki": "differs", - "range": "6h", + "loki_warm_p50": 0.011, + "parity_vs_loki": "same", + "range": "instant", "set": "unwrap-ungrouped-sum", - "shape": "U9 logfmt amount rate_counter (folded per series)", + "shape": "U7 json duration_ms sum_over_time, instant", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.74 - }, - { - "baseline": "base", - "baseline_status": "502", - "cold": 0.212, - "date": "2026-10-11", - "label": "unwrapsum-long", - "loki_cache_served": false, - "loki_cold": null, - "loki_warm_p50": null, - "parity_vs_loki": "n/a", - "range": "7d", - "set": "unwrap-ungrouped-sum", - "shape": "U9 logfmt amount rate_counter (folded per series)", - "slower_than_loki_cold": false, - "status": "502", - "valid": true, - "verdict_vs_baseline": "same", - "warm_p50": 0.182 + "warm_p50": 0.009 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.02, + "cold": 0.019, "date": "2026-10-22", - "label": "daily-unwrap-sum-without", + "label": "daily-unwrap-ungrouped-sum", "loki_cache_served": false, - "loki_cold": 0.04, + "loki_cold": 0.027, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -31978,8 +32094,8 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 0, - "entriesFound": 0, + "bytesReceived": 382, + "entriesFound": 2, "entriesRequested": 2 }, "volumeResult": { @@ -31988,11 +32104,11 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.011, - "parity_vs_loki": "differs", + "loki_warm_p50": 0.009, + "parity_vs_loki": "same", "range": "1h", - "set": "unwrap-sum-without", - "shape": "W1 sum without pod, max_over_time", + "set": "unwrap-ungrouped-sum", + "shape": "U8 logfmt amount rate by service (grouped)", "slower_than_loki_cold": false, "status": "200", "valid": true, @@ -32002,11 +32118,11 @@ { "baseline": "release", "baseline_status": "200", - "cold": 0.339, + "cold": 0.267, "date": "2026-10-22", - "label": "daily-unwrap-sum-without", + "label": "daily-unwrap-ungrouped-sum", "loki_cache_served": false, - "loki_cold": 0.527, + "loki_cold": 0.345, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -32029,7 +32145,7 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 7144, + "bytesReceived": 9982, "entriesFound": 26, "entriesRequested": 26 }, @@ -32039,25 +32155,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.017, - "parity_vs_loki": "differs", + "loki_warm_p50": 0.01, + "parity_vs_loki": "same", "range": "24h", - "set": "unwrap-sum-without", - "shape": "W1 sum without pod, max_over_time", + "set": "unwrap-ungrouped-sum", + "shape": "U8 logfmt amount rate by service (grouped)", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.341 + "warm_p50": 0.263 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.093, + "cold": 0.081, "date": "2026-10-22", - "label": "daily-unwrap-sum-without", + "label": "daily-unwrap-ungrouped-sum", "loki_cache_served": false, - "loki_cold": 0.135, + "loki_cold": 0.088, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -32080,8 +32196,8 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 1176, - "entriesFound": 4, + "bytesReceived": 2896, + "entriesFound": 8, "entriesRequested": 8 }, "volumeResult": { @@ -32090,44 +32206,44 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.017, - "parity_vs_loki": "differs", - "range": "6h", - "set": "unwrap-sum-without", - "shape": "W1 sum without pod, max_over_time", + "loki_warm_p50": 0.01, + "parity_vs_loki": "same", + "range": "6h", + "set": "unwrap-ungrouped-sum", + "shape": "U8 logfmt amount rate by service (grouped)", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.098 + "warm_p50": 0.08 }, { "baseline": "base", "baseline_status": "200", - "cold": 1.791, - "date": "2026-10-17", - "label": "unwrapwithout-long", + "cold": 0.143, + "date": "2026-10-11", + "label": "unwrapsum-long", "loki_cache_served": false, "loki_cold": null, "loki_warm_p50": null, "parity_vs_loki": "n/a", "range": "7d", - "set": "unwrap-sum-without", - "shape": "W1 sum without pod, max_over_time", + "set": "unwrap-ungrouped-sum", + "shape": "U8 logfmt amount rate by service (grouped)", "slower_than_loki_cold": false, "status": "200", "valid": true, - "verdict_vs_baseline": "faster", - "warm_p50": 1.714 + "verdict_vs_baseline": "same", + "warm_p50": 0.145 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.018, + "cold": 0.102, "date": "2026-10-22", - "label": "daily-unwrap-sum-without", - "loki_cache_served": false, - "loki_cold": 0.032, + "label": "daily-unwrap-ungrouped-sum", + "loki_cache_served": true, + "loki_cold": 0.103, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -32150,9 +32266,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 364, - "entriesFound": 2, - "entriesRequested": 2 + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 }, "volumeResult": { "bytesReceived": 0, @@ -32160,25 +32276,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.009, + "loki_warm_p50": 0.018, "parity_vs_loki": "differs", "range": "1h", - "set": "unwrap-sum-without", - "shape": "W2 sum without pod, sum_over_time", + "set": "unwrap-ungrouped-sum", + "shape": "U9 logfmt amount rate_counter (folded per series)", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.018 + "warm_p50": 0.1 }, { "baseline": "release", - "baseline_status": "200", - "cold": 0.312, + "baseline_status": "502", + "cold": 0.472, "date": "2026-10-22", - "label": "daily-unwrap-sum-without", - "loki_cache_served": false, - "loki_cold": 0.417, + "label": "daily-unwrap-ungrouped-sum", + "loki_cache_served": true, + "loki_cold": 1.656, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -32201,9 +32317,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 9748, - "entriesFound": 26, - "entriesRequested": 26 + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 }, "volumeResult": { "bytesReceived": 0, @@ -32211,25 +32327,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.012, + "loki_warm_p50": 0.019, "parity_vs_loki": "differs", "range": "24h", - "set": "unwrap-sum-without", - "shape": "W2 sum without pod, sum_over_time", + "set": "unwrap-ungrouped-sum", + "shape": "U9 logfmt amount rate_counter (folded per series)", "slower_than_loki_cold": false, - "status": "200", + "status": "502", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.315 + "warm_p50": 0.469 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.09, + "cold": 0.761, "date": "2026-10-22", - "label": "daily-unwrap-sum-without", - "loki_cache_served": false, - "loki_cold": 0.122, + "label": "daily-unwrap-ungrouped-sum", + "loki_cache_served": true, + "loki_cold": 0.709, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -32252,9 +32368,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 2360, - "entriesFound": 8, - "entriesRequested": 8 + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 }, "volumeResult": { "bytesReceived": 0, @@ -32262,44 +32378,44 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.011, + "loki_warm_p50": 0.018, "parity_vs_loki": "differs", "range": "6h", - "set": "unwrap-sum-without", - "shape": "W2 sum without pod, sum_over_time", + "set": "unwrap-ungrouped-sum", + "shape": "U9 logfmt amount rate_counter (folded per series)", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.092 + "warm_p50": 0.74 }, { "baseline": "base", - "baseline_status": "200", - "cold": 1.7, - "date": "2026-10-17", - "label": "unwrapwithout-long", + "baseline_status": "502", + "cold": 0.212, + "date": "2026-10-11", + "label": "unwrapsum-long", "loki_cache_served": false, "loki_cold": null, "loki_warm_p50": null, "parity_vs_loki": "n/a", "range": "7d", - "set": "unwrap-sum-without", - "shape": "W2 sum without pod, sum_over_time", + "set": "unwrap-ungrouped-sum", + "shape": "U9 logfmt amount rate_counter (folded per series)", "slower_than_loki_cold": false, - "status": "200", + "status": "502", "valid": true, - "verdict_vs_baseline": "faster", - "warm_p50": 1.704 + "verdict_vs_baseline": "same", + "warm_p50": 0.182 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.05, + "cold": 0.02, "date": "2026-10-22", "label": "daily-unwrap-sum-without", "loki_cache_served": false, - "loki_cold": 0.036, + "loki_cold": 0.04, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -32322,8 +32438,8 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 364, - "entriesFound": 2, + "bytesReceived": 0, + "entriesFound": 0, "entriesRequested": 2 }, "volumeResult": { @@ -32332,25 +32448,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.01, - "parity_vs_loki": "same", + "loki_warm_p50": 0.011, + "parity_vs_loki": "differs", "range": "1h", "set": "unwrap-sum-without", - "shape": "W3 sum without pod, rate", + "shape": "W1 sum without pod, max_over_time", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.05 + "warm_p50": 0.018 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.66, + "cold": 0.339, "date": "2026-10-22", "label": "daily-unwrap-sum-without", "loki_cache_served": false, - "loki_cold": 0.409, + "loki_cold": 0.527, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -32373,7 +32489,7 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 12344, + "bytesReceived": 7144, "entriesFound": 26, "entriesRequested": 26 }, @@ -32383,25 +32499,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.011, - "parity_vs_loki": "same", + "loki_warm_p50": 0.017, + "parity_vs_loki": "differs", "range": "24h", "set": "unwrap-sum-without", - "shape": "W3 sum without pod, rate", - "slower_than_loki_cold": true, + "shape": "W1 sum without pod, max_over_time", + "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.681 + "warm_p50": 0.341 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.249, + "cold": 0.093, "date": "2026-10-22", "label": "daily-unwrap-sum-without", "loki_cache_served": false, - "loki_cold": 0.239, + "loki_cold": 0.135, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -32424,8 +32540,8 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 2832, - "entriesFound": 8, + "bytesReceived": 1176, + "entriesFound": 4, "entriesRequested": 8 }, "volumeResult": { @@ -32434,21 +32550,21 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.011, - "parity_vs_loki": "same", + "loki_warm_p50": 0.017, + "parity_vs_loki": "differs", "range": "6h", "set": "unwrap-sum-without", - "shape": "W3 sum without pod, rate", + "shape": "W1 sum without pod, max_over_time", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.265 + "warm_p50": 0.098 }, { "baseline": "base", "baseline_status": "200", - "cold": 2.509, + "cold": 1.791, "date": "2026-10-17", "label": "unwrapwithout-long", "loki_cache_served": false, @@ -32457,12 +32573,12 @@ "parity_vs_loki": "n/a", "range": "7d", "set": "unwrap-sum-without", - "shape": "W3 sum without pod, rate", + "shape": "W1 sum without pod, max_over_time", "slower_than_loki_cold": false, "status": "200", "valid": true, - "verdict_vs_baseline": "same", - "warm_p50": 2.251 + "verdict_vs_baseline": "faster", + "warm_p50": 1.714 }, { "baseline": "release", @@ -32471,7 +32587,7 @@ "date": "2026-10-22", "label": "daily-unwrap-sum-without", "loki_cache_served": false, - "loki_cold": 0.034, + "loki_cold": 0.032, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -32508,7 +32624,7 @@ "parity_vs_loki": "differs", "range": "1h", "set": "unwrap-sum-without", - "shape": "W4 max without pod, max_over_time", + "shape": "W2 sum without pod, sum_over_time", "slower_than_loki_cold": false, "status": "200", "valid": true, @@ -32518,11 +32634,11 @@ { "baseline": "release", "baseline_status": "200", - "cold": 0.351, + "cold": 0.312, "date": "2026-10-22", "label": "daily-unwrap-sum-without", "loki_cache_served": false, - "loki_cold": 0.678, + "loki_cold": 0.417, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -32545,7 +32661,7 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 9836, + "bytesReceived": 9748, "entriesFound": 26, "entriesRequested": 26 }, @@ -32555,21 +32671,21 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.011, + "loki_warm_p50": 0.012, "parity_vs_loki": "differs", "range": "24h", "set": "unwrap-sum-without", - "shape": "W4 max without pod, max_over_time", + "shape": "W2 sum without pod, sum_over_time", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.323 + "warm_p50": 0.315 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.096, + "cold": 0.09, "date": "2026-10-22", "label": "daily-unwrap-sum-without", "loki_cache_served": false, @@ -32596,7 +32712,7 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 2840, + "bytesReceived": 2360, "entriesFound": 8, "entriesRequested": 8 }, @@ -32610,17 +32726,17 @@ "parity_vs_loki": "differs", "range": "6h", "set": "unwrap-sum-without", - "shape": "W4 max without pod, max_over_time", + "shape": "W2 sum without pod, sum_over_time", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.093 + "warm_p50": 0.092 }, { "baseline": "base", "baseline_status": "200", - "cold": 1.754, + "cold": 1.7, "date": "2026-10-17", "label": "unwrapwithout-long", "loki_cache_served": false, @@ -32629,21 +32745,21 @@ "parity_vs_loki": "n/a", "range": "7d", "set": "unwrap-sum-without", - "shape": "W4 max without pod, max_over_time", + "shape": "W2 sum without pod, sum_over_time", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "faster", - "warm_p50": 1.948 + "warm_p50": 1.704 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.005, + "cold": 0.05, "date": "2026-10-22", "label": "daily-unwrap-sum-without", "loki_cache_served": false, - "loki_cold": 0.008, + "loki_cold": 0.036, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -32658,7 +32774,7 @@ "result": { "bytesReceived": 0, "entriesFound": 0, - "entriesRequested": 0 + "entriesRequested": 2 }, "seriesResult": { "bytesReceived": 0, @@ -32666,9 +32782,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 182, - "entriesFound": 1, - "entriesRequested": 1 + "bytesReceived": 364, + "entriesFound": 2, + "entriesRequested": 2 }, "volumeResult": { "bytesReceived": 0, @@ -32676,25 +32792,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.008, + "loki_warm_p50": 0.01, "parity_vs_loki": "same", - "range": "instant", + "range": "1h", "set": "unwrap-sum-without", - "shape": "W5 sum without pod, max_over_time, instant", + "shape": "W3 sum without pod, rate", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.005 + "warm_p50": 0.05 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.005, + "cold": 0.66, "date": "2026-10-22", "label": "daily-unwrap-sum-without", "loki_cache_served": false, - "loki_cold": 0.008, + "loki_cold": 0.409, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -32709,7 +32825,7 @@ "result": { "bytesReceived": 0, "entriesFound": 0, - "entriesRequested": 0 + "entriesRequested": 25 }, "seriesResult": { "bytesReceived": 0, @@ -32717,9 +32833,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 182, - "entriesFound": 1, - "entriesRequested": 1 + "bytesReceived": 12344, + "entriesFound": 26, + "entriesRequested": 26 }, "volumeResult": { "bytesReceived": 0, @@ -32727,25 +32843,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.008, + "loki_warm_p50": 0.011, "parity_vs_loki": "same", - "range": "instant", + "range": "24h", "set": "unwrap-sum-without", - "shape": "W6 sum without pod, rate, instant", - "slower_than_loki_cold": false, + "shape": "W3 sum without pod, rate", + "slower_than_loki_cold": true, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.005 + "warm_p50": 0.681 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.019, + "cold": 0.249, "date": "2026-10-22", "label": "daily-unwrap-sum-without", "loki_cache_served": false, - "loki_cold": 0.034, + "loki_cold": 0.239, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -32760,7 +32876,7 @@ "result": { "bytesReceived": 0, "entriesFound": 0, - "entriesRequested": 2 + "entriesRequested": 7 }, "seriesResult": { "bytesReceived": 0, @@ -32768,9 +32884,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 364, - "entriesFound": 2, - "entriesRequested": 2 + "bytesReceived": 2832, + "entriesFound": 8, + "entriesRequested": 8 }, "volumeResult": { "bytesReceived": 0, @@ -32778,11 +32894,81 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.01, - "parity_vs_loki": "differs", - "range": "1h", - "set": "unwrap-sum-without", - "shape": "W7 topk 5 of sum without pod", + "loki_warm_p50": 0.011, + "parity_vs_loki": "same", + "range": "6h", + "set": "unwrap-sum-without", + "shape": "W3 sum without pod, rate", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.265 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 2.509, + "date": "2026-10-17", + "label": "unwrapwithout-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "7d", + "set": "unwrap-sum-without", + "shape": "W3 sum without pod, rate", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 2.251 + }, + { + "baseline": "release", + "baseline_status": "200", + "cold": 0.018, + "date": "2026-10-22", + "label": "daily-unwrap-sum-without", + "loki_cache_served": false, + "loki_cold": 0.034, + "loki_cold_cache": { + "instantMetricResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "labelResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "result": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 2 + }, + "seriesResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "statsResult": { + "bytesReceived": 364, + "entriesFound": 2, + "entriesRequested": 2 + }, + "volumeResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + } + }, + "loki_warm_p50": 0.009, + "parity_vs_loki": "differs", + "range": "1h", + "set": "unwrap-sum-without", + "shape": "W4 max without pod, max_over_time", "slower_than_loki_cold": false, "status": "200", "valid": true, @@ -32792,11 +32978,11 @@ { "baseline": "release", "baseline_status": "200", - "cold": 0.341, + "cold": 0.351, "date": "2026-10-22", "label": "daily-unwrap-sum-without", "loki_cache_served": false, - "loki_cold": 0.523, + "loki_cold": 0.678, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -32819,7 +33005,7 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 9880, + "bytesReceived": 9836, "entriesFound": 26, "entriesRequested": 26 }, @@ -32829,25 +33015,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.012, + "loki_warm_p50": 0.011, "parity_vs_loki": "differs", "range": "24h", "set": "unwrap-sum-without", - "shape": "W7 topk 5 of sum without pod", + "shape": "W4 max without pod, max_over_time", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.325 + "warm_p50": 0.323 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.092, + "cold": 0.096, "date": "2026-10-22", "label": "daily-unwrap-sum-without", "loki_cache_served": false, - "loki_cold": 0.118, + "loki_cold": 0.122, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -32870,7 +33056,7 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 2392, + "bytesReceived": 2840, "entriesFound": 8, "entriesRequested": 8 }, @@ -32880,21 +33066,21 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.013, + "loki_warm_p50": 0.011, "parity_vs_loki": "differs", "range": "6h", "set": "unwrap-sum-without", - "shape": "W7 topk 5 of sum without pod", + "shape": "W4 max without pod, max_over_time", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.094 + "warm_p50": 0.093 }, { "baseline": "base", "baseline_status": "200", - "cold": 1.779, + "cold": 1.754, "date": "2026-10-17", "label": "unwrapwithout-long", "loki_cache_served": false, @@ -32903,21 +33089,21 @@ "parity_vs_loki": "n/a", "range": "7d", "set": "unwrap-sum-without", - "shape": "W7 topk 5 of sum without pod", + "shape": "W4 max without pod, max_over_time", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "faster", - "warm_p50": 1.805 + "warm_p50": 1.948 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.313, + "cold": 0.005, "date": "2026-10-22", "label": "daily-unwrap-sum-without", "loki_cache_served": false, - "loki_cold": 1.464, + "loki_cold": 0.008, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -32932,7 +33118,7 @@ "result": { "bytesReceived": 0, "entriesFound": 0, - "entriesRequested": 2 + "entriesRequested": 0 }, "seriesResult": { "bytesReceived": 0, @@ -32940,9 +33126,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 354, - "entriesFound": 2, - "entriesRequested": 2 + "bytesReceived": 182, + "entriesFound": 1, + "entriesRequested": 1 }, "volumeResult": { "bytesReceived": 0, @@ -32950,82 +33136,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.537, - "parity_vs_loki": "differs", - "range": "1h", - "set": "unwrap-sum-without", - "shape": "W8 sum without pod, count_over_time (production)", - "slower_than_loki_cold": false, - "status": "200", - "valid": true, - "verdict_vs_baseline": "same", - "warm_p50": 0.31 - }, - { - "baseline": "release", - "baseline_status": "200", - "cold": 3.882, - "date": "2026-10-22", - "label": "daily-unwrap-sum-without", - "loki_cache_served": false, - "loki_cold": 180.09, - "loki_warm_p50": 180.044, - "parity_vs_loki": "differs", - "range": "24h", - "set": "unwrap-sum-without", - "shape": "W8 sum without pod, count_over_time (production)", - "slower_than_loki_cold": false, - "status": "200", - "valid": true, - "verdict_vs_baseline": "same", - "warm_p50": 3.735 - }, - { - "baseline": "release", - "baseline_status": "200", - "cold": 2.412, - "date": "2026-10-22", - "label": "daily-unwrap-sum-without", - "loki_cache_served": false, - "loki_cold": 180.1, - "loki_warm_p50": 180.089, - "parity_vs_loki": "differs", - "range": "6h", - "set": "unwrap-sum-without", - "shape": "W8 sum without pod, count_over_time (production)", - "slower_than_loki_cold": false, - "status": "200", - "valid": true, - "verdict_vs_baseline": "same", - "warm_p50": 2.55 - }, - { - "baseline": "base", - "baseline_status": "200", - "cold": 8.551, - "date": "2026-10-17", - "label": "unwrapwithout-long", - "loki_cache_served": false, - "loki_cold": null, - "loki_warm_p50": null, - "parity_vs_loki": "n/a", - "range": "7d", + "loki_warm_p50": 0.008, + "parity_vs_loki": "same", + "range": "instant", "set": "unwrap-sum-without", - "shape": "W8 sum without pod, count_over_time (production)", + "shape": "W5 sum without pod, max_over_time, instant", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 8.62 + "warm_p50": 0.005 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.003, + "cold": 0.005, "date": "2026-10-22", "label": "daily-unwrap-sum-without", "loki_cache_served": false, - "loki_cold": 0.034, + "loki_cold": 0.008, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -33048,7 +33177,7 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 177, + "bytesReceived": 182, "entriesFound": 1, "entriesRequested": 1 }, @@ -33058,27 +33187,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.034, + "loki_warm_p50": 0.008, "parity_vs_loki": "same", "range": "instant", "set": "unwrap-sum-without", - "shape": "W9 label_replace control, instant", + "shape": "W6 sum without pod, rate, instant", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.003 - } - ], - "operator-sum": [ + "warm_p50": 0.005 + }, { "baseline": "release", "baseline_status": "200", - "cold": 0.014, + "cold": 0.019, "date": "2026-10-22", - "label": "daily-control", + "label": "daily-unwrap-sum-without", "loki_cache_served": false, - "loki_cold": 0.337, + "loki_cold": 0.034, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -33101,7 +33228,7 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 354, + "bytesReceived": 364, "entriesFound": 2, "entriesRequested": 2 }, @@ -33111,25 +33238,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.066, - "parity_vs_loki": "same", + "loki_warm_p50": 0.01, + "parity_vs_loki": "differs", "range": "1h", - "set": "control", - "shape": "stream-label count by app", + "set": "unwrap-sum-without", + "shape": "W7 topk 5 of sum without pod", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.014 + "warm_p50": 0.018 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.132, + "cold": 0.341, "date": "2026-10-22", - "label": "daily-control", + "label": "daily-unwrap-sum-without", "loki_cache_served": false, - "loki_cold": 5.058, + "loki_cold": 0.523, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -33152,7 +33279,7 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 12434, + "bytesReceived": 9880, "entriesFound": 26, "entriesRequested": 26 }, @@ -33162,63 +33289,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.068, - "parity_vs_loki": "same", + "loki_warm_p50": 0.012, + "parity_vs_loki": "differs", "range": "24h", - "set": "control", - "shape": "stream-label count by app", + "set": "unwrap-sum-without", + "shape": "W7 topk 5 of sum without pod", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.117 + "warm_p50": 0.325 }, { - "baseline": "main", + "baseline": "release", "baseline_status": "200", - "cold": 0.019, - "date": "2026-10-06", - "label": "plain-name-control", + "cold": 0.092, + "date": "2026-10-22", + "label": "daily-unwrap-sum-without", "loki_cache_served": false, - "loki_cold": 0.185, - "loki_warm_p50": 0.005, - "parity_vs_loki": "same", - "range": "3h", - "set": "control", - "shape": "stream-label count by app", - "slower_than_loki_cold": false, - "status": "200", - "valid": true, - "verdict_vs_baseline": "same", - "warm_p50": 0.017 - }, - { - "baseline": "base", - "baseline_status": "200", - "cold": 0.172, - "date": "2026-10-17", - "label": "unwrapwithout-control-long", - "loki_cache_served": false, - "loki_cold": null, - "loki_warm_p50": null, - "parity_vs_loki": "n/a", - "range": "6h", - "set": "control", - "shape": "stream-label count by app", - "slower_than_loki_cold": false, - "status": "200", - "valid": true, - "verdict_vs_baseline": "same", - "warm_p50": 0.078 - }, - { - "baseline": "release", - "baseline_status": "200", - "cold": 0.006, - "date": "2026-10-22", - "label": "daily-control", - "loki_cache_served": false, - "loki_cold": 0.261, + "loki_cold": 0.118, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -33233,7 +33322,7 @@ "result": { "bytesReceived": 0, "entriesFound": 0, - "entriesRequested": 2 + "entriesRequested": 7 }, "seriesResult": { "bytesReceived": 0, @@ -33241,9 +33330,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 354, - "entriesFound": 2, - "entriesRequested": 2 + "bytesReceived": 2392, + "entriesFound": 8, + "entriesRequested": 8 }, "volumeResult": { "bytesReceived": 0, @@ -33251,25 +33340,44 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.077, - "parity_vs_loki": "same", - "range": "1h", - "set": "control", - "shape": "ungrouped rate", + "loki_warm_p50": 0.013, + "parity_vs_loki": "differs", + "range": "6h", + "set": "unwrap-sum-without", + "shape": "W7 topk 5 of sum without pod", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.006 + "warm_p50": 0.094 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 1.779, + "date": "2026-10-17", + "label": "unwrapwithout-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "7d", + "set": "unwrap-sum-without", + "shape": "W7 topk 5 of sum without pod", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 1.805 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.056, + "cold": 0.313, "date": "2026-10-22", - "label": "daily-control", + "label": "daily-unwrap-sum-without", "loki_cache_served": false, - "loki_cold": 5.301, + "loki_cold": 1.464, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -33284,7 +33392,7 @@ "result": { "bytesReceived": 0, "entriesFound": 0, - "entriesRequested": 25 + "entriesRequested": 2 }, "seriesResult": { "bytesReceived": 0, @@ -33292,9 +33400,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 12434, - "entriesFound": 26, - "entriesRequested": 26 + "bytesReceived": 354, + "entriesFound": 2, + "entriesRequested": 2 }, "volumeResult": { "bytesReceived": 0, @@ -33302,63 +33410,135 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.076, - "parity_vs_loki": "same", + "loki_warm_p50": 0.537, + "parity_vs_loki": "differs", + "range": "1h", + "set": "unwrap-sum-without", + "shape": "W8 sum without pod, count_over_time (production)", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.31 + }, + { + "baseline": "release", + "baseline_status": "200", + "cold": 3.882, + "date": "2026-10-22", + "label": "daily-unwrap-sum-without", + "loki_cache_served": false, + "loki_cold": 180.09, + "loki_warm_p50": 180.044, + "parity_vs_loki": "differs", "range": "24h", - "set": "control", - "shape": "ungrouped rate", + "set": "unwrap-sum-without", + "shape": "W8 sum without pod, count_over_time (production)", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.058 + "warm_p50": 3.735 }, { - "baseline": "main", + "baseline": "release", "baseline_status": "200", - "cold": 0.01, - "date": "2026-10-06", - "label": "plain-name-control", + "cold": 2.412, + "date": "2026-10-22", + "label": "daily-unwrap-sum-without", "loki_cache_served": false, - "loki_cold": 0.293, - "loki_warm_p50": 0.005, - "parity_vs_loki": "same", - "range": "3h", - "set": "control", - "shape": "ungrouped rate", + "loki_cold": 180.1, + "loki_warm_p50": 180.089, + "parity_vs_loki": "differs", + "range": "6h", + "set": "unwrap-sum-without", + "shape": "W8 sum without pod, count_over_time (production)", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.009 + "warm_p50": 2.55 }, { "baseline": "base", "baseline_status": "200", - "cold": 0.023, + "cold": 8.551, "date": "2026-10-17", - "label": "unwrapwithout-control-long", + "label": "unwrapwithout-long", "loki_cache_served": false, "loki_cold": null, "loki_warm_p50": null, "parity_vs_loki": "n/a", - "range": "6h", - "set": "control", - "shape": "ungrouped rate", + "range": "7d", + "set": "unwrap-sum-without", + "shape": "W8 sum without pod, count_over_time (production)", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.021 + "warm_p50": 8.62 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.029, + "cold": 0.003, + "date": "2026-10-22", + "label": "daily-unwrap-sum-without", + "loki_cache_served": false, + "loki_cold": 0.034, + "loki_cold_cache": { + "instantMetricResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "labelResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "result": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "seriesResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "statsResult": { + "bytesReceived": 177, + "entriesFound": 1, + "entriesRequested": 1 + }, + "volumeResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + } + }, + "loki_warm_p50": 0.034, + "parity_vs_loki": "same", + "range": "instant", + "set": "unwrap-sum-without", + "shape": "W9 label_replace control, instant", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.003 + } + ], + "operator-sum": [ + { + "baseline": "release", + "baseline_status": "200", + "cold": 0.014, "date": "2026-10-22", "label": "daily-control", "loki_cache_served": false, - "loki_cold": 0.278, + "loki_cold": 0.337, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -33391,25 +33571,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.08, + "loki_warm_p50": 0.066, "parity_vs_loki": "same", "range": "1h", "set": "control", - "shape": "bytes_over_time by app", + "shape": "stream-label count by app", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.029 + "warm_p50": 0.014 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.438, + "cold": 0.132, "date": "2026-10-22", "label": "daily-control", "loki_cache_served": false, - "loki_cold": 5.396, + "loki_cold": 5.058, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -33432,7 +33612,7 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 12456, + "bytesReceived": 12434, "entriesFound": 26, "entriesRequested": 26 }, @@ -33442,40 +33622,40 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.081, + "loki_warm_p50": 0.068, "parity_vs_loki": "same", "range": "24h", "set": "control", - "shape": "bytes_over_time by app", + "shape": "stream-label count by app", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.435 + "warm_p50": 0.117 }, { "baseline": "main", "baseline_status": "200", - "cold": 0.041, + "cold": 0.019, "date": "2026-10-06", "label": "plain-name-control", "loki_cache_served": false, - "loki_cold": 0.236, - "loki_warm_p50": 0.019, + "loki_cold": 0.185, + "loki_warm_p50": 0.005, "parity_vs_loki": "same", "range": "3h", "set": "control", - "shape": "bytes_over_time by app", + "shape": "stream-label count by app", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.095 + "warm_p50": 0.017 }, { "baseline": "base", "baseline_status": "200", - "cold": 0.289, + "cold": 0.172, "date": "2026-10-17", "label": "unwrapwithout-control-long", "loki_cache_served": false, @@ -33484,23 +33664,21 @@ "parity_vs_loki": "n/a", "range": "6h", "set": "control", - "shape": "bytes_over_time by app", + "shape": "stream-label count by app", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.328 - } - ], - "operator-topk": [ + "warm_p50": 0.078 + }, { "baseline": "release", "baseline_status": "200", - "cold": 0.02, + "cold": 0.006, "date": "2026-10-22", "label": "daily-control", "loki_cache_served": false, - "loki_cold": 0.284, + "loki_cold": 0.261, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -33533,25 +33711,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.084, - "parity_vs_loki": "differs", + "loki_warm_p50": 0.077, + "parity_vs_loki": "same", "range": "1h", "set": "control", - "shape": "topk rate by app", + "shape": "ungrouped rate", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.019 + "warm_p50": 0.006 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.479, + "cold": 0.056, "date": "2026-10-22", "label": "daily-control", "loki_cache_served": false, - "loki_cold": 5.519, + "loki_cold": 5.301, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -33574,7 +33752,7 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 12500, + "bytesReceived": 12434, "entriesFound": 26, "entriesRequested": 26 }, @@ -33584,40 +33762,40 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.077, - "parity_vs_loki": "differs", + "loki_warm_p50": 0.076, + "parity_vs_loki": "same", "range": "24h", "set": "control", - "shape": "topk rate by app", + "shape": "ungrouped rate", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.253 + "warm_p50": 0.058 }, { "baseline": "main", "baseline_status": "200", - "cold": 0.043, + "cold": 0.01, "date": "2026-10-06", "label": "plain-name-control", "loki_cache_served": false, - "loki_cold": 0.297, - "loki_warm_p50": 0.007, - "parity_vs_loki": "differs", + "loki_cold": 0.293, + "loki_warm_p50": 0.005, + "parity_vs_loki": "same", "range": "3h", "set": "control", - "shape": "topk rate by app", + "shape": "ungrouped rate", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.035 + "warm_p50": 0.009 }, { "baseline": "base", "baseline_status": "200", - "cold": 0.1, + "cold": 0.023, "date": "2026-10-17", "label": "unwrapwithout-control-long", "loki_cache_served": false, @@ -33626,23 +33804,21 @@ "parity_vs_loki": "n/a", "range": "6h", "set": "control", - "shape": "topk rate by app", + "shape": "ungrouped rate", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.114 - } - ], - "parsed-label-series-identity": [ + "warm_p50": 0.021 + }, { "baseline": "release", "baseline_status": "200", - "cold": 0.076, + "cold": 0.029, "date": "2026-10-22", - "label": "daily-json-filter-pushdown", + "label": "daily-control", "loki_cache_served": false, - "loki_cold": 0.267, + "loki_cold": 0.278, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -33665,9 +33841,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 176, - "entriesFound": 1, - "entriesRequested": 1 + "bytesReceived": 354, + "entriesFound": 2, + "entriesRequested": 2 }, "volumeResult": { "bytesReceived": 0, @@ -33675,25 +33851,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.061, + "loki_warm_p50": 0.08, "parity_vs_loki": "same", "range": "1h", - "set": "json-filter-pushdown", - "shape": "E field breakdown pipeline (explore)", + "set": "control", + "shape": "bytes_over_time by app", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.045 + "warm_p50": 0.029 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.592, + "cold": 0.438, "date": "2026-10-22", - "label": "daily-json-filter-pushdown", + "label": "daily-control", "loki_cache_served": false, - "loki_cold": 4.641, + "loki_cold": 5.396, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -33716,9 +33892,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 19998, - "entriesFound": 25, - "entriesRequested": 25 + "bytesReceived": 12456, + "entriesFound": 26, + "entriesRequested": 26 }, "volumeResult": { "bytesReceived": 0, @@ -33726,25 +33902,65 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.064, + "loki_warm_p50": 0.081, "parity_vs_loki": "same", "range": "24h", - "set": "json-filter-pushdown", - "shape": "E field breakdown pipeline (explore)", + "set": "control", + "shape": "bytes_over_time by app", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.358 + "warm_p50": 0.435 + }, + { + "baseline": "main", + "baseline_status": "200", + "cold": 0.041, + "date": "2026-10-06", + "label": "plain-name-control", + "loki_cache_served": false, + "loki_cold": 0.236, + "loki_warm_p50": 0.019, + "parity_vs_loki": "same", + "range": "3h", + "set": "control", + "shape": "bytes_over_time by app", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.095 }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.289, + "date": "2026-10-17", + "label": "unwrapwithout-control-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "control", + "shape": "bytes_over_time by app", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.328 + } + ], + "operator-topk": [ { "baseline": "release", "baseline_status": "200", - "cold": 0.093, + "cold": 0.02, "date": "2026-10-22", - "label": "daily-json-filter-pushdown", - "loki_cache_served": true, - "loki_cold": 0.401, + "label": "daily-control", + "loki_cache_served": false, + "loki_cold": 0.284, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -33757,9 +33973,9 @@ "entriesRequested": 0 }, "result": { - "bytesReceived": 43871, - "entriesFound": 2, - "entriesRequested": 4 + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 2 }, "seriesResult": { "bytesReceived": 0, @@ -33767,9 +33983,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 528, - "entriesFound": 3, - "entriesRequested": 3 + "bytesReceived": 354, + "entriesFound": 2, + "entriesRequested": 2 }, "volumeResult": { "bytesReceived": 0, @@ -33777,25 +33993,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.061, - "parity_vs_loki": "same", - "range": "3h", - "set": "json-filter-pushdown", - "shape": "E field breakdown pipeline (explore)", + "loki_warm_p50": 0.084, + "parity_vs_loki": "differs", + "range": "1h", + "set": "control", + "shape": "topk rate by app", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.069 + "warm_p50": 0.019 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.161, + "cold": 0.479, "date": "2026-10-22", - "label": "daily-json-filter-pushdown", + "label": "daily-control", "loki_cache_served": false, - "loki_cold": 1.198, + "loki_cold": 5.519, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -33810,7 +34026,7 @@ "result": { "bytesReceived": 0, "entriesFound": 0, - "entriesRequested": 7 + "entriesRequested": 25 }, "seriesResult": { "bytesReceived": 0, @@ -33818,9 +34034,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 4068, - "entriesFound": 7, - "entriesRequested": 7 + "bytesReceived": 12500, + "entriesFound": 26, + "entriesRequested": 26 }, "volumeResult": { "bytesReceived": 0, @@ -33828,25 +34044,65 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.062, - "parity_vs_loki": "same", - "range": "6h", - "set": "json-filter-pushdown", - "shape": "E field breakdown pipeline (explore)", + "loki_warm_p50": 0.077, + "parity_vs_loki": "differs", + "range": "24h", + "set": "control", + "shape": "topk rate by app", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.118 + "warm_p50": 0.253 + }, + { + "baseline": "main", + "baseline_status": "200", + "cold": 0.043, + "date": "2026-10-06", + "label": "plain-name-control", + "loki_cache_served": false, + "loki_cold": 0.297, + "loki_warm_p50": 0.007, + "parity_vs_loki": "differs", + "range": "3h", + "set": "control", + "shape": "topk rate by app", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.035 }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.1, + "date": "2026-10-17", + "label": "unwrapwithout-control-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "control", + "shape": "topk rate by app", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.114 + } + ], + "parsed-label-series-identity": [ { "baseline": "release", "baseline_status": "200", - "cold": 0.472, + "cold": 0.076, "date": "2026-10-22", "label": "daily-json-filter-pushdown", "loki_cache_served": false, - "loki_cold": 0.297, + "loki_cold": 0.267, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -33879,25 +34135,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.064, + "loki_warm_p50": 0.061, "parity_vs_loki": "same", "range": "1h", "set": "json-filter-pushdown", - "shape": "F field breakdown service_version (explore)", - "slower_than_loki_cold": true, + "shape": "E field breakdown pipeline (explore)", + "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.056 + "warm_p50": 0.045 }, { "baseline": "release", "baseline_status": "200", - "cold": 7.927, + "cold": 0.592, "date": "2026-10-22", "label": "daily-json-filter-pushdown", "loki_cache_served": false, - "loki_cold": 5.036, + "loki_cold": 4.641, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -33920,7 +34176,7 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 17402, + "bytesReceived": 19998, "entriesFound": 25, "entriesRequested": 25 }, @@ -33930,25 +34186,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.069, + "loki_warm_p50": 0.064, "parity_vs_loki": "same", "range": "24h", "set": "json-filter-pushdown", - "shape": "F field breakdown service_version (explore)", - "slower_than_loki_cold": true, + "shape": "E field breakdown pipeline (explore)", + "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.399 + "warm_p50": 0.358 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.867, + "cold": 0.093, "date": "2026-10-22", "label": "daily-json-filter-pushdown", "loki_cache_served": true, - "loki_cold": 0.468, + "loki_cold": 0.401, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -33961,7 +34217,7 @@ "entriesRequested": 0 }, "result": { - "bytesReceived": 129257, + "bytesReceived": 43871, "entriesFound": 2, "entriesRequested": 4 }, @@ -33981,25 +34237,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.066, + "loki_warm_p50": 0.061, "parity_vs_loki": "same", "range": "3h", "set": "json-filter-pushdown", - "shape": "F field breakdown service_version (explore)", + "shape": "E field breakdown pipeline (explore)", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.087 + "warm_p50": 0.069 }, { "baseline": "release", "baseline_status": "200", - "cold": 1.356, + "cold": 0.161, "date": "2026-10-22", "label": "daily-json-filter-pushdown", "loki_cache_served": false, - "loki_cold": 0.94, + "loki_cold": 1.198, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -34022,7 +34278,7 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 3596, + "bytesReceived": 4068, "entriesFound": 7, "entriesRequested": 7 }, @@ -34032,11 +34288,215 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.064, + "loki_warm_p50": 0.062, "parity_vs_loki": "same", "range": "6h", "set": "json-filter-pushdown", - "shape": "F field breakdown service_version (explore)", + "shape": "E field breakdown pipeline (explore)", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.118 + }, + { + "baseline": "release", + "baseline_status": "200", + "cold": 0.472, + "date": "2026-10-22", + "label": "daily-json-filter-pushdown", + "loki_cache_served": false, + "loki_cold": 0.297, + "loki_cold_cache": { + "instantMetricResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "labelResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "result": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 2 + }, + "seriesResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "statsResult": { + "bytesReceived": 176, + "entriesFound": 1, + "entriesRequested": 1 + }, + "volumeResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + } + }, + "loki_warm_p50": 0.064, + "parity_vs_loki": "same", + "range": "1h", + "set": "json-filter-pushdown", + "shape": "F field breakdown service_version (explore)", + "slower_than_loki_cold": true, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.056 + }, + { + "baseline": "release", + "baseline_status": "200", + "cold": 7.927, + "date": "2026-10-22", + "label": "daily-json-filter-pushdown", + "loki_cache_served": false, + "loki_cold": 5.036, + "loki_cold_cache": { + "instantMetricResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "labelResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "result": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 25 + }, + "seriesResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "statsResult": { + "bytesReceived": 17402, + "entriesFound": 25, + "entriesRequested": 25 + }, + "volumeResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + } + }, + "loki_warm_p50": 0.069, + "parity_vs_loki": "same", + "range": "24h", + "set": "json-filter-pushdown", + "shape": "F field breakdown service_version (explore)", + "slower_than_loki_cold": true, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.399 + }, + { + "baseline": "release", + "baseline_status": "200", + "cold": 0.867, + "date": "2026-10-22", + "label": "daily-json-filter-pushdown", + "loki_cache_served": true, + "loki_cold": 0.468, + "loki_cold_cache": { + "instantMetricResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "labelResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "result": { + "bytesReceived": 129257, + "entriesFound": 2, + "entriesRequested": 4 + }, + "seriesResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "statsResult": { + "bytesReceived": 528, + "entriesFound": 3, + "entriesRequested": 3 + }, + "volumeResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + } + }, + "loki_warm_p50": 0.066, + "parity_vs_loki": "same", + "range": "3h", + "set": "json-filter-pushdown", + "shape": "F field breakdown service_version (explore)", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.087 + }, + { + "baseline": "release", + "baseline_status": "200", + "cold": 1.356, + "date": "2026-10-22", + "label": "daily-json-filter-pushdown", + "loki_cache_served": false, + "loki_cold": 0.94, + "loki_cold_cache": { + "instantMetricResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "labelResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "result": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 7 + }, + "seriesResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "statsResult": { + "bytesReceived": 3596, + "entriesFound": 7, + "entriesRequested": 7 + }, + "volumeResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + } + }, + "loki_warm_p50": 0.064, + "parity_vs_loki": "same", + "range": "6h", + "set": "json-filter-pushdown", + "shape": "F field breakdown service_version (explore)", "slower_than_loki_cold": true, "status": "200", "valid": true, @@ -39102,17 +39562,15 @@ "valid": true, "verdict_vs_baseline": "same", "warm_p50": 11.608 - } - ], - "parser-json": [ + }, { - "baseline": "release", + "baseline": "base", "baseline_status": "200", - "cold": 0.076, + "cold": 0.032, "date": "2026-10-22", - "label": "daily-json-filter-pushdown", - "loki_cache_served": false, - "loki_cold": 0.267, + "label": "json-error-split-1h", + "loki_cache_served": true, + "loki_cold": 0.029, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -39125,8 +39583,8 @@ "entriesRequested": 0 }, "result": { - "bytesReceived": 0, - "entriesFound": 0, + "bytesReceived": 14752, + "entriesFound": 2, "entriesRequested": 2 }, "seriesResult": { @@ -39135,7 +39593,7 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 176, + "bytesReceived": 186, "entriesFound": 1, "entriesRequested": 1 }, @@ -39145,30 +39603,71 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.061, + "loki_warm_p50": 0.024, "parity_vs_loki": "same", "range": "1h", - "set": "json-filter-pushdown", - "shape": "E field breakdown pipeline (explore)", + "set": "json-error-split", + "shape": "J1 json no-error count by service_name, one service", "slower_than_loki_cold": false, + "slower_than_loki_warm": false, "status": "200", "valid": true, - "verdict_vs_baseline": "same", - "warm_p50": 0.045 + "verdict_vs_baseline": "faster", + "warm_p50": 0.032 }, { - "baseline": "release", + "baseline": "base", + "baseline_status": "502", + "cold": 1.914, + "date": "2026-10-22", + "label": "json-error-split-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-error-split", + "shape": "J1 json no-error count by service_name, one service", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 0.625 + }, + { + "baseline": "base", "baseline_status": "200", - "cold": 0.592, + "cold": 0.259, "date": "2026-10-22", - "label": "daily-json-filter-pushdown", + "label": "json-error-split-long", "loki_cache_served": false, - "loki_cold": 4.641, - "loki_cold_cache": { - "instantMetricResult": { - "bytesReceived": 0, - "entriesFound": 0, - "entriesRequested": 0 + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-error-split", + "shape": "J1 json no-error count by service_name, one service", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 0.142 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.064, + "date": "2026-10-22", + "label": "json-error-split-1h", + "loki_cache_served": true, + "loki_cold": 0.043, + "loki_cold_cache": { + "instantMetricResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 }, "labelResult": { "bytesReceived": 0, @@ -39176,9 +39675,9 @@ "entriesRequested": 0 }, "result": { - "bytesReceived": 0, - "entriesFound": 0, - "entriesRequested": 25 + "bytesReceived": 98051, + "entriesFound": 2, + "entriesRequested": 2 }, "seriesResult": { "bytesReceived": 0, @@ -39186,9 +39685,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 19998, - "entriesFound": 25, - "entriesRequested": 25 + "bytesReceived": 177, + "entriesFound": 1, + "entriesRequested": 1 }, "volumeResult": { "bytesReceived": 0, @@ -39196,25 +39695,66 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.064, + "loki_warm_p50": 0.042, "parity_vs_loki": "same", + "range": "1h", + "set": "json-error-split", + "shape": "J2 json no-error count by app, namespace", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 0.067 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 2.237, + "date": "2026-10-22", + "label": "json-error-split-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", "range": "24h", - "set": "json-filter-pushdown", - "shape": "E field breakdown pipeline (explore)", + "set": "json-error-split", + "shape": "J2 json no-error count by app, namespace", "slower_than_loki_cold": false, + "slower_than_loki_warm": false, "status": "200", "valid": true, - "verdict_vs_baseline": "same", - "warm_p50": 0.358 + "verdict_vs_baseline": "fixed", + "warm_p50": 1.46 }, { - "baseline": "release", + "baseline": "base", + "baseline_status": "502", + "cold": 0.62, + "date": "2026-10-22", + "label": "json-error-split-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-error-split", + "shape": "J2 json no-error count by app, namespace", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 0.476 + }, + { + "baseline": "base", "baseline_status": "200", - "cold": 0.093, + "cold": 0.029, "date": "2026-10-22", - "label": "daily-json-filter-pushdown", + "label": "json-error-split-1h", "loki_cache_served": true, - "loki_cold": 0.401, + "loki_cold": 0.016, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -39227,9 +39767,9 @@ "entriesRequested": 0 }, "result": { - "bytesReceived": 43871, + "bytesReceived": 70221, "entriesFound": 2, - "entriesRequested": 4 + "entriesRequested": 2 }, "seriesResult": { "bytesReceived": 0, @@ -39237,9 +39777,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 528, - "entriesFound": 3, - "entriesRequested": 3 + "bytesReceived": 186, + "entriesFound": 1, + "entriesRequested": 1 }, "volumeResult": { "bytesReceived": 0, @@ -39247,25 +39787,66 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.061, + "loki_warm_p50": 0.016, "parity_vs_loki": "same", - "range": "3h", - "set": "json-filter-pushdown", - "shape": "E field breakdown pipeline (explore)", + "range": "1h", + "set": "json-error-split", + "shape": "J3 json no-error count by parsed method", "slower_than_loki_cold": false, + "slower_than_loki_warm": false, "status": "200", "valid": true, - "verdict_vs_baseline": "same", - "warm_p50": 0.069 + "verdict_vs_baseline": "faster", + "warm_p50": 0.032 }, { - "baseline": "release", + "baseline": "base", + "baseline_status": "502", + "cold": 0.584, + "date": "2026-10-22", + "label": "json-error-split-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-error-split", + "shape": "J3 json no-error count by parsed method", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 0.611 + }, + { + "baseline": "base", "baseline_status": "200", - "cold": 0.161, + "cold": 0.209, "date": "2026-10-22", - "label": "daily-json-filter-pushdown", + "label": "json-error-split-long", "loki_cache_served": false, - "loki_cold": 1.198, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-error-split", + "shape": "J3 json no-error count by parsed method", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 0.159 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.075, + "date": "2026-10-22", + "label": "json-error-split-1h", + "loki_cache_served": true, + "loki_cold": 0.059, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -39278,9 +39859,9 @@ "entriesRequested": 0 }, "result": { - "bytesReceived": 0, - "entriesFound": 0, - "entriesRequested": 7 + "bytesReceived": 139905, + "entriesFound": 2, + "entriesRequested": 2 }, "seriesResult": { "bytesReceived": 0, @@ -39288,9 +39869,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 4068, - "entriesFound": 7, - "entriesRequested": 7 + "bytesReceived": 177, + "entriesFound": 1, + "entriesRequested": 1 }, "volumeResult": { "bytesReceived": 0, @@ -39298,25 +39879,66 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.062, + "loki_warm_p50": 0.053, "parity_vs_loki": "same", + "range": "1h", + "set": "json-error-split", + "shape": "J4 json no-error count by service_name, all services", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 0.072 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 3.142, + "date": "2026-10-22", + "label": "json-error-split-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-error-split", + "shape": "J4 json no-error count by service_name, all services", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 2.01 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 0.991, + "date": "2026-10-22", + "label": "json-error-split-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", "range": "6h", - "set": "json-filter-pushdown", - "shape": "E field breakdown pipeline (explore)", + "set": "json-error-split", + "shape": "J4 json no-error count by service_name, all services", "slower_than_loki_cold": false, + "slower_than_loki_warm": false, "status": "200", "valid": true, - "verdict_vs_baseline": "same", - "warm_p50": 0.118 + "verdict_vs_baseline": "fixed", + "warm_p50": 0.502 }, { - "baseline": "release", + "baseline": "base", "baseline_status": "200", - "cold": 0.017, + "cold": 0.091, "date": "2026-10-22", - "label": "daily-json-filter-pushdown", - "loki_cache_served": false, - "loki_cold": 0.287, + "label": "json-error-split-1h", + "loki_cache_served": true, + "loki_cold": 0.078, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -39329,8 +39951,8 @@ "entriesRequested": 0 }, "result": { - "bytesReceived": 0, - "entriesFound": 0, + "bytesReceived": 98033, + "entriesFound": 2, "entriesRequested": 2 }, "seriesResult": { @@ -39339,7 +39961,7 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 176, + "bytesReceived": 177, "entriesFound": 1, "entriesRequested": 1 }, @@ -39349,25 +39971,68 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.059, + "loki_warm_p50": 0.044, "parity_vs_loki": "same", "range": "1h", - "set": "json-filter-pushdown", - "shape": "G field breakdown pipeline (drilldown)", + "set": "json-error-split", + "shape": "J5 json no-error bytes_rate by app", "slower_than_loki_cold": false, + "slower_than_loki_warm": false, "status": "200", "valid": true, - "verdict_vs_baseline": "same", - "warm_p50": 0.017 + "verdict_vs_baseline": "faster", + "warm_p50": 0.07 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 3.001, + "date": "2026-10-22", + "label": "json-error-split-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-error-split", + "shape": "J5 json no-error bytes_rate by app", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 1.518 }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 0.503, + "date": "2026-10-22", + "label": "json-error-split-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-error-split", + "shape": "J5 json no-error bytes_rate by app", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 0.346 + } + ], + "parser-json": [ { "baseline": "release", "baseline_status": "200", - "cold": 0.325, + "cold": 0.076, "date": "2026-10-22", "label": "daily-json-filter-pushdown", "loki_cache_served": false, - "loki_cold": 4.653, + "loki_cold": 0.267, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -39382,7 +40047,7 @@ "result": { "bytesReceived": 0, "entriesFound": 0, - "entriesRequested": 25 + "entriesRequested": 2 }, "seriesResult": { "bytesReceived": 0, @@ -39390,9 +40055,9 @@ "entriesRequested": 0 }, "statsResult": { - "bytesReceived": 17446, - "entriesFound": 25, - "entriesRequested": 25 + "bytesReceived": 176, + "entriesFound": 1, + "entriesRequested": 1 }, "volumeResult": { "bytesReceived": 0, @@ -39400,25 +40065,25 @@ "entriesRequested": 0 } }, - "loki_warm_p50": 0.063, + "loki_warm_p50": 0.061, "parity_vs_loki": "same", - "range": "24h", + "range": "1h", "set": "json-filter-pushdown", - "shape": "G field breakdown pipeline (drilldown)", + "shape": "E field breakdown pipeline (explore)", "slower_than_loki_cold": false, "status": "200", "valid": true, "verdict_vs_baseline": "same", - "warm_p50": 0.327 + "warm_p50": 0.045 }, { "baseline": "release", "baseline_status": "200", - "cold": 0.045, + "cold": 0.592, "date": "2026-10-22", "label": "daily-json-filter-pushdown", - "loki_cache_served": true, - "loki_cold": 0.436, + "loki_cache_served": false, + "loki_cold": 4.641, "loki_cold_cache": { "instantMetricResult": { "bytesReceived": 0, @@ -39431,7 +40096,262 @@ "entriesRequested": 0 }, "result": { - "bytesReceived": 8167, + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 25 + }, + "seriesResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "statsResult": { + "bytesReceived": 19998, + "entriesFound": 25, + "entriesRequested": 25 + }, + "volumeResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + } + }, + "loki_warm_p50": 0.064, + "parity_vs_loki": "same", + "range": "24h", + "set": "json-filter-pushdown", + "shape": "E field breakdown pipeline (explore)", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.358 + }, + { + "baseline": "release", + "baseline_status": "200", + "cold": 0.093, + "date": "2026-10-22", + "label": "daily-json-filter-pushdown", + "loki_cache_served": true, + "loki_cold": 0.401, + "loki_cold_cache": { + "instantMetricResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "labelResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "result": { + "bytesReceived": 43871, + "entriesFound": 2, + "entriesRequested": 4 + }, + "seriesResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "statsResult": { + "bytesReceived": 528, + "entriesFound": 3, + "entriesRequested": 3 + }, + "volumeResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + } + }, + "loki_warm_p50": 0.061, + "parity_vs_loki": "same", + "range": "3h", + "set": "json-filter-pushdown", + "shape": "E field breakdown pipeline (explore)", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.069 + }, + { + "baseline": "release", + "baseline_status": "200", + "cold": 0.161, + "date": "2026-10-22", + "label": "daily-json-filter-pushdown", + "loki_cache_served": false, + "loki_cold": 1.198, + "loki_cold_cache": { + "instantMetricResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "labelResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "result": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 7 + }, + "seriesResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "statsResult": { + "bytesReceived": 4068, + "entriesFound": 7, + "entriesRequested": 7 + }, + "volumeResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + } + }, + "loki_warm_p50": 0.062, + "parity_vs_loki": "same", + "range": "6h", + "set": "json-filter-pushdown", + "shape": "E field breakdown pipeline (explore)", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.118 + }, + { + "baseline": "release", + "baseline_status": "200", + "cold": 0.017, + "date": "2026-10-22", + "label": "daily-json-filter-pushdown", + "loki_cache_served": false, + "loki_cold": 0.287, + "loki_cold_cache": { + "instantMetricResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "labelResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "result": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 2 + }, + "seriesResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "statsResult": { + "bytesReceived": 176, + "entriesFound": 1, + "entriesRequested": 1 + }, + "volumeResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + } + }, + "loki_warm_p50": 0.059, + "parity_vs_loki": "same", + "range": "1h", + "set": "json-filter-pushdown", + "shape": "G field breakdown pipeline (drilldown)", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.017 + }, + { + "baseline": "release", + "baseline_status": "200", + "cold": 0.325, + "date": "2026-10-22", + "label": "daily-json-filter-pushdown", + "loki_cache_served": false, + "loki_cold": 4.653, + "loki_cold_cache": { + "instantMetricResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "labelResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "result": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 25 + }, + "seriesResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "statsResult": { + "bytesReceived": 17446, + "entriesFound": 25, + "entriesRequested": 25 + }, + "volumeResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + } + }, + "loki_warm_p50": 0.063, + "parity_vs_loki": "same", + "range": "24h", + "set": "json-filter-pushdown", + "shape": "G field breakdown pipeline (drilldown)", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.327 + }, + { + "baseline": "release", + "baseline_status": "200", + "cold": 0.045, + "date": "2026-10-22", + "label": "daily-json-filter-pushdown", + "loki_cache_served": true, + "loki_cold": 0.436, + "loki_cold_cache": { + "instantMetricResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "labelResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "result": { + "bytesReceived": 8167, "entriesFound": 2, "entriesRequested": 4 }, @@ -46568,6 +47488,468 @@ "warm_p50": 1.325 } ], + "semantics/json-error-filter-metric-split-pushdown": [ + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.032, + "date": "2026-10-22", + "label": "json-error-split-1h", + "loki_cache_served": true, + "loki_cold": 0.029, + "loki_cold_cache": { + "instantMetricResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "labelResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "result": { + "bytesReceived": 14752, + "entriesFound": 2, + "entriesRequested": 2 + }, + "seriesResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "statsResult": { + "bytesReceived": 186, + "entriesFound": 1, + "entriesRequested": 1 + }, + "volumeResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + } + }, + "loki_warm_p50": 0.024, + "parity_vs_loki": "same", + "range": "1h", + "set": "json-error-split", + "shape": "J1 json no-error count by service_name, one service", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 0.032 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 1.914, + "date": "2026-10-22", + "label": "json-error-split-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-error-split", + "shape": "J1 json no-error count by service_name, one service", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 0.625 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.259, + "date": "2026-10-22", + "label": "json-error-split-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-error-split", + "shape": "J1 json no-error count by service_name, one service", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 0.142 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.064, + "date": "2026-10-22", + "label": "json-error-split-1h", + "loki_cache_served": true, + "loki_cold": 0.043, + "loki_cold_cache": { + "instantMetricResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "labelResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "result": { + "bytesReceived": 98051, + "entriesFound": 2, + "entriesRequested": 2 + }, + "seriesResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "statsResult": { + "bytesReceived": 177, + "entriesFound": 1, + "entriesRequested": 1 + }, + "volumeResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + } + }, + "loki_warm_p50": 0.042, + "parity_vs_loki": "same", + "range": "1h", + "set": "json-error-split", + "shape": "J2 json no-error count by app, namespace", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 0.067 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 2.237, + "date": "2026-10-22", + "label": "json-error-split-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-error-split", + "shape": "J2 json no-error count by app, namespace", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 1.46 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 0.62, + "date": "2026-10-22", + "label": "json-error-split-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-error-split", + "shape": "J2 json no-error count by app, namespace", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 0.476 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.029, + "date": "2026-10-22", + "label": "json-error-split-1h", + "loki_cache_served": true, + "loki_cold": 0.016, + "loki_cold_cache": { + "instantMetricResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "labelResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "result": { + "bytesReceived": 70221, + "entriesFound": 2, + "entriesRequested": 2 + }, + "seriesResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "statsResult": { + "bytesReceived": 186, + "entriesFound": 1, + "entriesRequested": 1 + }, + "volumeResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + } + }, + "loki_warm_p50": 0.016, + "parity_vs_loki": "same", + "range": "1h", + "set": "json-error-split", + "shape": "J3 json no-error count by parsed method", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 0.032 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 0.584, + "date": "2026-10-22", + "label": "json-error-split-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-error-split", + "shape": "J3 json no-error count by parsed method", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 0.611 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.209, + "date": "2026-10-22", + "label": "json-error-split-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-error-split", + "shape": "J3 json no-error count by parsed method", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 0.159 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.075, + "date": "2026-10-22", + "label": "json-error-split-1h", + "loki_cache_served": true, + "loki_cold": 0.059, + "loki_cold_cache": { + "instantMetricResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "labelResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "result": { + "bytesReceived": 139905, + "entriesFound": 2, + "entriesRequested": 2 + }, + "seriesResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "statsResult": { + "bytesReceived": 177, + "entriesFound": 1, + "entriesRequested": 1 + }, + "volumeResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + } + }, + "loki_warm_p50": 0.053, + "parity_vs_loki": "same", + "range": "1h", + "set": "json-error-split", + "shape": "J4 json no-error count by service_name, all services", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 0.072 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 3.142, + "date": "2026-10-22", + "label": "json-error-split-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-error-split", + "shape": "J4 json no-error count by service_name, all services", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 2.01 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 0.991, + "date": "2026-10-22", + "label": "json-error-split-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-error-split", + "shape": "J4 json no-error count by service_name, all services", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 0.502 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.091, + "date": "2026-10-22", + "label": "json-error-split-1h", + "loki_cache_served": true, + "loki_cold": 0.078, + "loki_cold_cache": { + "instantMetricResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "labelResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "result": { + "bytesReceived": 98033, + "entriesFound": 2, + "entriesRequested": 2 + }, + "seriesResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + }, + "statsResult": { + "bytesReceived": 177, + "entriesFound": 1, + "entriesRequested": 1 + }, + "volumeResult": { + "bytesReceived": 0, + "entriesFound": 0, + "entriesRequested": 0 + } + }, + "loki_warm_p50": 0.044, + "parity_vs_loki": "same", + "range": "1h", + "set": "json-error-split", + "shape": "J5 json no-error bytes_rate by app", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "faster", + "warm_p50": 0.07 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 3.001, + "date": "2026-10-22", + "label": "json-error-split-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "24h", + "set": "json-error-split", + "shape": "J5 json no-error bytes_rate by app", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 1.518 + }, + { + "baseline": "base", + "baseline_status": "502", + "cold": 0.503, + "date": "2026-10-22", + "label": "json-error-split-long", + "loki_cache_served": false, + "loki_cold": null, + "loki_warm_p50": null, + "parity_vs_loki": "n/a", + "range": "6h", + "set": "json-error-split", + "shape": "J5 json no-error bytes_rate by app", + "slower_than_loki_cold": false, + "slower_than_loki_warm": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "fixed", + "warm_p50": 0.346 + } + ], "semantics/json-extracted-metric-pushdown": [ { "baseline": "release", diff --git a/conformance/registry/generated/wiring.json b/conformance/registry/generated/wiring.json index d17041333..f57c81fd0 100644 --- a/conformance/registry/generated/wiring.json +++ b/conformance/registry/generated/wiring.json @@ -443,6 +443,10 @@ "internal/proxy/instant_result_type_test.go", "test/e2e-compat/instant_result_type_test.go" ], + "semantics/json-error-filter-metric-split-pushdown": [ + "internal/proxy/ordered_json_valid_split_test.go", + "test/e2e-compat/json_error_filter_split_test.go" + ], "semantics/json-extracted-metric-pushdown": [ "internal/proxy/ordered_json_extracted_differential_test.go", "internal/proxy/ordered_json_extracted_pushdown_test.go", @@ -554,7 +558,8 @@ ], "semantics/pipeline-error-text-parity": [ "internal/proxy/ordered_json_filter_pushdown_test.go", - "internal/proxy/ordered_json_metric_test.go" + "internal/proxy/ordered_json_metric_test.go", + "internal/proxy/ordered_json_valid_split_test.go" ], "semantics/plain-name-after-parser-reads-stream-label": [ "internal/proxy/plain_stream_labels_test.go", @@ -714,7 +719,7 @@ "test/e2e-compat/unwrap_ungrouped_sum_test.go" ] }, - "registry_items": 323, + "registry_items": 325, "routes_without_registry_item": [], "test_to_items": { "cmd/proxy/main_test.go": [ @@ -990,6 +995,10 @@ "internal/proxy/ordered_json_metric_test.go": [ "semantics/pipeline-error-text-parity" ], + "internal/proxy/ordered_json_valid_split_test.go": [ + "semantics/json-error-filter-metric-split-pushdown", + "semantics/pipeline-error-text-parity" + ], "internal/proxy/parenthesized_range_test.go": [ "loki_api_v1_query", "loki_api_v1_query_range", @@ -1275,6 +1284,9 @@ "test/e2e-compat/json_detected_level_test.go": [ "semantics/json-metric-detected-level-derivation" ], + "test/e2e-compat/json_error_filter_split_test.go": [ + "semantics/json-error-filter-metric-split-pushdown" + ], "test/e2e-compat/json_filter_pushdown_test.go": [ "parser-error-and-label-collision", "semantics/json-filter-pushdown-translated-label", @@ -1434,7 +1446,7 @@ "profiles/structured-metadata-keys-per-profile" ] }, - "tests_with_markers": 140, + "tests_with_markers": 142, "unknown_ids": {}, "unwired_items": [ "api_prom_label", @@ -1503,6 +1515,7 @@ "semantics/extracted-ungrouped-metric-raw-rows", "semantics/first-last-over-time-translation", "semantics/ip-filter-precision", + "semantics/json-empty-value-label-series", "semantics/label-filter-expression-syntax", "semantics/label-replace-new-label", "semantics/log-entries-differ-same-count", @@ -1537,5 +1550,5 @@ "versions/capabilities-floor-vs-matrix", "versions/rejected-stats-query-wording-vl-1.47" ], - "wired_items": 138 + "wired_items": 139 } diff --git a/conformance/reports/parity-gaps.md b/conformance/reports/parity-gaps.md index 95f9ddca5..c69836ee5 100644 --- a/conformance/reports/parity-gaps.md +++ b/conformance/reports/parity-gaps.md @@ -44,72 +44,73 @@ Last run: 2026-10-05T20:16:15Z on `fcaccfcc`, window [1791220800, 1791224400]. | 23 | `quality/labels-backfill-into-cached-bucket` — Rows backfilled into a past bucket the label inventory already cached with data appear in /labels only after its revalidation delay (open) | open | explore-visible | — | — | M | internal/proxy/metadata_inventory.go (bucket revalidation delay) | pkg/querier/querier.go Label (index read per request) | TestCompat_LabelsBackfillIntoCachedBucket (test/e2e-compat, group core) | | 24 | `semantics/drop-keep-plain-name-after-parser` — A log query's drop, keep or label_format source of a stream label's plain name after a parser acts on the stored field; Loki acts on the ... | open | explore-visible | — | — | S | internal/proxy stage-field exposure (drop and keep of a name a parser also reads) | pkg/logql/log/labels.go LabelsBuilder.Del / getWithCategory; pkg/logql/log/drop_labels.go, keep_labels.go | TestCompat_DropKeepPlainNameAfterParser (test/e2e-compat, group core) | | 25 | `semantics/extracted-ungrouped-metric-raw-rows` — An ungrouped range metric over a `\| json` pipeline filtered on name_extracted still reads raw rows and fails at the row limit (open) | open | explore-visible | — | — | L | internal/proxy bare-parser metric evaluator (fetchBareParserMetricSeries) and the ordered JSON pushdown (without grouping) | pkg/logql range aggregation keeps every label of the sample | TestCompat_ExtractedUngroupedMetricAtSixHours (test/e2e-compat, group core) | -| 26 | `semantics/label-replace-instant-vector` — label_replace around a vector at an instant adds no destination label (open) | open | explore-visible | — | — | S | internal/proxy applyLabelReplace (matrix only) | pkg/logql/evaluator.go:1275-1310 LabelReplaceEvaluator.Next | TestInstantLabelReplaceOnAVectorAddsNoLabel (internal/proxy); an e2e test vs Loki in group semantics when fixed | -| 27 | `semantics/label-replace-range-binary-marker` — label_replace around a binary expression on query_range sends the binary marker to VictoriaLogs and answers no series or an error (open) | open | explore-visible | — | — | S | internal/proxy handleQueryRange (stripLabelTransforms for a label transform around a binary expression) | pkg/logql/evaluator.go:1275-1310 LabelReplaceEvaluator.Next over the binary vector | TestRangeLabelReplaceOfABinaryExpressionSendsTheMarker (internal/proxy); an e2e test vs Loki in group semantics when fixed | -| 28 | `semantics/repeated-label-filter-replaced-across-label-format` — A second filter on the same label replaces the first even with a label_format between them (open) | open | explore-visible | — | — | S | internal/translator labelFilterLatest (latest filter per field replaces the earlier one in place) | pkg/logql/log/pipeline.go (stages run in order); label_filter.go | TestCompat_RepeatedLabelFilterAcrossLabelFormat (test/e2e-compat, group core) | -| 29 | `semantics/topk-bottomk-without-strips-label` — topk without and bottomk without drop the label and answer other values than Loki (open) | open | explore-visible | — | — | M | internal/translator tryTranslateMetricQueryWithout (topk and bottomk without); internal/proxy applyWithoutGrouping | pkg/logql/evaluator.go:451-466, :561-577 VectorAggEvaluator.Next (topk and bottomk keep the sample metric) | TestWithoutShapesStillDifferFromLoki (internal/proxy, topk and bottomk, range and instant), TestWithoutFoldIsLimitedToTheOperatorsItProve... | -| 30 | `semantics/two-parsers-keep-one` — A query with `\| json` and `\| logfmt` runs the one parser the data answers to (open) | open | explore-visible | — | — | M | internal/proxy/query_translation.go preferWorkingParser (probe-based removal of a parser stage) | pkg/logql/log/pipeline.go: every stage runs on every line; parser.go skips a key ParserHint.Extracted | TestCompat_TwoParsersOnMixedFormats (test/e2e-compat, group semantics) | -| 31 | `semantics/unwrap-bare-range-exposes-parsed-labels` — a range function with unwrap and no outer aggregation exposes every parsed label although the parser names one key (open) | open | explore-visible | — | — | M | internal/proxy raw-row range metric evaluator (the labels of a series built from the parsed fields) | pkg/logql/log/metrics_extraction.go:150-178 labelSampleExtractor (series = stream + the labels the parsers extract, less the unwrapped one) | TestCompat_UnwrapBareRangeSeriesLabels (test/e2e-compat, group core) over rate and the over-time functions, extraction list, range and in... | -| 32 | `semantics/unwrap-count-avg-without-after-parser` — count without and avg without over an unwrap function after a bare parser keep a series per remaining label set, like Loki (open) | open | explore-visible | — | — | L | internal/translator tryTranslateMetricQueryWithout (keepWithoutSeries covers sum, max, min); the raw evaluators series key | pkg/logql/evaluator.go:451-466 VectorAggEvaluator.Next (count and avg over the series of the inner range aggregation, HashWithoutLabels);... | TestCompat_UnwrapCountAvgWithoutAfterParser (test/e2e-compat, group core) over count and avg without, logfmt and json, range and instant | -| 33 | `semantics/unwrap-extraction-list-series-identity` — outside a without, an aggregation over a range function after an extraction list reads other series than Loki's (open) | open | explore-visible | — | — | M | internal/translator unwrapInnerGrouping and defaultRateInnerGrouping (parsedSeriesLabels is read for the without fold only); internal/pro... | pkg/logql/evaluator.go:435-529 VectorAggEvaluator.Next; pkg/logql/log/metrics_extraction.go:150-178 labelSampleExtractor (series = stream... | TestExtractionListSeriesIdentityStillDiffers and TestExtractionListOutsideWithoutKeepsItsGrouping (internal/proxy, emulator); an e2e test... | -| 34 | `semantics/unwrap-first-last-under-aggregation` — first_over_time and last_over_time with unwrap under sum fail with a VictoriaLogs 422 (open) | open | explore-visible | — | — | M | internal/translator first_over_time / last_over_time under a vector aggregation | pkg/logql/range_vector.go first/last | an e2e over sum(first_over_time\|last_over_time(... \| unwrap ...)) | -| 35 | `semantics/unwrap-over-time-range-windows` — sum, avg, max and min over time with unwrap answer the wrong window on query_range (open) | open | explore-visible | — | — | M | internal/proxy range_metric_compat.go shouldUseManualRangeMetricCompat (unwrap sum/avg/min/max) and metric_binary.go statsRateRangeEquals... | pkg/logql/range_vector.go:102-189 batchRangeVectorIterator (window (t-range, t]); range_vector.go:371-420 sumOverTime, avgOverTime, maxOv... | TestCompat_UnwrapOverTimeRangeWindows (test/e2e-compat, group core) over sum/avg/max/min x tumbling/sliding x logfmt/json | -| 36 | `semantics/unwrap-per-series-identity-raw-rows` — A function taken per series (avg, max, min over time, rate_counter) under sum or sum by uses the stream as the series (open) | open | explore-visible | — | — | L | internal/translator unwrapInnerGrouping (sum by over a per-series function) and the raw evaluators series key (collectRangeMetricSamples) | pkg/logql/evaluator.go:435-529 VectorAggEvaluator.Next; pkg/logql/log/metrics_extraction.go:150-178 labelSampleExtractor (series = stream... | TestCompat_UnwrapPerSeriesIdentity (test/e2e-compat, group core) over sum and sum by x avg/max/min/rate_counter, logfmt and json | -| 37 | `semantics/unwrap-quantile-sum-by-group-quantile` — sum by over quantile_over_time with unwrap adds the per-series quantiles instead of taking the group quantile, like Loki (open) | open | explore-visible | — | — | M | internal/translator unwrapInnerGrouping (an outer by moved into the quantile) and the raw evaluator of quantile_over_time | pkg/logql/range_vector.go:449-487 quantileOverTime; pkg/logql/syntax/extractor.go:21-60 (a range by groups the samples); pkg/logql/evalua... | TestCompat_UnwrapQuantileOuterSum (test/e2e-compat, group core) over sum by and sum without, logfmt and json, range and instant | -| 38 | `semantics/unwrap-quantile-ungrouped-sum-field-parse` — sum and sum without over quantile_over_time with unwrap answer a series, not an empty result, like Loki (open) | open | explore-visible | — | — | S | internal/proxy parseStatsCompatSpec (Field from the first "(" to the last ")") and the quantile route of shouldUseManualRangeMetricCompat | pkg/logql/range_vector.go:449-487 quantileOverTime; pkg/logql/evaluator.go:451-466 VectorAggEvaluator.Next (sum folds the per-series quan... | TestCompat_UnwrapQuantileOuterSum (test/e2e-compat, group core) over sum and sum without, logfmt and json, range and instant | -| 39 | `semantics/unwrap-range-last-point` — A grouped unwrap range metric misses the sample at `end` (open) | open | explore-visible | — | — | S | internal/proxy range-metric unwrap path (end bucket) | pkg/logql/range_vector.go step iteration start..end inclusive | TestCompat_UnwrapRangeEndSample (test/e2e-compat, group semantics) | -| 40 | `semantics/unwrap-sum-by-per-series-function-extraction-list` — sum by over max, min or avg over time with unwrap after an extraction list takes the maximum of the group, not the sum of the per-series ... | open | explore-visible | — | — | M | internal/translator unwrapInnerGrouping (sum by over a per-series function: the inner stats groups by the by labels only) | pkg/logql/evaluator.go:435-529 VectorAggEvaluator.Next; pkg/logql/log/metrics_extraction.go:150-178 (series = stream + parsed labels) | TestCompat_UnwrapSumByPerSeriesFunction (test/e2e-compat, group core) over sum by x avg/max/min, extraction list, range and instant | -| 41 | `semantics/without-comparison-inside-ignored` — sum without over a range function filtered by a comparison inside the aggregation ignores the filter (open) | open | explore-visible | — | — | M | internal/translator tryTranslateMetricQueryWithout (the comparison after the range aggregation is dropped) | pkg/logql/evaluator.go:451-466 VectorAggEvaluator.Next over the filtered vector of the comparison (vectorBinop) | TestWithoutShapesStillDifferFromLoki (internal/proxy, rate with the threshold, range and instant); an e2e test vs Loki in group semantics... | -| 42 | `semantics/without-empty-grouping-range-collapses` — sum without () over an over-time unwrap function on query_range answers one series, Loki keeps every series (open) | open | explore-visible | — | — | S | internal/translator extractWithoutLabels / withoutMarkerRE (a clause with no label records no fold); tryTranslateMetricQueryWithout | pkg/logql/evaluator.go:451-466 VectorAggEvaluator.Next (HashWithoutLabels with no names); pkg/logql/syntax/ast.go Grouping.Noop; pkg/logq... | TestWithoutShapesStillDifferFromLoki (internal/proxy, the series of the registered request); an e2e test vs Loki in group core when fixed | -| 43 | `semantics/without-instant-detected-level-unknown` — an instant without answer lacks the detected_level=unknown label Loki puts on a stream with no level (open) | open | explore-visible | — | — | S | internal/proxy instant stats answer labels (the stream-derived label set of an instant vector; the range path adds detected_level in buil... | detected_level is a label of every stream when log level discovery is on (unknown without a level) | TestCompat_UnwrapSumWithoutExtractionList compares the instant answers after dropping the label (test/e2e-compat); a parity case without ... | -| 44 | `semantics/without-nested-under-aggregation` — a without nested under an outer aggregation answers the wrong series and values (open) | open | explore-visible | — | — | M | internal/translator tryTranslateMetricQueryWithout (nested aggregations with a without clause); internal/proxy applyWithoutGrouping | pkg/logql/evaluator.go:451-466 VectorAggEvaluator.Next (the outer aggregation takes the vector the inner without produced) | TestWithoutShapesStillDifferFromLoki (internal/proxy, max and sum by over a without, range and instant), TestWithoutFoldIsLimitedToTheOpe... | -| 45 | `semantics/without-over-inner-aggregation-collapses` — sum without over an inner sum by or count_over_time aggregation answers one series, Loki keeps a series per remaining label set (open) | open | explore-visible | — | — | M | internal/translator tryTranslateMetricQueryWithout (an outer aggregation over an aggregation: the last stats pipe has no by) | pkg/logql/evaluator.go:451-466 VectorAggEvaluator.Next (the outer without groups the inner vector by every label but the excluded ones) | TestWithoutShapesStillDifferFromLoki (internal/proxy, rate and count_over_time, range and instant); an e2e test vs Loki in group core whe... | -| 46 | `drilldown/detected-field-values` — detected_field values differ: none for service_name, other values for numeric and id fields (open) | open | drilldown-visible | 33 | 3 | M | internal/proxy detected_field values (sampling window and stream-label fields) | pkg/querier detected field values (v3.7.7): values of the sampled lines | TestCompat_DetectedFieldValues (test/e2e-compat, group drilldown) | -| 47 | `semantics/detected-fields-extracted-suffix` — detected_fields omits Loki's level_extracted and version_extracted (open; service fields are owner-kept) | open | drilldown-visible | 15 | 2 | M | internal/proxy/label_handlers.go handleDetectedFields (parsed fields named like a stream label) | pkg/logql/log/parser.go:25 duplicateSuffix; detected fields come from the parsed labels of sampled entries | TestCompat_DetectedFieldsExtractedSuffix (test/e2e-compat, group drilldown) | -| 48 | `profiles/json-number-label-text` — JSON numbers become label values in a different text form ("14.0" vs "14") (open) | open | drilldown-visible | 12 | 6 | M | internal/proxy JSON number rendering of unpacked values (VictoriaLogs unpack_json normalises) | pkg/logql/log/parser.go readValue (raw number bytes) | TestCompat_JSONNumberLabelText (test/e2e-compat, group drilldown) | -| 49 | `severity/detected-level-filter-json-level` — `\| detected_level=` filters select different lines than Loki (JSON-level lines, stream level precedence) (open) | open | drilldown-visible | 12 | 8 | M | internal/translator/detected_level_logsql.go (filter predicate) vs internal/proxy/detected_level.go (response derivation) | pkg/distributor field detection (v3.7.7) stores detected_level as structured metadata; filter reads it | TestCompat_DetectedLevelFilterJSONLevel (test/e2e-compat, group drilldown) | -| 50 | `limits/drilldown-7d-breakdown-response-limit` — Metric queries fail on the 64 MiB buffered-response budget where Loki answers (open) | open | drilldown-visible | 8 | 4 | M | internal/proxy/range_metric_compat.go:767, execution_limits.go:11 (raw-row metric paths buffer rows; push down or stream) | pkg/logql engine series limit (max_query_series) with partial result for Drilldown | TestCompat_MetricBufferedBudget1h + TestE2E_Drilldown7dHighCardinalityBreakdown (test/e2e-compat, groups semantics and drilldown) | -| 51 | `quality/index-stats-streams-differ` — index/stats stream counts differ from Loki in both directions (open) | open | drilldown-visible | 6 | 2 | S | internal/proxy/label_handlers.go handleIndexStats (stream counting per window) | pkg/storage index stats (streams from the index for the window) | bench/parity rerun with the /series cross-check, then TestCompat_IndexStatsStreams (test/e2e-compat, group drilldown) | -| 52 | `drilldown/detected-fields-type-inference` — detected_fields infers a different type for some fields (open) | open | drilldown-visible | — | — | S | internal/proxy detected_fields type inference | pkg/querier detected fields type detection (v3.7.7): int before duration before string | TestCompat_DetectedFieldsTypes (test/e2e-compat, group drilldown) | -| 53 | `profiles/existence-filter-on-sanitized-key` — An existence filter on a sanitized key reads the stored dotted field, not the JSON line key (open) | open | drilldown-visible | — | — | M | internal/translator (label-filter key variants, parsed_key_variants.go); Drilldown single-field fast paths | pkg/logql/log/parser.go:141 sanitizeLabelKey on every extracted key; label filter reads the parsed label | TestCompat_ExistenceFilterSanitizedKey (test/e2e-compat, group semantics) | -| 54 | `quality/detected-metadata-proxy-cache` — detected_labels and detected_fields are served from proxy caches; Loki answers each request fresh (open) | open | drilldown-visible | — | — | S | internal/proxy/cache_keys.go:48,130 (detected_* cache TTL) + recent-tail freshness bypass | pkg/querier detected labels/fields: no results cache in v3.7.7 | TestE2E_DetectedMetadataFreshness (test/e2e-compat, group drilldown) | -| 55 | `quality/stats-volume-freshness` — index/stats and volume answers ending now are cached; Loki does not cache inside max_stats_cache_freshness (10m) (open) | open | drilldown-visible | — | — | S | internal/proxy/volume.go handleVolume, label_handlers.go handleIndexStats (cache freshness) | pkg/validation/limits.go:413-414 max_stats_cache_freshness default 10m | TestE2E_StatsVolumeFreshness (test/e2e-compat, group drilldown) | -| 56 | `semantics/parse-error-text-parity` — Invalid queries fail on both sides, with different error text or status (open) | open | grafana-datasource | 210 | 129 | L | internal/logql parser error messages (Loki's goyacc messages and positions) + VictoriaLogs error mapping | pkg/logql/syntax/parser.go ParseError (line, col) and syntax.y | TestCompat_ParseErrorText corpus (test/e2e-compat, bad_request_status_parity_test.go) | -| 57 | `semantics/proxy-rejects-valid-logql` — The proxy rejects LogQL Loki accepts: comments, negative and compound offsets, filters after a range, logfmt flags, ip() label filters (o... | open | api-only | 79 | 48 | M | internal/logql parser (lexer: comments, signed durations, flags) + translator | pkg/logql/syntax/lex.go:149 comments, :90-91 OpStrict/OpKeepEmpty; syntax.y offsetExpr; pkg/logql/log/ip.go | TestCompat_LogQLSurfaceAccepted (test/e2e-compat, group semantics), one subtest per family | -| 58 | `semantics/proxy-accepts-invalid-logql` — Queries Loki rejects with 400 are answered by the proxy (open) | open | api-only | 18 | 14 | S | internal/logql parser validation | pkg/logql/syntax/ast.go:1446,1579 grouping not allowed; duration parsing | TestCompat_InvalidLogQLRejected (test/e2e-compat, bad_request_status_parity_test.go) | -| 59 | `parsed-identity/regexp-partial-match-series-count` — A partially-matching \| regexp returns about 3x Loki's series count | open | api-only | 2 | 2 | M | internal/proxy regexp capture series identity (non-matching lines) | pkg/logql/log/parser.go RegexpParser (no capture label for a non-matching line) | TestCompat_RegexpPartialMatchSeries (test/e2e-compat, group semantics) | -| 60 | `profiles/json-extraction-list-missing-key` — A json extraction-list key absent from the line is dropped; Loki keeps it in the metric series (open) | open | api-only | 2 | 2 | S | internal/proxy stage-field exposure (extraction lists, metric path) | pkg/logql/log/parser.go:671-720 JSONExpressionParser.Process | TestCompat_JSONExtractionListMissingKey (test/e2e-compat, group semantics) | -| 61 | `semantics/unpack-non-string-values` — `\| unpack` extracts non-string JSON values that Loki ignores (open) | open | api-only | 2 | 2 | S | internal/translator unpack (string values only) | pkg/logql/log/parser.go:789-820 UnpackParser.unpack | TestCompat_UnpackStringValuesOnly (test/e2e-compat, group semantics) | -| 62 | `semantics/pattern-line-filter-invalid-captures` — A pattern line filter with consecutive or named captures is a 400 on Loki; the proxy answers it (open) | open | api-only | — | — | S | internal/translator/translator.go planPatternLineFilter (patterns are translated without validation) | pkg/logql/log/pattern/pattern.go:36-52 (ParseLineFilter, validateNoConsecutiveCaptures, validateNoNamedCaptures) | TestCompat_PatternLineFilterInvalidCaptures (test/e2e-compat, group core) | -| 63 | `semantics/detected-level-pipes-known-differences` — The detected_level pipes read level keys in a fixed order, do not decode logfmt like Loki and overwrite a detected_level stream label (open) | open | edge | — | — | M | internal/translator detected_level_logsql.go BuildDetectedLevelChain; internal/proxy native metric route | pkg/distributor/field_detection.go getLevelUsingJSONParser (document order, depth < 2), labels.go (detected_level_extracted) | an e2e on plain-selector probe lines once the route probes and falls back to a raw-row or keyed evaluation | -| 64 | `semantics/extracted-derived-labels-raw-evaluator` — service_name_extracted and detected_level_extracted metrics keep the raw-row evaluator (open) | open | edge | — | — | S | internal/proxy orderedJSONPushdownLabelOK (service_name, detected_level) | pkg/logql/log/parser.go duplicateSuffix; service_name set by discover_service_name at ingest | TestCompat_ExtractedLabelMetricPushdown rows for service_name_extracted (test/e2e-compat, group core) | -| 65 | `semantics/extracted-metric-known-differences` — Two name_extracted differences against Loki remain in the metric pushdown and the raw evaluator (open) | open | edge | — | — | M | internal/proxy raw evaluator parsed-label naming (processWithStreamLabels) and translator.ExtractedLabelPipes | pkg/logql/log/parser.go duplicateSuffix and ParserHint.Extracted (a key an earlier extraction produced is skipped) | TestCompat_ExtractedBothKeysInLineOrder (test/e2e-compat, group core) | -| 66 | `semantics/unwrap-conversion-error-parser-divergence` — Where VictoriaLogs' logfmt parser reads another value than Loki's, an unwrap misses Loki's SampleExtractionErr or drops samples Loki keep... | open | edge | — | — | L | internal/translator logfmt unpacking (unpack_logfmt keeps the last value); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:379-438 LogfmtParser.Process; pkg/logql/log/logfmt/decode.go | TestUnwrapConversionErrorDifferential counts the first case; TestUnwrapLokiLabels pins the decoder; TestCompat_UnwrapConversionErrorParit... | -| 67 | `semantics/unwrap-conversion-error-postfilter-forms` — An unwrap whose post filter keeps only the error, compares a label, or combines filters with and/or answers 200 where Loki may fail with ... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorPostFilters; the translator's post-filter placement | pkg/logql/log/label_filter.go:90-100 BinaryLabelFilter.Process, 306-340 NumericLabelFilter.Process, 418-424 labelValue | TestUnwrapErrorProbePlans and TestUnwrapConversionErrorDifferential (internal/proxy) assert the detection stays off for these forms; flip... | -| 68 | `semantics/unwrap-conversion-error-stored-value-overwritten` — An unwrap over a label stored as structured metadata and also parsed from the line misses Loki's SampleExtractionErr (open) | open | edge | — | — | M | internal/translator parsed-key collision with stored fields (unpack keep_original_fields); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:151,809 duplicateSuffix | TestUnwrapConversionErrorDifferential counts the first case; TestCompat_UnwrapConversionErrorParity asserts the 200 side | -| 69 | `semantics/unwrap-conversion-error-undetected-shapes` — A few query shapes answer 200 where Loki fails with SampleExtractionErr, because no VictoriaLogs query of theirs carries the conversion c... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorProbes (opaque expressions), unwrapCountingQuery (pipes after stats), unwrapLokiLabe... | pkg/logql/evaluator.go:726-733 | TestCompat_UnwrapConversionErrorParity open/* asserts Loki's 400 and skips naming this case (absent_over_time, label_replace); TestUnwrap... | -| 70 | `semantics/unwrap-gate-parsefloat-divergences` — The unwrap sample gate drops a few values Loki's conversions accept or reject (open) | open | edge | — | — | S | internal/translator/unwrap_gate.go (UnwrapNumberPattern, UnwrapDurationPattern, UnwrapBytesPattern) | pkg/logql/log/metrics_extraction.go:316-338 convertFloat, convertDuration, convertBytes | TestUnwrapNumberPatternMatchesParseFloat (internal/translator) pins the known differences | -| 71 | `semantics/unwrap-sum-without-extracted-label-collision` — a parsed key that is also a stream label is not renamed to key_extracted after a without fold (open) | open | edge | — | — | M | internal/proxy withParsedGroupLabels (a parsed group key that equals a stream label of the series); internal/translator parsedSeriesLabels | pkg/logql/log/labels.go and parser.go (duplicateSuffix "_extracted") | TestSumWithoutExtractedLabelCollision (internal/proxy, emulator); an e2e test vs Loki in group semantics when fixed | -| 72 | `metric-identity/structured-metadata-splits-series` — Loki splits a stream's metric series per structured-metadata set; the proxy keeps the stream (owner-accepted deviation) | owner-kept | explore-visible | 173 | 37 | L | internal/translator series identity (stats by stream); VictoriaLogs cannot group by a row's remaining fields | pkg/logql/log/metrics_extraction.go:102-120 builder.Add(StructuredMetadataLabel) -> GroupedLabels | differential run cluster count (bench/parity) | -| 73 | `parsed-identity/dynamic-key-parsers` — `\| json` and `\| logfmt` metric series keep the stream identity by default (owner-accepted deviation, opt-in flag) | owner-kept | explore-visible | 95 | 29 | L | internal/translator parser series identity; -exact-parser-series-identity | pkg/logql/log/parser.go label extraction into the sample labels | differential run cluster count (bench/parity) | -| 74 | `profiles/label-filter-on-line-field` — A label filter on a line key no earlier stage exposes matches like Loki | owner-kept | explore-visible | — | — | M | internal/proxy label-filter refill (-label-filter-refill-max-pages); metric queries on keys no stage exposes | pkg/logql/log/label_filter.go (filter reads only labels the pipeline set) | TestCompat_LabelFilterOnLineField (exists, profile tests) | -| 75 | `profiles/detected-fields-no-extracted-suffix` — detected_fields lists service / service.name without Loki's _extracted suffix (owner decision) | owner-kept | drilldown-visible | — | — | S | internal/proxy/label_handlers.go handleDetectedFields | pkg/logql/log/parser.go:25 duplicateSuffix | bench/parity diff rule (documented facet, service fields only) | -| 76 | `versions/stats-bucket-label-v1.45` — Stats-bucket metrics put a line on a bucket edge one bucket late on VictoriaLogs before v1.45 (unsupported versions) | owner-kept | edge | — | — | M | stats-bucket relabel (internal/proxy range_metric_compat.go setSlidingStatsRangeParams / slidingStatsBucket; sliding_window_stats.go) | pkg/logql range vector (T-range, T] | TestRangeMetricCompatibilityJSONFilterPushdown on every supported VictoriaLogs version (compat-vl vl-matrix job); TestSlidingRangeMetric_... | -| 77 | `profiles/json-line-stored-without-line` — A JSON line pushed without _msg is stored without its line (documented) | documented | explore-visible | — | — | L | client push configuration (disable_message_parsing=1); docs/KNOWN_ISSUES.md | pkg/distributor push keeps the line verbatim | cases/profiles/stage-field-exposure with disable_message_parsing pushes | -| 78 | `drilldown/detected-labels-sampled-cardinality` — detected_labels cardinality is sampled from the window's last minutes; Loki counts every stream its ingesters hold (documented) | documented | drilldown-visible | 20 | 2 | M | internal/proxy/patterns.go handleDetectedLabels | pkg/ingester/instance.go LabelsWithValues | bench/parity diff rule (documented facet) | -| 79 | `profiles/volume-detected-level-extension` — index/volume answers targetLabels=detected_level; Loki returns no volume for it (documented proxy extension) | documented | drilldown-visible | 17 | 2 | S | internal/proxy/volume.go | pkg/storage volume index (stream labels only) | differential run row volume by detected_level | -| 80 | `drilldown/patterns-mined-differently` — Patterns come from different miners on each side (documented) | documented | drilldown-visible | 15 | 3 | L | internal/proxy/patterns.go | pkg/pattern (ingester-side mining) | bench/visual history-dependent comparison (exists) | -| 81 | `profiles/volume-bytes-accounting` — Volume values count line bytes; Loki counts chunk bytes with structured metadata (documented) | documented | drilldown-visible | 4 | 3 | L | internal/proxy/volume.go; docs/KNOWN_ISSUES.md | pkg/storage volume index (chunk bytes incl. structured metadata) | bench/parity diff rule (documented facet) | -| 82 | `limits/drilldown-breakdown-keeps-busiest` — Over max_query_series a Drilldown breakdown keeps the busiest series; Loki keeps the first it meets (documented) | documented | drilldown-visible | 1 | 1 | S | internal/proxy series limit ranking | pkg/logql engine series limit | cases/limits/drilldown-partial-with-warning (exists) | -| 83 | `quality/index-stats-bytes-and-chunks` — index/stats bytes and chunks have no VictoriaLogs equivalent (documented) | documented | drilldown-visible | — | — | L | internal/proxy/label_handlers.go handleIndexStats; docs/KNOWN_ISSUES.md | pkg/storage index stats from chunk refs (bytes, chunks) | bench/parity diff rule (documented facet) | -| 84 | `profiles/grafana-stats-error-as-warning` — A failed stats query from Grafana answers 200 with warnings; Loki returns the error (documented) | documented | grafana-datasource | — | — | S | internal/proxy Grafana-sourced stats error handling | pkg/querier/queryrange error propagation (no warnings conversion) | TestCompat_GrafanaStatsErrorStatus (test/e2e-compat, fault injection) | -| 85 | `profiles/metadata-default-lookback` — Label endpoints without start/end default to a different lookback than Loki's 1h (documented) | documented | api-only | — | — | S | internal/config flag default -metadata-default-lookback | pkg/loghttp/params.go defaultSince = 1h | TestCompat_MetadataDefaultLookback (test/e2e-compat, group core) | -| 86 | `profiles/multi-tenant-volume-merge` — Multi-tenant volume returns per-tenant series with __tenant_id__; Loki sums the tenants' volumes (documented) | documented | api-only | — | — | S | internal/proxy/volume.go multi-tenant merge | pkg/querier/queryrange volume merge | TestCompat_MultiTenantVolume (test/e2e-compat, group tail-multitenancy) | -| 87 | `semantics/count-values-unsupported` — count_values() is rejected with 400 where Loki answers (documented) | documented | api-only | — | — | M | internal/translator + proxy-side post-aggregation of a stats matrix | pkg/logql/syntax OpTypeCountValues; vector evaluator | TestCompat_CountValues (test/e2e-compat, group semantics) | -| 88 | `semantics/eager-pipeline-validation` — Invalid pipeline stages fail with 400 regardless of data; Loki answers 200 empty over a range with no data (documented) | documented | edge | — | — | S | internal/translator validation | pkg/logql/log/fmt.go template build at pipeline construction | TestCompat_EagerValidationEmptyRange (test/e2e-compat, group semantics) | -| 89 | `semantics/ip-filter-precision` — ip() filters are approximated with regular expressions (documented) | documented | edge | — | — | M | internal/proxy/postprocess.go:84 ip filtering | pkg/logql/log/ip.go | TestCompat_IPFilterForms (test/e2e-compat, group semantics) | -| 90 | `drilldown/plugin-legend-sync-race` — Logs Drilldown crashes a breakdown page when a query answers before the panel module loads (upstream, not the proxy) | upstream | drilldown-visible | — | — | S | none (grafana-lokiexplore-app) | n/a (client race) | bench/visual capture waits for the panel module (already) | -| 91 | `quality/index-stats-loki-chunk-accounting` — Loki's index/stats counts chunks it holds twice; the proxy's counts equal Loki's own count_over_time (upstream) | upstream | drilldown-visible | — | — | S | none (Loki chunk index accounting) | pkg/storage stats from chunk refs (flushed and retained chunks both counted) | bench/parity cross-check (run.py loki_counts_itself) | +| 26 | `semantics/json-empty-value-label-series` — A parsed label with an empty value is its own series in Loki, merged into the label-less series by the proxy (open) | open | explore-visible | — | — | M | internal/proxy ordered_json_metric.go groupLabels (row evaluator); stats pushdown grouping (VictoriaLogs cannot tell an empty field from ... | pkg/logql/log/parser.go:137-211 parseLabelValue/readValue; the vector aggregation keeps the empty label | a row-evaluator unit case and an e2e on a corpus with empty values | +| 27 | `semantics/label-replace-instant-vector` — label_replace around a vector at an instant adds no destination label (open) | open | explore-visible | — | — | S | internal/proxy applyLabelReplace (matrix only) | pkg/logql/evaluator.go:1275-1310 LabelReplaceEvaluator.Next | TestInstantLabelReplaceOnAVectorAddsNoLabel (internal/proxy); an e2e test vs Loki in group semantics when fixed | +| 28 | `semantics/label-replace-range-binary-marker` — label_replace around a binary expression on query_range sends the binary marker to VictoriaLogs and answers no series or an error (open) | open | explore-visible | — | — | S | internal/proxy handleQueryRange (stripLabelTransforms for a label transform around a binary expression) | pkg/logql/evaluator.go:1275-1310 LabelReplaceEvaluator.Next over the binary vector | TestRangeLabelReplaceOfABinaryExpressionSendsTheMarker (internal/proxy); an e2e test vs Loki in group semantics when fixed | +| 29 | `semantics/repeated-label-filter-replaced-across-label-format` — A second filter on the same label replaces the first even with a label_format between them (open) | open | explore-visible | — | — | S | internal/translator labelFilterLatest (latest filter per field replaces the earlier one in place) | pkg/logql/log/pipeline.go (stages run in order); label_filter.go | TestCompat_RepeatedLabelFilterAcrossLabelFormat (test/e2e-compat, group core) | +| 30 | `semantics/topk-bottomk-without-strips-label` — topk without and bottomk without drop the label and answer other values than Loki (open) | open | explore-visible | — | — | M | internal/translator tryTranslateMetricQueryWithout (topk and bottomk without); internal/proxy applyWithoutGrouping | pkg/logql/evaluator.go:451-466, :561-577 VectorAggEvaluator.Next (topk and bottomk keep the sample metric) | TestWithoutShapesStillDifferFromLoki (internal/proxy, topk and bottomk, range and instant), TestWithoutFoldIsLimitedToTheOperatorsItProve... | +| 31 | `semantics/two-parsers-keep-one` — A query with `\| json` and `\| logfmt` runs the one parser the data answers to (open) | open | explore-visible | — | — | M | internal/proxy/query_translation.go preferWorkingParser (probe-based removal of a parser stage) | pkg/logql/log/pipeline.go: every stage runs on every line; parser.go skips a key ParserHint.Extracted | TestCompat_TwoParsersOnMixedFormats (test/e2e-compat, group semantics) | +| 32 | `semantics/unwrap-bare-range-exposes-parsed-labels` — a range function with unwrap and no outer aggregation exposes every parsed label although the parser names one key (open) | open | explore-visible | — | — | M | internal/proxy raw-row range metric evaluator (the labels of a series built from the parsed fields) | pkg/logql/log/metrics_extraction.go:150-178 labelSampleExtractor (series = stream + the labels the parsers extract, less the unwrapped one) | TestCompat_UnwrapBareRangeSeriesLabels (test/e2e-compat, group core) over rate and the over-time functions, extraction list, range and in... | +| 33 | `semantics/unwrap-count-avg-without-after-parser` — count without and avg without over an unwrap function after a bare parser keep a series per remaining label set, like Loki (open) | open | explore-visible | — | — | L | internal/translator tryTranslateMetricQueryWithout (keepWithoutSeries covers sum, max, min); the raw evaluators series key | pkg/logql/evaluator.go:451-466 VectorAggEvaluator.Next (count and avg over the series of the inner range aggregation, HashWithoutLabels);... | TestCompat_UnwrapCountAvgWithoutAfterParser (test/e2e-compat, group core) over count and avg without, logfmt and json, range and instant | +| 34 | `semantics/unwrap-extraction-list-series-identity` — outside a without, an aggregation over a range function after an extraction list reads other series than Loki's (open) | open | explore-visible | — | — | M | internal/translator unwrapInnerGrouping and defaultRateInnerGrouping (parsedSeriesLabels is read for the without fold only); internal/pro... | pkg/logql/evaluator.go:435-529 VectorAggEvaluator.Next; pkg/logql/log/metrics_extraction.go:150-178 labelSampleExtractor (series = stream... | TestExtractionListSeriesIdentityStillDiffers and TestExtractionListOutsideWithoutKeepsItsGrouping (internal/proxy, emulator); an e2e test... | +| 35 | `semantics/unwrap-first-last-under-aggregation` — first_over_time and last_over_time with unwrap under sum fail with a VictoriaLogs 422 (open) | open | explore-visible | — | — | M | internal/translator first_over_time / last_over_time under a vector aggregation | pkg/logql/range_vector.go first/last | an e2e over sum(first_over_time\|last_over_time(... \| unwrap ...)) | +| 36 | `semantics/unwrap-over-time-range-windows` — sum, avg, max and min over time with unwrap answer the wrong window on query_range (open) | open | explore-visible | — | — | M | internal/proxy range_metric_compat.go shouldUseManualRangeMetricCompat (unwrap sum/avg/min/max) and metric_binary.go statsRateRangeEquals... | pkg/logql/range_vector.go:102-189 batchRangeVectorIterator (window (t-range, t]); range_vector.go:371-420 sumOverTime, avgOverTime, maxOv... | TestCompat_UnwrapOverTimeRangeWindows (test/e2e-compat, group core) over sum/avg/max/min x tumbling/sliding x logfmt/json | +| 37 | `semantics/unwrap-per-series-identity-raw-rows` — A function taken per series (avg, max, min over time, rate_counter) under sum or sum by uses the stream as the series (open) | open | explore-visible | — | — | L | internal/translator unwrapInnerGrouping (sum by over a per-series function) and the raw evaluators series key (collectRangeMetricSamples) | pkg/logql/evaluator.go:435-529 VectorAggEvaluator.Next; pkg/logql/log/metrics_extraction.go:150-178 labelSampleExtractor (series = stream... | TestCompat_UnwrapPerSeriesIdentity (test/e2e-compat, group core) over sum and sum by x avg/max/min/rate_counter, logfmt and json | +| 38 | `semantics/unwrap-quantile-sum-by-group-quantile` — sum by over quantile_over_time with unwrap adds the per-series quantiles instead of taking the group quantile, like Loki (open) | open | explore-visible | — | — | M | internal/translator unwrapInnerGrouping (an outer by moved into the quantile) and the raw evaluator of quantile_over_time | pkg/logql/range_vector.go:449-487 quantileOverTime; pkg/logql/syntax/extractor.go:21-60 (a range by groups the samples); pkg/logql/evalua... | TestCompat_UnwrapQuantileOuterSum (test/e2e-compat, group core) over sum by and sum without, logfmt and json, range and instant | +| 39 | `semantics/unwrap-quantile-ungrouped-sum-field-parse` — sum and sum without over quantile_over_time with unwrap answer a series, not an empty result, like Loki (open) | open | explore-visible | — | — | S | internal/proxy parseStatsCompatSpec (Field from the first "(" to the last ")") and the quantile route of shouldUseManualRangeMetricCompat | pkg/logql/range_vector.go:449-487 quantileOverTime; pkg/logql/evaluator.go:451-466 VectorAggEvaluator.Next (sum folds the per-series quan... | TestCompat_UnwrapQuantileOuterSum (test/e2e-compat, group core) over sum and sum without, logfmt and json, range and instant | +| 40 | `semantics/unwrap-range-last-point` — A grouped unwrap range metric misses the sample at `end` (open) | open | explore-visible | — | — | S | internal/proxy range-metric unwrap path (end bucket) | pkg/logql/range_vector.go step iteration start..end inclusive | TestCompat_UnwrapRangeEndSample (test/e2e-compat, group semantics) | +| 41 | `semantics/unwrap-sum-by-per-series-function-extraction-list` — sum by over max, min or avg over time with unwrap after an extraction list takes the maximum of the group, not the sum of the per-series ... | open | explore-visible | — | — | M | internal/translator unwrapInnerGrouping (sum by over a per-series function: the inner stats groups by the by labels only) | pkg/logql/evaluator.go:435-529 VectorAggEvaluator.Next; pkg/logql/log/metrics_extraction.go:150-178 (series = stream + parsed labels) | TestCompat_UnwrapSumByPerSeriesFunction (test/e2e-compat, group core) over sum by x avg/max/min, extraction list, range and instant | +| 42 | `semantics/without-comparison-inside-ignored` — sum without over a range function filtered by a comparison inside the aggregation ignores the filter (open) | open | explore-visible | — | — | M | internal/translator tryTranslateMetricQueryWithout (the comparison after the range aggregation is dropped) | pkg/logql/evaluator.go:451-466 VectorAggEvaluator.Next over the filtered vector of the comparison (vectorBinop) | TestWithoutShapesStillDifferFromLoki (internal/proxy, rate with the threshold, range and instant); an e2e test vs Loki in group semantics... | +| 43 | `semantics/without-empty-grouping-range-collapses` — sum without () over an over-time unwrap function on query_range answers one series, Loki keeps every series (open) | open | explore-visible | — | — | S | internal/translator extractWithoutLabels / withoutMarkerRE (a clause with no label records no fold); tryTranslateMetricQueryWithout | pkg/logql/evaluator.go:451-466 VectorAggEvaluator.Next (HashWithoutLabels with no names); pkg/logql/syntax/ast.go Grouping.Noop; pkg/logq... | TestWithoutShapesStillDifferFromLoki (internal/proxy, the series of the registered request); an e2e test vs Loki in group core when fixed | +| 44 | `semantics/without-instant-detected-level-unknown` — an instant without answer lacks the detected_level=unknown label Loki puts on a stream with no level (open) | open | explore-visible | — | — | S | internal/proxy instant stats answer labels (the stream-derived label set of an instant vector; the range path adds detected_level in buil... | detected_level is a label of every stream when log level discovery is on (unknown without a level) | TestCompat_UnwrapSumWithoutExtractionList compares the instant answers after dropping the label (test/e2e-compat); a parity case without ... | +| 45 | `semantics/without-nested-under-aggregation` — a without nested under an outer aggregation answers the wrong series and values (open) | open | explore-visible | — | — | M | internal/translator tryTranslateMetricQueryWithout (nested aggregations with a without clause); internal/proxy applyWithoutGrouping | pkg/logql/evaluator.go:451-466 VectorAggEvaluator.Next (the outer aggregation takes the vector the inner without produced) | TestWithoutShapesStillDifferFromLoki (internal/proxy, max and sum by over a without, range and instant), TestWithoutFoldIsLimitedToTheOpe... | +| 46 | `semantics/without-over-inner-aggregation-collapses` — sum without over an inner sum by or count_over_time aggregation answers one series, Loki keeps a series per remaining label set (open) | open | explore-visible | — | — | M | internal/translator tryTranslateMetricQueryWithout (an outer aggregation over an aggregation: the last stats pipe has no by) | pkg/logql/evaluator.go:451-466 VectorAggEvaluator.Next (the outer without groups the inner vector by every label but the excluded ones) | TestWithoutShapesStillDifferFromLoki (internal/proxy, rate and count_over_time, range and instant); an e2e test vs Loki in group core whe... | +| 47 | `drilldown/detected-field-values` — detected_field values differ: none for service_name, other values for numeric and id fields (open) | open | drilldown-visible | 33 | 3 | M | internal/proxy detected_field values (sampling window and stream-label fields) | pkg/querier detected field values (v3.7.7): values of the sampled lines | TestCompat_DetectedFieldValues (test/e2e-compat, group drilldown) | +| 48 | `semantics/detected-fields-extracted-suffix` — detected_fields omits Loki's level_extracted and version_extracted (open; service fields are owner-kept) | open | drilldown-visible | 15 | 2 | M | internal/proxy/label_handlers.go handleDetectedFields (parsed fields named like a stream label) | pkg/logql/log/parser.go:25 duplicateSuffix; detected fields come from the parsed labels of sampled entries | TestCompat_DetectedFieldsExtractedSuffix (test/e2e-compat, group drilldown) | +| 49 | `profiles/json-number-label-text` — JSON numbers become label values in a different text form ("14.0" vs "14") (open) | open | drilldown-visible | 12 | 6 | M | internal/proxy JSON number rendering of unpacked values (VictoriaLogs unpack_json normalises) | pkg/logql/log/parser.go readValue (raw number bytes) | TestCompat_JSONNumberLabelText (test/e2e-compat, group drilldown) | +| 50 | `severity/detected-level-filter-json-level` — `\| detected_level=` filters select different lines than Loki (JSON-level lines, stream level precedence) (open) | open | drilldown-visible | 12 | 8 | M | internal/translator/detected_level_logsql.go (filter predicate) vs internal/proxy/detected_level.go (response derivation) | pkg/distributor field detection (v3.7.7) stores detected_level as structured metadata; filter reads it | TestCompat_DetectedLevelFilterJSONLevel (test/e2e-compat, group drilldown) | +| 51 | `limits/drilldown-7d-breakdown-response-limit` — Metric queries fail on the 64 MiB buffered-response budget where Loki answers (open) | open | drilldown-visible | 8 | 4 | M | internal/proxy/range_metric_compat.go:767, execution_limits.go:11 (raw-row metric paths buffer rows; push down or stream) | pkg/logql engine series limit (max_query_series) with partial result for Drilldown | TestCompat_MetricBufferedBudget1h + TestE2E_Drilldown7dHighCardinalityBreakdown (test/e2e-compat, groups semantics and drilldown) | +| 52 | `quality/index-stats-streams-differ` — index/stats stream counts differ from Loki in both directions (open) | open | drilldown-visible | 6 | 2 | S | internal/proxy/label_handlers.go handleIndexStats (stream counting per window) | pkg/storage index stats (streams from the index for the window) | bench/parity rerun with the /series cross-check, then TestCompat_IndexStatsStreams (test/e2e-compat, group drilldown) | +| 53 | `drilldown/detected-fields-type-inference` — detected_fields infers a different type for some fields (open) | open | drilldown-visible | — | — | S | internal/proxy detected_fields type inference | pkg/querier detected fields type detection (v3.7.7): int before duration before string | TestCompat_DetectedFieldsTypes (test/e2e-compat, group drilldown) | +| 54 | `profiles/existence-filter-on-sanitized-key` — An existence filter on a sanitized key reads the stored dotted field, not the JSON line key (open) | open | drilldown-visible | — | — | M | internal/translator (label-filter key variants, parsed_key_variants.go); Drilldown single-field fast paths | pkg/logql/log/parser.go:141 sanitizeLabelKey on every extracted key; label filter reads the parsed label | TestCompat_ExistenceFilterSanitizedKey (test/e2e-compat, group semantics) | +| 55 | `quality/detected-metadata-proxy-cache` — detected_labels and detected_fields are served from proxy caches; Loki answers each request fresh (open) | open | drilldown-visible | — | — | S | internal/proxy/cache_keys.go:48,130 (detected_* cache TTL) + recent-tail freshness bypass | pkg/querier detected labels/fields: no results cache in v3.7.7 | TestE2E_DetectedMetadataFreshness (test/e2e-compat, group drilldown) | +| 56 | `quality/stats-volume-freshness` — index/stats and volume answers ending now are cached; Loki does not cache inside max_stats_cache_freshness (10m) (open) | open | drilldown-visible | — | — | S | internal/proxy/volume.go handleVolume, label_handlers.go handleIndexStats (cache freshness) | pkg/validation/limits.go:413-414 max_stats_cache_freshness default 10m | TestE2E_StatsVolumeFreshness (test/e2e-compat, group drilldown) | +| 57 | `semantics/parse-error-text-parity` — Invalid queries fail on both sides, with different error text or status (open) | open | grafana-datasource | 210 | 129 | L | internal/logql parser error messages (Loki's goyacc messages and positions) + VictoriaLogs error mapping | pkg/logql/syntax/parser.go ParseError (line, col) and syntax.y | TestCompat_ParseErrorText corpus (test/e2e-compat, bad_request_status_parity_test.go) | +| 58 | `semantics/proxy-rejects-valid-logql` — The proxy rejects LogQL Loki accepts: comments, negative and compound offsets, filters after a range, logfmt flags, ip() label filters (o... | open | api-only | 79 | 48 | M | internal/logql parser (lexer: comments, signed durations, flags) + translator | pkg/logql/syntax/lex.go:149 comments, :90-91 OpStrict/OpKeepEmpty; syntax.y offsetExpr; pkg/logql/log/ip.go | TestCompat_LogQLSurfaceAccepted (test/e2e-compat, group semantics), one subtest per family | +| 59 | `semantics/proxy-accepts-invalid-logql` — Queries Loki rejects with 400 are answered by the proxy (open) | open | api-only | 18 | 14 | S | internal/logql parser validation | pkg/logql/syntax/ast.go:1446,1579 grouping not allowed; duration parsing | TestCompat_InvalidLogQLRejected (test/e2e-compat, bad_request_status_parity_test.go) | +| 60 | `parsed-identity/regexp-partial-match-series-count` — A partially-matching \| regexp returns about 3x Loki's series count | open | api-only | 2 | 2 | M | internal/proxy regexp capture series identity (non-matching lines) | pkg/logql/log/parser.go RegexpParser (no capture label for a non-matching line) | TestCompat_RegexpPartialMatchSeries (test/e2e-compat, group semantics) | +| 61 | `profiles/json-extraction-list-missing-key` — A json extraction-list key absent from the line is dropped; Loki keeps it in the metric series (open) | open | api-only | 2 | 2 | S | internal/proxy stage-field exposure (extraction lists, metric path) | pkg/logql/log/parser.go:671-720 JSONExpressionParser.Process | TestCompat_JSONExtractionListMissingKey (test/e2e-compat, group semantics) | +| 62 | `semantics/unpack-non-string-values` — `\| unpack` extracts non-string JSON values that Loki ignores (open) | open | api-only | 2 | 2 | S | internal/translator unpack (string values only) | pkg/logql/log/parser.go:789-820 UnpackParser.unpack | TestCompat_UnpackStringValuesOnly (test/e2e-compat, group semantics) | +| 63 | `semantics/pattern-line-filter-invalid-captures` — A pattern line filter with consecutive or named captures is a 400 on Loki; the proxy answers it (open) | open | api-only | — | — | S | internal/translator/translator.go planPatternLineFilter (patterns are translated without validation) | pkg/logql/log/pattern/pattern.go:36-52 (ParseLineFilter, validateNoConsecutiveCaptures, validateNoNamedCaptures) | TestCompat_PatternLineFilterInvalidCaptures (test/e2e-compat, group core) | +| 64 | `semantics/detected-level-pipes-known-differences` — The detected_level pipes read level keys in a fixed order, do not decode logfmt like Loki and overwrite a detected_level stream label (open) | open | edge | — | — | M | internal/translator detected_level_logsql.go BuildDetectedLevelChain; internal/proxy native metric route | pkg/distributor/field_detection.go getLevelUsingJSONParser (document order, depth < 2), labels.go (detected_level_extracted) | an e2e on plain-selector probe lines once the route probes and falls back to a raw-row or keyed evaluation | +| 65 | `semantics/extracted-derived-labels-raw-evaluator` — service_name_extracted and detected_level_extracted metrics keep the raw-row evaluator (open) | open | edge | — | — | S | internal/proxy orderedJSONPushdownLabelOK (service_name, detected_level) | pkg/logql/log/parser.go duplicateSuffix; service_name set by discover_service_name at ingest | TestCompat_ExtractedLabelMetricPushdown rows for service_name_extracted (test/e2e-compat, group core) | +| 66 | `semantics/extracted-metric-known-differences` — Two name_extracted differences against Loki remain in the metric pushdown and the raw evaluator (open) | open | edge | — | — | M | internal/proxy raw evaluator parsed-label naming (processWithStreamLabels) and translator.ExtractedLabelPipes | pkg/logql/log/parser.go duplicateSuffix and ParserHint.Extracted (a key an earlier extraction produced is skipped) | TestCompat_ExtractedBothKeysInLineOrder (test/e2e-compat, group core) | +| 67 | `semantics/unwrap-conversion-error-parser-divergence` — Where VictoriaLogs' logfmt parser reads another value than Loki's, an unwrap misses Loki's SampleExtractionErr or drops samples Loki keep... | open | edge | — | — | L | internal/translator logfmt unpacking (unpack_logfmt keeps the last value); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:379-438 LogfmtParser.Process; pkg/logql/log/logfmt/decode.go | TestUnwrapConversionErrorDifferential counts the first case; TestUnwrapLokiLabels pins the decoder; TestCompat_UnwrapConversionErrorParit... | +| 68 | `semantics/unwrap-conversion-error-postfilter-forms` — An unwrap whose post filter keeps only the error, compares a label, or combines filters with and/or answers 200 where Loki may fail with ... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorPostFilters; the translator's post-filter placement | pkg/logql/log/label_filter.go:90-100 BinaryLabelFilter.Process, 306-340 NumericLabelFilter.Process, 418-424 labelValue | TestUnwrapErrorProbePlans and TestUnwrapConversionErrorDifferential (internal/proxy) assert the detection stays off for these forms; flip... | +| 69 | `semantics/unwrap-conversion-error-stored-value-overwritten` — An unwrap over a label stored as structured metadata and also parsed from the line misses Loki's SampleExtractionErr (open) | open | edge | — | — | M | internal/translator parsed-key collision with stored fields (unpack keep_original_fields); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:151,809 duplicateSuffix | TestUnwrapConversionErrorDifferential counts the first case; TestCompat_UnwrapConversionErrorParity asserts the 200 side | +| 70 | `semantics/unwrap-conversion-error-undetected-shapes` — A few query shapes answer 200 where Loki fails with SampleExtractionErr, because no VictoriaLogs query of theirs carries the conversion c... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorProbes (opaque expressions), unwrapCountingQuery (pipes after stats), unwrapLokiLabe... | pkg/logql/evaluator.go:726-733 | TestCompat_UnwrapConversionErrorParity open/* asserts Loki's 400 and skips naming this case (absent_over_time, label_replace); TestUnwrap... | +| 71 | `semantics/unwrap-gate-parsefloat-divergences` — The unwrap sample gate drops a few values Loki's conversions accept or reject (open) | open | edge | — | — | S | internal/translator/unwrap_gate.go (UnwrapNumberPattern, UnwrapDurationPattern, UnwrapBytesPattern) | pkg/logql/log/metrics_extraction.go:316-338 convertFloat, convertDuration, convertBytes | TestUnwrapNumberPatternMatchesParseFloat (internal/translator) pins the known differences | +| 72 | `semantics/unwrap-sum-without-extracted-label-collision` — a parsed key that is also a stream label is not renamed to key_extracted after a without fold (open) | open | edge | — | — | M | internal/proxy withParsedGroupLabels (a parsed group key that equals a stream label of the series); internal/translator parsedSeriesLabels | pkg/logql/log/labels.go and parser.go (duplicateSuffix "_extracted") | TestSumWithoutExtractedLabelCollision (internal/proxy, emulator); an e2e test vs Loki in group semantics when fixed | +| 73 | `metric-identity/structured-metadata-splits-series` — Loki splits a stream's metric series per structured-metadata set; the proxy keeps the stream (owner-accepted deviation) | owner-kept | explore-visible | 173 | 37 | L | internal/translator series identity (stats by stream); VictoriaLogs cannot group by a row's remaining fields | pkg/logql/log/metrics_extraction.go:102-120 builder.Add(StructuredMetadataLabel) -> GroupedLabels | differential run cluster count (bench/parity) | +| 74 | `parsed-identity/dynamic-key-parsers` — `\| json` and `\| logfmt` metric series keep the stream identity by default (owner-accepted deviation, opt-in flag) | owner-kept | explore-visible | 95 | 29 | L | internal/translator parser series identity; -exact-parser-series-identity | pkg/logql/log/parser.go label extraction into the sample labels | differential run cluster count (bench/parity) | +| 75 | `profiles/label-filter-on-line-field` — A label filter on a line key no earlier stage exposes matches like Loki | owner-kept | explore-visible | — | — | M | internal/proxy label-filter refill (-label-filter-refill-max-pages); metric queries on keys no stage exposes | pkg/logql/log/label_filter.go (filter reads only labels the pipeline set) | TestCompat_LabelFilterOnLineField (exists, profile tests) | +| 76 | `profiles/detected-fields-no-extracted-suffix` — detected_fields lists service / service.name without Loki's _extracted suffix (owner decision) | owner-kept | drilldown-visible | — | — | S | internal/proxy/label_handlers.go handleDetectedFields | pkg/logql/log/parser.go:25 duplicateSuffix | bench/parity diff rule (documented facet, service fields only) | +| 77 | `versions/stats-bucket-label-v1.45` — Stats-bucket metrics put a line on a bucket edge one bucket late on VictoriaLogs before v1.45 (unsupported versions) | owner-kept | edge | — | — | M | stats-bucket relabel (internal/proxy range_metric_compat.go setSlidingStatsRangeParams / slidingStatsBucket; sliding_window_stats.go) | pkg/logql range vector (T-range, T] | TestRangeMetricCompatibilityJSONFilterPushdown on every supported VictoriaLogs version (compat-vl vl-matrix job); TestSlidingRangeMetric_... | +| 78 | `profiles/json-line-stored-without-line` — A JSON line pushed without _msg is stored without its line (documented) | documented | explore-visible | — | — | L | client push configuration (disable_message_parsing=1); docs/KNOWN_ISSUES.md | pkg/distributor push keeps the line verbatim | cases/profiles/stage-field-exposure with disable_message_parsing pushes | +| 79 | `drilldown/detected-labels-sampled-cardinality` — detected_labels cardinality is sampled from the window's last minutes; Loki counts every stream its ingesters hold (documented) | documented | drilldown-visible | 20 | 2 | M | internal/proxy/patterns.go handleDetectedLabels | pkg/ingester/instance.go LabelsWithValues | bench/parity diff rule (documented facet) | +| 80 | `profiles/volume-detected-level-extension` — index/volume answers targetLabels=detected_level; Loki returns no volume for it (documented proxy extension) | documented | drilldown-visible | 17 | 2 | S | internal/proxy/volume.go | pkg/storage volume index (stream labels only) | differential run row volume by detected_level | +| 81 | `drilldown/patterns-mined-differently` — Patterns come from different miners on each side (documented) | documented | drilldown-visible | 15 | 3 | L | internal/proxy/patterns.go | pkg/pattern (ingester-side mining) | bench/visual history-dependent comparison (exists) | +| 82 | `profiles/volume-bytes-accounting` — Volume values count line bytes; Loki counts chunk bytes with structured metadata (documented) | documented | drilldown-visible | 4 | 3 | L | internal/proxy/volume.go; docs/KNOWN_ISSUES.md | pkg/storage volume index (chunk bytes incl. structured metadata) | bench/parity diff rule (documented facet) | +| 83 | `limits/drilldown-breakdown-keeps-busiest` — Over max_query_series a Drilldown breakdown keeps the busiest series; Loki keeps the first it meets (documented) | documented | drilldown-visible | 1 | 1 | S | internal/proxy series limit ranking | pkg/logql engine series limit | cases/limits/drilldown-partial-with-warning (exists) | +| 84 | `quality/index-stats-bytes-and-chunks` — index/stats bytes and chunks have no VictoriaLogs equivalent (documented) | documented | drilldown-visible | — | — | L | internal/proxy/label_handlers.go handleIndexStats; docs/KNOWN_ISSUES.md | pkg/storage index stats from chunk refs (bytes, chunks) | bench/parity diff rule (documented facet) | +| 85 | `profiles/grafana-stats-error-as-warning` — A failed stats query from Grafana answers 200 with warnings; Loki returns the error (documented) | documented | grafana-datasource | — | — | S | internal/proxy Grafana-sourced stats error handling | pkg/querier/queryrange error propagation (no warnings conversion) | TestCompat_GrafanaStatsErrorStatus (test/e2e-compat, fault injection) | +| 86 | `profiles/metadata-default-lookback` — Label endpoints without start/end default to a different lookback than Loki's 1h (documented) | documented | api-only | — | — | S | internal/config flag default -metadata-default-lookback | pkg/loghttp/params.go defaultSince = 1h | TestCompat_MetadataDefaultLookback (test/e2e-compat, group core) | +| 87 | `profiles/multi-tenant-volume-merge` — Multi-tenant volume returns per-tenant series with __tenant_id__; Loki sums the tenants' volumes (documented) | documented | api-only | — | — | S | internal/proxy/volume.go multi-tenant merge | pkg/querier/queryrange volume merge | TestCompat_MultiTenantVolume (test/e2e-compat, group tail-multitenancy) | +| 88 | `semantics/count-values-unsupported` — count_values() is rejected with 400 where Loki answers (documented) | documented | api-only | — | — | M | internal/translator + proxy-side post-aggregation of a stats matrix | pkg/logql/syntax OpTypeCountValues; vector evaluator | TestCompat_CountValues (test/e2e-compat, group semantics) | +| 89 | `semantics/eager-pipeline-validation` — Invalid pipeline stages fail with 400 regardless of data; Loki answers 200 empty over a range with no data (documented) | documented | edge | — | — | S | internal/translator validation | pkg/logql/log/fmt.go template build at pipeline construction | TestCompat_EagerValidationEmptyRange (test/e2e-compat, group semantics) | +| 90 | `semantics/ip-filter-precision` — ip() filters are approximated with regular expressions (documented) | documented | edge | — | — | M | internal/proxy/postprocess.go:84 ip filtering | pkg/logql/log/ip.go | TestCompat_IPFilterForms (test/e2e-compat, group semantics) | +| 91 | `drilldown/plugin-legend-sync-race` — Logs Drilldown crashes a breakdown page when a query answers before the panel module loads (upstream, not the proxy) | upstream | drilldown-visible | — | — | S | none (grafana-lokiexplore-app) | n/a (client race) | bench/visual capture waits for the panel module (already) | +| 92 | `quality/index-stats-loki-chunk-accounting` — Loki's index/stats counts chunks it holds twice; the proxy's counts equal Loki's own count_over_time (upstream) | upstream | drilldown-visible | — | — | S | none (Loki chunk index accounting) | pkg/storage stats from chunk refs (flushed and retained chunks both counted) | bench/parity cross-check (run.py loki_counts_itself) | ## Fixed, awaiting the next run @@ -122,6 +123,7 @@ Last run: 2026-10-05T20:16:15Z on `fcaccfcc`, window [1791220800, 1791224400]. | `semantics/line-filter-or-alternatives` — Line filters with `or` alternatives match like Loki | #669 | 6 | 42 | | `semantics/plain-name-after-parser-reads-stream-label` — A filter or grouping on a name a parser also reads reads the parsed value; Loki reads the stream label (fixed) | #676 | 4 | 6 | | `semantics/pattern-line-filter` — An empty pattern line filter (`\|> ""`, `!> ""`) matches only the empty line, like Loki (fixed) | #743 | 2 | 2 | +| `semantics/json-error-filter-metric-split-pushdown` — A `\| json \| __error__=""` metric counts Loki's accepted lines from VictoriaLogs stats, reading only the doubtful lines raw (fixed) | #758 | 0 | 0 | | `semantics/json-extracted-metric-pushdown` — A metric filtered or grouped on a name_extracted label after `\| json` is answered from VictoriaLogs stats (fixed) | #690 | 0 | 0 | | `semantics/json-metric-detected-level-derivation` — sum by (detected_level) over a JSON pipeline follows the level Loki's ingest gave each line, by level key in document order, by keyword s... | #760 | 0 | 0 | | `semantics/labels-end-inclusive-like-loki` — /labels and /label/{name}/values include data at the request's end, so a zero-width request (Grafana "Show context") lists the row's labe... | #703 | 0 | 0 | diff --git a/conformance/reports/performance.md b/conformance/reports/performance.md index 9fbeedd6c..750fbfe8a 100644 --- a/conformance/reports/performance.md +++ b/conformance/reports/performance.md @@ -13,7 +13,7 @@ without counters are older and their Loki cold time is unverified. A query two s the first set's computed Loki cold (`loki_cold_reused_from`). Warm and refresh timings are compared with the same noise rules; a row Loki's cache answered is excluded from all three. -Runs: pr624-final-drilldown-breakdown-exact (2026-09-30), backfilled-empty-buckets-metadata-freshness (2026-10-01), backfilled-empty-buckets-metadata-inventory (2026-10-01), plain-name-control (2026-10-06), plain-name-drilldown-breakdown-exact (2026-10-06), unwrap-validity-24h (2026-10-06), unwraperr-conversion (2026-10-08), unwrapsum-long (2026-10-11), unwrapwithout-control-long (2026-10-17), unwrapwithout-long (2026-10-17), daily-control (2026-10-22), daily-drilldown-breakdown-exact (2026-10-22), daily-extracted-pushdown (2026-10-22), daily-extracted-suffix (2026-10-22), daily-json-detected-level (2026-10-22), daily-json-filter-pushdown (2026-10-22), daily-loki-profile-otel-metadata (2026-10-22), daily-metadata-freshness (2026-10-22), daily-metadata-inventory (2026-10-22), daily-metric-latency-gap (2026-10-22), daily-parenthesized-range-or-filters (2026-10-22), daily-pattern-line-filter (2026-10-22), daily-pattern-line-filter-anchoring (2026-10-22), daily-plain-name-after-parser (2026-10-22), daily-stage-field-exposure (2026-10-22), daily-unwrap-conversion-error (2026-10-22), daily-unwrap-duration-seconds (2026-10-22), daily-unwrap-sum-without (2026-10-22), daily-unwrap-ungrouped-sum (2026-10-22), daily-unwrap-validity (2026-10-22). +Runs: pr624-final-drilldown-breakdown-exact (2026-09-30), backfilled-empty-buckets-metadata-freshness (2026-10-01), backfilled-empty-buckets-metadata-inventory (2026-10-01), plain-name-control (2026-10-06), plain-name-drilldown-breakdown-exact (2026-10-06), unwrap-validity-24h (2026-10-06), unwraperr-conversion (2026-10-08), unwrapsum-long (2026-10-11), unwrapwithout-control-long (2026-10-17), unwrapwithout-long (2026-10-17), daily-control (2026-10-22), daily-drilldown-breakdown-exact (2026-10-22), daily-extracted-pushdown (2026-10-22), daily-extracted-suffix (2026-10-22), daily-json-detected-level (2026-10-22), daily-json-filter-pushdown (2026-10-22), daily-loki-profile-otel-metadata (2026-10-22), daily-metadata-freshness (2026-10-22), daily-metadata-inventory (2026-10-22), daily-metric-latency-gap (2026-10-22), daily-parenthesized-range-or-filters (2026-10-22), daily-pattern-line-filter (2026-10-22), daily-pattern-line-filter-anchoring (2026-10-22), daily-plain-name-after-parser (2026-10-22), daily-stage-field-exposure (2026-10-22), daily-unwrap-conversion-error (2026-10-22), daily-unwrap-duration-seconds (2026-10-22), daily-unwrap-sum-without (2026-10-22), daily-unwrap-ungrouped-sum (2026-10-22), daily-unwrap-validity (2026-10-22), json-error-split-1h (2026-10-22), json-error-split-long (2026-10-22). ## Slower than Loki on first load @@ -101,12 +101,12 @@ None measured. | `loki_api_v1_label_name_values` | 6 × 4 ranges | 24 same | worst 0.71s / 0.00s (24h) | 0.08s / 0.09s | differs | | `loki_api_v1_labels` | 6 × 4 ranges | 24 same | worst 0.73s / 0.00s (24h) | 0.04s / 0.04s | differs | | `loki_api_v1_query` | 14 × 1 ranges | 14 same | worst 0.20s / 0.19s (instant) | 0.51s / 0.48s | same | -| `loki_api_v1_query_range` | 137 × 6 ranges | 8 faster, 4 fixed, 437 same, 1 slower | worst 22.48s / 2.63s (24h) | 4.70s / 0.04s | differs; 49 slower than Loki cold | +| `loki_api_v1_query_range` | 142 × 6 ranges | 15 faster, 12 fixed, 437 same, 1 slower | worst 22.48s / 2.63s (24h) | 4.70s / 0.04s | differs; 49 slower than Loki cold | | `metric-series-identity` | 19 × 6 ranges | 6 faster, 2 fixed, 50 same | worst 8.55s / 8.62s (7d) | — / — | differs; 1 slower than Loki cold | | `operator-sum` | 3 × 4 ranges | 12 same | worst 0.44s / 0.43s (24h) | 5.40s / 0.08s | same | | `operator-topk` | 1 × 4 ranges | 4 same | worst 0.48s / 0.25s (24h) | 5.52s / 0.08s | differs | | `parsed-label-series-identity` | 6 × 4 ranges | 24 same | worst 7.93s / 0.40s (24h) | 5.04s / 0.07s | same; 3 slower than Loki cold | -| `parser-error-and-label-collision` | 32 × 6 ranges | 2 faster, 2 fixed, 98 same | worst 22.48s / 2.63s (24h) | 4.70s / 0.04s | differs; 13 slower than Loki cold | +| `parser-error-and-label-collision` | 37 × 6 ranges | 9 faster, 10 fixed, 98 same | worst 22.48s / 2.63s (24h) | 4.70s / 0.04s | differs; 13 slower than Loki cold | | `parser-json` | 8 × 6 ranges | 31 same | worst **502** / **502** (7d) | 155.58s / 1.36s | differs | | `parser-logfmt` | 5 × 5 ranges | 17 same | worst 5.67s / 5.33s (24h) | 8.19s / 0.66s | differs; 1 slower than Loki cold | | `profiles/dotted-name-parse-error` | 2 × 4 ranges | 8 same | worst **400** / **400** (1h) | 0.00s / 0.00s | same | @@ -118,6 +118,7 @@ None measured. | `range_function-count-over-time` | 1 × 4 ranges | 4 same | worst 0.17s / 0.08s (6h) | — / — | same | | `range_function-rate` | 3 × 5 ranges | 9 same | worst 0.48s / 0.25s (24h) | 5.52s / 0.08s | differs | | `semantics/extracted-suffix-collision` | 13 × 5 ranges | 30 same | worst 22.48s / 2.63s (24h) | 4.70s / 0.04s | same; 4 slower than Loki cold | +| `semantics/json-error-filter-metric-split-pushdown` | 5 × 3 ranges | 7 faster, 8 fixed | worst 3.14s / 2.01s (24h) | — / — | same | | `semantics/json-extracted-metric-pushdown` | 5 × 3 ranges | 15 same | worst 22.48s / 2.63s (24h) | 4.70s / 0.04s | same; 3 slower than Loki cold | | `semantics/json-filter-pushdown-translated-label` | 1 × 4 ranges | 4 same | worst 8.05s / 0.37s (24h) | 4.78s / 0.06s | same; 3 slower than Loki cold | | `semantics/json-filter-pushdown-underscore-label` | 3 × 4 ranges | 12 same | worst 8.05s / 0.37s (24h) | 4.78s / 0.06s | same; 6 slower than Loki cold | diff --git a/internal/proxy/ordered_json_detected_level_chain_test.go b/internal/proxy/ordered_json_detected_level_chain_test.go index bee1abb4b..80048de0d 100644 --- a/internal/proxy/ordered_json_detected_level_chain_test.go +++ b/internal/proxy/ordered_json_detected_level_chain_test.go @@ -217,6 +217,9 @@ func chainStart(pipe string) bool { if strings.HasPrefix(pipe, "copy ") || strings.HasPrefix(pipe, "filter (__dl_d:*") { return false // the stored-level copies of the cheap plan and its filters } + if strings.HasPrefix(pipe, "format if (") && (strings.HasSuffix(pipe, `" as __dl_d`) || strings.HasSuffix(pipe, `" as __dl_l`)) { + return false // a split plan blanks the stored-level copies of the lines it reads as raw rows + } return strings.Contains(pipe, "__dl") || strings.Contains(pipe, `result_prefix "__j_"`) } diff --git a/internal/proxy/ordered_json_filter_pushdown_test.go b/internal/proxy/ordered_json_filter_pushdown_test.go index 7663e5bf7..c27690f3f 100644 --- a/internal/proxy/ordered_json_filter_pushdown_test.go +++ b/internal/proxy/ordered_json_filter_pushdown_test.go @@ -15,6 +15,8 @@ import ( "sync" "testing" "time" + "unicode" + "unicode/utf8" "github.com/ReliablyObserve/Loki-VL-proxy/internal/translator" ) @@ -42,8 +44,6 @@ type pushdownFakeVL struct { stats []string guards []string raw int - // rawQueries are the queries of the raw-row fetches. - rawQueries []string // presence answers the level presence probe from the rows; otherwise the // probe gets no counts (the proxy then keeps the full chain). presenceFail // makes it a backend error, presenceLine replaces the honest answer. @@ -53,12 +53,30 @@ type pushdownFakeVL struct { presenceCalls []string // presenceGate, when set, holds every presence probe until it is closed. presenceGate chan struct{} + // rawQueries holds the query and time range of every raw-row request. + rawQueries []fakeRawQuery + // jsonUnpack replaces vlUnpackJSONFields (vlFastjsonUnpackFields mirrors + // VictoriaLogs' parser exactly). + jsonUnpack func(string) map[string]string + // onStats, when set, runs at the start of every stats_query_range request. + onStats func(r *http.Request, query string) +} + +type fakeRawQuery struct { + query string + start, end int64 } var pushdownLiteralFormatRE = regexp.MustCompile("^format if \\((.+)\\) \"([^<\"]+)\" as (\\S+)$") var pushdownFormatRE = regexp.MustCompile("^format if \\((.+)\\) \"<([^>]+)>\" as (\\S+)$") +// pushdownFormatBlankRE matches format if () "" as dst. +var pushdownFormatBlankRE = regexp.MustCompile("^format if \\((.+)\\) \"\" as (\\S+)$") + +// pushdownFormatMarkRE matches format if () "1" as dst. +var pushdownFormatMarkRE = regexp.MustCompile("^format if \\((.+)\\) \"1\" as (\\S+)$") + // vlUnpackJSONFields mirrors unpack_json: a line that is one JSON object after // trimming whitespace yields its keys raw, nested objects flattened with a // dot, arrays kept as their JSON text; anything else yields nothing. @@ -244,6 +262,49 @@ func (f *fakeLogsQLFilter) quoted() string { } func (f *fakeLogsQLFilter) fieldFilter() bool { + f.skipSpace() + if strings.HasPrefix(f.src[f.pos:], "{") { + // A stream filter {label=~"re"}: the row's _stream holds the label + // with a value the regexp matches in full. + f.pos++ + var name string + if strings.HasPrefix(f.src[f.pos:], `"`) { + name = f.quoted() + } else { + end := f.pos + strings.IndexAny(f.src[f.pos:], "=!") + name, f.pos = f.src[f.pos:end], end + } + op := "" + for _, candidate := range []string{"=~", "!=", "="} { + if strings.HasPrefix(f.src[f.pos:], candidate) { + op = candidate + break + } + } + if op == "" { + f.t.Errorf("fake VL: unsupported stream filter at %d in %q", f.pos, f.src) + return false + } + f.pos += len(op) + pattern := f.quoted() + if !strings.HasPrefix(f.src[f.pos:], "}") { + f.t.Errorf("fake VL: expected } at %d in %q", f.pos, f.src) + return false + } + f.pos++ + // A label the stream lacks reads as the empty string. + value := "" + if m := fakeRegexp(`[{,]` + regexp.QuoteMeta(name) + `="((?:[^"\\]|\\.)*)"`).FindStringSubmatch(f.vals["_stream"]); m != nil { + value, _ = strconv.Unquote(`"` + m[1] + `"`) + } + switch op { + case "=": + return value == pattern + case "!=": + return value != pattern + } + return fakeRegexp("^(?:" + pattern + ")$").MatchString(value) + } name := f.quoted() if !strings.HasPrefix(f.src[f.pos:], ":") { f.t.Errorf("fake VL: expected : after %q at %d in %q", name, f.pos, f.src) @@ -283,7 +344,7 @@ func (f *fakeLogsQLFilter) fieldFilter() bool { return value == f.quoted() case strings.HasPrefix(f.src[f.pos:], "~"): f.pos++ - return regexp.MustCompile(f.quoted()).MatchString(value) + return fakeRegexp(f.quoted()).MatchString(value) case strings.HasPrefix(f.src[f.pos:], "!"): f.pos++ return value != f.quoted() @@ -310,15 +371,95 @@ func (f *fakeLogsQLFilter) fieldFilter() bool { // A word or phrase, or with a trailing * a word prefix. VictoriaLogs' // tokenizer takes every Unicode letter and digit and the underscore // as word runes (isTokenRune). + // The boundary is checked only on a side where the phrase starts or + // ends with a word rune (getPhrasePos): a phrase of punctuation is a + // substring. word := f.quoted() + isWordRune := func(r rune) bool { return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r) } + before := "" + if first, _ := utf8.DecodeRuneInString(word); isWordRune(first) { + before = `(^|[^\p{L}\p{Nd}_])` + } if strings.HasPrefix(f.src[f.pos:], "*") { f.pos++ - return regexp.MustCompile(`(^|[^\p{L}\p{Nd}_])` + regexp.QuoteMeta(word)).MatchString(value) + return fakeRegexp(before + regexp.QuoteMeta(word)).MatchString(value) } return vlPhrasePos(value, word) >= 0 } } +var fakeRegexps sync.Map + +// fakeRegexp compiles a pattern once for every evaluation of the fake. +func fakeRegexp(pattern string) *regexp.Regexp { + if re, ok := fakeRegexps.Load(pattern); ok { + return re.(*regexp.Regexp) + } + re := regexp.MustCompile(pattern) + fakeRegexps.Store(pattern, re) + return re +} + +// fakeStatsFunc is one `fn() [if (cond)] as name` of a stats pipe. +type fakeStatsFunc struct { + name, cond string + sumLen bool +} + +// fakeStatsFuncs parses the functions of a stats pipe: count() and +// sum_len(_msg), each with an optional if (...) filter. +func fakeStatsFuncs(t testing.TB, statsPipe string) []fakeStatsFunc { + rest := strings.TrimPrefix(statsPipe, "| stats ") + if strings.HasPrefix(rest, "by (") { + rest = rest[strings.IndexByte(rest, ')')+2:] + } + var funcs []fakeStatsFunc + for rest != "" { + var fn fakeStatsFunc + switch { + case strings.HasPrefix(rest, "count() "): + rest = strings.TrimPrefix(rest, "count() ") + case strings.HasPrefix(rest, "sum_len(_msg) "): + fn.sumLen = true + rest = strings.TrimPrefix(rest, "sum_len(_msg) ") + default: + t.Errorf("fake VL: unsupported stats function in %q", statsPipe) + return nil + } + if strings.HasPrefix(rest, "if (") { + depth, end := 0, 3 + var quote byte + scan: + for ; end < len(rest); end++ { + c := rest[end] + switch { + case quote != 0: + switch c { + case '\\': + end++ + case quote: + quote = 0 + } + case c == '"': + quote = c + case c == '(': + depth++ + case c == ')': + if depth--; depth == 0 { + break scan + } + } + } + fn.cond = rest[4:end] + rest = rest[end+2:] + } + rest = strings.TrimPrefix(rest, "as ") + fn.name, rest, _ = strings.Cut(rest, ", ") + funcs = append(funcs, fn) + } + return funcs +} + func fakeLogsQLMatch(t testing.TB, expr string, values map[string]string) bool { f := &fakeLogsQLFilter{t: t, src: expr, vals: values} result := f.expr() @@ -381,11 +522,20 @@ func (f *pushdownFakeVL) applyPipes(t testing.TB, query string, row pushdownRow) } switch { case strings.HasPrefix(pipe, "unpack_json") || strings.HasPrefix(pipe, "unpack_logfmt"): + source := "_msg" + if _, args, found := strings.Cut(pipe, " "); found { + if rest, ok := strings.CutPrefix(args, "from "); ok { + source, _, _ = strings.Cut(rest, " ") + } + } var unpacked map[string]string - if strings.HasPrefix(pipe, "unpack_json") { - unpacked = vlUnpackJSONFields(values["_msg"]) - } else { - unpacked = slidingLogfmtFields(values["_msg"]) + switch { + case strings.HasPrefix(pipe, "unpack_json") && f.jsonUnpack != nil: + unpacked = f.jsonUnpack(values[source]) + case strings.HasPrefix(pipe, "unpack_json"): + unpacked = vlUnpackJSONFields(values[source]) + default: + unpacked = slidingLogfmtFields(values[source]) } if i := strings.Index(pipe, " if ("); i >= 0 && i < strings.Index(pipe, "fields (") { // unpack_json if (cond) ...: rows failing the condition are not parsed. @@ -419,6 +569,41 @@ func (f *pushdownFakeVL) applyPipes(t testing.TB, query string, row pushdownRow) values[prefix+field] = "" } } + case strings.HasPrefix(pipe, "format \"<") && strings.HasSuffix(pipe, " keep_original_fields"): + // format "" as dst keep_original_fields: dst takes src where dst is empty. + src, dst, _ := strings.Cut(strings.TrimSuffix(strings.TrimPrefix(pipe, "format \"<"), " keep_original_fields"), ">\" as ") + if values[dst] == "" { + values[dst] = values[src] + } + case strings.HasPrefix(pipe, "format \"") && strings.HasSuffix(pipe, " keep_original_fields"): + value, dst, _ := strings.Cut(strings.TrimSuffix(strings.TrimPrefix(pipe, "format \""), " keep_original_fields"), "\" as ") + if values[dst] == "" { + values[dst] = value + } + case strings.HasPrefix(pipe, "coalesce("): + // coalesce(a, b) default "v" as dst: the first non-empty field, else v. + list, rest, _ := strings.Cut(strings.TrimPrefix(pipe, "coalesce("), ") default ") + fallback, dst, _ := strings.Cut(rest, " as ") + value, _ := strconv.Unquote(fallback) + for _, field := range strings.Split(list, ", ") { + if v := values[strings.Trim(field, "\"`")]; v != "" { + value = v + break + } + } + values[dst] = value + case strings.HasPrefix(pipe, "format if (") && pushdownFormatBlankRE.MatchString(pipe): + // format if () "" as dst: dst is empty where the filter holds. + m := pushdownFormatBlankRE.FindStringSubmatch(pipe) + if fakeLogsQLMatch(t, m[1], values) { + values[m[2]] = "" + } + case strings.HasPrefix(pipe, "format if (") && pushdownFormatMarkRE.MatchString(pipe): + // format if () "1" as dst: dst is 1 where the filter holds. + m := pushdownFormatMarkRE.FindStringSubmatch(pipe) + if fakeLogsQLMatch(t, m[1], values) { + values[m[2]] = "1" + } case strings.HasPrefix(pipe, "format if ("): if lit := pushdownLiteralFormatRE.FindStringSubmatch(pipe); lit != nil { // format if () "" as dst: dst is the text where the filter holds. @@ -466,6 +651,17 @@ func (f *pushdownFakeVL) applyPipes(t testing.TB, query string, row pushdownRow) if !fakeLogsQLMatch(t, strings.TrimPrefix(pipe, "filter "), values) { return values, false } + case strings.HasPrefix(pipe, "fields "): + kept := map[string]string{} + for _, field := range strings.Split(strings.TrimPrefix(pipe, "fields "), ", ") { + field = strings.Trim(field, "`") + for name, value := range values { + if prefix, ok := strings.CutSuffix(field, "*"); (ok && strings.HasPrefix(name, prefix)) || name == field { + kept[name] = value + } + } + } + values = kept case strings.HasPrefix(pipe, "stats "), strings.HasPrefix(pipe, "limit "): default: t.Errorf("fake VL: unsupported pipe %q", pipe) @@ -497,6 +693,9 @@ func newPushdownFakeVL(t testing.TB, rows []pushdownRow, stored func(string) str fake.mu.Lock() fake.stats = append(fake.stats, query) fake.mu.Unlock() + if fake.onStats != nil { + fake.onStats(r, query) + } start, end, step, offset, ok := slidingBucketParams(t, r) if !ok { return @@ -506,7 +705,7 @@ func newPushdownFakeVL(t testing.TB, rows []pushdownRow, stored func(string) str if !strings.Contains(statsPipe, "stats by (") { groupBy = nil } - bytesMetric := strings.Contains(statsPipe, "sum_len(_msg) as c") + funcs := fakeStatsFuncs(t, statsPipe) type point struct { name string labels string @@ -535,11 +734,16 @@ func newPushdownFakeVL(t testing.TB, rows []pushdownRow, stored func(string) str key := canonicalLabelsKey(labels) labelSets[key] = labels bucket := vlTruncate(ts, step, offset) - if bytesMetric { - add(point{"c", key}, bucket, float64(len(row.msg))) - add(point{"__sample_count", key}, bucket, 1) - } else { - add(point{"c", key}, bucket, 1) + for _, fn := range funcs { + if fn.cond != "" && !fakeLogsQLMatch(t, fn.cond, values) { + add(point{fn.name, key}, bucket, 0) // every stats column is emitted for every group row, as logsql.go writes them + continue + } + if fn.sumLen { + add(point{fn.name, key}, bucket, float64(len(values["_msg"]))) + } else { + add(point{fn.name, key}, bucket, 1) + } } } keys := make([]point, 0, len(points)) @@ -612,7 +816,7 @@ func newPushdownFakeVL(t testing.TB, rows []pushdownRow, stored func(string) str fake.guards = append(fake.guards, query) } else { fake.raw++ - fake.rawQueries = append(fake.rawQueries, query) + fake.rawQueries = append(fake.rawQueries, fakeRawQuery{query, start, end}) } fake.mu.Unlock() w.Header().Set("Content-Type", "application/x-ndjson") @@ -635,6 +839,20 @@ func newPushdownFakeVL(t testing.TB, rows []pushdownRow, stored func(string) str for k, v := range row.vl { encoded[k] = v } + if !guard && strings.Contains(query, " | filter ") { + // The raw rows of a split plan: what the pipes leave of + // each selected row. + values, kept := fake.applyPipes(t, query, row) + if !kept { + continue + } + encoded = map[string]string{"_time": encoded["_time"]} + for k, v := range values { + if v != "" { + encoded[k] = v + } + } + } line, _ := json.Marshal(encoded) _, _ = w.Write(append(line, '\n')) if guard { @@ -1112,7 +1330,8 @@ func TestOrderedJSONFilterPushdownEligibility(t *testing.T) { {`sum by (k8s_pod_name) (count_over_time({env="production"} | json | drop __error__ | k8s_pod_name!="" [1m]))`, true, []string{"k8s_pod_name"}, nil}, {`sum(count_over_time({env="production"} | json | drop __error__ [1m]))`, false, nil, nil}, {`sum by (__error__) (count_over_time({env="production"} | json | pipeline="x" [1m]))`, false, nil, nil}, - {`sum by (level) (count_over_time({env="production"} | json | __error__="" [1m]))`, false, nil, nil}, + // `__error__=""` keeps the lines the parser accepts: the split pushdown (ordered_json_valid_split.go). + {`sum by (level) (count_over_time({env="production"} | json | __error__="" [1m]))`, true, []string{"level"}, nil}, // name_extracted: the key name where the stream has the label name, the key name_extracted otherwise. {`sum by (level_extracted) (count_over_time({env="production"} | json | drop __error__ [1m]))`, true, []string{"level_extracted"}, nil}, {`sum by (level) (count_over_time({env="production"} | json | level_extracted!="" [1m]))`, true, []string{"level"}, []string{"level_extracted"}}, @@ -1476,7 +1695,7 @@ func TestOrderedJSONDetectedLevelFilterBeforeParserRawEvaluator(t *testing.T) { if fake.raw != 1 || len(fake.rawQueries) != 1 { t.Fatalf("expected the raw-row evaluator after the probe: raw=%d", fake.raw) } - raw := fake.rawQueries[0] + raw := fake.rawQueries[0].query if strings.Contains(raw, "unpack_logfmt") || !strings.HasPrefix(raw, `app:="api" | copy detected_level as __lr_d, level as __lr_l | filter (__lr_d:* OR -__lr_l:*`) || !strings.Contains(raw, "| delete __lr_d, __lr_l") { t.Fatalf("the raw fetch is the selector narrowed by the stored-level filter: %s", raw) } @@ -1761,7 +1980,7 @@ func TestOrderedJSONPipelineErrorMatchesLokiText(t *testing.T) { t.Fatalf("got %d %q\nwant 400 %q", code, msg, want) } fake.mu.Lock() - if fake.raw != 0 || len(fake.guards) != 1 || !strings.HasSuffix(fake.guards[0], ` | filter -_msg:~"^\\s*\\{" | limit 1`) { + if fake.raw != 0 || len(fake.guards) != 1 || !strings.HasSuffix(fake.guards[0], ` | filter -_msg:~"^[ \\t\\n\\r]*\\{" | limit 1`) { t.Fatalf("expected the error from one lookup without a raw scan: raw=%d guards=%q", fake.raw, fake.guards) } fake.mu.Unlock() diff --git a/internal/proxy/ordered_json_logs_volume_bench_test.go b/internal/proxy/ordered_json_logs_volume_bench_test.go index 4c31fbd05..5fbe0e2e0 100644 --- a/internal/proxy/ordered_json_logs_volume_bench_test.go +++ b/internal/proxy/ordered_json_logs_volume_bench_test.go @@ -15,6 +15,16 @@ import ( // BenchmarkOrderedJSONLogsVolume measures Grafana's `{...} | json` logs volume // query over one hour of lines (one every 200ms) against the fake backend. func BenchmarkOrderedJSONLogsVolume(b *testing.B) { + benchmarkOrderedJSONQuery(b, `sum by (level, detected_level) (count_over_time({app="api"} | json | drop __error__[1m]))`) +} + +// BenchmarkOrderedJSONRowEvaluator measures the row evaluator (a `without` +// grouping has no stats pushdown) over the same hour of lines. +func BenchmarkOrderedJSONRowEvaluator(b *testing.B) { + benchmarkOrderedJSONQuery(b, `sum without (n, counter, msg) (count_over_time({app="api"} | json | drop __error__[1m]))`) +} + +func benchmarkOrderedJSONQuery(b *testing.B, query string) { s0 := time.Unix(1700000400, 0).UTC() lines := make([]jsonVolumeLine, 0, 18000) for i := 0; i < 18000; i++ { @@ -28,7 +38,7 @@ func BenchmarkOrderedJSONLogsVolume(b *testing.B) { b.Cleanup(func() { _ = p.Shutdown(context.Background()) }) p.storeBackendVersion("v1.50.0", "v1.50.0") params := url.Values{ - "query": {`sum by (level, detected_level) (count_over_time({app="api"} | json | drop __error__[1m]))`}, + "query": {query}, "start": {strconv.FormatInt(s0.Add(time.Minute).Unix(), 10)}, "end": {strconv.FormatInt(s0.Add(time.Hour).Unix(), 10)}, "step": {"60"}, diff --git a/internal/proxy/ordered_json_metric.go b/internal/proxy/ordered_json_metric.go index 0ce7bfaa5..4d3d40253 100644 --- a/internal/proxy/ordered_json_metric.go +++ b/internal/proxy/ordered_json_metric.go @@ -108,6 +108,16 @@ type orderedJSONMetricPlan struct { // evaluator then gives every row the value Loki's ingest set // (withDerivedLevel) before any stage runs. derivesLevel bool + // validOnly reports a `| json | __error__=""` plan: only the lines Loki's + // parser accepts count, so the pushdown splits the lines VictoriaLogs can + // vouch for from the ones read as raw rows (ordered_json_valid_split.go). + validOnly bool + // groupServiceName reports a validOnly plan grouped by service_name, the + // stream label Loki assigns at ingest from the other stream labels (a + // parsed service_name key becomes service_name_extracted): VictoriaLogs + // derives it from the stream labels before any unpack + // (serviceNameFromStreamPipes). + groupServiceName bool } // handleOrderedJSONMetric uses the existing metric accumulators, but executes @@ -857,13 +867,28 @@ func (plan *orderedJSONMetricPlan) setStatsPushdown() { if !plan.aggregated || (plan.grouping != nil && plan.grouping.Without) { return } - filters, errorFilters, ok := plan.pushdownLabelFilters() + filters, errorFilters, validOnly, ok := plan.pushdownLabelFilters() if !ok { return } var fields []string + groupServiceName := false + for _, filter := range filters { + if validOnly && strings.HasPrefix(filter.Field, "_") { + return + } + } if plan.grouping != nil { for _, label := range plan.grouping.Labels { + if validOnly && strings.HasPrefix(label, "_") { + // _msg, _time, _stream: the split blanks the grouped fields of + // the lines it reads as raw rows, which these must not suffer. + return + } + if label == "service_name" && validOnly { + groupServiceName = true + continue + } if label == "detected_level" { // Computed by VictoriaLogs from the whole row // (translator.BuildDetectedLevelChain), not from one key. @@ -877,10 +902,12 @@ func (plan *orderedJSONMetricPlan) setStatsPushdown() { } } } - if len(fields) == 0 && len(filters) == 0 && !plan.derivesLevel { + if len(fields) == 0 && len(filters) == 0 && !plan.derivesLevel && !validOnly { return } plan.pushdown = true + plan.validOnly = validOnly + plan.groupServiceName = groupServiceName plan.unpackFields = fields plan.pushdownFilters = filters plan.pushdownErrorFilters = errorFilters @@ -898,7 +925,15 @@ func (plan *orderedJSONMetricPlan) setStatsPushdown() { // returned as errorFilters, because a line may still pass one when Loki // extracted the key before the syntax error (the partial-parse probe) or the // key is a stored label of the line (the stored-field probe). -func (plan *orderedJSONMetricPlan) pushdownLabelFilters() (filters []translator.DropCondition, errorFilters []string, ok bool) { +// +// A JSON parser followed by `__error__=""` (with no drop of the error label in +// between) keeps only the lines the parser accepts; validOnly reports it, and +// such a plan needs neither dropped errors nor error filters. A split plan may +// group by detected_level (the stored level fields decide it, see +// orderedJSONStatsBucketsWithReason) but not filter on it: the filter would have +// to run on the stored fields for the counted lines and on the derived level +// for the lines read as raw rows, so that plan stays with the row evaluator. +func (plan *orderedJSONMetricPlan) pushdownLabelFilters() (filters []translator.DropCondition, errorFilters []string, validOnly, ok bool) { parsed, errorCleared := false, false // A filter on detected_level before the parser holds the value it holds // after it (the parser cannot set it), so it is rendered like one after @@ -912,45 +947,80 @@ func (plan *orderedJSONMetricPlan) pushdownLabelFilters() (filters []translator. switch { case stage.parser: if parsed { - return nil, nil, false + return nil, nil, false, false } parsed, errorCleared = true, false case stage.line != nil: continue - case stage.filter != nil: - // Loki's detected_level is on every line, parsed or not: a filter - // on it compares the derived value (it never rejects an unparsed - // line, so it cannot stand for the error drop) and `!= ""` always - // holds. - level := stage.filter.Field == detectedLevelLabel && plan.derivesLevel - if !parsed || !level && (!orderedJSONPushdownLabelOK(stage.filter.Field) || plan.parser == "" && strings.HasSuffix(stage.filter.Field, "_extracted")) { - return nil, nil, false - } - if level { - if stage.filter.Op != "!=" || stage.filter.Value != "" { - filters = append(filters, *stage.filter) - } - continue + case stage.filter != nil && stage.filter.Field == "__error__" && stage.filter.Op == "=" && stage.filter.Value == "": + // After a drop of __error__ the filter passes every line. + if !parsed || plan.parser != "json" { + return nil, nil, false, false } - filters = append(filters, *stage.filter) - if !stage.filter.Matches("") && !containsString(errorFilters, stage.filter.Field) { - errorFilters = append(errorFilters, stage.filter.Field) + validOnly = validOnly || !errorCleared + case stage.filter != nil: + if !plan.addPushdownFilter(stage.filter, parsed, &filters, &errorFilters) { + return nil, nil, false, false } case stage.keep || len(stage.match) != 0: - return nil, nil, false + return nil, nil, false, false default: for field := range stage.fields { if field != "__error__" && field != "__error_details__" { - return nil, nil, false + return nil, nil, false, false } } errorCleared = errorCleared || stage.fields["__error__"] } } - if errorCleared { - return filters, nil, true + if validOnly && filtersOnLevel(filters) { + return nil, nil, false, false } - return filters, errorFilters, len(errorFilters) > 0 + if errorCleared || validOnly { + return filters, nil, validOnly, true + } + return filters, errorFilters, false, len(errorFilters) > 0 +} + +// addPushdownFilter adds a string label filter stage to the pushdown's filters, +// and to the error filters when it rejects the empty value; false when the +// stats pushdown cannot apply it. +// +// Loki's detected_level is on every line, parsed or not: a filter on it +// compares the derived value (it never rejects an unparsed line, so it cannot +// stand for the error drop) and `!= ""` always holds. +func (plan *orderedJSONMetricPlan) addPushdownFilter(filter *translator.DropCondition, parsed bool, filters *[]translator.DropCondition, errorFilters *[]string) bool { + level := filter.Field == detectedLevelLabel && plan.derivesLevel + if !parsed || !level && !plan.pushdownFilterLabelOK(filter.Field) { + return false + } + if level { + if filter.Op != "!=" || filter.Value != "" { + *filters = append(*filters, *filter) + } + return true + } + *filters = append(*filters, *filter) + if !filter.Matches("") && !containsString(*errorFilters, filter.Field) { + *errorFilters = append(*errorFilters, filter.Field) + } + return true +} + +// pushdownFilterLabelOK reports whether the stats pushdown can filter on a +// parsed label. +func (plan *orderedJSONMetricPlan) pushdownFilterLabelOK(label string) bool { + return orderedJSONPushdownLabelOK(label) && (plan.parser != "" || !strings.HasSuffix(label, "_extracted")) +} + +// filtersOnLevel reports a filter on detected_level among filters. +func filtersOnLevel(filters []translator.DropCondition) bool { + for _, filter := range filters { + if filter.Field == detectedLevelLabel { + return true + } + } + return false } // logsQLLabelFilter renders a Loki string label filter as a LogsQL filter pipe @@ -1018,8 +1088,22 @@ var errLevelNotStored = errors.New("rows without a stored level") // from stats_query_range buckets on the anchored sliding grid instead of raw // rows. An instant query (instant, start == end) asks for the one bucket // (t-window, t] and answers a vector. served is false when the raw evaluator -// must answer: no bucket grid, a possible series-limit overflow, or a line a -// probe found to be read differently by the two parsers (parseDisagreement). +// must answer: no bucket grid, a possible series-limit overflow (plain +// pushdowns), a line a probe found to be read differently by the two parsers +// (parseDisagreement), and for a split plan (validOnly) a stats read past the +// memory guard of twice the series limit in groups, a stats response too large +// to read, or a Drilldown request. A validOnly plan counts the lines +// VictoriaLogs vouches for in the same scan that counts the other brace-opened +// lines per bucket, and reads only those as raw rows; Loki's series limit +// applies to its merged answer only. +// +// A split plan grouped by detected_level asks for the stored fields alone and +// never for the chain: it is served only when the stored level fields decide +// the level of every counted line exactly as Loki's ingest does +// (storedLevelSeries). A window with a counted line they do not decide, a read +// past the group guard, or a response too large to read leaves the query to +// the row evaluator, which derives the level of every line itself; so do the +// lines read as raw rows. // // A plan reading detected_level first asks for the stored fields alone: the // buckets are grouped by the stored detected_level and level fields, and the @@ -1033,6 +1117,14 @@ func (p *Proxy) orderedJSONStatsBucketsWithReason(ctx context.Context, plan *ord if !plan.pushdown || p.labelTranslator == nil { return nil, false, false, nil } + if plan.validOnly && (instant || requestKeepsPartialSeries(ctx)) { + // A Drilldown request keeps the busiest series of an over-limit answer + // instead of failing, which the split's merged answer does not rank, + // and an instant query has no bucket grid to add the raw lines to + // (buildStatsInstantVector reads the one bucket): both take the row + // evaluator, as they did before the split. + return nil, false, false, nil + } if plan.derivesLevel && !p.detectedLevelBodyScan { // VictoriaLogs pipes always read the line; without the body scan only // the stored fields decide, which the raw evaluator derives. A plan @@ -1045,6 +1137,9 @@ func (p *Proxy) orderedJSONStatsBucketsWithReason(ctx context.Context, plan *ord } if plan.derivesLevel { body, served, parseDisagreement, err = p.orderedJSONStatsBuckets(ctx, plan, start, end, step, true, instant) + if plan.validOnly && (errors.Is(err, errLevelNotStored) || errors.Is(err, errStatsGroupGuard)) { + return nil, false, false, nil + } if !errors.Is(err, errLevelNotStored) && !errors.Is(err, errStatsGroupGuard) { return body, served, parseDisagreement, err } @@ -1087,39 +1182,11 @@ func (p *Proxy) orderedJSONStatsBuckets(ctx context.Context, plan *orderedJSONMe windowStart := start.Add(-plan.window) statsCtx, cancelStats := context.WithCancel(ctx) defer cancelStats() - groupBy := plan.statsGroupBy(storedLevel) - statsAggFunc := "count() as c" - if plan.function == "bytes_rate" || plan.function == "bytes_over_time" { - statsAggFunc = "sum_len(_msg) as c, count() as __sample_count" - } - var required []string - for _, field := range plan.requiredFields(base) { - if _, ok := orderedJSONExtractedBase(field); !ok { - required = append(required, field) - } - } - remember, compute := translator.ExtractedLabelPipes("unpack_"+plan.parser, extracted, stored) - beforeUnpack := plan.levelPipes(storedLevel) - shared := plan.sharedLevelUnpack(storedLevel, allKeys, extracted, stored) - if shared != nil { - // One parse of every line: the chain's unpack reads the plan's labels too. - beforeUnpack = " " + translator.BuildDetectedLevelChainShared(translator.DetectedLevelFieldNames, translator.DetectedLevelChainGrouping, nil, shared) - statsCtx = withLevelSharedUnpack(statsCtx, shared) - } - query := base + remember + orderedJSONUnpackPipesShared(plan.parser, allKeys, stored, required, beforeUnpack, shared != nil) + compute - // A filter on detected_level compares the derived value: after the chain in - // VictoriaLogs, or on the stored spellings (levelPipes) and again on the - // normalised series below. - for _, filter := range plan.pushdownFilters { - if storedLevel && filter.Field == detectedLevelLabel { - continue - } - if filter.Field == detectedLevelLabel { - query += levelChainFilter(filter) - continue - } - query += logsQLLabelFilter(filter) + pipes := p.orderedJSONStatsPipeline(plan, base, storedLevel, allKeys, extracted, stored) + if pipes.shared != nil { + statsCtx = withLevelSharedUnpack(statsCtx, pipes.shared) } + groupBy, lokiGroupBy, bytesMetric, statsAggFunc, query := pipes.groupBy, pipes.lokiGroupBy, pipes.bytesMetric, pipes.aggFunc, pipes.query if storedLevel && (p.levelNeedsChain(ctx, base) || p.levelNeedsChain(ctx, base+levelShapeSeparator+query)) { return nil, false, false, errLevelNotStored } @@ -1137,32 +1204,174 @@ func (p *Proxy) orderedJSONStatsBuckets(ctx context.Context, plan *orderedJSONMe risky, err := p.pushdownParseRisk(riskCtx, plan, base+plan.levelRestriction(), plain, extracted, extractedKeys, stored, storedLevel, windowStart, end) riskDone <- riskResult{risky, err} }() - type statsResult struct { - series map[string]manualSeriesSamples - err error + // A split plan's stored level is only Loki's when no other stored level + // field can outrank it (storedLevelForeignFieldRisk). + foreignDone := make(chan riskResult, 1) + if plan.validOnly && storedLevel { + go func() { + foreign, err := p.storedLevelForeignFieldRisk(riskCtx, base, windowStart, end) + foreignDone <- riskResult{foreign, err} + }() + } else { + foreignDone <- riskResult{} } - statsDone := make(chan statsResult, 1) - go func() { - readCtx := statsCtx + hazardDone := make(chan riskResult, 1) + if plan.validOnly { + go func() { + hazard, err := p.jsonLokiRejectHazardRisk(riskCtx, base, windowStart, end) + hazardDone <- riskResult{hazard, err} + }() + } else { + hazardDone <- riskResult{} + } + runStats := func(ctx context.Context, hazard bool) orderedJSONStatsResult { + q, agg, aux := query, statsAggFunc, "" + if plan.validOnly { + q, agg = plan.splitStatsQuery(query, bytesMetric, hazard, groupBy) + aux = jsonInvalidRowsName + } if storedLevel { // The spellings of one level are groups here, series in the answer. - readCtx = guardStatsGroups(statsCtx) + ctx = guardStatsGroups(ctx) } - series, err := p.collectRangeMetricHits(readCtx, query, groupBy, groupBy, false, statsAggFunc, windowStart, end, bucket) - statsDone <- statsResult{series, err} - }() + series, rowEdges, err := p.collectRangeMetricHitsAux(ctx, q, groupBy, lokiGroupBy, false, agg, windowStart, end, bucket, aux) + return orderedJSONStatsResult{series, rowEdges, err} + } + statsDone := make(chan orderedJSONStatsResult, 1) + go func() { statsDone <- runStats(statsCtx, false) }() risk := <-riskDone if risk.err != nil || risk.risky { cancelStats() return nil, false, risk.risky, risk.err } - stats := <-statsDone + foreign := <-foreignDone + if foreign.err != nil { + cancelStats() + return nil, false, false, foreign.err + } + if foreign.risky { + // Not provably Loki's level: the row evaluator derives it. + cancelStats() + return nil, false, false, nil + } + hazard := <-hazardDone + if hazard.err != nil { + cancelStats() + return nil, false, false, hazard.err + } + var stats orderedJSONStatsResult + if hazard.risky { + // A line Loki's parser may reject among the ones VictoriaLogs parses: + // count again with such lines moved to the raw rows, without waiting + // for the scan that would be discarded. + cancelStats() + <-statsDone + stats = runStats(ctx, true) + } else { + stats = <-statsDone + } + return p.finishOrderedJSONStats(ctx, plan, stats, base, query, storedLevel, hazard.risky, bucket, start, end, step, instant) +} + +// orderedJSONStatsPipes is the query text of the stats pushdown and what its +// stats group by. +type orderedJSONStatsPipes struct { + query string + aggFunc string + groupBy, lokiGroupBy []string + bytesMetric bool + shared *translator.SharedUnpack +} + +// orderedJSONStatsPipeline builds the pipes of one attempt of the stats +// pushdown (see orderedJSONStatsBuckets). +func (p *Proxy) orderedJSONStatsPipeline(plan *orderedJSONMetricPlan, base string, storedLevel bool, allKeys, extracted []string, stored map[string]string) orderedJSONStatsPipes { + groupBy := plan.statsGroupBy(storedLevel) + lokiGroupBy := groupBy + if plan.groupServiceName { + groupBy = append(append([]string(nil), groupBy...), streamServiceNameField) + lokiGroupBy = append(append([]string(nil), lokiGroupBy...), "service_name") + } + bytesMetric := plan.function == "bytes_rate" || plan.function == "bytes_over_time" + statsAggFunc := "count() as c" + if bytesMetric { + statsAggFunc = "sum_len(_msg) as c, count() as __sample_count" + } + var required []string + if !plan.validOnly { + // A validOnly plan counts every brace-opened line it cannot vouch for, + // so no line is dropped before the parse. + for _, field := range plan.requiredFields(base) { + if _, ok := orderedJSONExtractedBase(field); !ok { + required = append(required, field) + } + } + } + remember, compute := translator.ExtractedLabelPipes("unpack_"+plan.parser, extracted, stored) + beforeUnpack := plan.levelPipes(storedLevel) + shared := plan.sharedLevelUnpack(storedLevel, allKeys, extracted, stored) + if shared != nil { + // One parse of every line: the chain's unpack reads the plan's labels too. + beforeUnpack = " " + translator.BuildDetectedLevelChainShared(translator.DetectedLevelFieldNames, translator.DetectedLevelChainGrouping, nil, shared) + } + query := base + if plan.validOnly { + // The copies of the stored level fields come first: nothing of the + // split (the splice, the marker, the blanked groups) may touch them. + query += beforeUnpack + beforeUnpack = "" + // The brace filter and the sentinel splice, then the service_name + // derivation from the stream labels, before any unpack. + query += " | filter " + jsonObjectStartFilter + jsonValidityPipes(bytesMetric) + if plan.groupServiceName { + query += serviceNameFromStreamPipes() + } + } + query += remember + orderedJSONUnpackPipesShared(plan.parser, allKeys, stored, required, beforeUnpack, shared != nil) + compute + // A filter on detected_level compares the derived value: after the chain in + // VictoriaLogs, or on the stored spellings (levelPipes) and again on the + // normalised series below. A split plan puts its filters into the counted + // condition instead (splitStatsQuery). + for _, filter := range plan.pushdownFilters { + if plan.validOnly || storedLevel && filter.Field == detectedLevelLabel { + continue + } + if filter.Field == detectedLevelLabel { + query += levelChainFilter(filter) + continue + } + query += logsQLLabelFilter(filter) + } + return orderedJSONStatsPipes{query: query, aggFunc: statsAggFunc, groupBy: groupBy, lokiGroupBy: lokiGroupBy, bytesMetric: bytesMetric, shared: shared} +} + +// orderedJSONStatsResult is one stats_query_range read of the pushdown: the +// series, and for a split plan the bucket edges holding lines read as raw rows. +type orderedJSONStatsResult struct { + series map[string]manualSeriesSamples + rowEdges []int64 + err error +} + +// finishOrderedJSONStats turns the stats read of orderedJSONStatsBuckets into the +// answer: the guards that leave the query to another route, the groups merged +// under their Loki labels, the series limit applied to the merged answer, and +// for a split plan the raw rows added. +func (p *Proxy) finishOrderedJSONStats(ctx context.Context, plan *orderedJSONMetricPlan, stats orderedJSONStatsResult, base, query string, storedLevel, hazard bool, bucket time.Duration, start, end time.Time, step time.Duration, instant bool) (body []byte, served bool, parseDisagreement bool, err error) { series, err := stats.series, stats.err - if errors.Is(err, errStatsGroupGuard) { + var ok bool + if storedLevel && errors.Is(err, errStatsGroupGuard) { // More groups than the guard reads: the chain's groups are the answer's // series, so it counts them and answers Loki's series limit itself. p.markLevelNeedsChain(ctx, base+levelShapeSeparator+query) } + if plan.validOnly && (errors.Is(err, errBodyTooLarge) || errors.Is(err, errStatsGroupGuard)) { + // A split read too large to hold says nothing about the series of the + // answer (the groups merge into fewer series): the row evaluator + // answers, as it did before the split, with its own limits and Loki's + // series limit error. + return nil, false, false, nil + } if err != nil { return nil, false, false, err } @@ -1171,17 +1380,35 @@ func (p *Proxy) orderedJSONStatsBuckets(ctx context.Context, plan *orderedJSONMe p.markLevelNeedsChain(ctx, base) return nil, false, false, errLevelNotStored } - } else if series, err = capSeriesForRequest(ctx, series, p.resolvedMaxStatsQuerySeries(ctx)); err != nil { - // Only above the limit: a query with exactly the limit's series passes. - return nil, false, false, err + } else if !plan.validOnly { + if series, err = capSeriesForRequest(ctx, series, p.resolvedMaxStatsQuerySeries(ctx)); err != nil { + // Only above the limit: a query with exactly the limit's series passes. + return nil, false, false, err + } } merged := plan.mergeGroups(series) - if storedLevel { + switch { + case plan.validOnly: + // The series limit applies to the answer: the series with a point, + // after the groups are merged under their Loki labels, and again once + // the raw rows are added. + dropSeriesWithoutPoints(merged, start, end, step, plan.window) + if len(stats.rowEdges) > 0 { + // Raw rows only add series: over the limit already, fail before reading them. + if _, err = capSeriesForRequest(ctx, merged, p.resolvedMaxStatsQuerySeries(ctx)); err != nil { + return nil, false, false, err + } + merged, err = p.addOrderedJSONRowLines(ctx, plan, merged, base, stats.rowEdges, hazard, bucket, start, end, step) + } else { + merged, err = capSeriesForRequest(ctx, merged, p.resolvedMaxStatsQuerySeries(ctx)) + } + case storedLevel: // The spellings of one level are one series, as in Loki: only the // merged answer meets the limit. - if merged, err = capSeriesForRequest(ctx, merged, p.resolvedMaxStatsQuerySeries(ctx)); err != nil { - return nil, false, false, err - } + merged, err = capSeriesForRequest(ctx, merged, p.resolvedMaxStatsQuerySeries(ctx)) + } + if err != nil { + return nil, false, false, err } if instant { body, err = buildStatsInstantVector(plan, merged, start, p.limits().BufferedBackendBodyBytes) @@ -1251,6 +1478,9 @@ func (plan *orderedJSONMetricPlan) mergeGroups(series map[string]manualSeriesSam metric := make(map[string]string, len(grouped)) for _, label := range grouped { value := entry.Metric[label] + if label == "service_name" && plan.groupServiceName && value == "" { + value = unknownServiceName + } if value != "" { metric[label] = value } @@ -1269,6 +1499,39 @@ func (plan *orderedJSONMetricPlan) mergeGroups(series map[string]manualSeriesSam return merged } +// splitStatsQuery completes the pipes of a validOnly plan with its label +// filters and returns the stats functions. It marks the lines it reads as raw +// rows once (jsonRawMarker, hazard moves the lines jsonLokiRejectHazard matches +// there) and puts the filters into the counted condition, so the lines read as +// raw rows are counted whatever VictoriaLogs makes of them. They also lose the +// fields the stats group by: they are counted per bucket whatever their labels +// (the row evaluator reads them and derives the labels itself), so they form +// one group instead of one per label set. +func (plan *orderedJSONMetricPlan) splitStatsQuery(query string, bytesMetric, hazard bool, groupBy []string) (string, string) { + query += jsonRawMarker(hazard) + conds := []string{jsonRawRowCondition(false)} + var filters []string + for _, filter := range plan.pushdownFilters { + filters = append(filters, logsQLLabelCondition(filter)) + } + if len(filters) > 0 { + // VictoriaLogs emits a group, with zero counts, for every row that + // reaches stats: the lines the filters drop must not form groups. The + // lines read as raw rows stay, so their buckets are still counted. + query += " | filter (" + jsonRawRowCondition(true) + " or (" + strings.Join(filters, " ") + "))" + conds = append(conds, filters...) + } + for _, field := range groupBy { + query += " | format if (" + jsonRawRowCondition(true) + ") \"\" as " + quoteLogsQLIdent(field) + } + counted := "if (" + strings.Join(conds, " ") + ")" + rows := "count() if (" + jsonRawRowCondition(true) + ") as " + jsonInvalidRowsName + if bytesMetric { + return query, "sum_len(_msg) " + counted + " as c, count() " + counted + " as __sample_count, " + rows + } + return query, "count() " + counted + " as c, " + rows +} + // statsGroupBy lists the fields the pushdown's stats group by: the grouped // labels it unpacks, and the field(s) detected_level comes from. func (plan *orderedJSONMetricPlan) statsGroupBy(storedLevel bool) []string { @@ -1352,6 +1615,9 @@ func splitPushdownKeys(unpack []string) (plain, allKeys, extracted, extractedKey // value of name deleted, because the scratch unpack reads the body whatever // the stream holds. func (p *Proxy) pushdownParseRisk(ctx context.Context, plan *orderedJSONMetricPlan, base string, plain, extracted, extractedKeys []string, stored map[string]string, storedLevel bool, start, end time.Time) (bool, error) { + if plan.validOnly { + ctx = withRiskRestriction(ctx, jsonValidOnlyRestriction) + } type result struct { risky bool err error @@ -1524,7 +1790,7 @@ func orderedJSONUnpackPipesShared(parser string, unpack []string, stored map[str } } sb.WriteString(beforeUnpack) - if parser != "" && !unpacked && (len(unpack) > 0 || beforeUnpack == "") { + if parser != "" && !unpacked && len(unpack) > 0 { sb.WriteString(" | unpack_" + parser) if len(required) != 0 { sb.WriteString(" if (" + orderedJSONUnpackCond(unpack, stored) + ")") @@ -1893,7 +2159,7 @@ func (p *Proxy) cachedStatsPushdownRisk(ctx context.Context, parser, base string } } labels := func(ctx context.Context) (bool, error) { - key := "stats-pushdown-risk:v5:" + getOrgID(ctx) + ":" + parser + ":" + strings.Join(spelled, ",") + ":" + strings.Join(errorFilters, ",") + ":" + base + key := "stats-pushdown-risk:v5:" + getOrgID(ctx) + ":" + parser + ":" + strings.Join(spelled, ",") + ":" + strings.Join(errorFilters, ",") + ":" + base + riskRestriction(ctx) return p.rememberedRisk(ctx, key, start, end, func(from, to time.Time) []func(context.Context) (bool, error) { probes := []func(context.Context) (bool, error){ func(ctx context.Context) (bool, error) { @@ -1934,7 +2200,7 @@ func (p *Proxy) cachedStatsPushdownRisk(ctx context.Context, parser, base string return labels(ctx) } levels := func(ctx context.Context) (bool, error) { - key := "stats-pushdown-risk:v5:level:" + getOrgID(ctx) + ":" + base + key := "stats-pushdown-risk:v5:level:" + getOrgID(ctx) + ":" + base + riskRestriction(ctx) return p.rememberedRisk(ctx, key, start, end, func(from, to time.Time) []func(context.Context) (bool, error) { return []func(context.Context) (bool, error){ func(ctx context.Context) (bool, error) { return p.detectedLevelStreamRisk(ctx, base, from, to) }, @@ -1985,6 +2251,11 @@ func anyRisk(ctx context.Context, probes []func(context.Context) (bool, error)) // statsPushdownRiskExists runs a `| limit 1` risk query over [start, end] and // reports whether it matched a line. func (p *Proxy) statsPushdownRiskExists(ctx context.Context, query string, start, end time.Time) (bool, error) { + if restriction := riskRestriction(ctx); restriction != "" { + if head, ok := strings.CutSuffix(query, " | limit 1"); ok { + query = head + restriction + " | limit 1" + } + } params := url.Values{"query": {query}, "start": {start.UTC().Format(time.RFC3339Nano)}, "end": {end.Add(time.Nanosecond).UTC().Format(time.RFC3339Nano)}} resp, err := p.vlPost(ctx, "/select/logsql/query", params) if err != nil { @@ -2249,7 +2520,7 @@ func (p *Proxy) unparsedLinePipelineError(ctx context.Context, plan *orderedJSON } last := start.Add(end.Sub(start) / step * step) // Loki's window is left-open, right-closed; VictoriaLogs' end is exclusive. - params := url.Values{"query": {query + ` | filter -_msg:~"^\\s*\\{" | limit 1`}, + params := url.Values{"query": {query + " | filter -" + jsonObjectStartFilter + " | limit 1"}, "start": {start.Add(-plan.window).Add(time.Nanosecond).UTC().Format(time.RFC3339Nano)}, "end": {last.Add(time.Nanosecond).UTC().Format(time.RFC3339Nano)}} resp, err := p.vlPost(ctx, "/select/logsql/query", params) if err != nil { @@ -2524,12 +2795,9 @@ func (p *Proxy) collectOrderedJSONMetric(ctx context.Context, plan *orderedJSONM return nil, err } query += plan.levelRestriction() - limit := p.rangeMetricRowLimit - if limit <= 0 { - limit = 1_000_000 - } - if limit == math.MaxInt { - return nil, fmt.Errorf("manual range metric row limit is too large") + reader, err := p.newOrderedJSONRowReader() + if err != nil { + return nil, err } if pipelineErr, err := p.unparsedLinePipelineError(ctx, plan, query, start, end, step); err != nil || pipelineErr != nil { if err != nil { @@ -2537,81 +2805,119 @@ func (p *Proxy) collectOrderedJSONMetric(ctx context.Context, plan *orderedJSONM } return nil, pipelineErr } + if err := p.readOrderedJSONMetricRows(ctx, plan, reader, query+p.orderedJSONRowProjection(plan), start.Add(-plan.window), end, start, end, step); err != nil { + return nil, err + } + return capSeriesForRequest(ctx, reader.series, p.resolvedMaxStatsQuerySeries(ctx)) +} + +// orderedJSONRowReader accumulates the samples of raw rows read by one +// evaluation, possibly over several queries, under one row limit +// (-manual-range-metric-row-limit) and one byte limit +// (-ordered-json-metric-max-bytes). +type orderedJSONRowReader struct { + series map[string]manualSeriesSamples + streamLabels map[string]map[string]string + descriptors map[string]cachedLogQueryStreamDescriptor + exposures map[string][]metadataFieldExposure + levels logRowLevels + rows, limit int + read int64 + maxBytes int64 +} + +func (p *Proxy) newOrderedJSONRowReader() (*orderedJSONRowReader, error) { + limit := p.rangeMetricRowLimit + if limit <= 0 { + limit = 1_000_000 + } + if limit == math.MaxInt { + return nil, fmt.Errorf("manual range metric row limit is too large") + } + return &orderedJSONRowReader{ + series: make(map[string]manualSeriesSamples), + streamLabels: make(map[string]map[string]string), + descriptors: make(map[string]cachedLogQueryStreamDescriptor), + exposures: make(map[string][]metadataFieldExposure), + levels: p.newLogRowLevels(), + limit: limit, + maxBytes: p.orderedJSONMetricMaxBytes(), + }, nil +} + +// readOrderedJSONMetricRows reads the rows query selects in [from, to] (both +// inclusive) and adds the samples of the evaluations at start, start+step, ... +// up to end to reader. +func (p *Proxy) readOrderedJSONMetricRows(ctx context.Context, plan *orderedJSONMetricPlan, reader *orderedJSONRowReader, query string, from, to, start, end time.Time, step time.Duration) error { // The query argument limit asks VL to sort by timestamp. A final limit pipe // bounds raw collection without that sort; local window evaluation sorts // the complete accepted rows and rejects overflow instead of truncating. - query += " | limit " + strconv.Itoa(limit+1) + query += " | limit " + strconv.Itoa(reader.limit-reader.rows+1) // VL's end is exclusive; Loki evaluation includes a line exactly at end. - params := url.Values{"query": {query}, "start": {start.Add(-plan.window).UTC().Format(time.RFC3339Nano)}, "end": {end.Add(time.Nanosecond).UTC().Format(time.RFC3339Nano)}} + params := url.Values{"query": {query}, "start": {from.UTC().Format(time.RFC3339Nano)}, "end": {to.Add(time.Nanosecond).UTC().Format(time.RFC3339Nano)}} resp, err := p.vlPost(ctx, "/select/logsql/query", params) if err != nil { - return nil, err + return err } defer resp.Body.Close() if resp.StatusCode >= 400 { body, _ := readBodyLimited(resp.Body, maxUpstreamErrorBodyBytes) - return nil, p.redactedBackendStatusError("backend returned", resp.StatusCode, body) + return p.redactedBackendStatusError("backend returned", resp.StatusCode, body) } - maxBytes := p.orderedJSONMetricMaxBytes() - readLimit := maxBytes + readLimit := reader.maxBytes - reader.read if readLimit < math.MaxInt64 { readLimit++ // one byte past the cap detects overflow } limited := &io.LimitedReader{R: resp.Body, N: readLimit} + defer func() { reader.read += readLimit - limited.N }() scanner := bufio.NewScanner(limited) scanner.Buffer(make([]byte, 64<<10), 8<<20) - series := make(map[string]manualSeriesSamples) - streamLabels := make(map[string]map[string]string) - descriptors := make(map[string]cachedLogQueryStreamDescriptor) - exposures := make(map[string][]metadataFieldExposure) - levels := p.newLogRowLevels() - rows := 0 for scanner.Scan() { if err := ctx.Err(); err != nil { - return nil, err + return err } if limited.N <= 0 { - return nil, orderedJSONResponseLimitError(maxBytes) + return orderedJSONResponseLimitError(reader.maxBytes) } - rows++ - if rows > limit { - return nil, fmt.Errorf("manual range metric row limit exceeded (%d)", limit) + reader.rows++ + if reader.rows > reader.limit { + return fmt.Errorf("manual range metric row limit exceeded (%d)", reader.limit) } var row map[string]string if err := json.Unmarshal(scanner.Bytes(), &row); err != nil { - return nil, fmt.Errorf("invalid backend metric row: %w", err) + return fmt.Errorf("invalid backend metric row: %w", err) } ts, ok := parseFlexibleUnixNanos(row["_time"]) if !ok { - return nil, fmt.Errorf("invalid backend metric timestamp") + return fmt.Errorf("invalid backend metric timestamp") } if !orderedJSONSampleVisible(ts, start.UnixNano(), end.UnixNano(), int64(step), int64(plan.window)) { continue } - desc := p.logQueryStreamDescriptor(row["_stream"], row["level"], streamLabels, descriptors) - base, err := p.orderedJSONBaseLabels(row, desc, exposures) + desc := p.logQueryStreamDescriptor(row["_stream"], row["level"], reader.streamLabels, reader.descriptors) + base, err := p.orderedJSONBaseLabels(row, desc, reader.exposures) if err != nil { - return nil, err + return err } - if !plan.acceptDerivedLevel(base, &levels, row, desc) { + if !plan.acceptDerivedLevel(base, &reader.levels, row, desc) { continue } labels, keep, err := plan.processWithContext(ctx, row["_msg"], base, desc.translatedLabels) if err != nil { - return nil, err + return err } if !keep { continue } if labels["__error__"] != "" && labels["__preserve_error__"] != "true" { - return nil, lokiPipelineError(labels) + return lokiPipelineError(labels) } labels = plan.groupLabels(labels) key := canonicalLabelsKey(labels) - entry, exists := series[key] + entry, exists := reader.series[key] if !exists { - if err := seriesLimitCollecting(ctx, len(series), p.resolvedMaxStatsQuerySeries(ctx)); err != nil { - return nil, err + if err := seriesLimitCollecting(ctx, len(reader.series), p.resolvedMaxStatsQuerySeries(ctx)); err != nil { + return err } entry.Metric = labels } @@ -2620,18 +2926,15 @@ func (p *Proxy) collectOrderedJSONMetric(ctx context.Context, plan *orderedJSONM value = float64(len(row["_msg"])) } entry.Samples = append(entry.Samples, rangeMetricSample{ts: ts, value: value}) - series[key] = entry + reader.series[key] = entry } if err := scanner.Err(); err != nil { - return nil, err + return err } if limited.N <= 0 { - return nil, orderedJSONResponseLimitError(maxBytes) - } - if err := ctx.Err(); err != nil { - return nil, err + return orderedJSONResponseLimitError(reader.maxBytes) } - return capSeriesForRequest(ctx, series, p.resolvedMaxStatsQuerySeries(ctx)) + return ctx.Err() } // Raw VL rows have not executed unpack_json: regular fields outside _stream diff --git a/internal/proxy/ordered_json_valid_split.go b/internal/proxy/ordered_json_valid_split.go new file mode 100644 index 000000000..8e36ea195 --- /dev/null +++ b/internal/proxy/ordered_json_valid_split.go @@ -0,0 +1,407 @@ +package proxy + +import ( + "context" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" + + "github.com/ReliablyObserve/Loki-VL-proxy/internal/translator" +) + +// A `| json | __error__=""` metric counts the lines Loki's JSON parser accepts +// (pkg/logql/log/parser.go JSONParser.Process: jsonparser.ObjectEach over the +// line, nested objects walked only where a required label starts with their +// prefix, parser_hints.go ShouldExtractPrefix). VictoriaLogs attaches no parser +// error, so the stats pushdown splits the selected lines in three: +// +// - lines that do not open with a brace after JSON whitespace: ObjectEach +// fails on them (grafana/jsonparser nextToken skips only space, tab, line +// feed and carriage return), so Loki drops them; +// - lines VictoriaLogs parses as one JSON object and that hold none of the +// constructs its parser accepts and Loki's rejects (jsonValidityPipes, +// jsonLokiRejectHazard): Loki accepts every such line under any parser +// hints, so VictoriaLogs stats count them; +// - the rest open with a brace but VictoriaLogs cannot vouch for them; they +// are read as raw rows, only from the buckets holding them, and evaluated +// by the row evaluator, whose parser follows Loki's hints. +// +// The guards of the row evaluator (nesting depth, field count, line and value +// size) therefore apply to the third set only. + +// jsonObjectStartFilter keeps the lines Loki's JSON parser may accept. +var jsonObjectStartFilter = "_msg:~" + strconv.Quote(`^[ \t\n\r]*\{`) + +// jsonValidField is the sentinel key jsonValidityPipes splices into a line. +const jsonValidField = "__vlp" + +// jsonInvalidRowsName names the stats result counting the lines of the third +// set (jsonInvalidCondition) in each bucket. +const jsonInvalidRowsName = "__json_rows" + +// jsonValidityPipes sets __vlp exactly on the lines VictoriaLogs parses as one +// JSON object: the sentinel member is spliced in after the opening brace and +// unpack_json asked for it. The splice keeps a line's validity (it adds one +// leading member and cannot complete a broken line, unlike wrapping the line +// in a new object, which trailing text could close) except for the empty +// object `{}`, which it turns invalid and so only sends to the raw rows. With +// onCopy the splice works on a copy, leaving _msg (and sum_len(_msg)) intact. +func jsonValidityPipes(onCopy bool) string { + at, from := "_msg", "" + if onCopy { + at, from = "__vlm", " from __vlm" + } + pipes := ` | replace_regexp ("^\\{", "{\"` + jsonValidField + `\":1,") at ` + at + ` | unpack_json` + from + ` fields (` + jsonValidField + `)` + if onCopy { + return ` | copy _msg as __vlm` + pipes + ` | delete __vlm` + } + return pipes +} + +// jsonLokiRejectHazard matches every line VictoriaLogs' parser (fastjson) +// accepts while Loki's (grafana/jsonparser) may reject it. A key with a +// backslash escape: jsonparser unescapes keys and fails on an invalid escape or +// a lone surrogate (ObjectEach, MalformedStringEscapeError) where fastjson +// keeps the key raw. A member value starting with one of `. e E +`, or NaN or +// Inf in any case: fastjson reads them as numbers (parseRawNumber, parseValue), +// jsonparser's getType rejects them (UnknownValueTypeError). Each regexp runs +// behind a filter VictoriaLogs answers from its token index or a substring +// search where one exists, and it matches anywhere in the line, which is a +// superset: a value inside a string or an array only sends the line to the +// raw rows. +var jsonLokiRejectHazard = func() string { + var words []string + for _, word := range []string{"nan", "inf"} { + for mask := 0; mask < 8; mask++ { + b := []byte(word) + for i := range b { + if mask&(1<" as ` + streamServiceNameField) + } + return b.String() +} + +// orderedJSONRowSpans turns the left edges of the buckets holding raw-row +// lines into the time ranges to read: each bucket is widened to both of its +// edges (a bucket is (e, e+b] with the stats offset and [e, e+b) without), +// touching buckets are merged, and more than maxOrderedJSONRowSpans ranges +// collapse into their hull, one query instead of many. +func orderedJSONRowSpans(edges []int64, bucket time.Duration) [][2]time.Time { + if len(edges) == 0 { + return nil + } + sorted := append([]int64(nil), edges...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] }) + size := int64(bucket) + var spans [][2]int64 + for _, edge := range sorted { + if n := len(spans); n > 0 && edge <= spans[n-1][1] { + if edge+size > spans[n-1][1] { + spans[n-1][1] = edge + size + } + continue + } + spans = append(spans, [2]int64{edge, edge + size}) + } + if len(spans) > maxOrderedJSONRowSpans { + spans = [][2]int64{{spans[0][0], spans[len(spans)-1][1]}} + } + out := make([][2]time.Time, len(spans)) + for i, span := range spans { + out[i] = [2]time.Time{time.Unix(0, span[0]), time.Unix(0, span[1])} + } + return out +} + +// maxOrderedJSONRowSpans bounds the raw-row queries of one evaluation. +const maxOrderedJSONRowSpans = 16 + +// mergeOrderedJSONRowSeries adds the samples the row evaluator found to the +// stats buckets at the evaluation points: a line at t counts in the window +// (T-window, T] exactly when t-1ns falls in the [T-window, T) range the bucket +// labels are summed over (buildHitsRangeMetricMatrix), so each window sums the +// bucket counts and the raw lines before metricWindowValue runs once. +func mergeOrderedJSONRowSeries(merged, rows map[string]manualSeriesSamples, bytesMetric bool) { + for key, entry := range rows { + target := merged[key] + if target.Metric == nil { + target.Metric = entry.Metric + } + for _, sample := range entry.Samples { + target.Samples = append(target.Samples, rangeMetricSample{ts: sample.ts - 1, value: sample.value}) + if bytesMetric { + target.PresentBuckets = append(target.PresentBuckets, sample.ts-1) + } + } + sort.Slice(target.Samples, func(i, j int) bool { return target.Samples[i].ts < target.Samples[j].ts }) + if bytesMetric { + sort.Slice(target.PresentBuckets, func(i, j int) bool { return target.PresentBuckets[i] < target.PresentBuckets[j] }) + } + merged[key] = target + } +} + +// orderedJSONRowProjection returns the `| fields` pipe that keeps what the row +// evaluator reads from a raw row: _time, _stream, _msg, level (stream +// descriptor and detected_level) and every stored field that can expose one of +// the labels the plan requires (orderedJSONBaseLabels turns stored fields into +// labels that rename colliding parsed keys, answer filters and group). A label +// is read from the field of its own name, its query spelling (ToVL), a custom +// mapping, or any field it sanitizes from, which starts with the label's +// leading letters and digits. Empty when the plan needs every label (no hints) +// or a label has no such prefix. Also empty for a plan that derives +// detected_level: the derivation reads every stored level field (and the OTLP +// severity fields), which no label of the plan names. +func (p *Proxy) orderedJSONRowProjection(plan *orderedJSONMetricPlan) string { + if plan.required == nil || plan.derivesLevel { + return "" + } + fields := []string{"_time", "_stream", "_msg", "level"} + add := func(name string) { + if name != "" && !containsString(fields, name) { + fields = append(fields, name) + } + } + labels := make([]string, 0, len(plan.required)) + for label := range plan.required { + labels = append(labels, label) + } + sort.Strings(labels) + for _, label := range labels { + add(label) + if p.labelTranslator == nil { + continue + } + add(p.labelTranslator.ToVL(label)) + for vl, loki := range p.labelTranslator.vlToLoki { + if loki == label { + add(vl) + } + } + if p.labelTranslator.style != LabelStyleUnderscores { + continue + } + if strings.Contains(label, "__") || strings.HasSuffix(label, "_") { + // SanitizeLabelName never yields consecutive or trailing underscores, + // so only the field of the label's own name exposes it. + continue + } + lead := label + if i := strings.IndexByte(label, '_'); i >= 0 { + lead = label[:i] + } + if lead == "" || lead == "key" || label == "key_empty" || !orderedJSONASCIIAlnum(lead) { + return "" // sanitized from a field without a common prefix + } + add(lead + "*") + } + quoted := make([]string, len(fields)) + for i, field := range fields { + if prefix, ok := strings.CutSuffix(field, "*"); ok { + quoted[i] = quoteLogsQLIdent(prefix) + "*" + } else { + quoted[i] = quoteLogsQLIdent(field) + } + } + return " | fields " + strings.Join(quoted, ", ") +} + +func orderedJSONASCIIAlnum(s string) bool { + for i := 0; i < len(s); i++ { + c := s[i] + if c >= utf8.RuneSelf || (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') && (c < '0' || c > '9') { + return false + } + } + return true +} + +// addOrderedJSONRowLines reads the brace-opened lines VictoriaLogs cannot +// vouch for from the buckets holding them (rowEdges), evaluates them with the +// row evaluator and adds them to the stats series. +func (p *Proxy) addOrderedJSONRowLines(ctx context.Context, plan *orderedJSONMetricPlan, merged map[string]manualSeriesSamples, base string, rowEdges []int64, hazard bool, bucket time.Duration, start, end time.Time, step time.Duration) (map[string]manualSeriesSamples, error) { + rowQuery := base + " | filter " + jsonObjectStartFilter + jsonValidityPipes(true) + " | filter " + jsonInvalidCondition(hazard) + " | delete " + jsonValidField + p.orderedJSONRowProjection(plan) + rows, err := p.collectOrderedJSONRowSpans(ctx, plan, rowQuery, orderedJSONRowSpans(rowEdges, bucket), start, end, step) + if err != nil { + return nil, err + } + mergeOrderedJSONRowSeries(merged, rows, plan.function == "bytes_rate" || plan.function == "bytes_over_time") + return capSeriesForRequest(ctx, merged, p.resolvedMaxStatsQuerySeries(ctx)) +} + +// collectOrderedJSONRowSpans evaluates the raw-row lines of a split plan: +// rowQuery over each span, sharing the row and byte limits of one raw read. +func (p *Proxy) collectOrderedJSONRowSpans(ctx context.Context, plan *orderedJSONMetricPlan, rowQuery string, spans [][2]time.Time, start, end time.Time, step time.Duration) (map[string]manualSeriesSamples, error) { + reader, err := p.newOrderedJSONRowReader() + if err != nil { + return nil, err + } + for _, span := range spans { + from, to := span[0], span[1] + if windowStart := start.Add(-plan.window); from.Before(windowStart) { + from = windowStart + } + if to.After(end) { + to = end + } + if to.Before(from) { + continue + } + if err := p.readOrderedJSONMetricRows(ctx, plan, reader, rowQuery, from, to, start, end, step); err != nil { + return nil, err + } + } + return reader.series, nil +} + +// dropSeriesWithoutPoints removes the series none of whose counted buckets falls +// in an evaluation window (T-window, T], T = start+k*step up to end: Loki's +// answer holds no series without a point, so such a series (a line in the gap +// between windows when step exceeds the window) is no series for the limit +// either. Buckets sit on the grid of gcd(step, window) anchored at +// start-window, so a bucket is inside a window or outside it entirely; a bucket +// at left edge e is the sample at e+1ns of orderedJSONSampleVisible. +func dropSeriesWithoutPoints(series map[string]manualSeriesSamples, start, end time.Time, step, window time.Duration) { + startNS, endNS := start.UnixNano(), end.UnixNano() + for key, entry := range series { + visible := false + if entry.PresentBuckets != nil { + // A byte series counts where a line is present, even an empty one. + for _, edge := range entry.PresentBuckets { + if visible = orderedJSONSampleVisible(edge+1, startNS, endNS, int64(step), int64(window)); visible { + break + } + } + } else { + for _, sample := range entry.Samples { + if sample.value != 0 && orderedJSONSampleVisible(sample.ts+1, startNS, endNS, int64(step), int64(window)) { + visible = true + break + } + } + } + if !visible { + delete(series, key) + } + } +} + +// storedLevelForeignFieldRisk reports whether a selected line holds a stored +// level field other than detected_level and level (severity, lvl, the OTLP +// severity fields, ...). The stored level of a split plan is the copy of level, +// which cannot tell a stream label from structured metadata, and Loki reads +// the stream's level fields before the metadata's: a stream severity=error with +// metadata level=info is error for Loki and info for the copy. Such a window is +// left to the row evaluator, which derives the level like Loki's ingest. +func (p *Proxy) storedLevelForeignFieldRisk(ctx context.Context, base string, start, end time.Time) (bool, error) { + var has []string + for _, name := range translator.DetectedLevelFieldNames { + if name != detectedLevelLabel && name != "level" { + has = append(has, quoteLogsQLIdent(name)+":*") + } + } + key := "json-split-level-foreign:v1:" + getOrgID(ctx) + ":" + base + _, risky, err := p.coveredWindowProbe(ctx, key, start, end, func(from, to time.Time, _ string) (string, bool, error) { + risky, err := p.statsPushdownRiskExists(ctx, base+" | filter ("+strings.Join(has, " OR ")+") | limit 1", from, to) + return "", risky, err + }) + return risky, err +} diff --git a/internal/proxy/ordered_json_valid_split_test.go b/internal/proxy/ordered_json_valid_split_test.go new file mode 100644 index 000000000..315661c54 --- /dev/null +++ b/internal/proxy/ordered_json_valid_split_test.go @@ -0,0 +1,1939 @@ +package proxy + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math/rand" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "runtime" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/grafana/jsonparser" + "github.com/valyala/fastjson" +) + +// vlFastjsonUnpackFields mirrors VictoriaLogs' unpack_json exactly +// v1.52 (lib/logstorage/pipe_unpack_json.go, json_parser.go): JSON whitespace +// around the value is trimmed (trimJSONWhitespace; v1.47 and v1.40 do not trim); +// a value whose first byte is then not `{` yields nothing; otherwise fastjson parses the whole value +// (trailing text fails it), nested objects are flattened with a dot, nulls are +// skipped, numbers, booleans and arrays keep their JSON text and strings are +// decoded. +func vlFastjsonUnpackFields(msg string) map[string]string { + msg = strings.Trim(msg, " \t\n\r") + if msg == "" || msg[0] != '{' { + return nil + } + var parser fastjson.Parser + v, err := parser.Parse(msg) + if err != nil { + return nil + } + o, err := v.Object() + if err != nil { + return nil + } + out := map[string]string{} + var walk func(prefix string, o *fastjson.Object) + walk = func(prefix string, o *fastjson.Object) { + o.Visit(func(k []byte, v *fastjson.Value) { + switch v.Type() { + case fastjson.TypeNull: + case fastjson.TypeObject: + child, _ := v.Object() + walk(prefix+string(k)+".", child) + case fastjson.TypeString: + out[prefix+string(k)] = string(v.GetStringBytes()) + default: + out[prefix+string(k)] = string(v.MarshalTo(nil)) + } + }) + } + walk("", o) + return out +} + +// lokiJSONAccepts mirrors Loki v3.7.7's JSONParser.Process acceptance +// (pkg/logql/log/parser.go:74-135): jsonparser.ObjectEach over the line, +// descending into a nested object only when one of the required labels +// starts with its sanitized key path (parser_hints.go ShouldExtractPrefix, +// parser.go buildSanitizedPrefixFromBuffer; nil required walks every object). +// The required labels hold __error__, so parsing never stops early. +func lokiJSONAccepts(line string, required []string) bool { + var walk func(data []byte, path [][]byte) error + walk = func(data []byte, path [][]byte) error { + return jsonparser.ObjectEach(data, func(key, value []byte, kind jsonparser.ValueType, _ int) error { + if kind != jsonparser.Object { + return nil + } + next := append(append([][]byte(nil), path...), append([]byte(nil), key...)) + var sanitized []byte + for i, part := range next { + part = []byte(strings.TrimSpace(string(part))) + if len(part) == 0 { + continue + } + if i > 0 && len(sanitized) > 0 { + sanitized = append(sanitized, '_') + } + if len(sanitized) == 0 && part[0] >= '0' && part[0] <= '9' { + sanitized = append(sanitized, '_') + } + for _, r := range string(part) { + if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_' { + sanitized = append(sanitized, byte(r)) + } else { + sanitized = append(sanitized, '_') + } + } + } + descend := required == nil + for _, label := range required { + descend = descend || strings.HasPrefix(label, string(sanitized)) + } + if !descend { + return nil + } + return walk(value, next) + }) + } + return walk([]byte(line), nil) == nil +} + +// jsonSplitClass reports how the split pushdown files a line: counted by +// VictoriaLogs (the spliced line parses and holds no hazard), read as a raw +// row (it opens with a brace after JSON whitespace), or dropped. The +// predicates are the LogsQL the pushdown sends, evaluated with VictoriaLogs +// semantics. +func jsonSplitClass(t testing.TB, line string) (counted, rawRow bool) { + values := map[string]string{"_msg": line} + if !fakeLogsQLMatch(t, jsonObjectStartFilter, values) { + return false, false + } + spliced := line + if strings.HasPrefix(line, "{") { + spliced = `{"` + jsonValidField + `":1,` + line[1:] + } + valid := vlFastjsonUnpackFields(spliced)[jsonValidField] != "" + if valid && !fakeLogsQLMatch(t, jsonLokiRejectHazard, values) { + return true, false + } + return false, true +} + +// jsonSplitHintSets are the parser hints the differential checks: every object +// walked (the strictest), a query with no parsed label, and two that reach +// into nested objects. +var jsonSplitHintSets = [][]string{nil, {"__error__"}, {"a", "__error__"}, {"a_b", "__error__"}} + +var jsonSplitSeeds = []string{ + `{"service": {"name": "api-gateway"}, "method": "PATCH", "status": 502, "ok": true, "x": null, "arr": [1, "a", {"b": 2}], "level": "error"}`, + `{"a":1}`, `{}`, `{ }`, `{"a":{"b":{"c":"d"}}}`, `{"a":"b\"c","d\\e":1}`, `{"kéy":"v"}`, + `{"a":NaN}`, `{"a":Infinity}`, `{"a":-Infinity}`, `{"a":.5}`, `{"a":+1}`, `{"a":e5}`, `{"a":-}`, `{"a":1e5}`, + `{"a":"\ud800"}`, `{"\ud800":1}`, `{"\udc00x":1}`, `{"a\qb":1}`, `{"a":1,}`, `{"a":1} trailing`, ` {"a":1}`, + `{"a":[NaN, .5, +1]}`, `{"a":{"b":NaN}}`, `{"":1}`, `{"a":1,"a":2}`, "{\"a\":\"\xff\"}", "{\"a\":\"x\ny\"}", + `{"a":tru}`, `{"a":nul}`, `{"a":nan}`, `{"a":nAn}`, `{"a":inf}`, `{"a":-nan}`, `{"a":1.2.3}`, `{"a":--1}`, + "\f{\"a\":1}", "{\"a\":\fNaN}", `{"a_b":{"c":NaN}}`, `{"a":{"b":.5}}`, `{"x":{"y":+1}}`, +} + +var ( + jsonSplitTokens = []string{"1", "-1", "0.5", "1e5", "-0", ".5", "+1", "e5", "E1", "NaN", "nan", "nAn", "-NaN", "Infinity", "inf", "-inf", "+inf", "INF", "-", "1.2.3", "--1", + "true", "false", "null", `"s"`, `"a\"b"`, `"\ud800"`, `"A"`, `"\q"`, `"�"`, `[]`, `[1,NaN]`, `[.5]`, `{}`, `{"x":1}`} + jsonSplitKeys = []string{"a", "b", "a_b", "", " a ", `a\"b`, `a\\b`, `A`, `\ud800`, `\udc00`, `a\qb`, "é", "1a", "__vlp", "a.b"} + jsonSplitSpaces = []string{"", " ", " ", "\t", "\n", "\r", "\f"} + jsonSplitAlpha = []string{"{", "}", "[", "]", `"`, ":", ",", `\`, ".", "e", "E", "+", "-", "0", "1", "9", " ", "\t", "\n", "\r", "\f", + "t", "r", "u", "f", "a", "l", "s", "n", "N", "I", "i", "x", `\u`, `\ud800`, `\udc00`, `A`, "NaN", "Inf", "inf", "nan", + "true", "false", "null", "�", "é", `\"`, `\\`, `":`, `":1`, `,"k":`, `{"`, `"}`} +) + +func jsonSplitObject(rng *rand.Rand, depth int) string { + var b strings.Builder + b.WriteString("{") + for i, n := 0, rng.Intn(5); i < n; i++ { + if i > 0 { + b.WriteString(",") + } + b.WriteString(jsonSplitSpaces[rng.Intn(len(jsonSplitSpaces))] + `"` + jsonSplitKeys[rng.Intn(len(jsonSplitKeys))] + `"`) + b.WriteString(jsonSplitSpaces[rng.Intn(len(jsonSplitSpaces))] + ":" + jsonSplitSpaces[rng.Intn(len(jsonSplitSpaces))]) + if depth < 3 && rng.Intn(4) == 0 { + b.WriteString(jsonSplitObject(rng, depth+1)) + } else { + b.WriteString(jsonSplitTokens[rng.Intn(len(jsonSplitTokens))]) + } + } + if rng.Intn(20) == 0 { + b.WriteString(",") + } + b.WriteString(jsonSplitSpaces[rng.Intn(len(jsonSplitSpaces))] + "}") + return b.String() +} + +func jsonSplitMutate(rng *rand.Rand, s string) string { + for i, n := 0, 1+rng.Intn(4); i < n; i++ { + pos := rng.Intn(len(s) + 1) + switch rng.Intn(3) { + case 0: + s = s[:pos] + jsonSplitAlpha[rng.Intn(len(jsonSplitAlpha))] + s[pos:] + case 1: + if pos < len(s) { + s = s[:pos] + s[min(len(s), pos+1+rng.Intn(3)):] + } + default: + if pos < len(s) { + s = s[:pos] + jsonSplitAlpha[rng.Intn(len(jsonSplitAlpha))] + s[pos+1:] + } + } + } + return s +} + +// jsonSplitLine draws one line: a generated object, possibly mutated, with +// leading space or trailing text now and then, or a mutated seed. +func jsonSplitLine(rng *rand.Rand) string { + if rng.Intn(3) == 0 { + return jsonSplitMutate(rng, jsonSplitSeeds[rng.Intn(len(jsonSplitSeeds))]) + } + s := jsonSplitObject(rng, 0) + if rng.Intn(4) == 0 { + s = jsonSplitMutate(rng, s) + } + if rng.Intn(10) == 0 { + s = jsonSplitSpaces[rng.Intn(len(jsonSplitSpaces))] + s + } + if rng.Intn(10) == 0 { + s += []string{" trailing", "}", ",", "\n", " "}[rng.Intn(5)] + } + return s +} + +// checkJSONSplitLine asserts the split's soundness for one line: a line +// VictoriaLogs counts is accepted by Loki's parser under every hint set (so +// Loki counts it once and VictoriaLogs' count is exact), and a line the split +// drops is rejected under every hint set (so Loki drops it too). +func checkJSONSplitLine(t testing.TB, line string) (counted, rawRow bool) { + counted, rawRow = jsonSplitClass(t, line) + for _, hints := range jsonSplitHintSets { + accepted := lokiJSONAccepts(line, hints) + switch { + case counted && !accepted: + t.Errorf("VictoriaLogs counts %q, Loki's parser rejects it (hints %q)", line, hints) + case !counted && !rawRow && accepted: + t.Errorf("the split drops %q, Loki's parser accepts it (hints %q)", line, hints) + } + } + return counted, rawRow +} + +// The lines the split pushdown lets VictoriaLogs count are a subset of the +// lines Loki's JSON parser accepts under any parser hints, and the lines it +// drops are a subset of the lines Loki rejects; everything else is read as a +// raw row and evaluated by the row evaluator. Checked over a seeded corpus of +// generated and mutated JSON (non-standard numbers, NaN and Inf, escaped and +// broken keys, trailing text, leading whitespace, nesting), and with the +// hazard exclusion removed the same corpus must find lines VictoriaLogs +// parses and Loki rejects, so the corpus exercises the exclusion. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitPredicateDifferential(t *testing.T) { + n := 200_000 + if testing.Short() { + n = 40_000 + } + rng := rand.New(rand.NewSource(20261012)) + counted, rawRows, unguarded := 0, 0, 0 + for i := 0; i < n; i++ { + line := jsonSplitSeeds[i%len(jsonSplitSeeds)] + if i >= len(jsonSplitSeeds) { + line = jsonSplitLine(rng) + } + c, r := checkJSONSplitLine(t, line) + if c { + counted++ + } + if r { + rawRows++ + spliced := line + if strings.HasPrefix(line, "{") { + spliced = `{"` + jsonValidField + `":1,` + line[1:] + } + if vlFastjsonUnpackFields(spliced)[jsonValidField] != "" && !lokiJSONAccepts(line, nil) { + unguarded++ // only the hazard exclusion keeps this line from VictoriaLogs' count + } + } + if t.Failed() { + return + } + } + t.Logf("lines=%d counted by VictoriaLogs=%d raw rows=%d hazard-excluded Loki rejects=%d", n, counted, rawRows, unguarded) + if counted < n/50 || unguarded < n/1000 { + t.Fatalf("the corpus no longer exercises the split: counted=%d hazard-excluded=%d", counted, unguarded) + } +} + +// FuzzJSONValidSplitPredicate runs the differential over fuzzed lines. +func FuzzJSONValidSplitPredicate(f *testing.F) { + for _, seed := range jsonSplitSeeds { + f.Add(seed) + } + f.Fuzz(func(t *testing.T, line string) { + checkJSONSplitLine(t, line) + }) +} + +// The hazard filter flags each construct VictoriaLogs' parser accepts and +// Loki's rejects, anywhere in the line, and leaves ordinary JSON alone. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONLokiRejectHazardTable(t *testing.T) { + for _, tc := range []struct { + line string + hazard, counts bool + }{ + {`{"a":1,"b":"x","c":true,"d":null,"e":[1,2],"f":{"g":-1.5e3}}`, false, true}, + {`{"a":NaN}`, true, false}, + {`{"a": nan}`, true, false}, + {`{"a":INF}`, true, false}, + {`{"a":-NaN}`, false, true}, // jsonparser reads any token starting with '-' as a number + {`{"a":"NaN"}`, false, true}, + {`{"a":.5}`, true, false}, + {`{"a":e5}`, true, false}, + {`{"a":E5}`, true, false}, + {`{"a":+1}`, true, false}, + {"{\"a\":\t+1}", true, false}, + {`{"a\qb":1}`, true, false}, + {`{"\ud800":1}`, true, false}, + {`{"a\"b":1}`, true, false}, + {`{"a":{"b":NaN}}`, true, false}, + {`{"msg":"took 5ms: e"}`, true, false}, // inside a string: a superset, read as a raw row + {`{"a":"\\"}`, false, true}, + {`{"a":"\""}`, false, true}, + {`{"url":"http://x/a+b"}`, false, true}, + {`{"n":"inf"}`, false, true}, + {`{"level":"info"}`, false, true}, + } { + t.Run(tc.line, func(t *testing.T) { + if got := fakeLogsQLMatch(t, jsonLokiRejectHazard, map[string]string{"_msg": tc.line}); got != tc.hazard { + t.Fatalf("hazard=%v, want %v", got, tc.hazard) + } + if counted, _ := checkJSONSplitLine(t, tc.line); counted != tc.counts { + t.Fatalf("counted by VictoriaLogs=%v, want %v", counted, tc.counts) + } + }) + } +} + +// Which `| json | __error__=""` metrics take the split pushdown. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitEligibility(t *testing.T) { + for _, tc := range []struct { + query string + pushdown, validOnly bool + }{ + {`sum by (app) (count_over_time({env="production"} | json | __error__="" [5m]))`, true, true}, + {`sum(count_over_time({env="production"} | json | __error__="" [5m]))`, true, true}, + {`sum by (method) (rate({env="production"} | json | __error__="" [5m]))`, true, true}, + {`sum by (level, detected_level) (bytes_over_time({env="production"} | json | __error__="" [5m]))`, true, true}, + {`sum by (method) (count_over_time({env="production"} |= "GET" | json | __error__="" | status="500" [5m]))`, true, true}, + {`sum by (method) (count_over_time({env="production"} | json | __error__="" | drop __error_details__ [5m]))`, true, true}, + // detected_level may be grouped by (the stored level fields decide it), + // not filtered on: the filter would need the derived level for the + // lines read as raw rows. + {`sum by (detected_level) (count_over_time({env="production"} | json | __error__="" [5m]))`, true, true}, + {`sum by (app, detected_level) (rate({env="production"} | json | __error__="" [5m]))`, true, true}, + {`sum by (method) (count_over_time({env="production"} | json | __error__="" | detected_level="info" [5m]))`, false, false}, + {`sum by (detected_level) (count_over_time({env="production"} | json | __error__="" | detected_level!="debug" [5m]))`, false, false}, + // service_name is the stream label Loki assigns, derived before any unpack. + {`sum by (service_name) (count_over_time({service_name="api-gateway"} | json | __error__="" [5m]))`, true, true}, + {`sum by (service_name, method) (count_over_time({env="production"} | json | __error__="" [5m]))`, true, true}, + // After a drop of __error__ the filter passes every line: the plain pushdown. + {`sum by (method) (count_over_time({env="production"} | json | drop __error__ | __error__="" [5m]))`, true, false}, + {`sum by (method) (count_over_time({env="production"} | json | __error__!="" [5m]))`, false, false}, + {`sum by (method) (count_over_time({env="production"} | json | __error__="JSONParserErr" [5m]))`, false, false}, + {`sum by (method) (count_over_time({env="production"} | logfmt | __error__="" [5m]))`, false, false}, + {`sum without (method) (count_over_time({env="production"} | json | __error__="" [5m]))`, false, false}, + {`count_over_time({env="production"} | json | __error__="" [5m])`, false, false}, + // The split blanks the grouped fields of the lines it reads as raw rows: + // _msg, _time and _stream are not fields to blank, so these keep the + // row evaluator they always had. + {`sum by (_msg) (count_over_time({env="production"} | json | __error__="" [5m]))`, false, false}, + {`sum by (_stream) (count_over_time({env="production"} | json | __error__="" [5m]))`, false, false}, + {`sum by (app, _time) (count_over_time({env="production"} | json | __error__="" [5m]))`, false, false}, + {`sum by (app) (count_over_time({env="production"} | json | __error__="" | _msg="x" [5m]))`, false, false}, + } { + t.Run(tc.query, func(t *testing.T) { + plan, ok := compileOrderedJSONMetric(tc.query) + if !ok { + if tc.pushdown { + t.Fatal("expected an ordered JSON plan") + } + return + } + if plan.pushdown != tc.pushdown || plan.validOnly != tc.validOnly { + t.Fatalf("pushdown=%v validOnly=%v, want %v %v", plan.pushdown, plan.validOnly, tc.pushdown, tc.validOnly) + } + }) + } +} + +// jsonSplitRows is a seeded fixture of the shapes the split must answer like +// Loki: valid JSON of several apps (one with the level in structured +// metadata), lines VictoriaLogs parses and Loki rejects, truncated and +// trailing-text JSON, leading whitespace, plain and logfmt lines. +func jsonSplitRows(rng *rand.Rand, s0 time.Time, n int, hazards bool) []pushdownRow { + statuses := []string{"200", "404", "500"} + var rows []pushdownRow + for i := 0; i < n; i++ { + ts := s0.Add(time.Duration(rng.Intn(600_000)) * time.Millisecond) + if i%25 == 0 { // on an evaluation time or a window edge + ts = s0.Add(time.Duration(rng.Intn(11)) * time.Minute) + } + app := []string{"api", "web", "worker"}[rng.Intn(3)] + row := pushdownRow{ts: ts, stream: map[string]string{"app": app, "env": "prod"}} + status := statuses[rng.Intn(len(statuses))] + switch k := rng.Intn(20); { + case k < 12: + row.msg = `{"status":"` + status + `","method":"GET","n":` + strconv.Itoa(i) + `}` + case k == 12: + row.msg = `{"status":` + status + `,"nested":{"a":{"b":1}},"arr":[1,2]}` + case k == 13: + row.msg = `{"status":"` + status + `","msg":"truncated` + case k == 14: + row.msg = `{"status":"` + status + `"} trailing` + case k == 15: + row.msg = " {\"status\":\"" + status + "\"}" + case k == 16: + row.msg = "plain text " + strconv.Itoa(i) + case k == 17: + row.msg = "status=" + status + " method=GET" + case k == 18 && hazards: + row.msg = []string{`{"status":"` + status + `","ratio":NaN}`, `{"status":"` + status + `","v":.5}`, `{"st\qatus":"` + status + `"}`}[rng.Intn(3)] + default: + row.stream["level"] = "info" + row.msg = `{"status":"` + status + `","level":"error"}` + } + rows = append(rows, row) + } + return rows +} + +func runSplitQuery(t *testing.T, p *Proxy, query string, start, end time.Time, step time.Duration) (int, map[string]map[int64]string, string) { + t.Helper() + params := url.Values{"query": {query}, "start": {strconv.FormatInt(start.UnixNano(), 10)}, "end": {strconv.FormatInt(end.UnixNano(), 10)}, "step": {strconv.FormatFloat(step.Seconds(), 'f', -1, 64)}} + rec := httptest.NewRecorder() + p.handleQueryRange(rec, httptest.NewRequest(http.MethodGet, "/loki/api/v1/query_range?"+params.Encode(), nil)) + if rec.Code != http.StatusOK { + return rec.Code, nil, rec.Body.String() + } + return rec.Code, decodeExtractedMatrix(t, rec.Body.Bytes()), "" +} + +var jsonSplitQueries = []string{ + `sum(count_over_time({env="prod"} | json | __error__="" [1m]))`, + `sum by (app) (count_over_time({env="prod"} | json | __error__="" [1m]))`, + `sum by (status) (count_over_time({env="prod"} | json | __error__="" [1m]))`, + `sum by (app) (rate({env="prod"} | json | __error__="" [2m]))`, + `sum by (status) (bytes_over_time({env="prod"} | json | __error__="" [1m]))`, + `sum by (app) (bytes_rate({env="prod"} | json | __error__="" [1m]))`, + `sum by (app) (count_over_time({env="prod"} | json | __error__="" | status="500" [1m]))`, + `sum by (level, detected_level) (count_over_time({env="prod"} | json | __error__="" [1m]))`, + `sum by (service_name) (count_over_time({env="prod"} | json | __error__="" [1m]))`, + `sum by (service_name, status) (bytes_over_time({env="prod"} | json | __error__="" [1m]))`, +} + +// withStoredLevels gives every row a level the way a stream label stores it, +// in several spellings of the same few levels. +func withStoredLevels(rows []pushdownRow) []pushdownRow { + spellings := []string{"info", "INFO", "error", "Warning", "inf", "debug"} + out := make([]pushdownRow, len(rows)) + for i, row := range rows { + stream := map[string]string{} + for k, v := range row.stream { + stream[k] = v + } + stream["level"] = spellings[i%len(spellings)] + row.stream = stream + out[i] = row + } + return out +} + +// assertStoredLevelStats fails when a stats query sent for a split plan +// grouped by detected_level is not the stored-level form: the copies of the +// stored fields, and none of the pipes, probes or flags of the derivation chain. +func assertStoredLevelStats(t *testing.T, fake *pushdownFakeVL, label string) { + t.Helper() + fake.mu.Lock() + defer fake.mu.Unlock() + if len(fake.stats) == 0 { + t.Fatalf("%s: no stats query", label) + } + for _, stats := range fake.stats { + if !strings.Contains(stats, "copy detected_level as "+storedDetectedLevelField+", level as "+storedLevelField) { + t.Fatalf("%s: stats query does not read the stored level fields: %s", label, stats) + } + if strings.Contains(stats, `result_prefix "__j_"`) || strings.Contains(stats, levelWitnessField) || strings.Contains(stats, "__dl_h") { + t.Fatalf("%s: stats query carries the derivation chain: %s", label, stats) + } + if strings.Index(stats, "copy detected_level") > strings.Index(stats, jsonValidField) { + t.Fatalf("%s: the stored level fields are copied after the sentinel splice: %s", label, stats) + } + } + if len(fake.presenceCalls) != 0 { + t.Fatalf("%s: %d level presence probes sent", label, len(fake.presenceCalls)) + } +} + +// rowEvaluatorMatrix answers plan with the proxy's row evaluator over every +// selected row, the path these shapes took before the split. +func rowEvaluatorMatrix(t *testing.T, p *Proxy, plan *orderedJSONMetricPlan, start, end time.Time, step time.Duration) map[string]map[int64]string { + t.Helper() + series, err := p.collectOrderedJSONMetric(context.Background(), plan, start, end, step) + if err != nil { + t.Fatalf("row evaluator: %v", err) + } + body, err := buildOrderedJSONMetric(context.Background(), plan, series, start, end, step, true, 1<<30) + if err != nil { + t.Fatalf("row evaluator: %v", err) + } + return decodeExtractedMatrix(t, body) +} + +// The split pushdown answers every `| json | __error__=""` shape exactly as +// the row evaluator (whose parser follows Loki's), over a seeded fixture with +// VictoriaLogs' own parser semantics in the fake backend: one stats scan, a +// raw read only of the buckets holding lines VictoriaLogs cannot vouch for, +// and a second scan only when the hazard probe finds a line. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitMatchesRawEvaluator(t *testing.T) { + seeds := 10 + if testing.Short() { + seeds = 4 + } + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(10*time.Minute) + for seed := int64(1); seed <= int64(seeds); seed++ { + rng := rand.New(rand.NewSource(seed)) + hazards := seed%2 == 0 + mixed := jsonSplitRows(rng, s0, 400, hazards) + for _, query := range jsonSplitQueries { + // A query grouped by detected_level is served by the split only when + // the stored level fields decide the level of every counted line; + // the fixture with body-only levels leaves it to the row evaluator. + // Either way the answer is the row evaluator's. + variants := []bool{false} + if strings.Contains(query, "detected_level") { + variants = []bool{true, false} + } + for _, stored := range variants { + rows := mixed + if stored { + rows = withStoredLevels(mixed) + } + served := stored || !strings.Contains(query, "detected_level") + label := fmt.Sprintf("seed %d stored=%v: %s", seed, stored, query) + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + plan, ok := compileOrderedJSONMetric(query) + if !ok || !plan.pushdown || !plan.validOnly { + t.Fatalf("%s: expected a split pushdown plan", label) + } + want := rowEvaluatorMatrix(t, p, plan, start, end, time.Minute) + fake.mu.Lock() + fake.raw, fake.rawQueries, fake.stats, fake.guards = 0, nil, nil, nil + fake.mu.Unlock() + code, got, body := runSplitQuery(t, p, query, start, end, time.Minute) + if code != http.StatusOK { + t.Fatalf("%s: status %d: %s", label, code, body) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("%s: differs from the row evaluator\n got: %v\nwant: %v", label, got, want) + } + fake.mu.Lock() + stats, rawQueries := len(fake.stats), fake.rawQueries + fake.mu.Unlock() + if !served { + // The stored fields do not decide every line: the row evaluator + // read every selected row and derived each level itself. + full := false + for _, raw := range rawQueries { + full = full || !strings.Contains(raw.query, " | filter "+jsonInvalidCondition(hazards)) + } + if !full { + t.Fatalf("%s: expected the row evaluator over every row, got %d bounded raw reads", label, len(rawQueries)) + } + continue + } + wantStats := 1 + if hazards { + wantStats = 2 + } + if stats != wantStats || len(rawQueries) == 0 || len(rawQueries) > maxOrderedJSONRowSpans { + t.Fatalf("%s: stats=%d raw=%d, want %d stats scans and a bounded raw read", label, stats, len(rawQueries), wantStats) + } + for _, raw := range rawQueries { + if !strings.Contains(raw.query, " | filter "+jsonInvalidCondition(hazards)) { + t.Fatalf("%s: raw read %q is not limited to the lines VictoriaLogs cannot vouch for", label, raw.query) + } + } + if strings.Contains(query, "detected_level") { + assertStoredLevelStats(t, fake, label) + } + } + } + } +} + +// With only lines VictoriaLogs vouches for, the split is one stats scan: +// no raw row is read, and the raw read of an invalid line covers only the +// buckets around it and keeps only the fields the evaluator reads. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitReadsOnlyRawRowBuckets(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(10*time.Minute) + var rows []pushdownRow + for i := 0; i < 120; i++ { + rows = append(rows, pushdownRow{ts: s0.Add(time.Duration(i)*5*time.Second + time.Millisecond), stream: map[string]string{"app": "api", "env": "prod"}, msg: `{"status":"200"}`}) + } + query := `sum by (status) (count_over_time({env="prod"} | json | __error__="" [1m]))` + run := func(rows []pushdownRow) *pushdownFakeVL { + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + plan, _ := compileOrderedJSONMetric(query) + want := lokiPushdownReference(t, plan, rows, start, end, time.Minute) + code, got, body := runSplitQuery(t, p, query, start, end, time.Minute) + if code != http.StatusOK || !reflect.DeepEqual(got, want) { + t.Fatalf("status %d %s\n got: %v\nwant: %v", code, body, got, want) + } + return fake + } + if fake := run(rows); fake.raw != 0 || len(fake.stats) != 1 { + t.Fatalf("expected one stats scan and no raw rows: stats=%d raw=%d", len(fake.stats), fake.raw) + } + broken := pushdownRow{ts: s0.Add(5*time.Minute + 7*time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: `{"status":"500"} trailing`} + fake := run(append(append([]pushdownRow(nil), rows...), broken)) + if len(fake.rawQueries) != 1 { + t.Fatalf("expected one raw read, got %d", len(fake.rawQueries)) + } + raw := fake.rawQueries[0] + if span := time.Duration(raw.end - raw.start); span > 2*time.Minute || raw.start > broken.ts.UnixNano() || raw.end <= broken.ts.UnixNano() { + t.Fatalf("raw read [%d, %d) is not the buckets around the line at %d", raw.start, raw.end, broken.ts.UnixNano()) + } + if !strings.Contains(raw.query, " | fields _time, _stream, _msg, level, ") { + t.Fatalf("raw read %q keeps every field", raw.query) + } +} + +// A line VictoriaLogs vouches for is counted however wide and up to 300 levels +// deep (fastjson MaxDepth) it is: the +// row evaluator's guards (nesting depth, field count, sizes) now apply only to +// the raw rows. Loki has no such guard, so the count matches Loki. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitCountsDeepLinesLikeLoki(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + deep := strings.Repeat(`{"a":`, 80) + "1" + strings.Repeat("}", 80) + rows := []pushdownRow{ + {ts: s0.Add(30 * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: deep}, + {ts: s0.Add(40 * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: `{"b":1}`}, + } + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + p := newFilterPushdownProxy(t, srv.URL) + if !lokiJSONAccepts(deep, []string{"app", "__error__"}) { + t.Fatal("Loki's parser must accept the deep line") + } + code, got, body := runSplitQuery(t, p, `sum by (app) (count_over_time({env="prod"} | json | __error__="" [1m]))`, s0.Add(time.Minute), s0.Add(time.Minute), time.Minute) + if code != http.StatusOK { + t.Fatalf("status %d: %s", code, body) + } + if want := map[string]map[int64]string{`{app="api"}`: {s0.Add(time.Minute).Unix(): "2"}}; !reflect.DeepEqual(got, want) { + t.Fatalf("got %v, want %v", got, want) + } +} + +// The raw rows keep the fields the row evaluator reads: the stream, the line, +// level and every stored field that can expose a required label. +func TestOrderedJSONRowProjection(t *testing.T) { + p := &Proxy{labelTranslator: NewLabelTranslator(LabelStyleUnderscores, nil)} + for _, tc := range []struct{ query, want string }{ + {`sum by (app) (count_over_time({env="prod"} | json | __error__="" [1m]))`, + " | fields _time, _stream, _msg, level, __error__, app, app*"}, + {`sum by (service_version) (count_over_time({env="prod"} | json | __error__="" [1m]))`, + " | fields _time, _stream, _msg, level, __error__, service_version, `service.version`, service*"}, + {`sum by (level_extracted) (count_over_time({env="prod"} | json | __error__="" [1m]))`, + " | fields _time, _stream, _msg, level, __error__, level*, level_extracted"}, + // No hints: every label can matter. + {`sum without (app) (count_over_time({env="prod"} | json | __error__="" [1m]))`, ""}, + // Sanitized from a field without a common prefix (a leading digit or symbol). + {`sum by (key_1a) (count_over_time({env="prod"} | json | __error__="" [1m]))`, ""}, + {`sum by (_x) (count_over_time({env="prod"} | json | __error__="" [1m]))`, ""}, + } { + t.Run(tc.query, func(t *testing.T) { + plan, ok := compileOrderedJSONMetric(tc.query) + if !ok { + t.Fatal("expected an ordered JSON plan") + } + if got := p.orderedJSONRowProjection(plan); got != tc.want { + t.Fatalf("got %q, want %q", got, tc.want) + } + }) + } +} + +// Loki's JSON parser skips only space, tab, line feed and carriage return +// before the opening brace, so a line starting with a form feed fails the +// query like any other unparsed line, found by the one-row lookup. +// conformance: semantics/pipeline-error-text-parity +func TestOrderedJSONPipelineErrorFormFeedLine(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + rows := []pushdownRow{ + {ts: s0.Add(10 * time.Second), stream: map[string]string{"app": "api"}, msg: `{"status":"200"}`}, + {ts: s0.Add(20 * time.Second), stream: map[string]string{"app": "api"}, msg: "\f{\"status\":\"500\"}"}, + } + if lokiJSONAccepts(rows[1].msg, nil) { + t.Fatal("Loki's parser must reject a line opening with a form feed") + } + srv, fake := newPushdownFakeVL(t, rows, nil) + p := newFilterPushdownProxy(t, srv.URL) + params := url.Values{"query": {`sum by (status) (count_over_time({app="api"} | json [1m]))`}, "start": {strconv.FormatInt(s0.Add(time.Minute).UnixNano(), 10)}, "end": {strconv.FormatInt(s0.Add(time.Minute).UnixNano(), 10)}, "step": {"60"}} + rec := httptest.NewRecorder() + p.handleQueryRange(rec, httptest.NewRequest(http.MethodGet, "/loki/api/v1/query_range?"+params.Encode(), nil)) + if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "JSONParserErr") { + t.Fatalf("got %d %s, want Loki's pipeline error", rec.Code, rec.Body) + } + fake.mu.Lock() + defer fake.mu.Unlock() + if fake.raw != 0 { + t.Fatalf("expected the error from the one-row lookup, read %d raw scans", fake.raw) + } +} + +// orderedJSONRowSpans merges touching buckets and collapses many spans. +func TestOrderedJSONRowSpans(t *testing.T) { + b := 10 * time.Second + s := int64(b) + if got := orderedJSONRowSpans([]int64{3 * s, s, 2 * s, 7 * s}, b); len(got) != 2 || got[0][0].UnixNano() != s || got[0][1].UnixNano() != 4*s || got[1][0].UnixNano() != 7*s || got[1][1].UnixNano() != 8*s { + t.Fatalf("got %v", got) + } + var many []int64 + for i := 0; i < 3*maxOrderedJSONRowSpans; i++ { + many = append(many, int64(i)*3*s) + } + if got := orderedJSONRowSpans(many, b); len(got) != 1 || got[0][0].UnixNano() != 0 || got[0][1].UnixNano() != many[len(many)-1]+s { + t.Fatalf("got %v", got) + } +} + +// service_name derived for a stream whose source field is structured metadata +// only: the stats half and the raw-row half must agree with the row evaluator. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitServiceNameFromMetadata(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(5*time.Minute) + var rows []pushdownRow + for i := 0; i < 30; i++ { + ts := s0.Add(time.Duration(i)*7*time.Second + time.Millisecond) + rows = append(rows, + pushdownRow{ts: ts, stream: map[string]string{"env": "prod"}, vl: map[string]string{"app": "meta-app"}, loki: map[string]string{"app": "meta-app"}, msg: `{"status":"200"}`}, + pushdownRow{ts: ts.Add(time.Second), stream: map[string]string{"env": "prod"}, vl: map[string]string{"app": "meta-app"}, loki: map[string]string{"app": "meta-app"}, msg: `{"status":"500"} trailing`}, + pushdownRow{ts: ts.Add(2 * time.Second), stream: map[string]string{"env": "prod", "app": "stream-app"}, msg: `{"status":"200"}`}, + ) + } + for _, query := range []string{ + `sum by (service_name) (count_over_time({env="prod"} | json | __error__="" [1m]))`, + `sum by (service_name, status) (count_over_time({env="prod"} | json | __error__="" [1m]))`, + } { + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + plan, _ := compileOrderedJSONMetric(query) + want := rowEvaluatorMatrix(t, p, plan, start, end, time.Minute) + code, got, body := runSplitQuery(t, p, query, start, end, time.Minute) + if code != http.StatusOK || !reflect.DeepEqual(got, want) { + t.Fatalf("%s: status %d %s\n got: %v\nwant: %v", query, code, body, got, want) + } + } +} + +// Groups made only of lines the label filter drops must not exist: VictoriaLogs +// emits a group (with zero counts) for every row reaching stats, so the filter +// is applied before stats and those groups never count against the series +// limit. 20 distinct paths overall, 2 with status 500, limit 5: Loki answers +// the two series. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitFilteredOutGroupsStayUnderSeriesLimit(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(5*time.Minute) + var rows []pushdownRow + for i := 0; i < 20; i++ { + status := "200" + if i < 2 { + status = "500" + } + rows = append(rows, pushdownRow{ts: s0.Add(time.Duration(100+i) * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: fmt.Sprintf(`{"path":"/p%d","status":"%s"}`, i, status)}) + } + for _, function := range []string{"count_over_time", "bytes_over_time"} { + query := `sum by (path) (` + function + `({env="prod"} | json | __error__="" | status="500" [5m]))` + t.Run(function, func(t *testing.T) { runSplitFilteredOutGroups(t, rows, query, start, end) }) + } +} + +func runSplitFilteredOutGroups(t *testing.T, rows []pushdownRow, query string, start, end time.Time) { + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + p.maxStatsQuerySeries = 5 + plan, _ := compileOrderedJSONMetric(query) + if !plan.validOnly { + t.Fatal("plan must be a split plan") + } + want := rowEvaluatorMatrix(t, p, plan, start, end, time.Minute) + if len(want) != 2 { + t.Fatalf("fixture: row evaluator answers %d series, want 2", len(want)) + } + code, got, body := runSplitQuery(t, p, query, start, end, time.Minute) + if code != http.StatusOK { + t.Fatalf("split status %d while Loki answers: %s", code, body) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("split %v, row evaluator %v", got, want) + } +} + +// A line VictoriaLogs cannot parse forms an extra empty-label group (zero +// counts, rows only): it is not a series Loki returns, so it does not count +// against the series limit. Exactly 5 real paths with limit 5 answer. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitRawOnlyGroupIsNotASeries(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(5*time.Minute) + var rows []pushdownRow + for i := 0; i < 5; i++ { + rows = append(rows, pushdownRow{ts: s0.Add(time.Duration(100+i) * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: fmt.Sprintf(`{"path":"/p%d"}`, i)}) + } + rows = append(rows, pushdownRow{ts: s0.Add(150 * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: `{"path":"/p0"} trailing`}) + query := `sum by (path) (count_over_time({env="prod"} | json | __error__="" [5m]))` + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + p.maxStatsQuerySeries = 5 + plan, _ := compileOrderedJSONMetric(query) + want := rowEvaluatorMatrix(t, p, plan, start, end, time.Minute) + if len(want) != 5 { + t.Fatalf("fixture: row evaluator answers %d series, want 5", len(want)) + } + code, got, body := runSplitQuery(t, p, query, start, end, time.Minute) + if code != http.StatusOK { + t.Fatalf("split status %d while Loki answers: %s", code, body) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("split %v, row evaluator %v", got, want) + } + // One more real path is a real overflow, with Loki's error. + rows = append(rows, pushdownRow{ts: s0.Add(160 * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: `{"path":"/p9"}`}) + fake.rows = rows + if code, _, body := runSplitQuery(t, p, query, start, end, time.Minute); code != http.StatusBadRequest || !strings.Contains(body, "maximum number of series") { + t.Fatalf("six real series: status %d body %s, want the series limit error", code, body) + } +} + +// The service name VictoriaLogs derives from the stream labels is the first +// source field that is non-empty (deriveServiceName), whatever the value holds: +// a label value with a line feed is a name too. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitServiceNameFromStreamLabelWithNewline(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(5*time.Minute) + rows := []pushdownRow{ + {ts: s0.Add(100 * time.Second), stream: map[string]string{"app": "api\nx", "env": "prod"}, msg: `{"a":1}`}, + {ts: s0.Add(110 * time.Second), stream: map[string]string{"app": "web", "env": "prod"}, msg: `{"a":1}`}, + {ts: s0.Add(120 * time.Second), stream: map[string]string{"env": "prod"}, msg: `{"a":1}`}, + } + query := `sum by (service_name) (count_over_time({env="prod"} | json | __error__="" [5m]))` + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + plan, _ := compileOrderedJSONMetric(query) + if !plan.validOnly || !plan.groupServiceName { + t.Fatal("plan must be a split plan grouped by service_name") + } + want := rowEvaluatorMatrix(t, p, plan, start, end, time.Minute) + code, got, body := runSplitQuery(t, p, query, start, end, time.Minute) + if code != http.StatusOK || !reflect.DeepEqual(got, want) { + t.Fatalf("split %d %v %s, row evaluator %v", code, got, body, want) + } + if len(want) != 3 { + t.Fatalf("fixture: want 3 service names, got %v", want) + } +} + +// The hazard probe covers the lines that open as the split reads them (a brace +// after JSON whitespace), the same filter the split applies. VictoriaLogs +// v1.52 trims JSON whitespace before unpack_json's brace check where v1.47 and +// v1.40 do not, so a probe anchored at the first byte would be right on one +// and wrong on the other. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitHazardProbeUsesTheSplitBraceFilter(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(5*time.Minute) + rows := []pushdownRow{ + {ts: s0.Add(100 * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: `{"a":1}`}, + {ts: s0.Add(110 * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: " \t{\"a\":.5}"}, + } + query := `sum(count_over_time({env="prod"} | json | __error__="" [5m]))` + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + plan, _ := compileOrderedJSONMetric(query) + want := rowEvaluatorMatrix(t, p, plan, start, end, time.Minute) + code, got, body := runSplitQuery(t, p, query, start, end, time.Minute) + if code != http.StatusOK || !reflect.DeepEqual(got, want) { + t.Fatalf("split %d %v %s, row evaluator %v", code, got, body, want) + } + probes := 0 + for _, guard := range fake.guards { + if strings.Contains(guard, jsonLokiRejectHazard) { + probes++ + if !strings.Contains(guard, "| filter "+jsonObjectStartFilter+" ") { + t.Fatalf("hazard probe %q does not use the split's brace filter", guard) + } + } + } + if probes == 0 { + t.Fatal("no hazard probe ran") + } +} + +// VictoriaLogs v1.52 trims JSON whitespace around a value before unpack_json +// checks for the opening brace (pipe_unpack_json.go trimJSONWhitespace). +func TestVLFastjsonUnpackFieldsTrimsJSONWhitespace(t *testing.T) { + if got := vlFastjsonUnpackFields(" \t\r\n{\"a\":1} \n"); !reflect.DeepEqual(got, map[string]string{"a": "1"}) { + t.Fatalf("got %v", got) + } + if got := vlFastjsonUnpackFields("\f{\"a\":1}"); got != nil { + t.Fatalf("a form feed is not JSON whitespace, got %v", got) + } +} + +// A positive hazard probe discards the first scan, so the scan is cancelled +// instead of awaited: the answer comes from the second one without waiting for +// the first to finish. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitHazardProbeCancelsTheFirstScan(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(5*time.Minute) + rows := []pushdownRow{ + {ts: s0.Add(100 * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: `{"a":1}`}, + {ts: s0.Add(110 * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: `{"a":.5}`}, + } + query := `sum(count_over_time({env="prod"} | json | __error__="" [5m]))` + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + cancelled := make(chan struct{}) + var sent atomic.Int64 + fake.onStats = func(r *http.Request, q string) { + if strings.Contains(q, jsonLokiRejectHazard) { + return + } + // The first scan, without the hazard exclusion: it never finishes on + // its own. + sent.Add(1) + select { + case <-r.Context().Done(): + close(cancelled) + case <-time.After(3 * time.Second): + } + } + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + plan, _ := compileOrderedJSONMetric(query) + want := rowEvaluatorMatrix(t, p, plan, start, end, time.Minute) + began := time.Now() + code, got, body := runSplitQuery(t, p, query, start, end, time.Minute) + if code != http.StatusOK || !reflect.DeepEqual(got, want) { + t.Fatalf("split %d %v %s, row evaluator %v", code, got, body, want) + } + if elapsed := time.Since(began); elapsed > 2500*time.Millisecond { + t.Fatalf("answered in %s: the first scan was awaited", elapsed) + } + select { + case <-cancelled: // the server sees the closed connection asynchronously + case <-time.After(2 * time.Second): + // The scan may also be cancelled before its request leaves the proxy. + if sent.Load() != 0 { + t.Fatal("the first scan was not cancelled") + } + } +} + +// The series limit applies to the merged answer, never to the groups +// VictoriaLogs returns: 8 groups under a limit of 5 (at most twice the limit, +// so the read holds them all) merge into 8 series and end with Loki's error +// before any raw row is read, with or without lines read as raw rows. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitMergedAnswerOverTheLimitReturnsLokiError(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(5*time.Minute) + for _, withRaw := range []bool{false, true} { + var rows []pushdownRow + for i := 0; i < 8; i++ { + rows = append(rows, pushdownRow{ts: s0.Add(time.Duration(100+i) * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: fmt.Sprintf(`{"trace_id":"t%d"}`, i)}) + } + if withRaw { + // A line only Loki's parser accepts, so the plan also reads raw rows. + rows = append(rows, pushdownRow{ts: s0.Add(150 * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: `{"trace_id":"t0"} trailing`}) + } + for _, function := range []string{"count_over_time", "bytes_over_time", "bytes_rate"} { + query := `sum by (trace_id) (` + function + `({env="prod"} | json | __error__="" [5m]))` + t.Run(fmt.Sprintf("raw=%v/%s", withRaw, function), func(t *testing.T) { + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + p.maxStatsQuerySeries = 5 + code, _, body := runSplitQuery(t, p, query, start, end, time.Minute) + if code != http.StatusBadRequest || !strings.Contains(body, "maximum number of series (5) reached for a single query") { + t.Fatalf("status %d body %.200s, want Loki's series limit error", code, body) + } + if fake.raw != 0 { + t.Fatalf("%d raw-row queries issued: the error comes from the split's merged answer", fake.raw) + } + }) + } + } +} + +// More than twice the series limit of groups is only a memory guard: the split +// does not answer and the row evaluator does, whatever the groups merge into. +// It ends with Loki's series limit error when its own series are over the +// limit, and answers when they are not. The response size cap leads to the +// same place. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitOverTwiceTheLimitGroupsFallBackToRowEvaluator(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(5*time.Minute) + var rows []pushdownRow + for i := 0; i < 60; i++ { + rows = append(rows, pushdownRow{ts: s0.Add(time.Duration(100+i) * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: fmt.Sprintf(`{"trace_id":"t%d"}`, i)}) + } + query := `sum by (trace_id) (count_over_time({env="prod"} | json | __error__="" [5m]))` + for _, tc := range []struct { + name string + bytes int64 + }{ + {"group guard", maxStatsBucketResponseBytes}, + {"group guard before the size cap", 2500}, + {"too large to read", 600}, + {"smaller than a group", 200}, + } { + t.Run(tc.name, func(t *testing.T) { + old := maxStatsBucketResponseBytes + maxStatsBucketResponseBytes = tc.bytes + t.Cleanup(func() { maxStatsBucketResponseBytes = old }) + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + p.maxStatsQuerySeries = 5 + code, _, body := runSplitQuery(t, p, query, start, end, time.Minute) + if code != http.StatusBadRequest || !strings.Contains(body, "maximum number of series (5) reached for a single query") { + t.Fatalf("status %d body %.200s, want Loki's series limit error", code, body) + } + if fake.raw == 0 { + t.Fatal("the row evaluator did not answer") + } + // The limit the row evaluator's series fit in answers. + p.maxStatsQuerySeries = 60 + plan, _ := compileOrderedJSONMetric(query) + want := rowEvaluatorMatrix(t, p, plan, start, end, time.Minute) + if code, got, body := runSplitQuery(t, p, query, start, end, time.Minute); code != http.StatusOK || !reflect.DeepEqual(got, want) || len(want) != 60 { + t.Fatalf("limit 60: status %d %.200s, got %d series, want %d", code, body, len(got), len(want)) + } + }) + } +} + +// Level spellings Loki folds into one detected_level (info, INFO, inf and +// information are four VictoriaLogs groups, one series) are separate +// VictoriaLogs groups, which the series limit never counts: 8 spellings under +// a limit of 5 are 2 series. Loki derives detected_level at ingest, normalized +// (distributor/field_detection.go normalizeLogLevel), from a stream label or +// the JSON body; the expected answer counts the lines that way, with the level +// in the body or as a stream label and with or without a raw candidate (a line +// only Loki's parser accepts, which the row evaluator derives the level of +// too). Past the memory guard (8 groups under a limit of 3) the row evaluator +// answers, with the same series. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitLevelSpellingsMergeUnderTheLimit(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(5*time.Minute) + levels := []string{"info", "INFO", "Info", "inf", "information", "warn", "WARN", "warning"} + query := `sum by (detected_level) (count_over_time({env="prod"} | json | __error__="" [5m]))` + for _, inStream := range []bool{false, true} { + for _, withRaw := range []bool{false, true} { + t.Run(fmt.Sprintf("stream=%v/raw=%v", inStream, withRaw), func(t *testing.T) { + var rows []pushdownRow + for i, level := range levels { + row := pushdownRow{ts: s0.Add(time.Duration(100+i) * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: fmt.Sprintf(`{"level":%q,"n":%d}`, level, i)} + if inStream { + row.stream["level"] = level + row.msg = fmt.Sprintf(`{"n":%d}`, i) + } + rows = append(rows, row) + } + if withRaw { + raw := pushdownRow{ts: s0.Add(150 * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: `{"level":"INFO"} trailing`} + if inStream { + raw.stream["level"] = "Info" + raw.msg = `{"n":9} trailing` + } + rows = append(rows, raw) + } + want := map[string]map[int64]string{} + for at := start; !at.After(end); at = at.Add(time.Minute) { + counts := map[string]int{} + for i, row := range rows { + if row.ts.After(at.Add(-5*time.Minute)) && !row.ts.After(at) { + level := "info" // the raw candidate + if i < len(levels) { + level = canonicalLevel([]byte(levels[i])) + } + counts[level]++ + } + } + for level, n := range counts { + key := canonicalLabelsKey(map[string]string{"detected_level": level}) + if want[key] == nil { + want[key] = map[int64]string{} + } + want[key][at.Unix()] = strconv.Itoa(n) + } + } + if len(want) != 2 { + t.Fatalf("fixture: %d expected series, want 2 (info, warn)", len(want)) + } + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + plan, _ := compileOrderedJSONMetric(query) + if !plan.validOnly || !plan.pushdown { + t.Fatal("plan must be a split plan") + } + if got := rowEvaluatorMatrix(t, p, plan, start, end, time.Minute); !reflect.DeepEqual(got, want) { + t.Fatalf("row evaluator %v, want %v", got, want) + } + // 8 VictoriaLogs groups (9 with a raw candidate) under a limit of + // 5 or 10: more groups than the limit, 2 series. The read holds + // them (the guard is twice the limit), so the split answers. + for _, limit := range []int{5, 10} { + p.maxStatsQuerySeries = limit + fake.raw = 0 + code, got, body := runSplitQuery(t, p, query, start, end, time.Minute) + if code != http.StatusOK || !reflect.DeepEqual(got, want) { + t.Fatalf("limit %d: status %d %.200s, got %v, want %v", limit, code, body, got, want) + } + if inStream && !withRaw && fake.raw != 0 { + t.Fatalf("limit %d: %d raw-row queries issued", limit, fake.raw) + } + if !inStream && fake.raw == 0 { + t.Fatalf("limit %d: a level held only by the body was counted from stats", limit) + } + } + if inStream { + assertStoredLevelStats(t, fake, "stored") + } + // Past the guard the row evaluator answers, the same series. + p.maxStatsQuerySeries = 3 + fake.raw = 0 + code, got, body := runSplitQuery(t, p, query, start, end, time.Minute) + if code != http.StatusOK || !reflect.DeepEqual(got, want) { + t.Fatalf("limit 3: status %d %.200s, got %v, want %v", code, body, got, want) + } + if fake.raw == 0 { + t.Fatal("limit 3: the row evaluator did not answer") + } + }) + } + } +} + +// A stats response too large to read does not fail a query within the series +// limit either: the raw evaluator answers it, as it did before the split. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitResponseTooLargeFallsBackToRawEvaluator(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(5*time.Minute) + var rows []pushdownRow + for i := 0; i < 12; i++ { + rows = append(rows, pushdownRow{ts: s0.Add(time.Duration(100+i) * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: fmt.Sprintf(`{"trace_id":"t%d"}`, i)}) + } + query := `sum by (trace_id) (count_over_time({env="prod"} | json | __error__="" [5m]))` + old := maxStatsBucketResponseBytes + maxStatsBucketResponseBytes = 600 + t.Cleanup(func() { maxStatsBucketResponseBytes = old }) + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + plan, _ := compileOrderedJSONMetric(query) + want := rowEvaluatorMatrix(t, p, plan, start, end, time.Minute) + fake.raw = 0 + code, got, body := runSplitQuery(t, p, query, start, end, time.Minute) + if code != http.StatusOK || !reflect.DeepEqual(got, want) || len(want) != 12 { + t.Fatalf("status %d %.200s, got %v, want %v", code, body, got, want) + } + if fake.raw == 0 { + t.Fatal("the raw evaluator did not answer") + } +} + +// Groups the label filter drops stay out of the series count whatever the +// response size cap does: the filter runs before stats, so even a response +// stopped early by a small cap holds only real series. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitFilteredOutGroupsUnderSizeCap(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(5*time.Minute) + var rows []pushdownRow + for i := 0; i < 20; i++ { + status := "200" + if i < 2 { + status = "500" + } + rows = append(rows, pushdownRow{ts: s0.Add(time.Duration(100+i) * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: fmt.Sprintf(`{"path":"/p%d","status":"%s"}`, i, status)}) + } + for _, size := range []int64{1200, 2000, 4000} { + for _, function := range []string{"count_over_time", "bytes_over_time"} { + query := `sum by (path) (` + function + `({env="prod"} | json | __error__="" | status="500" [5m]))` + t.Run(fmt.Sprintf("%s/%d", function, size), func(t *testing.T) { + old := maxStatsBucketResponseBytes + maxStatsBucketResponseBytes = size + t.Cleanup(func() { maxStatsBucketResponseBytes = old }) + runSplitFilteredOutGroups(t, rows, query, start, end) + }) + } + } +} + +// With the real series exactly at the limit plus the empty-label group the +// lines VictoriaLogs cannot parse are blanked into, the response read passes +// and the series merged after it are over the limit: the query ends with +// Loki's error before any raw row is read. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitSeriesLimitFailsBeforeRawRows(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(5*time.Minute) + stream := map[string]string{"app": "api", "env": "prod"} + var rows []pushdownRow + for i := 0; i < 5; i++ { + rows = append(rows, pushdownRow{ts: s0.Add(time.Duration(100+i) * time.Second), stream: stream, msg: fmt.Sprintf(`{"trace_id":"t%d"}`, i)}) + } + // A valid line without the key is the series with no labels; the line + // only Loki's parser accepts makes the plan read raw rows. + rows = append(rows, + pushdownRow{ts: s0.Add(110 * time.Second), stream: stream, msg: `{"other":1}`}, + pushdownRow{ts: s0.Add(150 * time.Second), stream: stream, msg: `{"trace_id":"t0"} trailing`}) + for _, function := range []string{"count_over_time", "bytes_over_time"} { + query := `sum by (trace_id) (` + function + `({env="prod"} | json | __error__="" [5m]))` + t.Run(function, func(t *testing.T) { + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + p.maxStatsQuerySeries = 5 + code, _, body := runSplitQuery(t, p, query, start, end, time.Minute) + if code != http.StatusBadRequest || !strings.Contains(body, "maximum number of series (5) reached for a single query") { + t.Fatalf("status %d body %s, want Loki's series limit error", code, body) + } + if fake.raw != 0 { + t.Fatalf("%d raw-row queries issued for a query over the series limit", fake.raw) + } + // One series fewer is within the limit. + p.maxStatsQuerySeries = 6 + plan, _ := compileOrderedJSONMetric(query) + want := rowEvaluatorMatrix(t, p, plan, start, end, time.Minute) + if code, got, body := runSplitQuery(t, p, query, start, end, time.Minute); code != http.StatusOK || !reflect.DeepEqual(got, want) { + t.Fatalf("limit 6: status %d %s, got %v, want %v", code, body, got, want) + } + }) + } +} + +// A stats response is read only up to a bound, never whole: far more groups +// than the series limit end the read early instead of holding the response (a +// 300k group response is parsed into a 300k entry map otherwise). A plain +// pushdown stops at the series limit with Loki's error, a split read stops at +// twice the limit as a memory guard and says nothing of the series, because +// its groups merge into fewer series, and the query then goes to the row +// evaluator, which answers. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitReadStopsAtTwiceTheSeriesLimit(t *testing.T) { + start := time.Unix(1700000400, 0).UTC() + end := start.Add(5 * time.Minute) + for _, tc := range []struct { + name string + aux string + want error + }{{"non-split", "", nil}, {"split", jsonInvalidRowsName, errStatsGroupGuard}} { + t.Run(tc.name, func(t *testing.T) { + body := statsResponseBody(300000, tc.aux != "", start.Unix()) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(body) })) + defer srv.Close() + p := newFilterPushdownProxy(t, srv.URL) + p.maxStatsQuerySeries = 500 + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + _, _, err := p.collectRangeMetricHitsAux(context.Background(), "*", []string{"trace_id"}, []string{"trace_id"}, false, "count() as c", start, end, time.Minute, tc.aux) + runtime.ReadMemStats(&after) + if tc.want == nil && !isSeriesLimitError(err) || tc.want != nil && !errors.Is(err, tc.want) { + t.Fatalf("err = %v, want %v", err, tc.want) + } + allocated := after.TotalAlloc - before.TotalAlloc + t.Logf("allocated %.2f MiB reading a %.1f MiB response", float64(allocated)/(1<<20), float64(len(body))/(1<<20)) + if allocated > 8<<20 { + t.Fatalf("allocated %.1f MiB reading a %.1f MiB response over the limit", float64(allocated)/(1<<20), float64(len(body))/(1<<20)) + } + }) + } +} + +// A split query whose stats response holds 300k groups is answered by the row +// evaluator, with bounded memory on the stats read and no series limit error +// from the group count: the rows it reads here are none, so the answer is empty. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitHugeResponseFallsBackWithBoundedMemory(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(5*time.Minute) + body := statsResponseBody(300000, true, s0.Unix()) + var rawReads atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/select/logsql/stats_query_range" { + _, _ = w.Write(body) + return + } + rawReads.Add(1) // the probes and the row evaluator find no row + })) + defer srv.Close() + p := newFilterPushdownProxy(t, srv.URL) + p.maxStatsQuerySeries = 500 + query := `sum by (trace_id) (count_over_time({env="prod"} | json | __error__="" [5m]))` + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + code, got, rsp := runSplitQuery(t, p, query, start, end, time.Minute) + runtime.ReadMemStats(&after) + if code != http.StatusOK || len(got) != 0 { + t.Fatalf("status %d %.200s, got %v: want the row evaluator's empty answer", code, rsp, got) + } + if rawReads.Load() == 0 { + t.Fatal("the row evaluator did not answer") + } + if allocated := after.TotalAlloc - before.TotalAlloc; allocated > 24<<20 { + t.Fatalf("allocated %.1f MiB for a %.1f MiB stats response", float64(allocated)/(1<<20), float64(len(body))/(1<<20)) + } +} + +// statsResponseBody renders a stats_query_range response of groups trace_id +// groups, each with a count result and, with aux, an aux result. +func statsResponseBody(groups int, aux bool, ts int64) []byte { + var b strings.Builder + b.WriteString(`{"status":"success","data":{"resultType":"matrix","result":[`) + for i := 0; i < groups; i++ { + if i > 0 { + b.WriteByte(',') + } + fmt.Fprintf(&b, `{"metric":{"__name__":"c","trace_id":"t%07d"},"values":[[%d,"1"]]}`, i, ts) + if aux { + fmt.Fprintf(&b, `,{"metric":{"__name__":%q,"trace_id":"t%07d"},"values":[[%d,"0"]]}`, jsonInvalidRowsName, i, ts) + } + } + b.WriteString(`]}}`) + return []byte(b.String()) +} + +// The lines read as raw rows carry no labels in the stats query: whatever +// service name, level or parsed key they hold, they form one group, because +// the raw read evaluates them itself and only their buckets are used. Fifty +// lines of which Loki rejects all (a number without a leading digit) leave the +// three real series within a limit of ten, and the stats response holds one +// group for them. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitRawCandidatesFormOneGroup(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(5*time.Minute) + var rows []pushdownRow + for i := 0; i < 3; i++ { + rows = append(rows, pushdownRow{ts: s0.Add(time.Duration(100+i) * time.Second), stream: map[string]string{"app": fmt.Sprintf("good%d", i), "level": "info", "env": "prod"}, msg: fmt.Sprintf(`{"trace_id":"ok%d"}`, i)}) + } + for i := 0; i < 50; i++ { + rows = append(rows, pushdownRow{ts: s0.Add(time.Duration(110+i) * time.Second), stream: map[string]string{"app": fmt.Sprintf("bad%d", i), "level": []string{"error", "warn", "debug"}[i%3], "env": "prod"}, msg: fmt.Sprintf(`{"trace_id":"bad%d","x":.5}`, i)}) + } + query := `sum by (trace_id, service_name, detected_level) (count_over_time({env="prod"} | json | __error__="" [5m]))` + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + var replay url.Values + fake.onStats = func(r *http.Request, q string) { + if replay == nil || strings.Contains(q, jsonLokiRejectHazard) { + replay = r.Form + } + } + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + p.maxStatsQuerySeries = 10 + plan, _ := compileOrderedJSONMetric(query) + if !plan.validOnly { + t.Fatal("plan must be a split plan") + } + want := rowEvaluatorMatrix(t, p, plan, start, end, time.Minute) + if len(want) != 3 { + t.Fatalf("fixture: row evaluator answers %d series, want 3", len(want)) + } + code, got, body := runSplitQuery(t, p, query, start, end, time.Minute) + if code != http.StatusOK || !reflect.DeepEqual(got, want) { + t.Fatalf("split %d %s, got %v, want %v", code, body, got, want) + } + rawGroups, allGroups := replayStatsGroups(t, srv.URL, replay) + if len(rawGroups) != 1 { + t.Fatalf("raw candidates form %d groups, want 1: %v", len(rawGroups), rawGroups) + } + if len(allGroups) != 4 { + t.Fatalf("stats response holds %d groups, want the 3 series and the raw group: %v", len(allGroups), allGroups) + } +} + +// replayStatsGroups sends a stats_query_range request the proxy made to the +// fake again and returns the groups holding raw candidates (the aux result is +// above zero) and all groups of the response. +func replayStatsGroups(t *testing.T, backend string, params url.Values) (raw, all map[string]bool) { + t.Helper() + resp, err := http.PostForm(backend+"/select/logsql/stats_query_range", params) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var decoded struct { + Data struct { + Result []struct { + Metric map[string]string `json:"metric"` + Values [][]any `json:"values"` + } `json:"result"` + } `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil { + t.Fatal(err) + } + raw, all = map[string]bool{}, map[string]bool{} + for _, res := range decoded.Data.Result { + labels := map[string]string{} + for k, v := range res.Metric { + if k != "__name__" { + labels[k] = v + } + } + key := canonicalLabelsKey(labels) + all[key] = true + if res.Metric["__name__"] != jsonInvalidRowsName { + continue + } + for _, point := range res.Values { + if point[1] != "0" { + raw[key] = true + } + } + } + return raw, all +} + +// The same holds without a hazard rescan: lines VictoriaLogs cannot parse and +// Loki rejects too (truncated JSON) over twenty distinct stream label sets are +// raw candidates in the first scan. They form one group, so they neither count +// against the series limit (the two real services under a limit of three +// answer) nor grow the response. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitRawCandidatesOverManyStreamsFormOneGroup(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(5*time.Minute) + var rows []pushdownRow + for i := 0; i < 2; i++ { + rows = append(rows, pushdownRow{ts: s0.Add(time.Duration(100+i) * time.Second), stream: map[string]string{"app": fmt.Sprintf("good%d", i), "env": "prod"}, msg: `{"a":1}`}) + } + for i := 0; i < 20; i++ { + rows = append(rows, pushdownRow{ts: s0.Add(time.Duration(110+i) * time.Second), stream: map[string]string{"app": fmt.Sprintf("bad%d", i), "env": "prod"}, msg: `{"a":1`}) + } + query := `sum by (service_name) (count_over_time({env="prod"} | json | __error__="" [5m]))` + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + var replay url.Values + fake.onStats = func(r *http.Request, q string) { replay = r.Form } + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + p.maxStatsQuerySeries = 3 + plan, _ := compileOrderedJSONMetric(query) + want := rowEvaluatorMatrix(t, p, plan, start, end, time.Minute) + if len(want) != 2 { + t.Fatalf("fixture: row evaluator answers %d series, want 2", len(want)) + } + code, got, body := runSplitQuery(t, p, query, start, end, time.Minute) + if code != http.StatusOK || !reflect.DeepEqual(got, want) { + t.Fatalf("split %d %s, got %v, want %v", code, body, got, want) + } + if len(fake.stats) != 1 { + t.Fatalf("expected one stats scan, no hazard rescan: %q", fake.stats) + } + rawGroups, allGroups := replayStatsGroups(t, srv.URL, replay) + if len(rawGroups) != 1 || len(allGroups) != 3 { + t.Fatalf("raw candidates form %d groups of %d, want 1 of 3: %v", len(rawGroups), len(allGroups), allGroups) + } +} + +// Six valid series with no line read as raw rows under a limit of 5 end with +// Loki's error from the merged answer, and with a limit of 6 answer. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitLimitPlusOneSeriesWithoutRawRows(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(5*time.Minute) + var rows []pushdownRow + for i := 0; i < 6; i++ { + rows = append(rows, pushdownRow{ts: s0.Add(time.Duration(100+i) * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: fmt.Sprintf(`{"trace_id":"t%d"}`, i)}) + } + query := `sum by (trace_id) (count_over_time({env="prod"} | json | __error__="" [5m]))` + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + p.maxStatsQuerySeries = 5 + code, _, body := runSplitQuery(t, p, query, start, end, time.Minute) + if code != http.StatusBadRequest || !strings.Contains(body, "maximum number of series (5) reached for a single query") { + t.Fatalf("status %d body %.200s, want Loki's series limit error", code, body) + } + if fake.raw != 0 { + t.Fatalf("%d raw-row queries issued: the error comes from the split's merged answer", fake.raw) + } + p.maxStatsQuerySeries = 6 + plan, _ := compileOrderedJSONMetric(query) + want := rowEvaluatorMatrix(t, p, plan, start, end, time.Minute) + if code, got, body := runSplitQuery(t, p, query, start, end, time.Minute); code != http.StatusOK || !reflect.DeepEqual(got, want) || len(want) != 6 { + t.Fatalf("limit 6: status %d %.200s, got %v, want %v", code, body, got, want) + } +} + +// A response too large to read is the split's own fallback: a plain pushdown +// plan keeps the error it always returned for it, and the split plan hands the +// query to the row evaluator without one. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitOversizedResponseOnlyFallsBackForSplitPlans(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(5*time.Minute) + var rows []pushdownRow + for i := 0; i < 12; i++ { + rows = append(rows, pushdownRow{ts: s0.Add(time.Duration(100+i) * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: fmt.Sprintf(`{"trace_id":"t%d","status":"500"}`, i)}) + } + old := maxStatsBucketResponseBytes + maxStatsBucketResponseBytes = 400 + t.Cleanup(func() { maxStatsBucketResponseBytes = old }) + for _, tc := range []struct { + name, query string + split bool + }{ + {"split", `sum by (trace_id) (count_over_time({env="prod"} | json | __error__="" [5m]))`, true}, + {"plain", `sum by (trace_id) (count_over_time({env="prod"} | json | drop __error__ | status="500" [5m]))`, false}, + } { + t.Run(tc.name, func(t *testing.T) { + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + plan, ok := compileOrderedJSONMetric(tc.query) + if !ok || !plan.pushdown || plan.validOnly != tc.split { + t.Fatalf("plan pushdown=%v validOnly=%v, want a pushdown with validOnly=%v", plan != nil && plan.pushdown, plan != nil && plan.validOnly, tc.split) + } + _, served, _, err := p.orderedJSONStatsBucketsWithReason(context.Background(), plan, start, end, time.Minute, false) + switch { + case tc.split && (served || err != nil): + t.Fatalf("split plan: served=%v err=%v, want the row evaluator to answer", served, err) + case !tc.split && !errors.Is(err, errBodyTooLarge): + t.Fatalf("plain plan: served=%v err=%v, want the size error as before", served, err) + } + }) + } +} + +// Grafana Drilldown keeps the busiest series of an over-limit answer instead of +// failing, which the split's merged answer does not rank: its requests take the +// row evaluator, and no split query reaches VictoriaLogs. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitDrilldownRequestsKeepTheRowEvaluator(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(5*time.Minute) + var rows []pushdownRow + for i := 0; i < 8; i++ { + rows = append(rows, pushdownRow{ts: s0.Add(time.Duration(100+i) * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: fmt.Sprintf(`{"trace_id":"t%d"}`, i)}) + } + query := `sum by (trace_id) (count_over_time({env="prod"} | json | __error__="" [5m]))` + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + p.maxStatsQuerySeries = 5 + params := url.Values{"query": {query}, "start": {strconv.FormatInt(start.UnixNano(), 10)}, "end": {strconv.FormatInt(end.UnixNano(), 10)}, "step": {"60"}} + r := httptest.NewRequest(http.MethodGet, "/loki/api/v1/query_range?"+params.Encode(), nil) + r.Header.Set("X-Query-Tags", "Source=grafana-lokiexplore-app") + rec := httptest.NewRecorder() + p.handleQueryRange(rec, r) + if rec.Code != http.StatusOK { + t.Fatalf("status %d %.200s, want the busiest series with a warning", rec.Code, rec.Body.String()) + } + if len(fake.stats) != 0 || fake.raw == 0 { + t.Fatalf("%d stats queries, %d raw-row queries: the row evaluator alone must answer", len(fake.stats), fake.raw) + } + if got := len(decodeExtractedMatrix(t, rec.Body.Bytes())); got != 5 { + t.Fatalf("%d series, want the 5 busiest", got) + } +} + +// An instant query has one evaluation time and no bucket grid to add the raw +// lines to (the instant stats route reads the one bucket): a split plan takes +// the row evaluator for it, as it did before the split, and no stats query +// reaches VictoriaLogs. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitInstantQueriesKeepTheRowEvaluator(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + at := s0.Add(5 * time.Minute) + rows := []pushdownRow{ + {ts: s0.Add(100 * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: `{"trace_id":"a"}`}, + {ts: s0.Add(110 * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: `{"trace_id":"b"}`}, + {ts: s0.Add(120 * time.Second), stream: map[string]string{"app": "api", "env": "prod"}, msg: `{"trace_id":"b",`}, + } + query := `sum by (trace_id) (count_over_time({env="prod"} | json | __error__="" [5m]))` + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + got, rec := runJSONVolumeQueryInstant(t, p, query, at) + if rec.Code != http.StatusOK { + t.Fatalf("status %d %.200s", rec.Code, rec.Body.String()) + } + if len(fake.stats) != 0 || fake.raw == 0 { + t.Fatalf("%d stats queries, %d raw-row queries: the row evaluator alone must answer", len(fake.stats), fake.raw) + } + if want := (map[string]string{`{trace_id="a"}`: "1", `{trace_id="b"}`: "1"}); !reflect.DeepEqual(got.values, want) { + t.Fatalf("got %v, want %v", got.values, want) + } +} + +// The raw-candidate marker is set once and the later conditions test it: the +// regexp conditions of jsonInvalidCondition appear in the query only in the +// marker, it never becomes a grouped field, and an answer's labels never carry +// it. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitMarksRawCandidatesOnce(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(5*time.Minute) + rows := []pushdownRow{ + {ts: s0.Add(100 * time.Second), stream: map[string]string{"app": "api", "env": "prod", "level": "info"}, msg: `{"path":"/a","status":"500","x":.5}`}, + {ts: s0.Add(110 * time.Second), stream: map[string]string{"app": "web", "env": "prod", "level": "error"}, msg: `{"path":"/b","status":"500"}`}, + {ts: s0.Add(120 * time.Second), stream: map[string]string{"app": "web", "env": "prod", "level": "WARN"}, msg: `{"path":"/c","status":"200"}`}, + } + for _, query := range []string{ + `sum by (path, service_name, detected_level) (count_over_time({env="prod"} | json | __error__="" [5m]))`, + `sum by (path) (bytes_over_time({env="prod"} | json | __error__="" | status="500" [5m]))`, + } { + t.Run(query, func(t *testing.T) { + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + plan, _ := compileOrderedJSONMetric(query) + want := rowEvaluatorMatrix(t, p, plan, start, end, time.Minute) + fake.raw = 0 + code, got, body := runSplitQuery(t, p, query, start, end, time.Minute) + if code != http.StatusOK || !reflect.DeepEqual(got, want) { + t.Fatalf("status %d %.200s, got %v, want %v", code, body, got, want) + } + if fake.raw == 0 { + t.Fatal("fixture: no line was read as a raw row") + } + for key := range got { + if strings.Contains(key, jsonRawField) { + t.Fatalf("series %s carries the marker", key) + } + } + if len(fake.stats) == 0 { + t.Fatal("no stats query") + } + for _, stats := range fake.stats { + marker := jsonRawMarker(strings.Contains(stats, jsonLokiRejectHazard)) + if strings.Count(stats, marker) != 1 { + t.Fatalf("stats query does not set the marker once: %s", stats) + } + if rest := strings.Replace(stats, marker, "", 1); strings.Contains(rest, jsonValidField+":*") { + t.Fatalf("stats query tests the line again after the marker: %s", stats) + } + if strings.Contains(stats, "stats by ("+jsonRawField) || strings.Contains(stats, ", "+jsonRawField+",") || strings.Contains(stats, ", "+jsonRawField+")") { + t.Fatalf("stats query groups by the marker: %s", stats) + } + } + }) + } +} + +// A step wider than the window leaves gaps between the evaluation windows. The +// stats scan covers the gaps too, but a line in a gap belongs to no sample, so +// its group is no series of Loki's answer and does not count against the limit +// (3 visible trace ids plus 4 only in the gaps, limit 5, step 5m, window 1m). +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitGapGroupsAreNoSeries(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(12*time.Minute) + stream := map[string]string{"app": "api", "env": "prod"} + var rows []pushdownRow + for i := 0; i < 3; i++ { + rows = append(rows, pushdownRow{ts: s0.Add(90*time.Second + time.Duration(i)*time.Second), stream: stream, msg: fmt.Sprintf(`{"trace_id":"seen%d"}`, i)}) + } + for i := 0; i < 4; i++ { + rows = append(rows, pushdownRow{ts: s0.Add(3*time.Minute + time.Duration(i)*20*time.Second), stream: stream, msg: fmt.Sprintf(`{"trace_id":"gap%d"}`, i)}) + } + // A line only Loki's parser accepts in a gap makes the plan read raw rows. + rows = append(rows, pushdownRow{ts: s0.Add(4 * time.Minute), stream: stream, msg: `{"trace_id":"gap9"} trailing`}) + for _, function := range []string{"count_over_time", "bytes_over_time"} { + query := `sum by (trace_id) (` + function + `({env="prod"} | json | __error__="" [1m]))` + t.Run(function, func(t *testing.T) { + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + p.maxStatsQuerySeries = 5 + plan, _ := compileOrderedJSONMetric(query) + want := rowEvaluatorMatrix(t, p, plan, start, end, 5*time.Minute) + if len(want) != 3 { + t.Fatalf("fixture: row evaluator answers %d series, want 3", len(want)) + } + code, got, body := runSplitQuery(t, p, query, start, end, 5*time.Minute) + if code != http.StatusOK || !reflect.DeepEqual(got, want) { + t.Fatalf("status %d %.200s, got %v, want %v", code, body, got, want) + } + }) + } +} + +// A split plan with no label to read (an ungrouped sum, no filter) has nothing +// to unpack: the sentinel splice is its only parse, and no `unpack_json fields +// ()` is sent. The answer is the row evaluator's. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitWithoutLabelsSendsNoEmptyUnpack(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(5*time.Minute) + stream := map[string]string{"app": "api", "env": "prod"} + rows := []pushdownRow{ + {ts: s0.Add(100 * time.Second), stream: stream, msg: `{"a":1}`}, + {ts: s0.Add(110 * time.Second), stream: stream, msg: `{"a":2} trailing`}, + {ts: s0.Add(120 * time.Second), stream: stream, msg: `not json`}, + {ts: s0.Add(130 * time.Second), stream: stream, msg: `{"a":3,"x":.5}`}, + } + for _, function := range []string{"count_over_time", "bytes_over_time"} { + query := `sum(` + function + `({env="prod"} | json | __error__="" [5m]))` + t.Run(function, func(t *testing.T) { + srv, fake := newPushdownFakeVL(t, rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + plan, _ := compileOrderedJSONMetric(query) + if !plan.validOnly || !plan.pushdown || len(plan.unpackKeys()) != 0 { + t.Fatalf("plan must be a split plan without labels: %+v", plan) + } + want := rowEvaluatorMatrix(t, p, plan, start, end, time.Minute) + code, got, body := runSplitQuery(t, p, query, start, end, time.Minute) + if code != http.StatusOK || !reflect.DeepEqual(got, want) { + t.Fatalf("status %d %.200s, got %v, want %v", code, body, got, want) + } + if len(fake.stats) == 0 { + t.Fatal("no stats query") + } + for _, stats := range fake.stats { + if strings.Contains(stats, "fields ()") || strings.Contains(stats, "keep_original_fields") { + t.Fatalf("stats query unpacks no label but says so: %s", stats) + } + if !strings.Contains(stats, jsonValidField) { + t.Fatalf("stats query lost the sentinel splice: %s", stats) + } + } + }) + } +} + +// A plan that derives detected_level reads every stored level field of a raw +// row (the stored level fields, the OTLP severity_text, severity_number and +// severity), none of which a label of the plan names: its raw read keeps every +// field instead of projecting to the plan's labels. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestOrderedJSONRowProjectionKeepsTheStoredLevelFields(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(5*time.Minute) + stream := map[string]string{"app": "api", "env": "prod"} + rows := []pushdownRow{ + {ts: s0.Add(100 * time.Second), stream: stream, vl: map[string]string{"severity": "error"}, msg: `{"n":1}`}, + {ts: s0.Add(110 * time.Second), stream: stream, vl: map[string]string{"severity_text": "Warn", "severity_number": "13"}, msg: `{"n":2}`}, + {ts: s0.Add(120 * time.Second), stream: stream, vl: map[string]string{"level": "info"}, msg: `{"n":3}`}, + } + for _, query := range []string{ + `sum by (detected_level) (count_over_time({env="prod"} | json | drop __error__ [5m]))`, + `sum by (app, detected_level) (count_over_time({env="prod"} | json | drop __error__ [5m]))`, + } { + t.Run(query, func(t *testing.T) { + srv, fake := newPushdownFakeVL(t, rows, nil) + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + plan, ok := compileOrderedJSONMetric(query) + if !ok || !plan.derivesLevel { + t.Fatal("expected a plan deriving detected_level") + } + if got := p.orderedJSONRowProjection(plan); got != "" { + t.Fatalf("projection %q hides the stored level fields from the derivation", got) + } + got := rowEvaluatorMatrix(t, p, plan, start, end, time.Minute) + for _, level := range []string{"error", "warn", "info"} { + found := false + for key := range got { + found = found || strings.Contains(key, `detected_level="`+level+`"`) + } + if !found { + t.Fatalf("no %s series in %v", level, got) + } + } + for _, raw := range fake.rawQueries { + if strings.Contains(raw.query, " | fields ") { + t.Fatalf("raw read projects the rows: %s", raw.query) + } + } + }) + } +} + +// A split plan grouped by detected_level is served by the stats only when the +// stored level fields decide the level of every counted line as Loki's ingest +// does; in every other window the row evaluator answers. Whichever answers, the +// matrix is the row evaluator's, which derives the level of each line itself: +// a window mixing stored and body-only levels, a stored detected_level label, +// a stored spelling Loki does not know, the level in structured metadata, only +// an OTLP severity, and lines read as raw rows with and without a stored level. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestJSONValidSplitDetectedLevelWindowsMatchTheRowEvaluator(t *testing.T) { + s0 := time.Unix(1700000400, 0).UTC() + start, end := s0.Add(2*time.Minute), s0.Add(5*time.Minute) + row := func(sec int, stream, vl map[string]string, msg string) pushdownRow { + labels := map[string]string{"app": "api", "env": "prod"} + for k, v := range stream { + labels[k] = v + } + return pushdownRow{ts: s0.Add(time.Duration(sec) * time.Second), stream: labels, vl: vl, msg: msg} + } + lvl := func(level string) map[string]string { return map[string]string{"level": level} } + const ( + split = "split" + fallback = "fallback" + ) + for _, tc := range []struct { + name string + want string + rows []pushdownRow + }{ + {"stored on every line", split, []pushdownRow{ + row(100, lvl("info"), nil, `{"n":1}`), row(110, lvl("INFO"), nil, `{"n":2}`), row(120, lvl("warning"), nil, `{"n":3}`), row(130, lvl("error"), nil, `{"n":4,"level":"debug"}`), + }}, + {"stored in structured metadata", split, []pushdownRow{ + row(100, nil, lvl("warn"), `{"n":1}`), row(110, nil, lvl("Information"), `{"n":2}`), + }}, + {"stored spellings Loki does not normalise", split, []pushdownRow{ + row(100, lvl("notice"), nil, `{"n":1}`), row(110, lvl("NOTICE"), nil, `{"n":2}`), row(120, lvl("Trace"), nil, `{"n":3}`), row(130, lvl("fatal"), nil, `{"n":4}`), row(140, lvl("crit"), nil, `{"n":5}`), + }}, + {"stored and body-only", fallback, []pushdownRow{ + row(100, lvl("info"), nil, `{"n":1}`), row(110, nil, nil, `{"n":2,"level":"error"}`), row(120, nil, nil, `{"n":3}`), + }}, + {"body-only only", fallback, []pushdownRow{ + row(100, nil, nil, `{"level":"WARN"}`), row(110, nil, nil, `{"level":"notice"}`), row(120, nil, nil, `{"msg":"request failed with an error"}`), + }}, + {"stored detected_level label", fallback, []pushdownRow{ + row(100, map[string]string{"detected_level": "Custom"}, nil, `{"n":1}`), row(110, lvl("info"), nil, `{"n":2}`), + }}, + {"a stream severity beside metadata level", fallback, []pushdownRow{ + row(100, map[string]string{"severity": "error"}, lvl("info"), `{"n":1}`), row(110, lvl("info"), nil, `{"n":2}`), + }}, + {"a body severity key beside a stored level", split, []pushdownRow{ + row(100, lvl("info"), nil, `{"n":1,"severity":"error"}`), row(110, nil, lvl("warn"), `{"severity":"error","level":"debug"}`), + }}, + {"stored OTLP severity fields beside a stored level", fallback, []pushdownRow{ + row(100, nil, map[string]string{"level": "info", "severity_text": "Error", "severity_number": "17"}, `{"n":1}`), + }}, + {"only an OTLP severity", fallback, []pushdownRow{ + row(100, nil, map[string]string{"severity": "error"}, `{"n":1}`), row(110, nil, map[string]string{"severity_text": "Warn", "severity_number": "13"}, `{"n":2}`), + }}, + {"stored on every counted line, a raw line without one", split, []pushdownRow{ + row(100, lvl("info"), nil, `{"n":1}`), row(110, lvl("error"), nil, `{"n":2}`), row(120, nil, nil, `{"n":3,"level":"warn"} trailing`), + }}, + {"stored on every counted line, a raw line with another one", split, []pushdownRow{ + row(100, lvl("info"), nil, `{"n":1}`), row(110, lvl("error"), nil, `{"n":2}`), row(120, lvl("DEBUG"), nil, `{"n":3,"level":"warn"} trailing`), row(130, lvl("Info"), nil, `{"n":4,"x":.5}`), + }}, + {"a counted line without a stored level beside a raw one", fallback, []pushdownRow{ + row(100, lvl("info"), nil, `{"n":1}`), row(110, nil, nil, `{"n":2}`), row(120, lvl("error"), nil, `{"n":3} trailing`), + }}, + } { + for _, function := range []string{"count_over_time", "bytes_over_time"} { + query := `sum by (detected_level) (` + function + `({env="prod"} | json | __error__="" [5m]))` + t.Run(tc.name+"/"+function, func(t *testing.T) { + srv, fake := newPushdownFakeVL(t, tc.rows, nil) + fake.jsonUnpack = vlFastjsonUnpackFields + p := newFilterPushdownProxy(t, srv.URL) + fake.stored = p.labelTranslator.ToVL + plan, ok := compileOrderedJSONMetric(query) + if !ok || !plan.validOnly || !plan.pushdown { + t.Fatal("plan must be a split plan") + } + want := rowEvaluatorMatrix(t, p, plan, start, end, time.Minute) + if len(want) == 0 { + t.Fatal("fixture: the row evaluator answers no series") + } + fake.mu.Lock() + fake.raw, fake.rawQueries, fake.stats, fake.guards = 0, nil, nil, nil + fake.mu.Unlock() + code, got, body := runSplitQuery(t, p, query, start, end, time.Minute) + if code != http.StatusOK { + t.Fatalf("status %d: %s", code, body) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("differs from the row evaluator\n got: %v\nwant: %v", got, want) + } + fake.mu.Lock() + rawQueries := fake.rawQueries + fake.mu.Unlock() + full := false + for _, raw := range rawQueries { + bounded := strings.Contains(raw.query, " | filter "+jsonInvalidCondition(false)) || strings.Contains(raw.query, " | filter "+jsonInvalidCondition(true)) + full = full || !bounded + } + if gotPath := map[bool]string{true: fallback, false: split}[full]; gotPath != tc.want { + t.Fatalf("answered by the %s, want the %s", gotPath, tc.want) + } + if tc.want == split { + assertStoredLevelStats(t, fake, tc.name) + } + }) + } + } +} diff --git a/internal/proxy/range_metric_compat.go b/internal/proxy/range_metric_compat.go index dc72939c9..143e7e5df 100644 --- a/internal/proxy/range_metric_compat.go +++ b/internal/proxy/range_metric_compat.go @@ -1062,7 +1062,7 @@ func (c *seriesCountingReader) Read(b []byte) (int, error) { keep = len(c.window) } c.tail = append(c.tail[:0], c.window[len(c.window)-keep:]...) - if c.seen > c.limit { + if c.limit > 0 && c.seen > c.limit { return n, errSeriesCountExceeded } } @@ -1203,22 +1203,22 @@ func snapSlidingBucketNanos(ts int64, start time.Time, hitStep time.Duration) in // readStatsBucketBody reads a stats_query_range response, stopping with Loki's // series limit error once it holds more result entries than the limit allows // (two per series when byte presence counts ride along). A Drilldown request -// reads it all and keeps the busiest series instead. +// reads it all and keeps the busiest series instead. perSeries is the number +// of results each series carries. // -// With the groups of the response guarded (guardStatsGroups) a VictoriaLogs -// group is no series of the answer: groups merge into fewer series after the -// read (the spellings of one level), and Loki's limit applies to the merged -// answer only. The read then stops at splitGroupSlack times the limit as a -// memory guard, with errStatsGroupGuard instead of the series limit error, and -// the caller asks again in a form whose groups are its series. -func (p *Proxy) readStatsBucketBody(ctx context.Context, body io.Reader, limit int, withPresence bool) ([]byte, error) { +// With the groups of the response guarded (guardStatsGroups, and every read +// with an aux result, see collectRangeMetricHitsAux) a VictoriaLogs group is +// no series of the answer: groups merge into fewer series after the read (the +// spellings of one level, the empty-valued groups) and Loki's limit applies to +// the merged answer only. The read then stops at splitGroupSlack times the +// limit as a memory guard, with errStatsGroupGuard instead of the series limit +// error, and the caller asks again in a form whose groups are its series, or +// leaves the query to the row evaluator. +func (p *Proxy) readStatsBucketBody(ctx context.Context, body io.Reader, limit, perSeries int) ([]byte, error) { counter := &seriesCountingReader{r: body} guarded := statsGroupsGuarded(ctx) if !requestKeepsPartialSeries(ctx) { - counter.limit = limit - if withPresence { - counter.limit = 2 * limit - } + counter.limit = perSeries * limit if guarded { counter.limit *= splitGroupSlack } @@ -1252,6 +1252,15 @@ func guardStatsGroups(ctx context.Context) context.Context { return context.WithValue(ctx, statsGroupGuardKey{}, true) } +// guardAuxStatsGroups guards the groups of a read with an aux result: they are +// the groups of a split plan, no series of the answer. +func guardAuxStatsGroups(ctx context.Context, aux string) context.Context { + if aux != "" { + return guardStatsGroups(ctx) + } + return ctx +} + func statsGroupsGuarded(ctx context.Context) bool { guarded, _ := ctx.Value(statsGroupGuardKey{}).(bool) return guarded @@ -1276,9 +1285,32 @@ func (p *Proxy) collectRangeMetricHits( start, end time.Time, hitStep time.Duration, ) (map[string]manualSeriesSamples, error) { + series, _, err := p.collectRangeMetricHitsAux(ctx, baseQuery, groupBy, origGroupBy, byExplicit, statsAggFunc, start, end, hitStep, "") + return series, err +} + +// collectRangeMetricHitsAux is collectRangeMetricHits where statsAggFunc also +// computes the result aux: it is not a series, and the left edges of the +// buckets where it is above zero are returned instead. With an aux result (a +// split query) the groups whose values are all zero are dropped, and the read +// never ends with the series limit error: a group is no series of the answer, +// so the limit is applied by the caller to the merged answer +// (capSeriesForRequest), and the read stops at splitGroupSlack times the limit +// with errStatsGroupGuard (readStatsBucketBody). +func (p *Proxy) collectRangeMetricHitsAux( + ctx context.Context, + baseQuery string, + groupBy, origGroupBy []string, + byExplicit bool, + statsAggFunc string, + start, end time.Time, + hitStep time.Duration, + aux string, +) (map[string]manualSeriesSamples, []int64, error) { if hitStep <= 0 { hitStep = time.Minute } + ctx = guardAuxStatsGroups(ctx, aux) // Build LogsQL stats query so VL returns pre-aggregated bucket values. // stats_query_range understands stream labels (stored in _stream), unlike /hits. @@ -1305,26 +1337,26 @@ func (p *Proxy) collectRangeMetricHits( case <-sem: defer func() { sem <- struct{}{} }() case <-ctx.Done(): - return nil, ctx.Err() + return nil, nil, ctx.Err() } } resp, err := p.vlPost(ctx, "/select/logsql/stats_query_range", params) if err != nil { - return nil, err + return nil, nil, err } defer resp.Body.Close() if resp.StatusCode >= 400 { body, _ := readBodyLimited(resp.Body, maxUpstreamErrorBodyBytes) - return nil, p.redactedBackendStatusError("stats_query_range backend", resp.StatusCode, body) + return nil, nil, p.redactedBackendStatusError("stats_query_range backend", resp.StatusCode, body) } limit := p.resolvedMaxStatsQuerySeries(ctx) - withPresence := strings.Contains(statsAggFunc, ", count() as __sample_count") - body, err := p.readStatsBucketBody(ctx, resp.Body, limit, withPresence) + withPresence := strings.Contains(statsAggFunc, " as __sample_count") + body, err := p.readStatsBucketBody(ctx, resp.Body, limit, statsResultsPerSeries(withPresence, aux != "")) if err != nil { - return nil, err + return nil, nil, err } // Build VL→Loki label name mapping from groupBy↔origGroupBy. @@ -1341,10 +1373,10 @@ func (p *Proxy) collectRangeMetricHits( // {"status":"success","data":{"result":[{"metric":{...},"values":[[ts,"v"],...]}]}} v, parseErr := fj.ParseBytes(body) if parseErr != nil { - return nil, fmt.Errorf("parse stats_query_range response: %w", parseErr) + return nil, nil, fmt.Errorf("parse stats_query_range response: %w", parseErr) } if status := string(v.GetStringBytes("status")); status != "success" { - return nil, fmt.Errorf("stats_query_range non-success status: %s", status) + return nil, nil, fmt.Errorf("stats_query_range non-success status: %s", status) } results := v.GetArray("data", "result") @@ -1355,7 +1387,14 @@ func (p *Proxy) collectRangeMetricHits( streamGrouped = streamGrouped || g == "_stream" } seriesMap := make(map[string]manualSeriesSamples, len(results)) + var auxEdges []int64 for _, res := range results { + if aux != "" && string(res.GetStringBytes("metric", "__name__")) == aux { + var entry manualSeriesSamples + addPresentBuckets(&entry, res.GetArray("values"), start, hitStep) + auxEdges = append(auxEdges, entry.PresentBuckets...) + continue + } metricObj := res.GetObject("metric") var metric map[string]string if streamGrouped { @@ -1402,14 +1441,51 @@ func (p *Proxy) collectRangeMetricHits( seriesMap[seriesKey] = manualSeriesSamples{Metric: metric, Samples: samples} } } - if statsGroupsGuarded(ctx) { - // The groups are not the answer's series: the caller applies the limit - // to the merged answer. - return seriesMap, nil + series, err := capHitSeries(ctx, seriesMap, limit, aux) + return series, auxEdges, err +} + +// capHitSeries applies the series limit to the series read from stats buckets: +// more than -max-stats-query-series is Loki's error, or for Drilldown the +// busiest series (complete, with byte values and presence) and a warning. With +// an aux result the groups that counted nothing are dropped and the caller +// applies the limit, because the aux lines add to the series; with the groups +// guarded (guardStatsGroups) the groups are no series of the answer either, and +// the caller applies the limit to the merged answer. +func capHitSeries(ctx context.Context, series map[string]manualSeriesSamples, limit int, aux string) (map[string]manualSeriesSamples, error) { + if aux != "" { + dropZeroSeries(series) + } + if aux != "" || statsGroupsGuarded(ctx) { + return series, nil + } + return capSeriesForRequest(ctx, series, limit) +} + +// dropZeroSeries removes the groups that counted nothing. +func dropZeroSeries(series map[string]manualSeriesSamples) { + for key, entry := range series { + zero := len(entry.PresentBuckets) == 0 + for _, sample := range entry.Samples { + zero = zero && sample.value == 0 + } + if zero { + delete(series, key) + } + } +} + +// statsResultsPerSeries counts the stats results each series carries: the +// value, byte presence, and an aux result. +func statsResultsPerSeries(withPresence, withAux bool) int { + n := 1 + if withPresence { + n++ + } + if withAux { + n++ } - // More series than -max-stats-query-series: Loki's error, or for Drilldown the - // busiest series (complete, with byte values and presence) and a warning. - return capSeriesForRequest(ctx, seriesMap, limit) + return n } // statsGroupMetric returns the Loki labels of a stats result's group: its diff --git a/test/e2e-compat/json_error_filter_split_test.go b/test/e2e-compat/json_error_filter_split_test.go new file mode 100644 index 000000000..751932cb4 --- /dev/null +++ b/test/e2e-compat/json_error_filter_split_test.go @@ -0,0 +1,293 @@ +//go:build e2e + +package e2e_compat + +import ( + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "testing" + "time" +) + +// jsonErrorFilterSplitFixture holds the lines a `| json | __error__=""` +// metric must count like Loki's parser: valid JSON of two statuses, some with +// a level stream label, nested objects and arrays, lines VictoriaLogs' parser +// accepts and Loki's rejects (NaN, a leading dot, a plus sign, an invalid key +// escape), JSON with trailing text (Loki accepts it), truncated JSON, an empty +// object, a line with leading whitespace, plain and logfmt text. +func jsonErrorFilterSplitFixture(s0 time.Time) []jsonVolumeStreamLine { + shapes := []struct{ level, msg string }{ + {"info", `{"status":"200","method":"GET","n":%d}`}, + {"", `{"status":"500","method":"POST","n":%d}`}, + {"", `{"status":"200","nested":{"a":{"b":%d}},"arr":[1,2]}`}, + {"warn", `{"status":"500","level":"error","n":%d}`}, + {"", `{"status":"200","ratio":NaN,"n":%d}`}, + {"", `{"status":"500","v":.5,"n":%d}`}, + {"", `{"status":"200","v":+1,"n":%d}`}, + {"", `{"st\qatus":"500","n":%d}`}, + {"", `{"status":"500","n":%d} trailing`}, + {"", `{"status":"200","msg":"truncated %d`}, + {"", `{}`}, + {"", ` {"status":"500","n":%d}`}, + {"", `plain text %d`}, + {"info", `status=200 n=%d`}, + } + var lines []jsonVolumeStreamLine + for i := 0; i < 120; i++ { + shape := shapes[i%len(shapes)] + msg := shape.msg + if strings.Contains(msg, "%d") { + msg = fmt.Sprintf(msg, i) + } + lines = append(lines, jsonVolumeStreamLine{ts: s0.Add(time.Duration(i) * 5 * time.Second), level: shape.level, msg: msg}) + } + return lines +} + +// A `| json | __error__=""` metric is answered from one VictoriaLogs stats +// scan over the lines VictoriaLogs parses (a second scan once a probe finds a +// line Loki's parser rejects), plus a raw read of only the buckets holding the +// other brace-opened lines, and matches Loki: count, rate and bytes, without +// grouping, by a stream label and by a parsed label. On v2.14.2 these shapes +// read every selected line as a raw row (and failed at 24 h on the row and +// byte limits). +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestRangeMetricCompatibilityJSONErrorFilterSplit(t *testing.T) { + now := time.Now() + app := fmt.Sprintf("json-error-split-%d", now.UnixNano()) + s0 := now.Add(-9 * time.Hour).Truncate(time.Hour) + ingestJSONVolumeFixture(t, map[string][]jsonVolumeStreamLine{app: jsonErrorFilterSplitFixture(s0)}) + + route, recorder := newJSONVolumeRouteProxy(t) + targets := map[string]string{"proxy": proxyURL, "in-process": route} + start, end := s0.Add(-2*time.Minute), s0.Add(11*time.Minute) + selector := `{app="` + app + `"}` + for _, tc := range []struct { + name, query string + minimum int // Loki series expected for the fixture + }{ + {"ungrouped count", `sum(count_over_time(` + selector + ` | json | __error__="" [1m]))`, 1}, + {"count by stream label", `sum by (level) (count_over_time(` + selector + ` | json | __error__="" [1m]))`, 2}, + {"count by parsed label", `sum by (status) (count_over_time(` + selector + ` | json | __error__="" [1m]))`, 2}, + {"rate by parsed label", `sum by (status) (rate(` + selector + ` | json | __error__="" [2m]))`, 2}, + {"bytes by stream label", `sum by (level) (bytes_over_time(` + selector + ` | json | __error__="" [1m]))`, 2}, + {"ungrouped bytes rate", `sum(bytes_rate(` + selector + ` | json | __error__="" [1m]))`, 1}, + {"count by service_name", `sum by (service_name) (count_over_time(` + selector + ` | json | __error__="" [1m]))`, 1}, + {"count by three labels", `sum by (level, service_name, status) (count_over_time(` + selector + ` | json | __error__="" [1m]))`, 2}, + {"bytes by two labels with a filter", `sum by (level, status) (bytes_over_time(` + selector + ` | json | __error__="" | status="500" [1m]))`, 1}, + {"count with a label filter", `sum by (level) (count_over_time(` + selector + ` | json | __error__="" | status="500" [1m]))`, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + loki := slidingRangeSeries(t, lokiURL, tc.query, start, end, time.Minute, nil) + if len(loki) < tc.minimum { + t.Fatalf("Loki sanity: expected at least %d series, got %d: %v", tc.minimum, len(loki), loki) + } + for name, base := range targets { + recorder.reset() + before := parserMetricEvaluations(t, route) + assertSlidingParity(t, tc.query+" ["+name+"]", loki, slidingRangeSeries(t, base, tc.query, start, end, time.Minute, nil)) + if name != "in-process" { + continue + } + after := parserMetricEvaluations(t, route) + stats, raw := 0, 0 + for _, call := range recorder.reset() { + switch { + case strings.HasPrefix(call, "/select/logsql/stats_query_range "): + stats++ + case strings.HasPrefix(call, "/select/logsql/query ") && !strings.HasSuffix(call, " | limit 1"): + raw++ + if !strings.Contains(call, "__vlp:* or ") { + t.Fatalf("raw read %q is not limited to the lines VictoriaLogs cannot vouch for", call) + } + } + } + // The fixture holds a NaN line: the hazard probe finds it and the + // stats scan runs again with such lines read as raw rows. + if stats != 2 || raw == 0 || after["vl_stats_buckets/pushdown"]-before["vl_stats_buckets/pushdown"] != 1 { + t.Fatalf("expected two stats scans and a bounded raw read: stats=%d raw=%d", stats, raw) + } + } + }) + } +} + +// The series limit counts the series Loki's answer holds, never the groups +// VictoriaLogs makes: it emits one for every row reaching its stats pipe, +// including the lines a label filter drops and the lines its parser rejects +// while Loki's accepts. A filter that keeps 2 of 20 paths and exactly as many +// paths as the limit both answer as Loki does, while 20 paths under a limit of +// 5 are Loki's series limit error (the split hands a read of more than twice +// the limit in groups to the row evaluator, which ends with that error). +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestRangeMetricCompatibilityJSONErrorFilterSplitSeriesLimit(t *testing.T) { + const limit = 5 + now := time.Now() + s0 := now.Add(-9 * time.Hour).Truncate(time.Hour) + filtered := fmt.Sprintf("json-error-split-filtered-%d", now.UnixNano()) + exact := fmt.Sprintf("json-error-split-exact-%d", now.UnixNano()) + var filteredLines, exactLines []jsonVolumeStreamLine + for i := 0; i < 20; i++ { + status := "200" + if i < 2 { + status = "500" + } + filteredLines = append(filteredLines, jsonVolumeStreamLine{ts: s0.Add(time.Duration(i) * 5 * time.Second), msg: fmt.Sprintf(`{"path":"/p%d","status":"%s"}`, i, status)}) + } + for i := 0; i < limit; i++ { + exactLines = append(exactLines, jsonVolumeStreamLine{ts: s0.Add(time.Duration(i) * 5 * time.Second), msg: fmt.Sprintf(`{"path":"/p%d"}`, i)}) + } + // Trailing text: Loki's parser accepts the line, VictoriaLogs' does not. + exactLines = append(exactLines, jsonVolumeStreamLine{ts: s0.Add(30 * time.Second), msg: `{"path":"/p0"} trailing`}) + ingestJSONVolumeFixture(t, map[string][]jsonVolumeStreamLine{filtered: filteredLines, exact: exactLines}) + + route, _ := newJSONVolumeRouteProxyWithLimits(t, 0, limit) + start, end := s0.Add(-2*time.Minute), s0.Add(4*time.Minute) + for _, tc := range []struct{ name, query string }{ + {"filter keeps few of many groups", `sum by (path) (count_over_time({app="` + filtered + `"} | json | __error__="" | status="500" [1m]))`}, + {"filtered bytes", `sum by (path) (bytes_over_time({app="` + filtered + `"} | json | __error__="" | status="500" [1m]))`}, + {"unparseable line beside exactly limit paths", `sum by (path) (count_over_time({app="` + exact + `"} | json | __error__="" [1m]))`}, + } { + t.Run(tc.name, func(t *testing.T) { + loki := slidingRangeSeries(t, lokiURL, tc.query, start, end, time.Minute, nil) + if len(loki) == 0 || len(loki) > limit { + t.Fatalf("Loki sanity: expected 1 to %d series, got %d: %v", limit, len(loki), loki) + } + assertSlidingParity(t, tc.query+" [limit "+strconv.Itoa(limit)+"]", loki, slidingRangeSeries(t, route, tc.query, start, end, time.Minute, nil)) + }) + } + + t.Run("20 paths over the limit are Loki's series limit error", func(t *testing.T) { + query := `sum by (path) (count_over_time({app="` + filtered + `"} | json | __error__="" | status=~"200|500" [1m]))` + // Loki's own limit is higher than the one set on the proxy: it holds all 20. + if loki := slidingRangeSeries(t, lokiURL, query, start, end, time.Minute, nil); len(loki) != 20 { + t.Fatalf("Loki sanity: expected 20 series, got %d: %v", len(loki), loki) + } + // A step of the window: every evaluation point sees lines, as Loki's + // series limit is the count of series the answer holds. + params := url.Values{"query": {query}, "start": {strconv.FormatInt(start.UnixNano(), 10)}, "end": {strconv.FormatInt(end.UnixNano(), 10)}, "step": {"60"}} + status, body, _ := doJSONGET(t, route+"/loki/api/v1/query_range?"+params.Encode(), map[string]string{"X-Scope-OrgID": "0"}) + if status != http.StatusBadRequest || !strings.Contains(body, "maximum number of series") { + t.Fatalf("20 series with limit %d: status %d body %s", limit, status, body) + } + }) +} + +// detected_level is the level Loki derives at ingest, normalized, from a stream +// label or from the JSON body. The split serves it from the stored level fields +// alone, and only when they decide the level of every counted line; a level the +// body holds, a counted line without one, or a read past the group guard is left +// to the row evaluator, which derives the level of each line like Loki's +// ingest. Either way the answer is Loki's, VictoriaLogs groups (one per +// spelling) notwithstanding: the spellings of one level are one series, and the +// series limit counts series. The last variant holds lines only Loki's parser +// accepts (nested deeper than VictoriaLogs reads), which the split reads as raw +// rows and whose level the row evaluator derives from the body. +// conformance: semantics/json-error-filter-metric-split-pushdown +func TestRangeMetricCompatibilityJSONErrorFilterSplitDetectedLevel(t *testing.T) { + now := time.Now() + s0 := now.Add(-9 * time.Hour).Truncate(time.Hour) + body := fmt.Sprintf("json-error-split-lvl-body-%d", now.UnixNano()) + stream := fmt.Sprintf("json-error-split-lvl-stream-%d", now.UnixNano()) + gap := fmt.Sprintf("json-error-split-lvl-gap-%d", now.UnixNano()) + spellings := []string{"info", "INFO", "Info", "inf", "information", "warn", "WARN", "warning", "error", "ERR", "debug"} + var bodyLines, streamLines, gapLines, rawLines []jsonVolumeStreamLine + for i := 0; i < 66; i++ { + ts := s0.Add(time.Duration(i) * 5 * time.Second) + level := spellings[i%len(spellings)] + bodyLines = append(bodyLines, jsonVolumeStreamLine{ts: ts, msg: fmt.Sprintf(`{"level":%q,"n":%d}`, level, i)}) + streamLines = append(streamLines, jsonVolumeStreamLine{ts: ts, level: level, msg: fmt.Sprintf(`{"n":%d}`, i)}) + gapLines = append(gapLines, jsonVolumeStreamLine{ts: ts, level: level, msg: fmt.Sprintf(`{"n":%d}`, i)}) + // Nested deeper than VictoriaLogs' parser reads (300 levels): Loki's + // parser accepts the line, so the split reads it as a raw row. + rawLines = append(rawLines, jsonVolumeStreamLine{ts: ts, msg: fmt.Sprintf(`{"level":%q,"deep":%s1%s,"n":%d}`, level, strings.Repeat(`{"a":`, 320), strings.Repeat(`}`, 320), i)}) + } + // A line without any level, the last of the fixture. + bodyLines = append(bodyLines, jsonVolumeStreamLine{ts: s0.Add(331 * time.Second), msg: `{"n":-1}`}) + streamLines = append(streamLines, jsonVolumeStreamLine{ts: s0.Add(331 * time.Second), level: "info", msg: `{"n":-1}`}) + gapLines = append(gapLines, jsonVolumeStreamLine{ts: s0.Add(331 * time.Second), msg: `{"n":-1}`}) + raw := fmt.Sprintf("json-error-split-lvl-raw-%d", now.UnixNano()) + ingestJSONVolumeFixture(t, map[string][]jsonVolumeStreamLine{body: bodyLines, stream: streamLines, gap: gapLines, raw: rawLines}) + + route, routeRecorder := newJSONVolumeRouteProxyWithLimits(t, 0, 50) + // 11 spellings and the line without a level are 12 groups past twice the + // limit of 5, but 5 series: the row evaluator answers that read. + fallback, fallbackRecorder := newJSONVolumeRouteProxyWithLimits(t, 0, 5) + targets := map[string]string{"proxy": proxyURL, "in-process": route, "in-process limit 5": fallback} + start, end := s0.Add(-2*time.Minute), s0.Add(9*time.Minute) + // split: the stats answer from the stored level fields (the raw app reads + // its lines as raw rows too); otherwise the row evaluator answers. + for _, tc := range []struct { + name, app string + split bool + }{ + {"level in the body", body, false}, + {"level as a stream label with mixed spellings", stream, true}, + {"a counted line without a stored level among stored ones", gap, false}, + {"level in the body of lines nested deeper than VictoriaLogs reads", raw, true}, + } { + for _, function := range []string{"count_over_time", "bytes_over_time"} { + query := `sum by (detected_level) (` + function + `({app="` + tc.app + `"} | json | __error__="" [1m]))` + t.Run(tc.name+"/"+function, func(t *testing.T) { + loki := slidingRangeSeries(t, lokiURL, query, start, end, time.Minute, nil) + t.Logf("Loki: %d series", len(loki)) + if len(loki) < 2 { + t.Fatalf("Loki sanity: expected several detected_level series, got %d: %v", len(loki), loki) + } + for name, base := range targets { + fallbackRecorder.reset() + routeRecorder.reset() + assertSlidingParity(t, query+" ["+name+"]", loki, slidingRangeSeries(t, base, query, start, end, time.Minute, nil)) + if name == "in-process" { + assertDetectedLevelSplitRoute(t, routeRecorder.reset(), tc.split) + } + if name != "in-process limit 5" { + continue + } + rowReads := 0 // a read of the selected rows, not the split's raw candidates + for _, call := range fallbackRecorder.reset() { + if strings.HasPrefix(call, "/select/logsql/query ") && !strings.HasSuffix(call, " | limit 1") && !strings.Contains(call, "__vlp:*") { + rowReads++ + } + } + // The raw app reads its lines as raw rows only: one group. + if rowReads == 0 && tc.app != raw && tc.app != stream { + t.Errorf("limit 5: the row evaluator did not answer the read of past twice the limit in groups") + } + } + }) + } + } +} + +// assertDetectedLevelSplitRoute checks which route answered a detected_level +// query of the split: with split the stats read the stored level fields and no +// line is read as a row but the raw candidates (bounded by the sentinel); +// without it the row evaluator read the selected rows. The derivation chain is +// never part of a split plan. +func assertDetectedLevelSplitRoute(t *testing.T, calls []string, split bool) { + t.Helper() + statsCalls, rowReads := 0, 0 + for _, call := range calls { + switch { + case strings.HasPrefix(call, "/select/logsql/stats_query_range "): + if strings.Contains(call, `result_prefix "__j_"`) || strings.Contains(call, "__dl_h") { + t.Errorf("a split plan sent the detected_level chain: %.300s", call) + } + if strings.Contains(call, "copy detected_level as __dl_d, level as __dl_l") { + statsCalls++ + } + case strings.HasPrefix(call, "/select/logsql/query ") && !strings.HasSuffix(call, " | limit 1") && !strings.Contains(call, "__vlp:*"): + rowReads++ + } + } + if split && (statsCalls == 0 || rowReads != 0) { + t.Errorf("expected the stats over the stored level fields and no row read, got %d stats reads and %d row reads", statsCalls, rowReads) + } + if !split && rowReads == 0 { + t.Errorf("expected the row evaluator to answer, got %d stats reads and no row read", statsCalls) + } +} diff --git a/test/e2e-compat/json_logs_volume_test.go b/test/e2e-compat/json_logs_volume_test.go index b72c41546..84da093d4 100644 --- a/test/e2e-compat/json_logs_volume_test.go +++ b/test/e2e-compat/json_logs_volume_test.go @@ -64,7 +64,17 @@ func newJSONVolumeRouteProxy(t *testing.T) (string, *vlQueryRecorder) { // back to the raw-row evaluator over more rows fails with the limit error. func newJSONVolumeRouteProxyWithRowLimit(t *testing.T, rowLimit int) (string, *vlQueryRecorder) { t.Helper() - return newJSONVolumeRouteProxyWithConfig(t, func(cfg *proxy.Config) { cfg.RangeMetricRowLimit = rowLimit }) + return newJSONVolumeRouteProxyWithLimits(t, rowLimit, 0) +} + +// newJSONVolumeRouteProxyWithLimits is newJSONVolumeRouteProxyWithRowLimit with +// -max-stats-query-series set too (0: the default). +func newJSONVolumeRouteProxyWithLimits(t *testing.T, rowLimit, seriesLimit int) (string, *vlQueryRecorder) { + t.Helper() + return newJSONVolumeRouteProxyWithConfig(t, func(cfg *proxy.Config) { + cfg.RangeMetricRowLimit = rowLimit + cfg.MaxStatsQuerySeries = seriesLimit + }) } // newJSONVolumeRouteProxyWithConfig is newJSONVolumeRouteProxy with the proxy's