diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e410cbc8..0d05e9c60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- **`label_replace` adds its destination label to instant answers, reads an + escaped regexp, and runs under `topk`, `sort` and a binary expression (#744).** + `label_replace(rate({app="api-gateway"}[5m]), "app_short", "$1", "app", + "(.*)-.*")` came back from `/loki/api/v1/query` without `app_short` on every + series (also through nested `label_replace`), where Loki adds it + (`pkg/logql/evaluator.go` `LabelReplaceEvaluator.Next`): the proxy applied the + transform to range matrices only. It now rewrites the labels of a vector and + of a matrix alike, keeps the rest of the answer, and expands the replacement + the way Loki does (`$1`, `${1}`, `$name`, `$$`; an empty expansion removes the + destination), with the anchored regexp. Two series that end with one label + set stay two samples on an instant query and become one series on a range + query, as `JoinSampleVector` does, and no error is raised, as in Loki. Also + fixed: the string arguments are unquoted like the LogQL lexer does, so a + regexp written `"(\\w+)\\.(\\w+)"` matches (the backslashes used to be kept, + and the regexp never matched) and an escaped quote no longer ends the + argument; `topk`, `bottomk`, `sort` and `stddev` over a `label_replace` sent + the translator's marker to VictoriaLogs and now rank what the transform + produced; and a `label_replace` around a binary expression on `query_range` + sent the binary marker to VictoriaLogs, as the instant path used to. An + invalid regexp under those wrappers is Loki's `parse error : invalid regex in + label_replace` 400. `label_replace(sum without (level) (...), ...)` kept the + excluded label and lost the transform, because the `without` marker swallowed + the `label_replace` marker after it; and a call without the four string + arguments of Loki's grammar, or with an invalid escape in one, was taken for a + log query (an empty `streams` answer on `query_range`) and is now a 400 parse + error on both endpoints. + An instant `label_replace(count_over_time({env="production"}[5m]), ...)` (or + `bytes_over_time`, `rate`, ...) no longer comes back as one unlabeled total: the + grouping by stream was added only for a query that itself starts with the range + function, so the wrapper turned sixteen series into `{}`. A `label_replace` + around a metric over `| json` is answered by the evaluator that owns JSON parser + errors, as the metric is without the wrapper: a line that is not JSON fails the + query with Loki's `pipeline error: 'JSONParserErr'` 400 unless the pipeline + filters `__error__`, and `| json | __error__=""` keeps only the streams that + parse (the wrapper used to skip both and answer 200 with every stream). The + transform is applied to that answer. + ## [2.13.0] - 2026-10-12 ### Fixed diff --git a/bench/ab/results/2026-10-12-labelreplace-1h.json b/bench/ab/results/2026-10-12-labelreplace-1h.json new file mode 100644 index 000000000..79f7bf290 --- /dev/null +++ b/bench/ab/results/2026-10-12-labelreplace-1h.json @@ -0,0 +1,232 @@ +{ + "label": "labelreplace-1h", + "date": "2026-10-12", + "set": "label-replace", + "description": "label_replace adds its label to instant answers, to post-aggregated series and around a bare range function or a `| json` metric (PR for semantics/label-replace-new-label). L1 instant and L2 range are the registry row's form over a sum by; L3 is the transform under topk (ranked after the transform, as Loki evaluates it); L4 is an instant label_replace over a bare count_over_time (one series per stream, not one total); L5 and L6 are the transform over a `| json` metric that drops __error__ (answered by the evaluator that owns JSON parser errors). Run with the control set.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 5, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 5, + "slower": 1 + }, + "result_differs_from_reference": 1, + "result_differs_preexisting": 1, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "L1 label_replace over sum by rate, instant", + "range": "instant", + "p50": { + "base": 0.005, + "pr": 0.005, + "loki": 0.109 + }, + "p95": { + "base": 0.008, + "pr": 0.006, + "loki": 0.119 + }, + "cold": { + "base": 0.014, + "pr": 0.005, + "loki": 0.099 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 3.49, + "pr": 3.49, + "loki": 3.49 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.0 + }, + { + "shape": "L2 label_replace over sum by rate", + "range": "1h", + "p50": { + "base": 0.029, + "pr": 0.035, + "loki": 0.166 + }, + "p95": { + "base": 0.032, + "pr": 0.037, + "loki": 0.187 + }, + "cold": { + "base": 0.041, + "pr": 0.035, + "loki": 0.82 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 29.01, + "pr": 29.01, + "loki": 29.01 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.83 + }, + { + "shape": "L3 topk over label_replace over sum by count", + "range": "1h", + "p50": { + "base": 0.022, + "pr": 0.048, + "loki": 0.07 + }, + "p95": { + "base": 0.023, + "pr": 0.053, + "loki": 0.141 + }, + "cold": { + "base": 0.031, + "pr": 0.049, + "loki": 0.826 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 39.57, + "pr": 40.35, + "loki": 40.35 + }, + "parity_vs_reference": "differs", + "baseline_parity_vs_reference": "differs", + "parity_vs_baseline": "differs", + "verdict": "same", + "speedup": 0.46 + }, + { + "shape": "L4 label_replace over bare count_over_time, instant", + "range": "instant", + "p50": { + "base": 0.003, + "pr": 0.007, + "loki": 0.013 + }, + "p95": { + "base": 0.003, + "pr": 0.009, + "loki": 0.016 + }, + "cold": { + "base": 0.003, + "pr": 0.006, + "loki": 0.012 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 40.35, + "pr": 40.35, + "loki": 40.35 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "differs", + "parity_vs_baseline": "differs", + "verdict": "same", + "speedup": 0.43 + }, + { + "shape": "L5 label_replace over json sum by, instant", + "range": "instant", + "p50": { + "base": 0.018, + "pr": 0.05, + "loki": 0.014 + }, + "p95": { + "base": 0.022, + "pr": 0.056, + "loki": 0.015 + }, + "cold": { + "base": 0.028, + "pr": 0.057, + "loki": 0.015 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 36.58, + "pr": 36.58, + "loki": 36.58 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.36 + }, + { + "shape": "L6 label_replace over json sum by", + "range": "1h", + "p50": { + "base": 0.021, + "pr": 0.554, + "loki": 0.015 + }, + "p95": { + "base": 0.022, + "pr": 0.557, + "loki": 0.022 + }, + "cold": { + "base": 0.024, + "pr": 0.536, + "loki": 0.112 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 35.61, + "pr": 35.61, + "loki": 35.61 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "slower", + "speedup": 0.04 + } + ] +} \ No newline at end of file diff --git a/bench/ab/results/2026-10-12-labelreplace-control-1h.json b/bench/ab/results/2026-10-12-labelreplace-control-1h.json new file mode 100644 index 000000000..57fda1905 --- /dev/null +++ b/bench/ab/results/2026-10-12-labelreplace-control-1h.json @@ -0,0 +1,707 @@ +{ + "label": "labelreplace-control-1h", + "date": "2026-10-12", + "set": "control", + "description": "Shapes a change must not slow down or alter. Run on every performance-relevant PR.", + "baseline": "base", + "candidate": "pr", + "reference": "loki", + "runs": 5, + "valid": true, + "restart_before": "0", + "restart_after": "0", + "noise": 0.3, + "timing": "warm", + "verdicts": { + "same": 20 + }, + "result_differs_from_reference": 1, + "result_differs_preexisting": 1, + "stale_answers": { + "base": 0, + "pr": 0, + "loki": 0 + }, + "rows": [ + { + "shape": "logs plain", + "range": "1h", + "p50": { + "base": 0.026, + "pr": 0.026, + "loki": 0.021 + }, + "p95": { + "base": 0.032, + "pr": 0.032, + "loki": 0.03 + }, + "cold": { + "base": 0.044, + "pr": 0.024, + "loki": 0.023 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 17.46, + "pr": 17.46, + "loki": 17.46 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.0 + }, + { + "shape": "logs | json", + "range": "1h", + "p50": { + "base": 0.036, + "pr": 0.035, + "loki": 0.037 + }, + "p95": { + "base": 0.046, + "pr": 0.037, + "loki": 0.045 + }, + "cold": { + "base": 0.039, + "pr": 0.047, + "loki": 0.045 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 29.01, + "pr": 29.01, + "loki": 29.01 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.03 + }, + { + "shape": "logs | logfmt", + "range": "1h", + "p50": { + "base": 0.042, + "pr": 0.045, + "loki": 0.029 + }, + "p95": { + "base": 0.046, + "pr": 0.052, + "loki": 0.031 + }, + "cold": { + "base": 0.052, + "pr": 0.051, + "loki": 0.032 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 42.88, + "pr": 42.88, + "loki": 42.88 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.93 + }, + { + "shape": "json volume no filters", + "range": "1h", + "p50": { + "base": 0.047, + "pr": 0.045, + "loki": 0.024 + }, + "p95": { + "base": 0.051, + "pr": 0.05, + "loki": 0.05 + }, + "cold": { + "base": 0.039, + "pr": 0.057, + "loki": 0.37 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 53.36, + "pr": 53.36, + "loki": 53.36 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.04 + }, + { + "shape": "logfmt volume", + "range": "1h", + "p50": { + "base": 0.101, + "pr": 0.09, + "loki": 0.025 + }, + "p95": { + "base": 0.124, + "pr": 0.091, + "loki": 0.071 + }, + "cold": { + "base": 0.06, + "pr": 0.053, + "loki": 0.551 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 76.92, + "pr": 76.92, + "loki": 76.92 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.12 + }, + { + "shape": "plain volume", + "range": "1h", + "p50": { + "base": 0.008, + "pr": 0.007, + "loki": 0.016 + }, + "p95": { + "base": 0.021, + "pr": 0.01, + "loki": 0.057 + }, + "cold": { + "base": 0.006, + "pr": 0.008, + "loki": 0.306 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 76.92, + "pr": 5.43, + "loki": 6.71 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.14 + }, + { + "shape": "stream-label count by app", + "range": "1h", + "p50": { + "base": 0.013, + "pr": 0.011, + "loki": 0.026 + }, + "p95": { + "base": 0.014, + "pr": 0.013, + "loki": 0.044 + }, + "cold": { + "base": 0.012, + "pr": 0.012, + "loki": 0.326 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 6.71, + "pr": 6.71, + "loki": 6.71 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.18 + }, + { + "shape": "ungrouped rate", + "range": "1h", + "p50": { + "base": 0.005, + "pr": 0.006, + "loki": 0.041 + }, + "p95": { + "base": 0.007, + "pr": 0.018, + "loki": 0.056 + }, + "cold": { + "base": 0.005, + "pr": 0.006, + "loki": 0.316 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 6.04, + "pr": 5.07, + "loki": 5.07 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.83 + }, + { + "shape": "bytes_over_time by app", + "range": "1h", + "p50": { + "base": 0.026, + "pr": 0.024, + "loki": 0.041 + }, + "p95": { + "base": 0.036, + "pr": 0.027, + "loki": 0.057 + }, + "cold": { + "base": 0.034, + "pr": 0.028, + "loki": 0.321 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 19.76, + "pr": 19.76, + "loki": 19.76 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.08 + }, + { + "shape": "topk rate by app", + "range": "1h", + "p50": { + "base": 0.018, + "pr": 0.019, + "loki": 0.036 + }, + "p95": { + "base": 0.021, + "pr": 0.02, + "loki": 0.056 + }, + "cold": { + "base": 0.019, + "pr": 0.029, + "loki": 0.327 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 15.8, + "pr": 15.8, + "loki": 15.8 + }, + "parity_vs_reference": "differs", + "baseline_parity_vs_reference": "differs", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.95 + }, + { + "shape": "detected_level volume", + "range": "1h", + "p50": { + "base": 0.009, + "pr": 0.006, + "loki": 0.018 + }, + "p95": { + "base": 0.011, + "pr": 0.006, + "loki": 0.039 + }, + "cold": { + "base": 0.007, + "pr": 0.006, + "loki": 0.289 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 12.47, + "pr": 12.47, + "loki": 5.56 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.5 + }, + { + "shape": "by service_name", + "range": "1h", + "p50": { + "base": 0.015, + "pr": 0.013, + "loki": 0.022 + }, + "p95": { + "base": 0.034, + "pr": 0.015, + "loki": 0.052 + }, + "cold": { + "base": 0.014, + "pr": 0.013, + "loki": 0.294 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 15.23, + "pr": 15.23, + "loki": 15.23 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.15 + }, + { + "shape": "drilldown level volume", + "range": "1h", + "p50": { + "base": 0.008, + "pr": 0.006, + "loki": 0.011 + }, + "p95": { + "base": 0.017, + "pr": 0.008, + "loki": 0.037 + }, + "cold": { + "base": 0.013, + "pr": 0.006, + "loki": 0.304 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 15.23, + "pr": 5.5, + "loki": 8.22 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.33 + }, + { + "shape": "drilldown service landing", + "range": "1h", + "p50": { + "base": 0.01, + "pr": 0.014, + "loki": 0.018 + }, + "p95": { + "base": 0.011, + "pr": 0.019, + "loki": 0.034 + }, + "cold": { + "base": 0.01, + "pr": 0.018, + "loki": 0.292 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 10.1, + "pr": 10.1, + "loki": 10.1 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.71 + }, + { + "shape": "drilldown detected_level", + "range": "1h", + "p50": { + "base": 0.02, + "pr": 0.019, + "loki": 0.01 + }, + "p95": { + "base": 0.028, + "pr": 0.024, + "loki": 0.062 + }, + "cold": { + "base": 0.017, + "pr": 0.022, + "loki": 0.299 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 23.01, + "pr": 28.86, + "loki": 28.86 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.05 + }, + { + "shape": "json rate by level (no filter, no drop)", + "range": "1h", + "p50": { + "base": 0.003, + "pr": 0.002, + "loki": 0.059 + }, + "p95": { + "base": 0.003, + "pr": 0.002, + "loki": 0.068 + }, + "cold": { + "base": 0.002, + "pr": 0.002, + "loki": 0.061 + }, + "status": { + "base": "400", + "pr": "400", + "loki": "400" + }, + "vl_cpu_max": { + "base": 28.86, + "pr": 28.86, + "loki": 28.86 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "n/a", + "verdict": "same", + "speedup": null + }, + { + "shape": "logfmt count by level, filter, drop", + "range": "1h", + "p50": { + "base": 0.067, + "pr": 0.067, + "loki": 0.027 + }, + "p95": { + "base": 0.07, + "pr": 0.091, + "loki": 0.068 + }, + "cold": { + "base": 0.069, + "pr": 0.08, + "loki": 0.516 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 111.14, + "pr": 111.14, + "loki": 111.14 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.0 + }, + { + "shape": "instant ungrouped rate", + "range": "instant", + "p50": { + "base": 0.005, + "pr": 0.004, + "loki": 0.033 + }, + "p95": { + "base": 0.006, + "pr": 0.006, + "loki": 0.036 + }, + "cold": { + "base": 0.003, + "pr": 0.006, + "loki": 0.029 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 99.25, + "pr": 3.48, + "loki": 3.48 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 1.25 + }, + { + "shape": "instant json volume", + "range": "instant", + "p50": { + "base": 0.177, + "pr": 0.178, + "loki": 0.045 + }, + "p95": { + "base": 0.18, + "pr": 0.207, + "loki": 0.047 + }, + "cold": { + "base": 0.183, + "pr": 0.19, + "loki": 0.046 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 17.18, + "pr": 17.18, + "loki": 17.18 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.99 + }, + { + "shape": "instant json filtered", + "range": "instant", + "p50": { + "base": 0.15, + "pr": 0.163, + "loki": 0.033 + }, + "p95": { + "base": 0.156, + "pr": 0.171, + "loki": 0.035 + }, + "cold": { + "base": 0.156, + "pr": 0.159, + "loki": 0.038 + }, + "status": { + "base": "200", + "pr": "200", + "loki": "200" + }, + "vl_cpu_max": { + "base": 21.71, + "pr": 21.71, + "loki": 21.71 + }, + "parity_vs_reference": "same", + "baseline_parity_vs_reference": "same", + "parity_vs_baseline": "same", + "verdict": "same", + "speedup": 0.92 + } + ] +} \ No newline at end of file diff --git a/bench/ab/shapes.json b/bench/ab/shapes.json index 3bc4795fe..d179d4cb0 100644 --- a/bench/ab/shapes.json +++ b/bench/ab/shapes.json @@ -1592,6 +1592,70 @@ ] } ] + }, + "label-replace": { + "description": "label_replace adds its label to instant answers, to post-aggregated series and around a bare range function or a `| json` metric (PR for semantics/label-replace-new-label). L1 instant and L2 range are the registry row's form over a sum by; L3 is the transform under topk (ranked after the transform, as Loki evaluates it); L4 is an instant label_replace over a bare count_over_time (one series per stream, not one total); L5 and L6 are the transform over a `| json` metric that drops __error__ (answered by the evaluator that owns JSON parser errors). Run with the control set.", + "ranges": [ + "1h", + "6h", + "24h", + "7d" + ], + "shapes": [ + { + "name": "L1 label_replace over sum by rate, instant", + "query": "label_replace(sum by (service_name) (rate({env=\"production\"}[5m])), \"svc_short\", \"$1\", \"service_name\", \"(.*)-.*\")", + "covers": [ + "semantics/label-replace-new-label", + "loki_api_v1_query" + ], + "instant": true + }, + { + "name": "L2 label_replace over sum by rate", + "query": "label_replace(sum by (service_name) (rate({env=\"production\"}[5m])), \"svc_short\", \"$1\", \"service_name\", \"(.*)-.*\")", + "covers": [ + "semantics/label-replace-new-label", + "loki_api_v1_query_range" + ] + }, + { + "name": "L3 topk over label_replace over sum by count", + "query": "topk(3, label_replace(sum by (service_name) (count_over_time({env=\"production\"}[5m])), \"svc_short\", \"$1\", \"service_name\", \"(.*)-.*\"))", + "covers": [ + "semantics/label-replace-new-label", + "loki_api_v1_query_range" + ] + }, + { + "name": "L4 label_replace over bare count_over_time, instant", + "query": "label_replace(count_over_time({service_name=\"api-gateway\"}[5m]), \"svc_short\", \"$1\", \"service_name\", \"(.*)-.*\")", + "covers": [ + "semantics/label-replace-new-label", + "loki_api_v1_query" + ], + "instant": true + }, + { + "name": "L5 label_replace over json sum by, instant", + "query": "label_replace(sum by (service_name) (count_over_time({service_name=\"api-gateway\"} | json | __error__=\"\" [5m])), \"svc_short\", \"$1\", \"service_name\", \"(.*)-.*\")", + "covers": [ + "semantics/label-replace-new-label", + "semantics/pipeline-error-text-parity", + "loki_api_v1_query" + ], + "instant": true + }, + { + "name": "L6 label_replace over json sum by", + "query": "label_replace(sum by (service_name) (count_over_time({service_name=\"api-gateway\"} | json | __error__=\"\" [5m])), \"svc_short\", \"$1\", \"service_name\", \"(.*)-.*\")", + "covers": [ + "semantics/label-replace-new-label", + "semantics/pipeline-error-text-parity", + "loki_api_v1_query_range" + ] + } + ] } } } diff --git a/conformance/registry/cases/semantics/label-replace-aggregation-over-transform.yaml b/conformance/registry/cases/semantics/label-replace-aggregation-over-transform.yaml new file mode 100644 index 000000000..4f80cdb91 --- /dev/null +++ b/conformance/registry/cases/semantics/label-replace-aggregation-over-transform.yaml @@ -0,0 +1,40 @@ +id: semantics/label-replace-aggregation-over-transform +title: An aggregation over a label_replace fails on the proxy and answers on Loki (open) +proves: + - loki_api_v1_query + - loki_api_v1_query_range +consumer: explore +why: | + `sum by (svc_short) (label_replace(sum by (service_name) (rate({env="production"}[5m])), + "svc_short", "$1", "service_name", "(.*)-.*"))` is a valid Loki query: the vector + aggregation groups what the label_replace evaluator yields + (pkg/logql/evaluator.go VectorAggEvaluator over LabelReplaceEvaluator), and Loki + answers one series per `svc_short` on both endpoints. The proxy answers a 400 on + both: the instant query with "log queries are not supported as an instant query + type", query_range with a parse error from VictoriaLogs (the translated query + still carries the transform's marker). The base commit answers the same, so the + difference is not new: the transform is parsed for the outermost expression of a + query, and one that sits under another aggregation is not run in the proxy. + An aggregation that does not read the new label is not affected. +request: + method: GET + path: /loki/api/v1/query + params: + query: 'sum by (svc_short) (label_replace(sum by (service_name) (rate({env="production"}[5m])), "svc_short", "$1", "service_name", "(.*)-.*"))' + headers: + X-Scope-OrgID: "0" + X-Loki-Response-Encoding-Flags: categorize-labels +compare: + mode: observed + fields: [status, series labels, values] + expect: | + One series per svc_short, as Loki answers, on query and query_range. +gap: + status: open + impact: explore-visible + effort: M + area: 'internal/proxy proxy.go (handleQuery, handleQueryRange: markers are parsed once, for the outermost expression), vector_matching.go (labelTransforms)' + loki: 'pkg/logql/evaluator.go VectorAggEvaluator.Next over LabelReplaceEvaluator.Next' + proxy: '400 from the proxy (instant) or from VictoriaLogs (range): the aggregation is not evaluated over the transformed series' + planned_test: 'TestCompat_AggregationOverLabelReplace (test/e2e-compat, group semantics)' + clusters: [] diff --git a/conformance/registry/cases/semantics/label-replace-around-ranking-and-vector.yaml b/conformance/registry/cases/semantics/label-replace-around-ranking-and-vector.yaml new file mode 100644 index 000000000..d142731f7 --- /dev/null +++ b/conformance/registry/cases/semantics/label-replace-around-ranking-and-vector.yaml @@ -0,0 +1,40 @@ +id: semantics/label-replace-around-ranking-and-vector +title: label_replace around topk, sort or vector() fails on the proxy and answers on Loki (open) +proves: + - loki_api_v1_query + - loki_api_v1_query_range +consumer: explore +why: | + Loki evaluates `label_replace(topk(2, sum by (service_name) (rate({env="production"}[5m]))), + "svc_short", "$1", "service_name", "(.*)-.*")`, the same call around `sort(...)` and + `label_replace(vector(1), "a", "$1", "b", "(.*)")`: the label_replace evaluator + transforms the series of any vector expression below it (pkg/logql/evaluator.go + LabelReplaceEvaluator.Next). The proxy answers a 400 on query ("log queries are + not supported as an instant query type") and, on query_range, a 400 from + VictoriaLogs' parser for topk and sort and an empty `streams` answer for + vector(). The transform is applied to a stats answer, and these inner expressions + are not routed as stats queries by the label_replace path (the other direction, + a label_replace under topk or sort, is handled and compared by + TestCompat_LabelReplaceNewLabel). The base commit answers the same. +request: + method: GET + path: /loki/api/v1/query + params: + query: 'label_replace(topk(2, sum by (service_name) (rate({env="production"}[5m]))), "svc_short", "$1", "service_name", "(.*)-.*")' + headers: + X-Scope-OrgID: "0" + X-Loki-Response-Encoding-Flags: categorize-labels +compare: + mode: observed + fields: [status, series labels, values] + expect: | + The ranked series with svc_short added, as Loki answers. +gap: + status: open + impact: explore-visible + effort: M + area: 'internal/proxy proxy.go (handleQuery, handleQueryRange routing of a label_replace whose inner expression is topk, sort or vector)' + loki: 'pkg/logql/evaluator.go LabelReplaceEvaluator.Next over VectorAggEvaluator, vector()' + proxy: '400 on query; 400 (topk, sort) or empty streams (vector) on query_range' + planned_test: 'TestCompat_LabelReplaceAroundRankingAndVector (test/e2e-compat, group semantics)' + clusters: [] diff --git a/conformance/registry/cases/semantics/label-replace-duplicates-before-ranking.yaml b/conformance/registry/cases/semantics/label-replace-duplicates-before-ranking.yaml new file mode 100644 index 000000000..24131593d --- /dev/null +++ b/conformance/registry/cases/semantics/label-replace-duplicates-before-ranking.yaml @@ -0,0 +1,42 @@ +id: semantics/label-replace-duplicates-before-ranking +title: topk, bottomk and stddev over label sets a label_replace made equal rank merged series on query_range (open) +proves: + - loki_api_v1_query_range +consumer: explore +why: | + When a label_replace makes two series share a label set, Loki keeps both + samples at each step until the query ends: the vector aggregation above it + (topk, bottomk, stddev, stdvar) ranks or folds every sample, and only + JoinSampleVector files the survivors under one series (pkg/logql/evaluator.go + VectorAggEvaluator.Next, pkg/logql/engine.go vectorsToSeries). The proxy + transforms the series VictoriaLogs answered and merges the duplicates into one + series, with the points of each step in turn, before topk, bottomk or stddev + see them, so a step holds two points under one series and the ranking works on + that merged series (internal/proxy handleRangeMetricPostAggregation, + transformSeriesLabels). On an instant query the samples stay apart, so the + instant answer ranks as Loki does. +request: + method: GET + path: /loki/api/v1/query_range + params: + query: 'topk(1, label_replace(sum by (app) (count_over_time({env="production"}[5m])), "app", "all", "app", ".+"))' + start: 'now-1h' + end: 'now' + step: '60' + headers: + X-Scope-OrgID: "0" +compare: + mode: observed + fields: [series, values] + expect: | + One series per step holding the larger of the two samples, as Loki ranks + them before it files them. +gap: + status: open + impact: explore-visible + effort: S + area: 'internal/proxy metric_agg.go (handleRangeMetricPostAggregation) after transformSeriesLabels' + loki: 'pkg/logql/evaluator.go VectorAggEvaluator.Next; pkg/logql/engine.go:440-470 vectorsToSeries' + proxy: 'duplicates merged into one series before the ranking' + planned_test: 'TestCompat_LabelReplaceDuplicatesBeforeRanking (test/e2e-compat, group semantics)' + clusters: [] diff --git a/conformance/registry/cases/semantics/label-replace-new-label.yaml b/conformance/registry/cases/semantics/label-replace-new-label.yaml index e8a04867c..297a84d0b 100644 --- a/conformance/registry/cases/semantics/label-replace-new-label.yaml +++ b/conformance/registry/cases/semantics/label-replace-new-label.yaml @@ -1,20 +1,83 @@ id: semantics/label-replace-new-label -title: label_replace does not add a new destination label (open) +title: label_replace adds its destination label to every series, instant and range (fixed) proves: + - loki_api_v1_query - loki_api_v1_query_range consumer: explore why: | `label_replace(rate({app="api-gateway"}[5m]), "app_short", "$1", "app", "(.*)-.*")` adds `app_short` to every series on Loki; the proxy's series - lack the new label (also through nested label_replace). + lacked the new label (also through nested label_replace), in the instant + answers of the parity run (cluster a9d4f85c, endpoint query). The + query_range form of the row's query already carried the label on the base + commit (checked against a simulated VictoriaLogs); the instant form did not, + so the request below is the instant one. + + Loki v3.7.7: pkg/logql/syntax/ast.go mustNewLabelReplaceExpr compiles the + regexp as `^(?:regex)$` (an invalid one is the parse error "invalid regex in + label_replace: ..."); pkg/logql/evaluator.go LabelReplaceEvaluator.Next reads + the source label (empty when absent), leaves a series the regexp does not + match as it is, and otherwise expands the replacement with + Regexp.ExpandString ($1, ${1}, $name, ${name}, $$), deleting the destination + and setting it only when the expansion is not empty. It validates no label + name and raises no error for two series that end with one label set; pkg/ + logql/engine.go JoinSampleVector then keeps both samples of an instant vector + and files the points of a range query's duplicates under one series, those of + each step in turn; both then sort by label set (sort.Sort(result) for a + range, sort.Slice with labels.Compare for an instant vector unless the query + sorts by value). + + Causes, reproduced against a simulated VictoriaLogs + (internal/proxy/label_replace_semantics_test.go): + - The proxy applied the transform to a range matrix only (applyLabelReplace + decoded `resultType: matrix` only), so an instant vector was returned + untouched. It also re-encoded only the series, dropping the rest + of the answer, and it expanded the replacement with ReplaceAllString. + - The translator took the string arguments of label_replace raw: `"\\."` was + the regexp `\\.` (a backslash and any character), never matching, and an + escaped quote ended the argument. They are now unquoted as the LogQL lexer + does (double-quoted: escapes resolved; backtick: raw). + - topk, bottomk, sort and stddev over a label_replace sent the translator's + marker to VictoriaLogs as part of the LogsQL query; the transform now runs on + what VictoriaLogs answered, before the ranking. + - A label_replace around a binary expression on query_range sent the binary + marker to VictoriaLogs (the instant path was fixed by the instant-result-type + gap); the range path now routes by the expression under the transform too. + + - `label_replace(sum without (level) (...))` lost its transform and kept the + excluded label: the without marker's label list swallowed the label_replace + marker that follows it. + - A call without Loki's four string arguments, or with an invalid escape in a + string, was taken for a log query (200 with no streams on query_range, a + wrong 400 on query); it is now Loki's 400 parse error (the position and + wording of the message are Loki's: checked against its parser). + + Also fixed in the same area (found by the e2e against Loki): + - An instant `label_replace(count_over_time({env="production"}[5m]), ...)` (any + bare range function) was one unlabeled total: the grouping by stream was added + only for a query that itself starts with the function, so the wrapper turned + sixteen series into `{}`. + - A `label_replace` around a metric over `| json` skipped the evaluator that owns + JSON parser errors (semantics/pipeline-error-text-parity): a line that is not + JSON no longer failed the query, and `| json | __error__=""` kept every stream + instead of the ones that parse. It is now answered by that evaluator and the + transform is applied to its answer. The cost is the metric's own: the same + query without the wrapper reads the rows too. + + Not changed (separate shapes, registered apart): + semantics/label-replace-aggregation-over-transform (an aggregation over a + label_replace), semantics/label-replace-around-ranking-and-vector (a + label_replace around topk, sort or vector()), and + semantics/label-replace-duplicates-before-ranking (topk, bottomk and stddev + merge the duplicate label sets a label_replace produced before ranking, where + Loki ranks the samples first). `label_join` is accepted by the proxy but is not + part of Loki 3.7.7's grammar (its lexer has only `label_replace`; Loki answers + `parse error ... unexpected IDENTIFIER`); that is handled by #748, not here. request: method: GET - path: /loki/api/v1/query_range + path: /loki/api/v1/query params: query: 'label_replace(sum by (app) (rate({app="api-gateway"}[5m])), "app_short", "$1", "app", "(.*)-.*")' - start: 'now-1h' - end: 'now' - step: '60' headers: X-Scope-OrgID: "0" X-Loki-Response-Encoding-Flags: categorize-labels @@ -24,12 +87,15 @@ compare: expect: | app_short on every series. gap: - status: open + status: fixed + fixed_by: "#744" impact: explore-visible + visual: none + visual_reason: 'the visible difference is on instant queries; the visual harness captures Explore range panels (queryType range), where label_replace already matched, so the fix is proven by the instant tests and the e2e against Loki' effort: S - area: 'internal/proxy label_replace post-processing' - loki: 'pkg/logql/evaluator.go:387, :1253 label_replace evaluator' - proxy: 'destination label missing' - planned_test: 'TestCompat_LabelReplaceNewLabel (test/e2e-compat, group semantics)' + area: 'internal/proxy vector_matching.go (transformSeriesLabels, applyLabelReplace, applyLabelJoin), metric_agg.go (post-aggregation markers), internal/translator parseUnquotedStringArgs' + loki: 'pkg/logql/syntax/ast.go mustNewLabelReplaceExpr; pkg/logql/evaluator.go:1249-1315 LabelReplaceEvaluator; pkg/logql/engine.go:530-600 JoinSampleVector, vectorsToSeries' + proxy: 'the transform runs on the series VictoriaLogs aggregated, matrix and vector alike' + planned_test: 'TestCompat_LabelReplaceNewLabel (test/e2e-compat, group semantics); TestLabelReplaceEdgeTable, TestLabelReplaceOnInstantAndRange, TestLabelReplaceDifferential, TestLabelReplaceUnderPostAggregation, TestLabelReplaceInvalidRegexp, TestLabelReplaceOverBareRangeFunctionKeepsOneSeriesPerStream, TestLabelReplaceOverJSONMetricUsesTheOrderedJSONEvaluator (internal/proxy); TestLabelReplaceStringArgumentsAreUnquoted (internal/translator)' clusters: - a9d4f85c diff --git a/conformance/registry/generated/perf-evidence.json b/conformance/registry/generated/perf-evidence.json index bde2527bf..1450fd46c 100644 --- a/conformance/registry/generated/perf-evidence.json +++ b/conformance/registry/generated/perf-evidence.json @@ -5936,6 +5936,63 @@ "valid": true, "verdict_vs_baseline": "same", "warm_p50": 0.016 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.005, + "date": "2026-10-12", + "label": "labelreplace-1h", + "loki_cache_served": false, + "loki_cold": 0.099, + "loki_warm_p50": 0.109, + "parity_vs_loki": "same", + "range": "instant", + "set": "label-replace", + "shape": "L1 label_replace over sum by rate, instant", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.005 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.006, + "date": "2026-10-12", + "label": "labelreplace-1h", + "loki_cache_served": false, + "loki_cold": 0.012, + "loki_warm_p50": 0.013, + "parity_vs_loki": "same", + "range": "instant", + "set": "label-replace", + "shape": "L4 label_replace over bare count_over_time, instant", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.007 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.057, + "date": "2026-10-12", + "label": "labelreplace-1h", + "loki_cache_served": false, + "loki_cold": 0.015, + "loki_warm_p50": 0.014, + "parity_vs_loki": "same", + "range": "instant", + "set": "label-replace", + "shape": "L5 label_replace over json sum by, instant", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.05 } ], "loki_api_v1_query_range": [ @@ -12892,6 +12949,63 @@ "valid": true, "verdict_vs_baseline": "same", "warm_p50": 0.008 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.035, + "date": "2026-10-12", + "label": "labelreplace-1h", + "loki_cache_served": false, + "loki_cold": 0.82, + "loki_warm_p50": 0.166, + "parity_vs_loki": "same", + "range": "1h", + "set": "label-replace", + "shape": "L2 label_replace over sum by rate", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.035 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.049, + "date": "2026-10-12", + "label": "labelreplace-1h", + "loki_cache_served": false, + "loki_cold": 0.826, + "loki_warm_p50": 0.07, + "parity_vs_loki": "differs", + "range": "1h", + "set": "label-replace", + "shape": "L3 topk over label_replace over sum by count", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.048 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.536, + "date": "2026-10-12", + "label": "labelreplace-1h", + "loki_cache_served": false, + "loki_cold": 0.112, + "loki_warm_p50": 0.015, + "parity_vs_loki": "same", + "range": "1h", + "set": "label-replace", + "shape": "L6 label_replace over json sum by", + "slower_than_loki_cold": true, + "status": "200", + "valid": true, + "verdict_vs_baseline": "slower", + "warm_p50": 0.554 } ], "metric-series-identity": [ @@ -21532,6 +21646,122 @@ "warm_p50": 0.001 } ], + "semantics/label-replace-new-label": [ + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.005, + "date": "2026-10-12", + "label": "labelreplace-1h", + "loki_cache_served": false, + "loki_cold": 0.099, + "loki_warm_p50": 0.109, + "parity_vs_loki": "same", + "range": "instant", + "set": "label-replace", + "shape": "L1 label_replace over sum by rate, instant", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.005 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.035, + "date": "2026-10-12", + "label": "labelreplace-1h", + "loki_cache_served": false, + "loki_cold": 0.82, + "loki_warm_p50": 0.166, + "parity_vs_loki": "same", + "range": "1h", + "set": "label-replace", + "shape": "L2 label_replace over sum by rate", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.035 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.049, + "date": "2026-10-12", + "label": "labelreplace-1h", + "loki_cache_served": false, + "loki_cold": 0.826, + "loki_warm_p50": 0.07, + "parity_vs_loki": "differs", + "range": "1h", + "set": "label-replace", + "shape": "L3 topk over label_replace over sum by count", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.048 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.006, + "date": "2026-10-12", + "label": "labelreplace-1h", + "loki_cache_served": false, + "loki_cold": 0.012, + "loki_warm_p50": 0.013, + "parity_vs_loki": "same", + "range": "instant", + "set": "label-replace", + "shape": "L4 label_replace over bare count_over_time, instant", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.007 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.057, + "date": "2026-10-12", + "label": "labelreplace-1h", + "loki_cache_served": false, + "loki_cold": 0.015, + "loki_warm_p50": 0.014, + "parity_vs_loki": "same", + "range": "instant", + "set": "label-replace", + "shape": "L5 label_replace over json sum by, instant", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.05 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.536, + "date": "2026-10-12", + "label": "labelreplace-1h", + "loki_cache_served": false, + "loki_cold": 0.112, + "loki_warm_p50": 0.015, + "parity_vs_loki": "same", + "range": "1h", + "set": "label-replace", + "shape": "L6 label_replace over json sum by", + "slower_than_loki_cold": true, + "status": "200", + "valid": true, + "verdict_vs_baseline": "slower", + "warm_p50": 0.554 + } + ], "semantics/labels-end-inclusive-like-loki": [ { "baseline": "main", @@ -23708,6 +23938,46 @@ "warm_p50": 0.008 } ], + "semantics/pipeline-error-text-parity": [ + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.057, + "date": "2026-10-12", + "label": "labelreplace-1h", + "loki_cache_served": false, + "loki_cold": 0.015, + "loki_warm_p50": 0.014, + "parity_vs_loki": "same", + "range": "instant", + "set": "label-replace", + "shape": "L5 label_replace over json sum by, instant", + "slower_than_loki_cold": false, + "status": "200", + "valid": true, + "verdict_vs_baseline": "same", + "warm_p50": 0.05 + }, + { + "baseline": "base", + "baseline_status": "200", + "cold": 0.536, + "date": "2026-10-12", + "label": "labelreplace-1h", + "loki_cache_served": false, + "loki_cold": 0.112, + "loki_warm_p50": 0.015, + "parity_vs_loki": "same", + "range": "1h", + "set": "label-replace", + "shape": "L6 label_replace over json sum by", + "slower_than_loki_cold": true, + "status": "200", + "valid": true, + "verdict_vs_baseline": "slower", + "warm_p50": 0.554 + } + ], "semantics/plain-name-after-parser-reads-stream-label": [ { "baseline": "main", diff --git a/conformance/registry/generated/wiring.json b/conformance/registry/generated/wiring.json index 911293b5b..72e9f172c 100644 --- a/conformance/registry/generated/wiring.json +++ b/conformance/registry/generated/wiring.json @@ -182,6 +182,8 @@ ], "loki_api_v1_query": [ "internal/proxy/instant_result_type_test.go", + "internal/proxy/label_replace_routing_test.go", + "internal/proxy/label_replace_semantics_test.go", "internal/proxy/parenthesized_range_test.go", "internal/proxy/series_limit_routes_test.go", "internal/proxy/tenant_query_limits_test.go", @@ -189,6 +191,7 @@ "internal/proxy/unwrap_conversion_error_test.go", "test/e2e-compat/extracted_suffix_collision_test.go", "test/e2e-compat/instant_result_type_test.go", + "test/e2e-compat/label_replace_new_label_test.go", "test/e2e-compat/parenthesized_range_or_filters_test.go", "test/e2e-compat/parser_capture_identity_test.go", "test/e2e-compat/pattern_line_filter_test.go", @@ -202,6 +205,8 @@ "internal/proxy/drilldown_breakdown_exact_test.go", "internal/proxy/drilldown_breakdown_matrix_test.go", "internal/proxy/extracted_suffix_collision_test.go", + "internal/proxy/label_replace_routing_test.go", + "internal/proxy/label_replace_semantics_test.go", "internal/proxy/line_fields_without_parser_test.go", "internal/proxy/metadata_query_validation_test.go", "internal/proxy/parenthesized_range_test.go", @@ -216,6 +221,7 @@ "test/e2e-compat/drilldown_field_breakdown_test.go", "test/e2e-compat/extracted_suffix_collision_test.go", "test/e2e-compat/heavy_query_protection_test.go", + "test/e2e-compat/label_replace_new_label_test.go", "test/e2e-compat/line_fields_parity_test.go", "test/e2e-compat/parenthesized_range_or_filters_test.go", "test/e2e-compat/pattern_line_filter_test.go", @@ -423,6 +429,14 @@ "internal/proxy/label_values_cap_inventory_test.go", "internal/proxy/metadata_inventory_test.go" ], + "semantics/label-replace-duplicates-before-ranking": [ + "test/e2e-compat/label_replace_new_label_test.go" + ], + "semantics/label-replace-new-label": [ + "internal/proxy/label_replace_routing_test.go", + "internal/proxy/label_replace_semantics_test.go", + "test/e2e-compat/label_replace_new_label_test.go" + ], "semantics/labels-end-inclusive-like-loki": [ "internal/proxy/labels_inclusive_end_test.go", "test/e2e-compat/labels_zero_width_test.go" @@ -462,6 +476,7 @@ "test/e2e-compat/pattern_line_filter_test.go" ], "semantics/pipeline-error-text-parity": [ + "internal/proxy/label_replace_routing_test.go", "internal/proxy/ordered_json_filter_pushdown_test.go", "internal/proxy/ordered_json_metric_test.go" ], @@ -554,7 +569,7 @@ "test/e2e-compat/unwrap_ungrouped_sum_test.go" ] }, - "registry_items": 300, + "registry_items": 303, "routes_without_registry_item": [], "test_to_items": { "cmd/proxy/main_test.go": [ @@ -659,6 +674,17 @@ "loki_api_v1_series", "semantics/metadata-answers-include-last-24h-like-loki" ], + "internal/proxy/label_replace_routing_test.go": [ + "loki_api_v1_query", + "loki_api_v1_query_range", + "semantics/label-replace-new-label", + "semantics/pipeline-error-text-parity" + ], + "internal/proxy/label_replace_semantics_test.go": [ + "loki_api_v1_query", + "loki_api_v1_query_range", + "semantics/label-replace-new-label" + ], "internal/proxy/label_surface_test.go": [ "limits/label-values-not-truncated-by-default", "loki_api_v1_label_name_values", @@ -982,6 +1008,12 @@ "test/e2e-compat/label_freshness_e2e_test.go": [ "semantics/metadata-answers-include-last-24h-like-loki" ], + "test/e2e-compat/label_replace_new_label_test.go": [ + "loki_api_v1_query", + "loki_api_v1_query_range", + "semantics/label-replace-duplicates-before-ranking", + "semantics/label-replace-new-label" + ], "test/e2e-compat/labels_zero_width_test.go": [ "semantics/labels-end-inclusive-like-loki" ], @@ -1096,7 +1128,7 @@ "profiles/structured-metadata-keys-per-profile" ] }, - "tests_with_markers": 101, + "tests_with_markers": 104, "unknown_ids": {}, "unwired_items": [ "api_prom_label", @@ -1174,7 +1206,8 @@ "semantics/first-last-over-time-translation", "semantics/ip-filter-precision", "semantics/label-filter-expression-syntax", - "semantics/label-replace-new-label", + "semantics/label-replace-aggregation-over-transform", + "semantics/label-replace-around-ranking-and-vector", "semantics/log-entries-differ-same-count", "semantics/logfmt-existence-filter-ungrouped-sum", "semantics/non-sum-vector-aggregation-unreduced", @@ -1203,5 +1236,5 @@ "versions/capabilities-floor-vs-matrix", "versions/rejected-stats-query-wording-vl-1.47" ], - "wired_items": 108 + "wired_items": 110 } diff --git a/conformance/reports/gaps.md b/conformance/reports/gaps.md index 3efcda80f..4cfddf93c 100644 --- a/conformance/reports/gaps.md +++ b/conformance/reports/gaps.md @@ -137,6 +137,8 @@ cold request (bench/ab runs; details in [performance.md](performance.md)). | `parser-json` | 2 | 0.49s (E field breakdown pipeline (explore), 3h) | 0.03s | | `loki-compatible-profile` | 3 | 0.52s (E7 json count by level_extracted, 1h) | 0.09s | | `profiles/structured-metadata-label-collision` | 1 | 0.52s (E7 json count by level_extracted, 1h) | 0.09s | +| `semantics/label-replace-new-label` | 1 | 0.54s (L6 label_replace over json sum by, 1h) | 0.11s | +| `semantics/pipeline-error-text-parity` | 1 | 0.54s (L6 label_replace over json sum by, 1h) | 0.11s | | `semantics/json-filter-pushdown-translated-label` | 1 | 0.37s (A explore volume, 2 filters, drop, 3h) | 0.00s | | `semantics/json-filter-pushdown-underscore-label` | 3 | 0.37s (A explore volume, 2 filters, drop, 3h) | 0.00s | | `semantics/json-label-spelling-probe` | 2 | 0.30s (F field breakdown service_version (explore), 3h) | 0.03s | diff --git a/conformance/reports/parity-gaps.md b/conformance/reports/parity-gaps.md index 8ca05c2a6..c220b3bbc 100644 --- a/conformance/reports/parity-gaps.md +++ b/conformance/reports/parity-gaps.md @@ -34,68 +34,70 @@ Last run: 2026-10-05T20:16:15Z on `fcaccfcc`, window [1791220800, 1791224400]. | 13 | `semantics/parse-error-labels-on-log-lines` — A line a parser rejects carries __error__ and __error_details__ in Loki's log answer (open) | open | explore-visible | 10 | 2 | M | internal/proxy log-row assembly (stage-field exposure) + translator parse-error predicate | pkg/logql/log/parser.go:97 addErrLabel(errJSON, err) -> :777-787 SetErr/SetErrorDetails; pkg/logql/log/error.go:5 errJSON | TestCompat_ParseErrorLabelsOnLogLines (test/e2e-compat, group semantics) | | 14 | `semantics/absent-over-time-semantics` — absent_over_time answers when data exists and stays empty when it is absent, after a parser (open) | open | explore-visible | 8 | 6 | S | internal/translator absent_over_time (parser pipelines) | pkg/logql/evaluator.go:647-664, :1320 absentLabels | TestCompat_AbsentOverTimeWithParser (test/e2e-compat, group semantics) | | 15 | `semantics/stddev-stdvar-grouped-values` — stddev and stdvar by a label compute different values (open) | open | explore-visible | 7 | 5 | S | internal/translator stddev/stdvar outer aggregation (by-group fold) | pkg/logql/evaluator.go VectorAggEvaluator stddev/stdvar | TestCompat_StddevStdvarBy (test/e2e-compat, group semantics) | -| 16 | `semantics/label-replace-new-label` — label_replace does not add a new destination label (open) | open | explore-visible | 5 | 1 | S | internal/proxy label_replace post-processing | pkg/logql/evaluator.go:387, :1253 label_replace evaluator | TestCompat_LabelReplaceNewLabel (test/e2e-compat, group semantics) | -| 17 | `quality/label-values-chunk-window` — Loki lists label values of streams whose chunks overlap the window; the proxy only of streams with lines in it (open) | open | explore-visible | 2 | 2 | M | internal/proxy label values window (chunk-overlap semantics) | pkg/querier/querier.go Label (index lookups by chunk overlap) | TestCompat_LabelValuesWindowEdge (test/e2e-compat, group core) | -| 18 | `semantics/aggregation-over-binary-dropped` — An aggregation over a binary expression is translated to its first operand (open) | open | explore-visible | 2 | 1 | M | internal/translator extractOuterAggregation and internal/proxy handleQuery routing of an aggregation over a binary expression | pkg/logql/evaluator.go VectorAggEvaluator over a binary evaluator | TestCompat_AggregationOverBinary (test/e2e-compat, group semantics) | -| 19 | `semantics/log-entries-differ-same-count` — A log query returns the same number of lines with different entries (open, cause not isolated) | open | explore-visible | 1 | 1 | S | to be isolated (log query paging / numeric label filters) | pkg/logql/log/label_filter.go numeric filters; pkg/iter entry ordering | TestCompat_NumericLabelFilterLines (test/e2e-compat, group semantics) | -| 20 | `profiles/label-filter-regexp-anchoring` — A label-filter regexp after a parser is not anchored the way Loki's is (open) | open | explore-visible | — | — | S | internal/translator label-filter regexp emission (anchor ^(?:re)$); external PR #572 open | pkg/logql/log/label_filter.go (regexp matcher on the whole value) | TestCompat_LabelFilterRegexpAnchoring (test/e2e-compat, group semantics) | -| 21 | `profiles/parsed-key-colliding-with-structured-metadata` — A parsed key named like structured metadata loses the metadata value in VictoriaLogs (open) | open | explore-visible | — | — | L | internal/translator json / logfmt unpack (original stored value) and internal/proxy row classification | pkg/logql/log/parser.go:152 HasInCategory(key, StructuredMetadataLabel) then duplicateSuffix | TestCompat_ParsedKeyAgainstStructuredMetadata (test/e2e-compat, group otel-edge) | -| 22 | `quality/label-cache-aged-labels-kept` — Labels that age out of the window are never dropped from the merged label cache (open) | open | explore-visible | — | — | S | internal/proxy/label_metadata.go:1259 mergeLabelsIntoCache | pkg/querier/querier.go Label (window-bounded) | TestE2E_LabelsAgeOutOfWindow (test/e2e-compat, group core) | -| 23 | `quality/labels-backfill-into-cached-bucket` — Rows backfilled into a past bucket the label inventory already cached with data appear in /labels only after its revalidation delay (open) | open | explore-visible | — | — | M | internal/proxy/metadata_inventory.go (bucket revalidation delay) | pkg/querier/querier.go Label (index read per request) | TestCompat_LabelsBackfillIntoCachedBucket (test/e2e-compat, group core) | -| 24 | `semantics/drop-keep-plain-name-after-parser` — A log query's drop, keep or label_format source of a stream label's plain name after a parser acts on the stored field; Loki acts on the ... | open | explore-visible | — | — | S | internal/proxy stage-field exposure (drop and keep of a name a parser also reads) | pkg/logql/log/labels.go LabelsBuilder.Del / getWithCategory; pkg/logql/log/drop_labels.go, keep_labels.go | TestCompat_DropKeepPlainNameAfterParser (test/e2e-compat, group core) | -| 25 | `semantics/extracted-ungrouped-metric-raw-rows` — An ungrouped range metric over a `\| json` pipeline filtered on name_extracted still reads raw rows and fails at the row limit (open) | open | explore-visible | — | — | L | internal/proxy bare-parser metric evaluator (fetchBareParserMetricSeries) and the ordered JSON pushdown (without grouping) | pkg/logql range aggregation keeps every label of the sample | TestCompat_ExtractedUngroupedMetricAtSixHours (test/e2e-compat, group core) | -| 26 | `semantics/repeated-label-filter-replaced-across-label-format` — A second filter on the same label replaces the first even with a label_format between them (open) | open | explore-visible | — | — | S | internal/translator labelFilterLatest (latest filter per field replaces the earlier one in place) | pkg/logql/log/pipeline.go (stages run in order); label_filter.go | TestCompat_RepeatedLabelFilterAcrossLabelFormat (test/e2e-compat, group core) | -| 27 | `semantics/two-parsers-keep-one` — A query with `\| json` and `\| logfmt` runs the one parser the data answers to (open) | open | explore-visible | — | — | M | internal/proxy/query_translation.go preferWorkingParser (probe-based removal of a parser stage) | pkg/logql/log/pipeline.go: every stage runs on every line; parser.go skips a key ParserHint.Extracted | TestCompat_TwoParsersOnMixedFormats (test/e2e-compat, group semantics) | -| 28 | `semantics/unwrap-duration-seconds` — '`unwrap duration_seconds(label)` answers no data where Loki converts it like duration() (open)' | open | explore-visible | — | — | S | internal/translator extractUnwrapField and unwrapGateField (duration_seconds conversion) | pkg/logql/syntax/extractor.go:63-71 | TestCompat_UnwrapDurationSeconds (test/e2e-compat, group core) | -| 29 | `semantics/unwrap-first-last-under-aggregation` — first_over_time and last_over_time with unwrap under sum fail with a VictoriaLogs 422 (open) | open | explore-visible | — | — | M | internal/translator first_over_time / last_over_time under a vector aggregation | pkg/logql/range_vector.go first/last | an e2e over sum(first_over_time\|last_over_time(... \| unwrap ...)) | -| 30 | `semantics/unwrap-over-time-range-windows` — sum, avg, max and min over time with unwrap answer the wrong window on query_range (open) | open | explore-visible | — | — | M | internal/proxy range_metric_compat.go shouldUseManualRangeMetricCompat (unwrap sum/avg/min/max) and metric_binary.go statsRateRangeEquals... | pkg/logql/range_vector.go:102-189 batchRangeVectorIterator (window (t-range, t]); range_vector.go:371-420 sumOverTime, avgOverTime, maxOv... | TestCompat_UnwrapOverTimeRangeWindows (test/e2e-compat, group core) over sum/avg/max/min x tumbling/sliding x logfmt/json | -| 31 | `semantics/unwrap-per-series-identity-raw-rows` — A function taken per series (avg, max, min over time, rate_counter) under sum or sum by uses the stream as the series (open) | open | explore-visible | — | — | L | internal/translator unwrapInnerGrouping (sum by over a per-series function) and the raw evaluators series key (collectRangeMetricSamples) | pkg/logql/evaluator.go:435-529 VectorAggEvaluator.Next; pkg/logql/log/metrics_extraction.go:150-178 labelSampleExtractor (series = stream... | TestCompat_UnwrapPerSeriesIdentity (test/e2e-compat, group core) over sum and sum by x avg/max/min/rate_counter, logfmt and json | -| 32 | `semantics/unwrap-quantile-response-limit` — quantile_over_time over a JSON unwrap fails at the manual metric response limit from 6 hours (open) | open | explore-visible | — | — | L | internal/proxy raw-row quantile evaluator (manual metric response cap) | pkg/logql/range_vector.go quantile_over_time | bench/ab set unwrap-conversion-error E2 at 6h/24h/7d | -| 33 | `semantics/unwrap-range-last-point` — A grouped unwrap range metric misses the sample at `end` (open) | open | explore-visible | — | — | S | internal/proxy range-metric unwrap path (end bucket) | pkg/logql/range_vector.go step iteration start..end inclusive | TestCompat_UnwrapRangeEndSample (test/e2e-compat, group semantics) | -| 34 | `semantics/unwrap-sum-without-collapses` — sum without over an unwrap range function returns a single series (open) | open | explore-visible | — | — | M | internal/translator `without` grouping over unwrap range functions | pkg/logql/evaluator.go newVectorAggEvaluator (without) | an e2e over sum without (...) with unwrap on query_range and query | -| 35 | `drilldown/detected-field-values` — detected_field values differ: none for service_name, other values for numeric and id fields (open) | open | drilldown-visible | 33 | 3 | M | internal/proxy detected_field values (sampling window and stream-label fields) | pkg/querier detected field values (v3.7.7): values of the sampled lines | TestCompat_DetectedFieldValues (test/e2e-compat, group drilldown) | -| 36 | `semantics/detected-fields-extracted-suffix` — detected_fields omits Loki's level_extracted and version_extracted (open; service fields are owner-kept) | open | drilldown-visible | 15 | 2 | M | internal/proxy/label_handlers.go handleDetectedFields (parsed fields named like a stream label) | pkg/logql/log/parser.go:25 duplicateSuffix; detected fields come from the parsed labels of sampled entries | TestCompat_DetectedFieldsExtractedSuffix (test/e2e-compat, group drilldown) | -| 37 | `profiles/json-number-label-text` — JSON numbers become label values in a different text form ("14.0" vs "14") (open) | open | drilldown-visible | 12 | 6 | M | internal/proxy JSON number rendering of unpacked values (VictoriaLogs unpack_json normalises) | pkg/logql/log/parser.go readValue (raw number bytes) | TestCompat_JSONNumberLabelText (test/e2e-compat, group drilldown) | -| 38 | `severity/detected-level-filter-json-level` — `\| detected_level=` filters select different lines than Loki (JSON-level lines, stream level precedence) (open) | open | drilldown-visible | 12 | 8 | M | internal/translator/detected_level_logsql.go (filter predicate) vs internal/proxy/detected_level.go (response derivation) | pkg/distributor field detection (v3.7.7) stores detected_level as structured metadata; filter reads it | TestCompat_DetectedLevelFilterJSONLevel (test/e2e-compat, group drilldown) | -| 39 | `limits/drilldown-7d-breakdown-response-limit` — Metric queries fail on the 64 MiB buffered-response budget where Loki answers (open) | open | drilldown-visible | 8 | 4 | M | internal/proxy/range_metric_compat.go:767, execution_limits.go:11 (raw-row metric paths buffer rows; push down or stream) | pkg/logql engine series limit (max_query_series) with partial result for Drilldown | TestCompat_MetricBufferedBudget1h + TestE2E_Drilldown7dHighCardinalityBreakdown (test/e2e-compat, groups semantics and drilldown) | -| 40 | `quality/index-stats-streams-differ` — index/stats stream counts differ from Loki in both directions (open) | open | drilldown-visible | 6 | 2 | S | internal/proxy/label_handlers.go handleIndexStats (stream counting per window) | pkg/storage index stats (streams from the index for the window) | bench/parity rerun with the /series cross-check, then TestCompat_IndexStatsStreams (test/e2e-compat, group drilldown) | -| 41 | `drilldown/detected-fields-type-inference` — detected_fields infers a different type for some fields (open) | open | drilldown-visible | — | — | S | internal/proxy detected_fields type inference | pkg/querier detected fields type detection (v3.7.7): int before duration before string | TestCompat_DetectedFieldsTypes (test/e2e-compat, group drilldown) | -| 42 | `profiles/existence-filter-on-sanitized-key` — An existence filter on a sanitized key reads the stored dotted field, not the JSON line key (open) | open | drilldown-visible | — | — | M | internal/translator (label-filter key variants, parsed_key_variants.go); Drilldown single-field fast paths | pkg/logql/log/parser.go:141 sanitizeLabelKey on every extracted key; label filter reads the parsed label | TestCompat_ExistenceFilterSanitizedKey (test/e2e-compat, group semantics) | -| 43 | `quality/detected-metadata-proxy-cache` — detected_labels and detected_fields are served from proxy caches; Loki answers each request fresh (open) | open | drilldown-visible | — | — | S | internal/proxy/cache_keys.go:48,130 (detected_* cache TTL) + recent-tail freshness bypass | pkg/querier detected labels/fields: no results cache in v3.7.7 | TestE2E_DetectedMetadataFreshness (test/e2e-compat, group drilldown) | -| 44 | `quality/stats-volume-freshness` — index/stats and volume answers ending now are cached; Loki does not cache inside max_stats_cache_freshness (10m) (open) | open | drilldown-visible | — | — | S | internal/proxy/volume.go handleVolume, label_handlers.go handleIndexStats (cache freshness) | pkg/validation/limits.go:413-414 max_stats_cache_freshness default 10m | TestE2E_StatsVolumeFreshness (test/e2e-compat, group drilldown) | -| 45 | `semantics/parse-error-text-parity` — Invalid queries fail on both sides, with different error text or status (open) | open | grafana-datasource | 210 | 129 | L | internal/logql parser error messages (Loki's goyacc messages and positions) + VictoriaLogs error mapping | pkg/logql/syntax/parser.go ParseError (line, col) and syntax.y | TestCompat_ParseErrorText corpus (test/e2e-compat, bad_request_status_parity_test.go) | -| 46 | `semantics/proxy-rejects-valid-logql` — The proxy rejects LogQL Loki accepts: comments, negative and compound offsets, filters after a range, logfmt flags, ip() label filters (o... | open | api-only | 79 | 48 | M | internal/logql parser (lexer: comments, signed durations, flags) + translator | pkg/logql/syntax/lex.go:149 comments, :90-91 OpStrict/OpKeepEmpty; syntax.y offsetExpr; pkg/logql/log/ip.go | TestCompat_LogQLSurfaceAccepted (test/e2e-compat, group semantics), one subtest per family | -| 47 | `semantics/proxy-accepts-invalid-logql` — Queries Loki rejects with 400 are answered by the proxy (open) | open | api-only | 18 | 14 | S | internal/logql parser validation | pkg/logql/syntax/ast.go:1446,1579 grouping not allowed; duration parsing | TestCompat_InvalidLogQLRejected (test/e2e-compat, bad_request_status_parity_test.go) | -| 48 | `parsed-identity/regexp-partial-match-series-count` — A partially-matching \| regexp returns about 3x Loki's series count | open | api-only | 2 | 2 | M | internal/proxy regexp capture series identity (non-matching lines) | pkg/logql/log/parser.go RegexpParser (no capture label for a non-matching line) | TestCompat_RegexpPartialMatchSeries (test/e2e-compat, group semantics) | -| 49 | `profiles/json-extraction-list-missing-key` — A json extraction-list key absent from the line is dropped; Loki keeps it in the metric series (open) | open | api-only | 2 | 2 | S | internal/proxy stage-field exposure (extraction lists, metric path) | pkg/logql/log/parser.go:671-720 JSONExpressionParser.Process | TestCompat_JSONExtractionListMissingKey (test/e2e-compat, group semantics) | -| 50 | `semantics/unpack-non-string-values` — `\| unpack` extracts non-string JSON values that Loki ignores (open) | open | api-only | 2 | 2 | S | internal/translator unpack (string values only) | pkg/logql/log/parser.go:789-820 UnpackParser.unpack | TestCompat_UnpackStringValuesOnly (test/e2e-compat, group semantics) | -| 51 | `semantics/extracted-derived-labels-raw-evaluator` — service_name_extracted and detected_level_extracted metrics keep the raw-row evaluator (open) | open | edge | — | — | S | internal/proxy orderedJSONPushdownLabelOK (service_name, detected_level) | pkg/logql/log/parser.go duplicateSuffix; service_name set by discover_service_name at ingest | TestCompat_ExtractedLabelMetricPushdown rows for service_name_extracted (test/e2e-compat, group core) | -| 52 | `semantics/extracted-metric-known-differences` — Two name_extracted differences against Loki remain in the metric pushdown and the raw evaluator (open) | open | edge | — | — | M | internal/proxy raw evaluator parsed-label naming (processWithStreamLabels) and translator.ExtractedLabelPipes | pkg/logql/log/parser.go duplicateSuffix and ParserHint.Extracted (a key an earlier extraction produced is skipped) | TestCompat_ExtractedBothKeysInLineOrder (test/e2e-compat, group core) | -| 53 | `semantics/unwrap-conversion-error-parser-divergence` — Where VictoriaLogs' logfmt parser reads another value than Loki's, an unwrap misses Loki's SampleExtractionErr or drops samples Loki keep... | open | edge | — | — | L | internal/translator logfmt unpacking (unpack_logfmt keeps the last value); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:379-438 LogfmtParser.Process; pkg/logql/log/logfmt/decode.go | TestUnwrapConversionErrorDifferential counts the first case; TestUnwrapLokiLabels pins the decoder; TestCompat_UnwrapConversionErrorParit... | -| 54 | `semantics/unwrap-conversion-error-postfilter-forms` — An unwrap whose post filter keeps only the error, compares a label, or combines filters with and/or answers 200 where Loki may fail with ... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorPostFilters; the translator's post-filter placement | pkg/logql/log/label_filter.go:90-100 BinaryLabelFilter.Process, 306-340 NumericLabelFilter.Process, 418-424 labelValue | TestUnwrapErrorProbePlans and TestUnwrapConversionErrorDifferential (internal/proxy) assert the detection stays off for these forms; flip... | -| 55 | `semantics/unwrap-conversion-error-stored-value-overwritten` — An unwrap over a label stored as structured metadata and also parsed from the line misses Loki's SampleExtractionErr (open) | open | edge | — | — | M | internal/translator parsed-key collision with stored fields (unpack keep_original_fields); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:151,809 duplicateSuffix | TestUnwrapConversionErrorDifferential counts the first case; TestCompat_UnwrapConversionErrorParity asserts the 200 side | -| 56 | `semantics/unwrap-conversion-error-undetected-shapes` — A few query shapes answer 200 where Loki fails with SampleExtractionErr, because no VictoriaLogs query of theirs carries the conversion c... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorProbes (opaque expressions), unwrapCountingQuery (pipes after stats), unwrapLokiLabe... | pkg/logql/evaluator.go:726-733 | TestCompat_UnwrapConversionErrorParity open/* asserts Loki's 400 and skips naming this case (absent_over_time, label_replace); TestUnwrap... | -| 57 | `semantics/unwrap-gate-parsefloat-divergences` — The unwrap sample gate drops a few values Loki's conversions accept or reject (open) | open | edge | — | — | S | internal/translator/unwrap_gate.go (UnwrapNumberPattern, UnwrapDurationPattern, UnwrapBytesPattern) | pkg/logql/log/metrics_extraction.go:316-338 convertFloat, convertDuration, convertBytes | TestUnwrapNumberPatternMatchesParseFloat (internal/translator) pins the known differences | -| 58 | `versions/stats-bucket-label-v1.45` — Stats-bucket metrics put a line on a bucket edge one bucket late on VictoriaLogs before v1.45 (open) | open | edge | — | — | M | stats-bucket relabel (internal/proxy range_metric_compat.go setSlidingStatsRangeParams / slidingStatsBucket; sliding_window_stats.go) | pkg/logql range vector (T-range, T] | TestRangeMetricCompatibilityJSONFilterPushdown on VictoriaLogs v1.40-v1.44 (version matrix); fix by relabelling edge lines or declining t... | -| 59 | `metric-identity/structured-metadata-splits-series` — Loki splits a stream's metric series per structured-metadata set; the proxy keeps the stream (owner-accepted deviation) | owner-kept | explore-visible | 173 | 37 | L | internal/translator series identity (stats by stream); VictoriaLogs cannot group by a row's remaining fields | pkg/logql/log/metrics_extraction.go:102-120 builder.Add(StructuredMetadataLabel) -> GroupedLabels | differential run cluster count (bench/parity) | -| 60 | `parsed-identity/dynamic-key-parsers` — `\| json` and `\| logfmt` metric series keep the stream identity by default (owner-accepted deviation, opt-in flag) | owner-kept | explore-visible | 95 | 29 | L | internal/translator parser series identity; -exact-parser-series-identity | pkg/logql/log/parser.go label extraction into the sample labels | differential run cluster count (bench/parity) | -| 61 | `profiles/label-filter-on-line-field` — A label filter on a line key no earlier stage exposes matches like Loki | owner-kept | explore-visible | — | — | M | internal/proxy label-filter refill (-label-filter-refill-max-pages); metric queries on keys no stage exposes | pkg/logql/log/label_filter.go (filter reads only labels the pipeline set) | TestCompat_LabelFilterOnLineField (exists, profile tests) | -| 62 | `profiles/detected-fields-no-extracted-suffix` — detected_fields lists service / service.name without Loki's _extracted suffix (owner decision) | owner-kept | drilldown-visible | — | — | S | internal/proxy/label_handlers.go handleDetectedFields | pkg/logql/log/parser.go:25 duplicateSuffix | bench/parity diff rule (documented facet, service fields only) | -| 63 | `profiles/json-line-stored-without-line` — A JSON line pushed without _msg is stored without its line (documented) | documented | explore-visible | — | — | L | client push configuration (disable_message_parsing=1); docs/KNOWN_ISSUES.md | pkg/distributor push keeps the line verbatim | cases/profiles/stage-field-exposure with disable_message_parsing pushes | -| 64 | `drilldown/detected-labels-sampled-cardinality` — detected_labels cardinality is sampled from the window's last minutes; Loki counts every stream its ingesters hold (documented) | documented | drilldown-visible | 20 | 2 | M | internal/proxy/patterns.go handleDetectedLabels | pkg/ingester/instance.go LabelsWithValues | bench/parity diff rule (documented facet) | -| 65 | `profiles/volume-detected-level-extension` — index/volume answers targetLabels=detected_level; Loki returns no volume for it (documented proxy extension) | documented | drilldown-visible | 17 | 2 | S | internal/proxy/volume.go | pkg/storage volume index (stream labels only) | differential run row volume by detected_level | -| 66 | `drilldown/patterns-mined-differently` — Patterns come from different miners on each side (documented) | documented | drilldown-visible | 15 | 3 | L | internal/proxy/patterns.go | pkg/pattern (ingester-side mining) | bench/visual history-dependent comparison (exists) | -| 67 | `profiles/volume-bytes-accounting` — Volume values count line bytes; Loki counts chunk bytes with structured metadata (documented) | documented | drilldown-visible | 4 | 3 | L | internal/proxy/volume.go; docs/KNOWN_ISSUES.md | pkg/storage volume index (chunk bytes incl. structured metadata) | bench/parity diff rule (documented facet) | -| 68 | `limits/drilldown-breakdown-keeps-busiest` — Over max_query_series a Drilldown breakdown keeps the busiest series; Loki keeps the first it meets (documented) | documented | drilldown-visible | 1 | 1 | S | internal/proxy series limit ranking | pkg/logql engine series limit | cases/limits/drilldown-partial-with-warning (exists) | -| 69 | `quality/index-stats-bytes-and-chunks` — index/stats bytes and chunks have no VictoriaLogs equivalent (documented) | documented | drilldown-visible | — | — | L | internal/proxy/label_handlers.go handleIndexStats; docs/KNOWN_ISSUES.md | pkg/storage index stats from chunk refs (bytes, chunks) | bench/parity diff rule (documented facet) | -| 70 | `profiles/grafana-stats-error-as-warning` — A failed stats query from Grafana answers 200 with warnings; Loki returns the error (documented) | documented | grafana-datasource | — | — | S | internal/proxy Grafana-sourced stats error handling | pkg/querier/queryrange error propagation (no warnings conversion) | TestCompat_GrafanaStatsErrorStatus (test/e2e-compat, fault injection) | -| 71 | `profiles/metadata-default-lookback` — Label endpoints without start/end default to a different lookback than Loki's 1h (documented) | documented | api-only | — | — | S | internal/config flag default -metadata-default-lookback | pkg/loghttp/params.go defaultSince = 1h | TestCompat_MetadataDefaultLookback (test/e2e-compat, group core) | -| 72 | `profiles/multi-tenant-volume-merge` — Multi-tenant volume returns per-tenant series with __tenant_id__; Loki sums the tenants' volumes (documented) | documented | api-only | — | — | S | internal/proxy/volume.go multi-tenant merge | pkg/querier/queryrange volume merge | TestCompat_MultiTenantVolume (test/e2e-compat, group tail-multitenancy) | -| 73 | `semantics/count-values-unsupported` — count_values() is rejected with 400 where Loki answers (documented) | documented | api-only | — | — | M | internal/translator + proxy-side post-aggregation of a stats matrix | pkg/logql/syntax OpTypeCountValues; vector evaluator | TestCompat_CountValues (test/e2e-compat, group semantics) | -| 74 | `semantics/eager-pipeline-validation` — Invalid pipeline stages fail with 400 regardless of data; Loki answers 200 empty over a range with no data (documented) | documented | edge | — | — | S | internal/translator validation | pkg/logql/log/fmt.go template build at pipeline construction | TestCompat_EagerValidationEmptyRange (test/e2e-compat, group semantics) | -| 75 | `semantics/ip-filter-precision` — ip() filters are approximated with regular expressions (documented) | documented | edge | — | — | M | internal/proxy/postprocess.go:84 ip filtering | pkg/logql/log/ip.go | TestCompat_IPFilterForms (test/e2e-compat, group semantics) | -| 76 | `drilldown/plugin-legend-sync-race` — Logs Drilldown crashes a breakdown page when a query answers before the panel module loads (upstream, not the proxy) | upstream | drilldown-visible | — | — | S | none (grafana-lokiexplore-app) | n/a (client race) | bench/visual capture waits for the panel module (already) | -| 77 | `quality/index-stats-loki-chunk-accounting` — Loki's index/stats counts chunks it holds twice; the proxy's counts equal Loki's own count_over_time (upstream) | upstream | drilldown-visible | — | — | S | none (Loki chunk index accounting) | pkg/storage stats from chunk refs (flushed and retained chunks both counted) | bench/parity cross-check (run.py loki_counts_itself) | +| 16 | `quality/label-values-chunk-window` — Loki lists label values of streams whose chunks overlap the window; the proxy only of streams with lines in it (open) | open | explore-visible | 2 | 2 | M | internal/proxy label values window (chunk-overlap semantics) | pkg/querier/querier.go Label (index lookups by chunk overlap) | TestCompat_LabelValuesWindowEdge (test/e2e-compat, group core) | +| 17 | `semantics/aggregation-over-binary-dropped` — An aggregation over a binary expression is translated to its first operand (open) | open | explore-visible | 2 | 1 | M | internal/translator extractOuterAggregation and internal/proxy handleQuery routing of an aggregation over a binary expression | pkg/logql/evaluator.go VectorAggEvaluator over a binary evaluator | TestCompat_AggregationOverBinary (test/e2e-compat, group semantics) | +| 18 | `semantics/log-entries-differ-same-count` — A log query returns the same number of lines with different entries (open, cause not isolated) | open | explore-visible | 1 | 1 | S | to be isolated (log query paging / numeric label filters) | pkg/logql/log/label_filter.go numeric filters; pkg/iter entry ordering | TestCompat_NumericLabelFilterLines (test/e2e-compat, group semantics) | +| 19 | `profiles/label-filter-regexp-anchoring` — A label-filter regexp after a parser is not anchored the way Loki's is (open) | open | explore-visible | — | — | S | internal/translator label-filter regexp emission (anchor ^(?:re)$); external PR #572 open | pkg/logql/log/label_filter.go (regexp matcher on the whole value) | TestCompat_LabelFilterRegexpAnchoring (test/e2e-compat, group semantics) | +| 20 | `profiles/parsed-key-colliding-with-structured-metadata` — A parsed key named like structured metadata loses the metadata value in VictoriaLogs (open) | open | explore-visible | — | — | L | internal/translator json / logfmt unpack (original stored value) and internal/proxy row classification | pkg/logql/log/parser.go:152 HasInCategory(key, StructuredMetadataLabel) then duplicateSuffix | TestCompat_ParsedKeyAgainstStructuredMetadata (test/e2e-compat, group otel-edge) | +| 21 | `quality/label-cache-aged-labels-kept` — Labels that age out of the window are never dropped from the merged label cache (open) | open | explore-visible | — | — | S | internal/proxy/label_metadata.go:1259 mergeLabelsIntoCache | pkg/querier/querier.go Label (window-bounded) | TestE2E_LabelsAgeOutOfWindow (test/e2e-compat, group core) | +| 22 | `quality/labels-backfill-into-cached-bucket` — Rows backfilled into a past bucket the label inventory already cached with data appear in /labels only after its revalidation delay (open) | open | explore-visible | — | — | M | internal/proxy/metadata_inventory.go (bucket revalidation delay) | pkg/querier/querier.go Label (index read per request) | TestCompat_LabelsBackfillIntoCachedBucket (test/e2e-compat, group core) | +| 23 | `semantics/drop-keep-plain-name-after-parser` — A log query's drop, keep or label_format source of a stream label's plain name after a parser acts on the stored field; Loki acts on the ... | open | explore-visible | — | — | S | internal/proxy stage-field exposure (drop and keep of a name a parser also reads) | pkg/logql/log/labels.go LabelsBuilder.Del / getWithCategory; pkg/logql/log/drop_labels.go, keep_labels.go | TestCompat_DropKeepPlainNameAfterParser (test/e2e-compat, group core) | +| 24 | `semantics/extracted-ungrouped-metric-raw-rows` — An ungrouped range metric over a `\| json` pipeline filtered on name_extracted still reads raw rows and fails at the row limit (open) | open | explore-visible | — | — | L | internal/proxy bare-parser metric evaluator (fetchBareParserMetricSeries) and the ordered JSON pushdown (without grouping) | pkg/logql range aggregation keeps every label of the sample | TestCompat_ExtractedUngroupedMetricAtSixHours (test/e2e-compat, group core) | +| 25 | `semantics/label-replace-aggregation-over-transform` — An aggregation over a label_replace fails on the proxy and answers on Loki (open) | open | explore-visible | — | — | M | internal/proxy proxy.go (handleQuery, handleQueryRange: markers are parsed once, for the outermost expression), vector_matching.go (label... | pkg/logql/evaluator.go VectorAggEvaluator.Next over LabelReplaceEvaluator.Next | TestCompat_AggregationOverLabelReplace (test/e2e-compat, group semantics) | +| 26 | `semantics/label-replace-around-ranking-and-vector` — label_replace around topk, sort or vector() fails on the proxy and answers on Loki (open) | open | explore-visible | — | — | M | internal/proxy proxy.go (handleQuery, handleQueryRange routing of a label_replace whose inner expression is topk, sort or vector) | pkg/logql/evaluator.go LabelReplaceEvaluator.Next over VectorAggEvaluator, vector() | TestCompat_LabelReplaceAroundRankingAndVector (test/e2e-compat, group semantics) | +| 27 | `semantics/label-replace-duplicates-before-ranking` — topk, bottomk and stddev over label sets a label_replace made equal rank merged series on query_range (open) | open | explore-visible | — | — | S | internal/proxy metric_agg.go (handleRangeMetricPostAggregation) after transformSeriesLabels | pkg/logql/evaluator.go VectorAggEvaluator.Next; pkg/logql/engine.go:440-470 vectorsToSeries | TestCompat_LabelReplaceDuplicatesBeforeRanking (test/e2e-compat, group semantics) | +| 28 | `semantics/repeated-label-filter-replaced-across-label-format` — A second filter on the same label replaces the first even with a label_format between them (open) | open | explore-visible | — | — | S | internal/translator labelFilterLatest (latest filter per field replaces the earlier one in place) | pkg/logql/log/pipeline.go (stages run in order); label_filter.go | TestCompat_RepeatedLabelFilterAcrossLabelFormat (test/e2e-compat, group core) | +| 29 | `semantics/two-parsers-keep-one` — A query with `\| json` and `\| logfmt` runs the one parser the data answers to (open) | open | explore-visible | — | — | M | internal/proxy/query_translation.go preferWorkingParser (probe-based removal of a parser stage) | pkg/logql/log/pipeline.go: every stage runs on every line; parser.go skips a key ParserHint.Extracted | TestCompat_TwoParsersOnMixedFormats (test/e2e-compat, group semantics) | +| 30 | `semantics/unwrap-duration-seconds` — '`unwrap duration_seconds(label)` answers no data where Loki converts it like duration() (open)' | open | explore-visible | — | — | S | internal/translator extractUnwrapField and unwrapGateField (duration_seconds conversion) | pkg/logql/syntax/extractor.go:63-71 | TestCompat_UnwrapDurationSeconds (test/e2e-compat, group core) | +| 31 | `semantics/unwrap-first-last-under-aggregation` — first_over_time and last_over_time with unwrap under sum fail with a VictoriaLogs 422 (open) | open | explore-visible | — | — | M | internal/translator first_over_time / last_over_time under a vector aggregation | pkg/logql/range_vector.go first/last | an e2e over sum(first_over_time\|last_over_time(... \| unwrap ...)) | +| 32 | `semantics/unwrap-over-time-range-windows` — sum, avg, max and min over time with unwrap answer the wrong window on query_range (open) | open | explore-visible | — | — | M | internal/proxy range_metric_compat.go shouldUseManualRangeMetricCompat (unwrap sum/avg/min/max) and metric_binary.go statsRateRangeEquals... | pkg/logql/range_vector.go:102-189 batchRangeVectorIterator (window (t-range, t]); range_vector.go:371-420 sumOverTime, avgOverTime, maxOv... | TestCompat_UnwrapOverTimeRangeWindows (test/e2e-compat, group core) over sum/avg/max/min x tumbling/sliding x logfmt/json | +| 33 | `semantics/unwrap-per-series-identity-raw-rows` — A function taken per series (avg, max, min over time, rate_counter) under sum or sum by uses the stream as the series (open) | open | explore-visible | — | — | L | internal/translator unwrapInnerGrouping (sum by over a per-series function) and the raw evaluators series key (collectRangeMetricSamples) | pkg/logql/evaluator.go:435-529 VectorAggEvaluator.Next; pkg/logql/log/metrics_extraction.go:150-178 labelSampleExtractor (series = stream... | TestCompat_UnwrapPerSeriesIdentity (test/e2e-compat, group core) over sum and sum by x avg/max/min/rate_counter, logfmt and json | +| 34 | `semantics/unwrap-quantile-response-limit` — quantile_over_time over a JSON unwrap fails at the manual metric response limit from 6 hours (open) | open | explore-visible | — | — | L | internal/proxy raw-row quantile evaluator (manual metric response cap) | pkg/logql/range_vector.go quantile_over_time | bench/ab set unwrap-conversion-error E2 at 6h/24h/7d | +| 35 | `semantics/unwrap-range-last-point` — A grouped unwrap range metric misses the sample at `end` (open) | open | explore-visible | — | — | S | internal/proxy range-metric unwrap path (end bucket) | pkg/logql/range_vector.go step iteration start..end inclusive | TestCompat_UnwrapRangeEndSample (test/e2e-compat, group semantics) | +| 36 | `semantics/unwrap-sum-without-collapses` — sum without over an unwrap range function returns a single series (open) | open | explore-visible | — | — | M | internal/translator `without` grouping over unwrap range functions | pkg/logql/evaluator.go newVectorAggEvaluator (without) | an e2e over sum without (...) with unwrap on query_range and query | +| 37 | `drilldown/detected-field-values` — detected_field values differ: none for service_name, other values for numeric and id fields (open) | open | drilldown-visible | 33 | 3 | M | internal/proxy detected_field values (sampling window and stream-label fields) | pkg/querier detected field values (v3.7.7): values of the sampled lines | TestCompat_DetectedFieldValues (test/e2e-compat, group drilldown) | +| 38 | `semantics/detected-fields-extracted-suffix` — detected_fields omits Loki's level_extracted and version_extracted (open; service fields are owner-kept) | open | drilldown-visible | 15 | 2 | M | internal/proxy/label_handlers.go handleDetectedFields (parsed fields named like a stream label) | pkg/logql/log/parser.go:25 duplicateSuffix; detected fields come from the parsed labels of sampled entries | TestCompat_DetectedFieldsExtractedSuffix (test/e2e-compat, group drilldown) | +| 39 | `profiles/json-number-label-text` — JSON numbers become label values in a different text form ("14.0" vs "14") (open) | open | drilldown-visible | 12 | 6 | M | internal/proxy JSON number rendering of unpacked values (VictoriaLogs unpack_json normalises) | pkg/logql/log/parser.go readValue (raw number bytes) | TestCompat_JSONNumberLabelText (test/e2e-compat, group drilldown) | +| 40 | `severity/detected-level-filter-json-level` — `\| detected_level=` filters select different lines than Loki (JSON-level lines, stream level precedence) (open) | open | drilldown-visible | 12 | 8 | M | internal/translator/detected_level_logsql.go (filter predicate) vs internal/proxy/detected_level.go (response derivation) | pkg/distributor field detection (v3.7.7) stores detected_level as structured metadata; filter reads it | TestCompat_DetectedLevelFilterJSONLevel (test/e2e-compat, group drilldown) | +| 41 | `limits/drilldown-7d-breakdown-response-limit` — Metric queries fail on the 64 MiB buffered-response budget where Loki answers (open) | open | drilldown-visible | 8 | 4 | M | internal/proxy/range_metric_compat.go:767, execution_limits.go:11 (raw-row metric paths buffer rows; push down or stream) | pkg/logql engine series limit (max_query_series) with partial result for Drilldown | TestCompat_MetricBufferedBudget1h + TestE2E_Drilldown7dHighCardinalityBreakdown (test/e2e-compat, groups semantics and drilldown) | +| 42 | `quality/index-stats-streams-differ` — index/stats stream counts differ from Loki in both directions (open) | open | drilldown-visible | 6 | 2 | S | internal/proxy/label_handlers.go handleIndexStats (stream counting per window) | pkg/storage index stats (streams from the index for the window) | bench/parity rerun with the /series cross-check, then TestCompat_IndexStatsStreams (test/e2e-compat, group drilldown) | +| 43 | `drilldown/detected-fields-type-inference` — detected_fields infers a different type for some fields (open) | open | drilldown-visible | — | — | S | internal/proxy detected_fields type inference | pkg/querier detected fields type detection (v3.7.7): int before duration before string | TestCompat_DetectedFieldsTypes (test/e2e-compat, group drilldown) | +| 44 | `profiles/existence-filter-on-sanitized-key` — An existence filter on a sanitized key reads the stored dotted field, not the JSON line key (open) | open | drilldown-visible | — | — | M | internal/translator (label-filter key variants, parsed_key_variants.go); Drilldown single-field fast paths | pkg/logql/log/parser.go:141 sanitizeLabelKey on every extracted key; label filter reads the parsed label | TestCompat_ExistenceFilterSanitizedKey (test/e2e-compat, group semantics) | +| 45 | `quality/detected-metadata-proxy-cache` — detected_labels and detected_fields are served from proxy caches; Loki answers each request fresh (open) | open | drilldown-visible | — | — | S | internal/proxy/cache_keys.go:48,130 (detected_* cache TTL) + recent-tail freshness bypass | pkg/querier detected labels/fields: no results cache in v3.7.7 | TestE2E_DetectedMetadataFreshness (test/e2e-compat, group drilldown) | +| 46 | `quality/stats-volume-freshness` — index/stats and volume answers ending now are cached; Loki does not cache inside max_stats_cache_freshness (10m) (open) | open | drilldown-visible | — | — | S | internal/proxy/volume.go handleVolume, label_handlers.go handleIndexStats (cache freshness) | pkg/validation/limits.go:413-414 max_stats_cache_freshness default 10m | TestE2E_StatsVolumeFreshness (test/e2e-compat, group drilldown) | +| 47 | `semantics/parse-error-text-parity` — Invalid queries fail on both sides, with different error text or status (open) | open | grafana-datasource | 210 | 129 | L | internal/logql parser error messages (Loki's goyacc messages and positions) + VictoriaLogs error mapping | pkg/logql/syntax/parser.go ParseError (line, col) and syntax.y | TestCompat_ParseErrorText corpus (test/e2e-compat, bad_request_status_parity_test.go) | +| 48 | `semantics/proxy-rejects-valid-logql` — The proxy rejects LogQL Loki accepts: comments, negative and compound offsets, filters after a range, logfmt flags, ip() label filters (o... | open | api-only | 79 | 48 | M | internal/logql parser (lexer: comments, signed durations, flags) + translator | pkg/logql/syntax/lex.go:149 comments, :90-91 OpStrict/OpKeepEmpty; syntax.y offsetExpr; pkg/logql/log/ip.go | TestCompat_LogQLSurfaceAccepted (test/e2e-compat, group semantics), one subtest per family | +| 49 | `semantics/proxy-accepts-invalid-logql` — Queries Loki rejects with 400 are answered by the proxy (open) | open | api-only | 18 | 14 | S | internal/logql parser validation | pkg/logql/syntax/ast.go:1446,1579 grouping not allowed; duration parsing | TestCompat_InvalidLogQLRejected (test/e2e-compat, bad_request_status_parity_test.go) | +| 50 | `parsed-identity/regexp-partial-match-series-count` — A partially-matching \| regexp returns about 3x Loki's series count | open | api-only | 2 | 2 | M | internal/proxy regexp capture series identity (non-matching lines) | pkg/logql/log/parser.go RegexpParser (no capture label for a non-matching line) | TestCompat_RegexpPartialMatchSeries (test/e2e-compat, group semantics) | +| 51 | `profiles/json-extraction-list-missing-key` — A json extraction-list key absent from the line is dropped; Loki keeps it in the metric series (open) | open | api-only | 2 | 2 | S | internal/proxy stage-field exposure (extraction lists, metric path) | pkg/logql/log/parser.go:671-720 JSONExpressionParser.Process | TestCompat_JSONExtractionListMissingKey (test/e2e-compat, group semantics) | +| 52 | `semantics/unpack-non-string-values` — `\| unpack` extracts non-string JSON values that Loki ignores (open) | open | api-only | 2 | 2 | S | internal/translator unpack (string values only) | pkg/logql/log/parser.go:789-820 UnpackParser.unpack | TestCompat_UnpackStringValuesOnly (test/e2e-compat, group semantics) | +| 53 | `semantics/extracted-derived-labels-raw-evaluator` — service_name_extracted and detected_level_extracted metrics keep the raw-row evaluator (open) | open | edge | — | — | S | internal/proxy orderedJSONPushdownLabelOK (service_name, detected_level) | pkg/logql/log/parser.go duplicateSuffix; service_name set by discover_service_name at ingest | TestCompat_ExtractedLabelMetricPushdown rows for service_name_extracted (test/e2e-compat, group core) | +| 54 | `semantics/extracted-metric-known-differences` — Two name_extracted differences against Loki remain in the metric pushdown and the raw evaluator (open) | open | edge | — | — | M | internal/proxy raw evaluator parsed-label naming (processWithStreamLabels) and translator.ExtractedLabelPipes | pkg/logql/log/parser.go duplicateSuffix and ParserHint.Extracted (a key an earlier extraction produced is skipped) | TestCompat_ExtractedBothKeysInLineOrder (test/e2e-compat, group core) | +| 55 | `semantics/unwrap-conversion-error-parser-divergence` — Where VictoriaLogs' logfmt parser reads another value than Loki's, an unwrap misses Loki's SampleExtractionErr or drops samples Loki keep... | open | edge | — | — | L | internal/translator logfmt unpacking (unpack_logfmt keeps the last value); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:379-438 LogfmtParser.Process; pkg/logql/log/logfmt/decode.go | TestUnwrapConversionErrorDifferential counts the first case; TestUnwrapLokiLabels pins the decoder; TestCompat_UnwrapConversionErrorParit... | +| 56 | `semantics/unwrap-conversion-error-postfilter-forms` — An unwrap whose post filter keeps only the error, compares a label, or combines filters with and/or answers 200 where Loki may fail with ... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorPostFilters; the translator's post-filter placement | pkg/logql/log/label_filter.go:90-100 BinaryLabelFilter.Process, 306-340 NumericLabelFilter.Process, 418-424 labelValue | TestUnwrapErrorProbePlans and TestUnwrapConversionErrorDifferential (internal/proxy) assert the detection stays off for these forms; flip... | +| 57 | `semantics/unwrap-conversion-error-stored-value-overwritten` — An unwrap over a label stored as structured metadata and also parsed from the line misses Loki's SampleExtractionErr (open) | open | edge | — | — | M | internal/translator parsed-key collision with stored fields (unpack keep_original_fields); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:151,809 duplicateSuffix | TestUnwrapConversionErrorDifferential counts the first case; TestCompat_UnwrapConversionErrorParity asserts the 200 side | +| 58 | `semantics/unwrap-conversion-error-undetected-shapes` — A few query shapes answer 200 where Loki fails with SampleExtractionErr, because no VictoriaLogs query of theirs carries the conversion c... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorProbes (opaque expressions), unwrapCountingQuery (pipes after stats), unwrapLokiLabe... | pkg/logql/evaluator.go:726-733 | TestCompat_UnwrapConversionErrorParity open/* asserts Loki's 400 and skips naming this case (absent_over_time, label_replace); TestUnwrap... | +| 59 | `semantics/unwrap-gate-parsefloat-divergences` — The unwrap sample gate drops a few values Loki's conversions accept or reject (open) | open | edge | — | — | S | internal/translator/unwrap_gate.go (UnwrapNumberPattern, UnwrapDurationPattern, UnwrapBytesPattern) | pkg/logql/log/metrics_extraction.go:316-338 convertFloat, convertDuration, convertBytes | TestUnwrapNumberPatternMatchesParseFloat (internal/translator) pins the known differences | +| 60 | `versions/stats-bucket-label-v1.45` — Stats-bucket metrics put a line on a bucket edge one bucket late on VictoriaLogs before v1.45 (open) | open | edge | — | — | M | stats-bucket relabel (internal/proxy range_metric_compat.go setSlidingStatsRangeParams / slidingStatsBucket; sliding_window_stats.go) | pkg/logql range vector (T-range, T] | TestRangeMetricCompatibilityJSONFilterPushdown on VictoriaLogs v1.40-v1.44 (version matrix); fix by relabelling edge lines or declining t... | +| 61 | `metric-identity/structured-metadata-splits-series` — Loki splits a stream's metric series per structured-metadata set; the proxy keeps the stream (owner-accepted deviation) | owner-kept | explore-visible | 173 | 37 | L | internal/translator series identity (stats by stream); VictoriaLogs cannot group by a row's remaining fields | pkg/logql/log/metrics_extraction.go:102-120 builder.Add(StructuredMetadataLabel) -> GroupedLabels | differential run cluster count (bench/parity) | +| 62 | `parsed-identity/dynamic-key-parsers` — `\| json` and `\| logfmt` metric series keep the stream identity by default (owner-accepted deviation, opt-in flag) | owner-kept | explore-visible | 95 | 29 | L | internal/translator parser series identity; -exact-parser-series-identity | pkg/logql/log/parser.go label extraction into the sample labels | differential run cluster count (bench/parity) | +| 63 | `profiles/label-filter-on-line-field` — A label filter on a line key no earlier stage exposes matches like Loki | owner-kept | explore-visible | — | — | M | internal/proxy label-filter refill (-label-filter-refill-max-pages); metric queries on keys no stage exposes | pkg/logql/log/label_filter.go (filter reads only labels the pipeline set) | TestCompat_LabelFilterOnLineField (exists, profile tests) | +| 64 | `profiles/detected-fields-no-extracted-suffix` — detected_fields lists service / service.name without Loki's _extracted suffix (owner decision) | owner-kept | drilldown-visible | — | — | S | internal/proxy/label_handlers.go handleDetectedFields | pkg/logql/log/parser.go:25 duplicateSuffix | bench/parity diff rule (documented facet, service fields only) | +| 65 | `profiles/json-line-stored-without-line` — A JSON line pushed without _msg is stored without its line (documented) | documented | explore-visible | — | — | L | client push configuration (disable_message_parsing=1); docs/KNOWN_ISSUES.md | pkg/distributor push keeps the line verbatim | cases/profiles/stage-field-exposure with disable_message_parsing pushes | +| 66 | `drilldown/detected-labels-sampled-cardinality` — detected_labels cardinality is sampled from the window's last minutes; Loki counts every stream its ingesters hold (documented) | documented | drilldown-visible | 20 | 2 | M | internal/proxy/patterns.go handleDetectedLabels | pkg/ingester/instance.go LabelsWithValues | bench/parity diff rule (documented facet) | +| 67 | `profiles/volume-detected-level-extension` — index/volume answers targetLabels=detected_level; Loki returns no volume for it (documented proxy extension) | documented | drilldown-visible | 17 | 2 | S | internal/proxy/volume.go | pkg/storage volume index (stream labels only) | differential run row volume by detected_level | +| 68 | `drilldown/patterns-mined-differently` — Patterns come from different miners on each side (documented) | documented | drilldown-visible | 15 | 3 | L | internal/proxy/patterns.go | pkg/pattern (ingester-side mining) | bench/visual history-dependent comparison (exists) | +| 69 | `profiles/volume-bytes-accounting` — Volume values count line bytes; Loki counts chunk bytes with structured metadata (documented) | documented | drilldown-visible | 4 | 3 | L | internal/proxy/volume.go; docs/KNOWN_ISSUES.md | pkg/storage volume index (chunk bytes incl. structured metadata) | bench/parity diff rule (documented facet) | +| 70 | `limits/drilldown-breakdown-keeps-busiest` — Over max_query_series a Drilldown breakdown keeps the busiest series; Loki keeps the first it meets (documented) | documented | drilldown-visible | 1 | 1 | S | internal/proxy series limit ranking | pkg/logql engine series limit | cases/limits/drilldown-partial-with-warning (exists) | +| 71 | `quality/index-stats-bytes-and-chunks` — index/stats bytes and chunks have no VictoriaLogs equivalent (documented) | documented | drilldown-visible | — | — | L | internal/proxy/label_handlers.go handleIndexStats; docs/KNOWN_ISSUES.md | pkg/storage index stats from chunk refs (bytes, chunks) | bench/parity diff rule (documented facet) | +| 72 | `profiles/grafana-stats-error-as-warning` — A failed stats query from Grafana answers 200 with warnings; Loki returns the error (documented) | documented | grafana-datasource | — | — | S | internal/proxy Grafana-sourced stats error handling | pkg/querier/queryrange error propagation (no warnings conversion) | TestCompat_GrafanaStatsErrorStatus (test/e2e-compat, fault injection) | +| 73 | `profiles/metadata-default-lookback` — Label endpoints without start/end default to a different lookback than Loki's 1h (documented) | documented | api-only | — | — | S | internal/config flag default -metadata-default-lookback | pkg/loghttp/params.go defaultSince = 1h | TestCompat_MetadataDefaultLookback (test/e2e-compat, group core) | +| 74 | `profiles/multi-tenant-volume-merge` — Multi-tenant volume returns per-tenant series with __tenant_id__; Loki sums the tenants' volumes (documented) | documented | api-only | — | — | S | internal/proxy/volume.go multi-tenant merge | pkg/querier/queryrange volume merge | TestCompat_MultiTenantVolume (test/e2e-compat, group tail-multitenancy) | +| 75 | `semantics/count-values-unsupported` — count_values() is rejected with 400 where Loki answers (documented) | documented | api-only | — | — | M | internal/translator + proxy-side post-aggregation of a stats matrix | pkg/logql/syntax OpTypeCountValues; vector evaluator | TestCompat_CountValues (test/e2e-compat, group semantics) | +| 76 | `semantics/eager-pipeline-validation` — Invalid pipeline stages fail with 400 regardless of data; Loki answers 200 empty over a range with no data (documented) | documented | edge | — | — | S | internal/translator validation | pkg/logql/log/fmt.go template build at pipeline construction | TestCompat_EagerValidationEmptyRange (test/e2e-compat, group semantics) | +| 77 | `semantics/ip-filter-precision` — ip() filters are approximated with regular expressions (documented) | documented | edge | — | — | M | internal/proxy/postprocess.go:84 ip filtering | pkg/logql/log/ip.go | TestCompat_IPFilterForms (test/e2e-compat, group semantics) | +| 78 | `drilldown/plugin-legend-sync-race` — Logs Drilldown crashes a breakdown page when a query answers before the panel module loads (upstream, not the proxy) | upstream | drilldown-visible | — | — | S | none (grafana-lokiexplore-app) | n/a (client race) | bench/visual capture waits for the panel module (already) | +| 79 | `quality/index-stats-loki-chunk-accounting` — Loki's index/stats counts chunks it holds twice; the proxy's counts equal Loki's own count_over_time (upstream) | upstream | drilldown-visible | — | — | S | none (Loki chunk index accounting) | pkg/storage stats from chunk refs (flushed and retained chunks both counted) | bench/parity cross-check (run.py loki_counts_itself) | ## Fixed, awaiting the next run @@ -107,6 +109,7 @@ Last run: 2026-10-05T20:16:15Z on `fcaccfcc`, window [1791220800, 1791224400]. | `semantics/parenthesized-log-range` — A parenthesized log expression under a range function evaluates like the plain form | #669 | 41 | 77 | | `semantics/line-filter-or-alternatives` — Line filters with `or` alternatives match like Loki | #669 | 6 | 42 | | `semantics/plain-name-after-parser-reads-stream-label` — A filter or grouping on a name a parser also reads reads the parsed value; Loki reads the stream label (fixed) | #676 | 4 | 6 | +| `semantics/label-replace-new-label` — label_replace adds its destination label to every series, instant and range (fixed) | #744 | 1 | 5 | | `semantics/pattern-line-filter` — An empty pattern line filter (`\|> ""`, `!> ""`) matches only the empty line, like Loki (fixed) | #743 | 2 | 2 | | `semantics/json-extracted-metric-pushdown` — A metric filtered or grouped on a name_extracted label after `\| json` is answered from VictoriaLogs stats (fixed) | #690 | 0 | 0 | | `semantics/labels-end-inclusive-like-loki` — /labels and /label/{name}/values include data at the request's end, so a zero-width request (Grafana "Show context") lists the row's labe... | #703 | 0 | 0 | diff --git a/conformance/reports/performance.md b/conformance/reports/performance.md index 7931fe7e5..f9203fff6 100644 --- a/conformance/reports/performance.md +++ b/conformance/reports/performance.md @@ -7,7 +7,7 @@ Do not edit by hand. Timings are the candidate build's; Loki is compared on cold Loki's first run can still hit split-level caches warmed by earlier requests, so a Loki cold time near zero understates what a truly cold Loki takes. -Runs: pr611-latency-gap-json-filter-pushdown (2026-09-23), pr611-latency-gap-metric-latency-gap (2026-09-23), pr624-final-drilldown-breakdown-exact (2026-09-30), pr624-final-loki-profile-otel-metadata (2026-09-30), backfilled-empty-buckets-metadata-freshness (2026-10-01), backfilled-empty-buckets-metadata-inventory (2026-10-01), loki-parity-pages-loki-profile-otel-metadata (2026-10-01), ordered-json-control-long (2026-10-01), ordered-json-drilldown-breakdown-exact-long (2026-10-01), ordered-json-json-filter-pushdown-1h (2026-10-01), ordered-json-json-filter-pushdown-long (2026-10-01), ordered-json-metric-latency-gap-1h (2026-10-01), ordered-json-metric-latency-gap-long (2026-10-01), stage-field-exposure (2026-10-02), extracted-suffix (2026-10-06), paren-or-filters-parenthesized-range-or-filters (2026-10-06), plain-name-after-parser (2026-10-06), plain-name-control (2026-10-06), plain-name-drilldown-breakdown-exact (2026-10-06), unwrap-validity (2026-10-06), unwrap-validity-24h (2026-10-06), extmetric-extracted (2026-10-07), unwraperr-control (2026-10-08), unwraperr-conversion (2026-10-08), unwrapsum-1h (2026-10-11), unwrapsum-long (2026-10-11), patfilter-1h (2026-10-12), patfilter-control-1h (2026-10-12). +Runs: pr611-latency-gap-json-filter-pushdown (2026-09-23), pr611-latency-gap-metric-latency-gap (2026-09-23), pr624-final-drilldown-breakdown-exact (2026-09-30), pr624-final-loki-profile-otel-metadata (2026-09-30), backfilled-empty-buckets-metadata-freshness (2026-10-01), backfilled-empty-buckets-metadata-inventory (2026-10-01), loki-parity-pages-loki-profile-otel-metadata (2026-10-01), ordered-json-control-long (2026-10-01), ordered-json-drilldown-breakdown-exact-long (2026-10-01), ordered-json-json-filter-pushdown-1h (2026-10-01), ordered-json-json-filter-pushdown-long (2026-10-01), ordered-json-metric-latency-gap-1h (2026-10-01), ordered-json-metric-latency-gap-long (2026-10-01), stage-field-exposure (2026-10-02), extracted-suffix (2026-10-06), paren-or-filters-parenthesized-range-or-filters (2026-10-06), plain-name-after-parser (2026-10-06), plain-name-control (2026-10-06), plain-name-drilldown-breakdown-exact (2026-10-06), unwrap-validity (2026-10-06), unwrap-validity-24h (2026-10-06), extmetric-extracted (2026-10-07), unwraperr-control (2026-10-08), unwraperr-conversion (2026-10-08), unwrapsum-1h (2026-10-11), unwrapsum-long (2026-10-11), labelreplace-1h (2026-10-12), patfilter-1h (2026-10-12), patfilter-control-1h (2026-10-12). ## Slower than Loki on first load @@ -51,6 +51,7 @@ Runs: pr611-latency-gap-json-filter-pushdown (2026-09-23), pr611-latency-gap-met | F field breakdown service_version (explore) | 3h | 0.30s | 0.03s | 0.16s | 0.04s | `parsed-label-series-identity`, `semantics/json-filter-pushdown-underscore-label`, `semantics/json-label-spelling-probe` | | H field breakdown service_version (drilldown) | 3h | 0.28s | 0.03s | 0.15s | 0.02s | `parsed-label-series-identity`, `semantics/json-filter-pushdown-underscore-label`, `semantics/json-label-spelling-probe` | | I grafana volume of B | 3h | 0.14s | 0.02s | 0.19s | 0.02s | `explore-logs-volume-json`, `semantics/json-filter-pushdown-without-error-drop` | +| L6 label_replace over json sum by | 1h | 0.54s | 0.11s | 0.55s | 0.01s | `semantics/label-replace-new-label`, `semantics/pipeline-error-text-parity` | | O2 explore log volume (Grafana form), underscore structured-metadata filter | 3h | 0.18s | 0.02s | 0.11s | 0.02s | `explore-logs-volume-json`, `loki-compatible-profile`, `profiles/structured-metadata-keys-per-profile` | | drilldown landing volume | 24h | 0.10s | 0.04s | 0.10s | 0.02s | `backend-admission-and-heavy-query-queueing` | | drilldown landing volume | 7d | 0.12s | 0.03s | 0.10s | 0.00s | `backend-admission-and-heavy-query-queueing` | @@ -76,7 +77,7 @@ Runs: pr611-latency-gap-json-filter-pushdown (2026-09-23), pr611-latency-gap-met | E9 logfmt latency_ms sum by service, namespace prod | 7d | 9.86s | 0.03s | 11.61s | 0.01s | `parser-error-and-label-collision`, `semantics/unwrap-conversion-error` | | U13 logfmt latency_ms over every service | 1h | 0.10s | 0.00s | 0.09s | 0.00s | `parser-error-and-label-collision`, `semantics/unwrap-sample-validity` | -62 shape×range slower than Loki beyond noise (25% and 50 ms). +63 shape×range slower than Loki beyond noise (25% and 50 ms). ## By registry item @@ -96,8 +97,8 @@ Runs: pr611-latency-gap-json-filter-pushdown (2026-09-23), pr611-latency-gap-met | `loki_api_v1_index_volume` | 1 × 4 ranges | 4 same | worst 0.12s / 0.10s (7d) | 0.03s / 0.00s | differs; 2 slower than Loki cold | | `loki_api_v1_label_name_values` | 6 × 4 ranges | 24 same | worst 0.45s / 0.00s (7d) | 0.02s / 0.01s | differs | | `loki_api_v1_labels` | 6 × 4 ranges | 24 same | worst 0.26s / 0.00s (7d) | 0.04s / 0.02s | differs | -| `loki_api_v1_query` | 9 × 1 ranges | 2 fixed, 7 same | worst 0.25s / 0.13s (instant) | — / — | same; 2 slower than Loki cold | -| `loki_api_v1_query_range` | 114 × 6 ranges | 33 faster, 62 fixed, 269 same, 2 slower | worst 13.13s / 13.44s (7d) | 0.01s / 0.00s | differs; 58 slower than Loki cold | +| `loki_api_v1_query` | 12 × 1 ranges | 2 fixed, 10 same | worst 0.25s / 0.13s (instant) | — / — | same; 2 slower than Loki cold | +| `loki_api_v1_query_range` | 117 × 6 ranges | 33 faster, 62 fixed, 271 same, 3 slower | worst 13.13s / 13.44s (7d) | 0.01s / 0.00s | differs; 59 slower than Loki cold | | `metric-series-identity` | 10 × 6 ranges | 8 faster, 6 fixed, 17 same | worst 1.18s / 1.06s (7d) | — / — | differs; 1 slower than Loki cold | | `operator-sum` | 3 × 4 ranges | 1 faster, 11 same | worst 0.56s / 0.47s (24h) | 0.03s / 0.01s | differs; 3 slower than Loki cold | | `operator-topk` | 1 × 4 ranges | 4 same | worst 0.62s / 0.32s (24h) | 0.01s / 0.01s | differs; 1 slower than Loki cold | @@ -123,11 +124,13 @@ Runs: pr611-latency-gap-json-filter-pushdown (2026-09-23), pr611-latency-gap-met | `semantics/label-browser-latency-cold-and-refresh` | 8 × 4 ranges | 32 same | worst 0.45s / 0.00s (7d) | 0.02s / 0.01s | same | | `semantics/label-filter-before-parser-pushdown` | 2 × 4 ranges | 2 faster, 6 same | worst 3.33s / 1.08s (24h) | — / — | differs; 2 slower than Loki cold | | `semantics/label-inventory-exact-and-incremental` | 12 × 4 ranges | 48 same | worst 0.45s / 0.00s (7d) | 0.02s / 0.01s | differs | +| `semantics/label-replace-new-label` | 6 × 2 ranges | 5 same, 1 slower | worst 0.54s / 0.55s (1h) | 0.11s / 0.01s | differs; 1 slower than Loki cold | | `semantics/labels-end-inclusive-like-loki` | 12 × 4 ranges | 48 same | worst 0.45s / 0.00s (7d) | 0.02s / 0.01s | differs | | `semantics/line-filter-or-alternatives` | 6 × 5 ranges | 21 fixed | worst 0.06s / 0.04s (3h) | 0.01s / 0.02s | same | | `semantics/metadata-answers-include-last-24h-like-loki` | 4 × 4 ranges | 16 same | worst 0.02s / 0.00s (1h) | 0.00s / 0.00s | differs | | `semantics/parenthesized-log-range` | 7 × 5 ranges | 17 fixed, 8 same | worst 0.07s / 0.04s (6h) | 0.28s / 0.01s | same | | `semantics/pattern-line-filter` | 4 × 1 ranges | 4 same | worst 0.03s / 0.03s (1h) | 0.07s / 0.06s | same | +| `semantics/pipeline-error-text-parity` | 2 × 2 ranges | 1 same, 1 slower | worst 0.54s / 0.55s (1h) | 0.11s / 0.01s | same; 1 slower than Loki cold | | `semantics/plain-name-after-parser-reads-stream-label` | 9 × 3 ranges | 17 same | worst 0.12s / 0.06s (1h) | 0.09s / 0.07s | same; 1 slower than Loki cold | | `semantics/unwrap-conversion-error` | 9 × 5 ranges | 2 faster, 8 fixed, 23 same | worst 13.13s / 13.44s (7d) | 0.01s / 0.00s | differs; 15 slower than Loki cold | | `semantics/unwrap-rate-ungrouped-sum-per-stream` | 9 × 5 ranges | 8 faster, 6 fixed, 13 same | worst 1.18s / 1.06s (7d) | — / — | differs | diff --git a/internal/logql/semantic.go b/internal/logql/semantic.go index 01e69a639..a4933544e 100644 --- a/internal/logql/semantic.go +++ b/internal/logql/semantic.go @@ -5,6 +5,7 @@ import ( "regexp" "regexp/syntax" "strconv" + "unicode/utf8" ) // errEmptyCompatibleMatchers is Loki's syntax.errAtleastOneEqualityMatcherRequired, @@ -42,6 +43,9 @@ func validateSemantics(expr Expr, raw string) string { if errors.As(err, &lokiErr) { return lokiErr.Msg } + if err := labelReplaceArgumentsError(x.Raw); err != nil { + return err.Error() + } if inner != nil { return validateSemantics(inner, raw) } @@ -74,6 +78,52 @@ func opaqueFirstArgument(call string) (Expr, error) { return inner, nil } +// labelReplaceArgumentsError reports the parse error Loki raises for a +// label_replace call that does not hold exactly four string arguments after its +// expression (pkg/logql/syntax/syntax.y labelReplaceExpr), a string whose +// escapes are invalid (the LogQL lexer unquotes it) included. It is nil for +// any other call. +func labelReplaceArgumentsError(call string) error { + p := &parser{sc: newScanner(call)} + p.advance() + if p.cur.Typ != TokIdent || p.cur.Val != "label_replace" { + return nil + } + if p.advance(); p.cur.Typ != TokLParen { + return nil + } + p.advance() + if _, err := p.parseExpr(); err != nil { + return nil // reported by opaqueFirstArgument + } + for i := 0; i < 4; i++ { + if p.cur.Typ != TokComma { + if i == 0 { + return p.syntaxErrorAt("") // the expression could go on: Loki names nothing + } + return p.syntaxErrorAt(",") + } + p.advance() + switch p.cur.Typ { + case TokString, TokRawString: + case TokError: + if p.cur.Val == "invalid quoted string" { + // Loki's lexer reports the start of the string. + before := p.sc.src[:p.pos()] + return newParseError(1, utf8.RuneCountInString(before)+1, "invalid char escape") + } + return p.syntaxErrorAt("STRING") + default: + return p.syntaxErrorAt("STRING") + } + p.advance() + } + if p.cur.Typ != TokRParen { + return p.syntaxErrorAt(")") + } + return nil +} + // validateLogQuerySemantics validates a log query. lineIndependent is true // inside range aggregations whose result does not depend on the log line // (every op except bytes_rate/bytes_over_time): there Loki's removeLineformat diff --git a/internal/proxy/binary_logql.go b/internal/proxy/binary_logql.go index f5f1578ae..178c05134 100644 --- a/internal/proxy/binary_logql.go +++ b/internal/proxy/binary_logql.go @@ -36,6 +36,19 @@ func nextBinaryEvaluation(r *http.Request) (*http.Request, error) { // Resolve original operands through the normal handlers: VL stats buckets do // not implement Loki's trailing range windows or parser-error semantics. +// binaryRequestQuery is the request's own LogQL with Grafana tokens resolved +// and a uniform offset stripped, as binaryExprForRequest reads it. +func binaryRequestQuery(r *http.Request) string { + query := stripIncompleteUnwrapStubs(r.FormValue("query")) + query = resolveGrafanaRangeTemplateTokens(query, r.FormValue("start"), r.FormValue("end"), r.FormValue("step")) + // The outer handler has already shifted uniform offsets. Mixed offsets + // remain on their individual operands and are applied by each child handler. + if _, stripped, err := extractLogQLOffset(query); err == nil { + query = stripped + } + return query +} + func binaryExprForRequest(r *http.Request) *logqlpkg.BinOpExpr { query := resolveGrafanaRangeTemplateTokens(r.FormValue("query"), r.FormValue("start"), r.FormValue("end"), r.FormValue("step")) // The outer handler has already shifted uniform offsets. Mixed offsets diff --git a/internal/proxy/label_replace_routing_test.go b/internal/proxy/label_replace_routing_test.go new file mode 100644 index 000000000..6d24ed494 --- /dev/null +++ b/internal/proxy/label_replace_routing_test.go @@ -0,0 +1,123 @@ +package proxy + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" +) + +// A label_replace around a bare range function yields the series of the +// function under it (pkg/logql/evaluator.go LabelReplaceEvaluator passes the +// series through): one per stream on an instant query too. The grouping by the +// stream was added only for a query that itself starts with the function, so an +// instant `label_replace(count_over_time(...))` came back as one unlabeled +// total. +// conformance: semantics/label-replace-new-label, loki_api_v1_query, loki_api_v1_query_range +func TestLabelReplaceOverBareRangeFunctionKeepsOneSeriesPerStream(t *testing.T) { + streams := []string{`{app="api-gateway"}`, `{app="auth-service"}`, `{app="worker"}`} + var statsQueries []string + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = r.ParseForm() + q := r.FormValue("query") + statsQueries = append(statsQueries, q) + grouped := strings.Contains(q, "stats by (_stream") + var out []string + if grouped { + for i, s := range streams { + app := strings.Split(s, `"`)[1] + metric, _ := json.Marshal(map[string]string{"_stream": s, "app": app}) + if r.URL.Path == "/select/logsql/stats_query" { + out = append(out, fmt.Sprintf(`{"metric":%s,"value":[1704067200,"%d"]}`, metric, i+1)) + } else { + out = append(out, fmt.Sprintf(`{"metric":%s,"values":[[1704067200,"%d"]]}`, metric, i+1)) + } + } + } else if r.URL.Path == "/select/logsql/stats_query" { + out = append(out, `{"metric":{},"value":[1704067200,"6"]}`) + } else { + out = append(out, `{"metric":{},"values":[[1704067200,"6"]]}`) + } + kind := "vector" + if r.URL.Path != "/select/logsql/stats_query" { + kind = "matrix" + } + _, _ = fmt.Fprintf(w, `{"status":"success","data":{"resultType":%q,"result":[%s]}}`, kind, strings.Join(out, ",")) + })) + defer backend.Close() + p := newTestProxy(t, backend.URL) + + for _, fn := range []string{"count_over_time", "bytes_over_time", "rate"} { + for _, endpoint := range []string{"query", "query_range"} { + q := fmt.Sprintf(`label_replace(%s({env="production"}[5m]), "app_short", "$1", "app", "(.*)-.*")`, fn) + a := lrAsk(t, p, endpoint, q) + if a.code != 200 || len(a.series) != len(streams) { + t.Errorf("%s %s: HTTP %d, %d series, want one per stream (%d)\n%s", endpoint, fn, a.code, len(a.series), len(streams), a.body) + continue + } + for _, s := range a.series { + if want := map[string]string{"api-gateway": "api", "auth-service": "auth", "worker": ""}[s.Metric["app"]]; s.Metric["app_short"] != want { + t.Errorf("%s %s: app=%s app_short=%q, want %q", endpoint, fn, s.Metric["app"], s.Metric["app_short"], want) + } + } + } + } + if len(statsQueries) == 0 { + t.Fatal("no query reached the backend") + } +} + +// A label_replace around a metric over `| json` is answered by the evaluator +// that owns JSON parser errors, like the metric without it: a sample whose +// line is not JSON fails the query with Loki's pipeline error unless the +// pipeline drops `__error__`, and the transform is applied to what the +// evaluator answered. Through the general path the pipeline was ignored and +// the query answered 200 with every stream. +// conformance: semantics/label-replace-new-label, semantics/pipeline-error-text-parity, loki_api_v1_query, loki_api_v1_query_range +func TestLabelReplaceOverJSONMetricUsesTheOrderedJSONEvaluator(t *testing.T) { + var rowQueries, statsQueries atomic.Int32 + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/select/logsql/query": + rowQueries.Add(1) + fmt.Fprintln(w, `{"_time":"2024-01-01T00:00:30Z","_msg":"{\"k\":1}","_stream":"{app=\"api-gateway\"}","app":"api-gateway"}`) + fmt.Fprintln(w, `{"_time":"2024-01-01T00:00:30Z","_msg":"{\"k\":1}","_stream":"{app=\"auth-service\"}","app":"auth-service"}`) + fmt.Fprintln(w, `{"_time":"2024-01-01T00:00:30Z","_msg":"not json","_stream":"{app=\"worker\"}","app":"worker"}`) + case "/select/logsql/stats_query", "/select/logsql/stats_query_range": + statsQueries.Add(1) + fmt.Fprintln(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`) + default: + fmt.Fprintln(w, `{}`) + } + })) + defer backend.Close() + p := newTestProxy(t, backend.URL) + ask := func(endpoint, filter string) lrAnswer { + q := fmt.Sprintf(`label_replace(sum by (app) (count_over_time({env="production"} | json %s [5m])), "app_short", "$1", "app", "(.*)-.*")`, filter) + return lrAsk(t, p, endpoint, q) + } + for _, endpoint := range []string{"query", "query_range"} { + rowQueries.Store(0) + statsQueries.Store(0) + failed := ask(endpoint, "") + if failed.code != 400 || !strings.Contains(failed.body, "JSONParserErr") { + t.Errorf("%s without a filter: HTTP %d, want Loki's 400 JSONParserErr\n%s", endpoint, failed.code, failed.body) + } + if rowQueries.Load() == 0 || statsQueries.Load() != 0 { + t.Errorf("%s: the ordered JSON evaluator was not used (row queries %d, stats queries %d)", endpoint, rowQueries.Load(), statsQueries.Load()) + } + ok := ask(endpoint, `| __error__=""`) + if ok.code != 200 || len(ok.series) != 2 { + t.Errorf("%s with __error__=\"\": HTTP %d, %d series, want the two streams that parse\n%s", endpoint, ok.code, len(ok.series), ok.body) + continue + } + for _, s := range ok.series { + if want := map[string]string{"api-gateway": "api", "auth-service": "auth"}[s.Metric["app"]]; want == "" || s.Metric["app_short"] != want { + t.Errorf("%s: app=%q app_short=%q, want the label_replace destination", endpoint, s.Metric["app"], s.Metric["app_short"]) + } + } + } +} diff --git a/internal/proxy/label_replace_semantics_test.go b/internal/proxy/label_replace_semantics_test.go new file mode 100644 index 000000000..abdac9068 --- /dev/null +++ b/internal/proxy/label_replace_semantics_test.go @@ -0,0 +1,641 @@ +package proxy + +import ( + "encoding/json" + "fmt" + "math/rand" + "net/http" + "net/http/httptest" + "net/url" + "regexp" + "sort" + "strconv" + "strings" + "testing" + + "github.com/ReliablyObserve/Loki-VL-proxy/internal/translator" +) + +// label_replace adds, rewrites or removes one label of every series its +// expression yields. Loki v3.7.7: +// +// - pkg/logql/syntax/ast.go mustNewLabelReplaceExpr compiles the regexp as +// `^(?:regex)$`, so it must match the whole source value; an invalid one is +// the parse error "invalid regex in label_replace: ..."; +// - pkg/logql/evaluator.go LabelReplaceEvaluator.Next reads the source label +// (empty when absent), leaves the series as it is when the regexp does not +// match, otherwise expands the replacement with Regexp.ExpandString ($1, +// ${1}, $name, ${name}, $$) and deletes the destination label before +// setting it when the expansion is not empty. It checks no label name and +// raises no error for two series that end with one label set; +// - pkg/logql/engine.go JoinSampleVector keeps those series apart on an +// instant query (one sample each) and, on a range query, files their +// points under one series (vectorsToSeries keys by label hash, appending +// the points of each step in turn). +// +// The proxy applies the transform to the series VictoriaLogs aggregated, so an +// instant vector and a range matrix go through the same function. + +// lrSeries is one series of a decoded metric answer: labels and points, a +// matrix series' values or an instant vector's single value. +type lrSeries struct { + Metric map[string]string `json:"metric"` + Value []any `json:"value,omitempty"` + Values [][]any `json:"values,omitempty"` +} + +type lrAnswer struct { + code int + resultType string + series []lrSeries + body string +} + +func decodeLR(t *testing.T, code int, body []byte) lrAnswer { + t.Helper() + var resp struct { + Status string `json:"status"` + Data struct { + ResultType string `json:"resultType"` + Result []lrSeries `json:"result"` + } `json:"data"` + } + if err := json.Unmarshal(body, &resp); err != nil { + t.Fatalf("answer is not JSON (HTTP %d): %v\n%s", code, err, body) + } + return lrAnswer{code: code, resultType: resp.Data.ResultType, series: resp.Data.Result, body: string(body)} +} + +// lrKey is a canonical rendering of a label set. +func lrKey(m map[string]string) string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + var b strings.Builder + b.WriteByte('{') + for i, k := range keys { + if i > 0 { + b.WriteByte(',') + } + b.WriteString(k + "=" + strconv.Quote(m[k])) + } + b.WriteByte('}') + return b.String() +} + +// lrBackend is a VictoriaLogs that answers every stats query with series +// carrying exactly the given label sets, two points each on a range query. +func lrBackend(t *testing.T, sets []map[string]string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = r.ParseForm() + if q := r.FormValue("query"); strings.Contains(q, "__lvp_lr:") || strings.Contains(q, "__lvp_lj:") || strings.Contains(q, "__binary__") || strings.Contains(q, "|||") { + t.Errorf("an internal marker reached VictoriaLogs: %s", q) + } + var out []string + switch r.URL.Path { + case "/select/logsql/stats_query_range": + for i, m := range sets { + mb, _ := json.Marshal(m) + out = append(out, fmt.Sprintf(`{"metric":%s,"values":[[1704067200,"%d"],[1704067260,"%d"]]}`, mb, i+1, i+2)) + } + _, _ = fmt.Fprintf(w, `{"status":"success","data":{"resultType":"matrix","result":[%s]}}`, strings.Join(out, ",")) + case "/select/logsql/stats_query": + for i, m := range sets { + mb, _ := json.Marshal(m) + out = append(out, fmt.Sprintf(`{"metric":%s,"value":[1704067200,"%d"]}`, mb, i+1)) + } + _, _ = fmt.Fprintf(w, `{"status":"success","data":{"resultType":"vector","result":[%s]}}`, strings.Join(out, ",")) + default: + _, _ = w.Write([]byte(`{"status":"success","data":{"result":[]}}`)) + } + })) + t.Cleanup(srv.Close) + return srv +} + +func lrAsk(t *testing.T, p *Proxy, endpoint, q string) lrAnswer { + t.Helper() + return lrAskAs(t, p, endpoint, q, nil) +} + +func lrAskAs(t *testing.T, p *Proxy, endpoint, q string, headers map[string]string) lrAnswer { + t.Helper() + v := url.Values{"query": {q}, "start": {"1704067200"}, "end": {"1704067260"}, "step": {"60"}, "time": {"1704067260"}} + req := httptest.NewRequest("GET", "/loki/api/v1/"+endpoint+"?"+v.Encode(), nil) + req.Header.Set("X-Scope-OrgID", "0") + req.Header.Set("X-Loki-Response-Encoding-Flags", "categorize-labels") + for k, val := range headers { + req.Header.Set(k, val) + } + rec := httptest.NewRecorder() + if endpoint == "query" { + p.handleQuery(rec, req) + } else { + p.handleQueryRange(rec, req) + } + return decodeLR(t, rec.Code, rec.Body.Bytes()) +} + +// lrReference is Loki's label_replace written apart from the proxy: the +// evaluator step (LabelReplaceEvaluator.Next) on one label set. +func lrReference(in map[string]string, dst, repl, src, regex string) map[string]string { + re := regexp.MustCompile("^(?:" + regex + ")$") + out := make(map[string]string, len(in)+1) + for k, v := range in { + out[k] = v + } + s := in[src] + idx := re.FindStringSubmatchIndex(s) + if idx == nil { + return out + } + res := re.ExpandString(nil, repl, s, idx) + delete(out, dst) + if len(res) > 0 { + out[dst] = string(res) + } + return out +} + +// lrReferenceJoin is JoinSampleVector for the series of a transformed answer: +// an instant vector keeps every sample, a range matrix files the points of the +// series that now share a label set under one series, step by step. +func lrReferenceJoin(base []lrSeries, instant bool, f func(map[string]string) map[string]string) map[string][]string { + out := map[string][]string{} + if instant { + for _, s := range base { + k := lrKey(f(s.Metric)) + out[k] = append(out[k], fmt.Sprint(s.Value)) + } + for k := range out { + sort.Strings(out[k]) + } + return out + } + steps := map[string]bool{} + for _, s := range base { + for _, v := range s.Values { + steps[fmt.Sprint(v[0])] = true + } + } + order := make([]string, 0, len(steps)) + for ts := range steps { + order = append(order, ts) + } + sort.Slice(order, func(i, j int) bool { + a, _ := strconv.ParseFloat(order[i], 64) + b, _ := strconv.ParseFloat(order[j], 64) + return a < b + }) + for _, ts := range order { + for _, s := range base { + for _, v := range s.Values { + if fmt.Sprint(v[0]) == ts { + k := lrKey(f(s.Metric)) + out[k] = append(out[k], fmt.Sprint(v)) + } + } + } + } + return out +} + +func lrGot(a lrAnswer, instant bool) map[string][]string { + out := map[string][]string{} + for _, s := range a.series { + k := lrKey(s.Metric) + if instant { + out[k] = append(out[k], fmt.Sprint(s.Value)) + continue + } + for _, v := range s.Values { + out[k] = append(out[k], fmt.Sprint(v)) + } + } + if instant { + for k := range out { + sort.Strings(out[k]) + } + } + return out +} + +// The edge table: every rule of the evaluator step, on both answer shapes. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestLabelReplaceEdgeTable(t *testing.T) { + type tc struct { + name string + in map[string]string + dst, repl, src, r string + want map[string]string + } + cases := []tc{ + {"adds the destination", map[string]string{"app": "api-gateway"}, "app_short", "$1", "app", "(.*)-.*", map[string]string{"app": "api-gateway", "app_short": "api"}}, + {"regexp is anchored: a prefix does not match", map[string]string{"app": "api-gateway"}, "d", "x", "app", "api", map[string]string{"app": "api-gateway"}}, + {"regexp is anchored: a suffix does not match", map[string]string{"app": "api-gateway"}, "d", "x", "app", "gateway", map[string]string{"app": "api-gateway"}}, + {"regexp is anchored: the whole value matches", map[string]string{"app": "api"}, "d", "x", "app", "api", map[string]string{"app": "api", "d": "x"}}, + {"alternation is anchored as a whole", map[string]string{"app": "bc"}, "d", "x", "app", "a|bc", map[string]string{"app": "bc", "d": "x"}}, + {"alternation does not leak the anchors", map[string]string{"app": "abc"}, "d", "x", "app", "a|bc", map[string]string{"app": "abc"}}, + {"no match keeps the series and an existing destination", map[string]string{"app": "web", "d": "old"}, "d", "$1", "app", "(.*)-.*", map[string]string{"app": "web", "d": "old"}}, + {"empty replacement removes the destination", map[string]string{"app": "web", "d": "old"}, "d", "", "app", ".*", map[string]string{"app": "web"}}, + {"empty replacement without a destination changes nothing", map[string]string{"app": "web"}, "d", "", "app", ".*", map[string]string{"app": "web"}}, + {"an empty expansion removes the destination", map[string]string{"app": "web", "d": "old"}, "d", "$1", "app", "web()", map[string]string{"app": "web"}}, + {"braced group", map[string]string{"app": "api-gateway"}, "d", "${1}_x", "app", "(.*)-.*", map[string]string{"app": "api-gateway", "d": "api_x"}}, + {"bare group name swallows the suffix", map[string]string{"app": "api-gateway"}, "d", "$1x", "app", "(.*)-.*", map[string]string{"app": "api-gateway"}}, + {"named group", map[string]string{"app": "api-gateway"}, "d", "${head}", "app", "(?P[a-z]+)-.*", map[string]string{"app": "api-gateway", "d": "api"}}, + {"bare named group", map[string]string{"app": "api-gateway"}, "d", "$head.", "app", "(?P[a-z]+)-.*", map[string]string{"app": "api-gateway", "d": "api."}}, + {"two groups", map[string]string{"app": "api-gateway"}, "d", "$2/$1", "app", "([a-z]+)-([a-z]+)", map[string]string{"app": "api-gateway", "d": "gateway/api"}}, + {"unknown group expands to nothing", map[string]string{"app": "api", "d": "old"}, "d", "$2", "app", "(.*)", map[string]string{"app": "api"}}, + {"dollar escape", map[string]string{"app": "api"}, "d", "$$1", "app", "(.*)", map[string]string{"app": "api", "d": "$1"}}, + {"constant replacement", map[string]string{"app": "api"}, "d", "const", "app", ".*", map[string]string{"app": "api", "d": "const"}}, + {"missing source is empty: a regexp that matches empty applies", map[string]string{"app": "api"}, "d", "dflt", "nope", "(.*)", map[string]string{"app": "api", "d": "dflt"}}, + {"missing source is empty: a regexp that needs a character does not", map[string]string{"app": "api"}, "d", "dflt", "nope", ".+", map[string]string{"app": "api"}}, + {"empty regexp matches an empty source", map[string]string{"app": "api"}, "d", "dflt", "nope", "", map[string]string{"app": "api", "d": "dflt"}}, + {"destination equal to the source", map[string]string{"app": "api-gateway"}, "app", "$1", "app", "(.*)-.*", map[string]string{"app": "api"}}, + {"destination equal to the source, removed", map[string]string{"app": "api", "b": "1"}, "app", "", "app", ".*", map[string]string{"b": "1"}}, + {"dot does not match a newline", map[string]string{"app": "a\nb"}, "d", "x", "app", ".*", map[string]string{"app": "a\nb"}}, + {"flag lets dot match a newline", map[string]string{"app": "a\nb"}, "d", "x", "app", "(?s).*", map[string]string{"app": "a\nb", "d": "x"}}, + {"label names are not validated", map[string]string{"app": "api"}, "not a label", "x", "app", ".*", map[string]string{"app": "api", "not a label": "x"}}, + {"dotted destination", map[string]string{"app": "api"}, "service.name", "$0", "app", ".*", map[string]string{"app": "api", "service.name": "api"}}, + {"group zero is the whole match", map[string]string{"app": "api"}, "d", "<$0>", "app", "a(p)i", map[string]string{"app": "api", "d": ""}}, + {"unicode value", map[string]string{"app": "żółć-1"}, "d", "$1", "app", "(.*)-.*", map[string]string{"app": "żółć-1", "d": "żółć"}}, + } + for _, c := range cases { + spec := translator.LabelReplaceSpec{DstLabel: c.dst, Replacement: c.repl, SrcLabel: c.src, Regex: c.r} + for _, shape := range []string{"matrix", "vector"} { + t.Run(c.name+"/"+shape, func(t *testing.T) { + mb, _ := json.Marshal(c.in) + var body string + if shape == "matrix" { + body = fmt.Sprintf(`{"status":"success","data":{"resultType":"matrix","result":[{"metric":%s,"values":[[1704067200,"1"]]}],"stats":{"x":1}}}`, mb) + } else { + body = fmt.Sprintf(`{"status":"success","data":{"resultType":"vector","result":[{"metric":%s,"value":[1704067200,"1"]}],"stats":{"x":1}}}`, mb) + } + out := applyLabelReplace([]byte(body), spec) + a := decodeLR(t, 200, out) + if len(a.series) != 1 || lrKey(a.series[0].Metric) != lrKey(c.want) { + t.Fatalf("labels %s, want %s\n%s", lrKey(a.series[0].Metric), lrKey(c.want), out) + } + if a.resultType != shape { + t.Errorf("resultType %q, want %q", a.resultType, shape) + } + if !strings.Contains(string(out), `"stats":{"x":1}`) { + t.Errorf("the rest of the answer is not kept: %s", out) + } + if ref := lrReference(c.in, c.dst, c.repl, c.src, c.r); lrKey(ref) != lrKey(c.want) { + t.Fatalf("test table disagrees with the reference: %s vs %s", lrKey(ref), lrKey(c.want)) + } + }) + } + } +} + +// Two series that end with one label set: no error (the evaluator checks +// nothing); an instant vector holds both samples, a range matrix one series +// with the points of each step in turn. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestLabelReplaceDuplicateLabelSets(t *testing.T) { + spec := translator.LabelReplaceSpec{DstLabel: "app", Replacement: "all", SrcLabel: "app", Regex: ".*"} + matrix := `{"status":"success","data":{"resultType":"matrix","result":[` + + `{"metric":{"app":"a"},"values":[[1,"1"],[2,"2"]]},{"metric":{"app":"b"},"values":[[1,"10"],[2,"20"]]},{"metric":{"app":"c","k":"v"},"values":[[1,"7"]]}]}}` + got := decodeLR(t, 200, applyLabelReplace([]byte(matrix), spec)) + if len(got.series) != 2 { + t.Fatalf("range: %d series, want the two duplicates filed under one\n%s", len(got.series), got.body) + } + for _, s := range got.series { + if s.Metric["app"] == "all" && len(s.Metric) == 1 { + if v := fmt.Sprint(s.Values); v != "[[1 1] [1 10] [2 2] [2 20]]" { + t.Errorf("merged points %s, want the points of each step in turn", v) + } + } + } + vector := `{"status":"success","data":{"resultType":"vector","result":[` + + `{"metric":{"app":"a"},"value":[1,"1"]},{"metric":{"app":"b"},"value":[1,"10"]}]}}` + got = decodeLR(t, 200, applyLabelReplace([]byte(vector), spec)) + if len(got.series) != 2 || lrKey(got.series[0].Metric) != lrKey(got.series[1].Metric) { + t.Fatalf("instant: want both samples under the shared label set\n%s", got.body) + } +} + +// The registry shape and the shapes around it, on both endpoints. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestLabelReplaceOnInstantAndRange(t *testing.T) { + sets := []map[string]string{{"app": "api-gateway"}, {"app": "web"}} + p := newTestProxy(t, lrBackend(t, sets).URL) + count := `count_over_time({app="api-gateway"}[5m])` + agg := `sum by (app) (` + count + `)` + lr := func(inner, dst, repl, src, re string) string { + return fmt.Sprintf(`label_replace(%s, %q, %q, %q, %q)`, inner, dst, repl, src, re) + } + cases := []struct { + name string + query string + want []map[string]string + }{ + {"over an aggregation", lr(agg, "app_short", "$1", "app", "(.*)-.*"), []map[string]string{{"app": "api-gateway", "app_short": "api"}, {"app": "web"}}}, + {"nested", lr(lr(agg, "a1", "$1", "app", "(.*)-.*"), "a2", "x$1", "a1", "(.*)"), []map[string]string{{"app": "api-gateway", "a1": "api", "a2": "xapi"}, {"app": "web", "a2": "x"}}}, + {"nested, the outer reads what the inner removed", lr(lr(agg, "app", "", "app", "web"), "w", "gone", "app", ""), []map[string]string{{"app": "api-gateway"}, {"w": "gone"}}}, + {"escaped regexp", lr(agg, "d", "$1", "app", `([a-z]+)\-(\w+)`), []map[string]string{{"app": "api-gateway", "d": "api"}, {"app": "web"}}}, + {"escaped dot", lr(agg, "d", "$1", "app", `(\w+)\.(\w+)`), []map[string]string{{"app": "api-gateway"}, {"app": "web"}}}, + {"over a binary expression", lr(agg+` * 2`, "app_short", "$1", "app", "(.*)-.*"), []map[string]string{{"app": "api-gateway", "app_short": "api"}, {"app": "web"}}}, + {"over a binary of two aggregations", lr(agg+` / `+agg, "app_short", "$1", "app", "(.*)-.*"), []map[string]string{{"app": "api-gateway", "app_short": "api"}, {"app": "web"}}}, + } + for _, c := range cases { + for _, endpoint := range []string{"query_range", "query"} { + t.Run(c.name+"/"+endpoint, func(t *testing.T) { + a := lrAsk(t, p, endpoint, c.query) + if a.code != 200 { + t.Fatalf("HTTP %d: %s", a.code, a.body) + } + want := "matrix" + if endpoint == "query" { + want = "vector" + } + if a.resultType != want { + t.Fatalf("resultType %q, want %q\n%s", a.resultType, want, a.body) + } + got, exp := map[string]bool{}, map[string]bool{} + for _, s := range a.series { + got[lrKey(s.Metric)] = true + } + for _, m := range c.want { + exp[lrKey(m)] = true + } + if fmt.Sprint(got) != fmt.Sprint(exp) { + t.Fatalf("label sets %v, want %v\n%s", got, exp, a.body) + } + }) + } + } +} + +// topk, bottomk, sort and stddev rank or fold the series label_replace +// produced: the transform runs on what VictoriaLogs answered, then the +// ranking. The translator's marker never reaches VictoriaLogs. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestLabelReplaceUnderPostAggregation(t *testing.T) { + sets := []map[string]string{{"app": "api-gateway"}, {"app": "web"}} + p := newTestProxy(t, lrBackend(t, sets).URL) + lr := `label_replace(sum by (app) (count_over_time({app="x"}[5m])), "app_short", "$1", "app", "(.*)-.*")` + for _, tc := range []struct { + query string + want []string + }{ + {`topk(2, ` + lr + `)`, []string{`{app="api-gateway",app_short="api"}`, `{app="web"}`}}, + {`bottomk(1, ` + lr + `)`, []string{`{app="api-gateway",app_short="api"}`}}, + {`sort(` + lr + `)`, []string{`{app="api-gateway",app_short="api"}`, `{app="web"}`}}, + {`sort_desc(` + lr + `)`, []string{`{app="api-gateway",app_short="api"}`, `{app="web"}`}}, + } { + for _, endpoint := range []string{"query_range", "query"} { + if endpoint == "query_range" && (strings.HasPrefix(tc.query, "topk") || strings.HasPrefix(tc.query, "bottomk")) { + continue // ranks per step from window statistics, which this backend does not emulate + } + a := lrAsk(t, p, endpoint, tc.query) + got := map[string]bool{} + for _, s := range a.series { + got[lrKey(s.Metric)] = true + } + if a.code != 200 || len(got) != len(tc.want) { + t.Errorf("%s %s: HTTP %d, series %v, want %v\n%s", endpoint, tc.query, a.code, got, tc.want, a.body) + continue + } + for _, w := range tc.want { + if !got[w] { + t.Errorf("%s %s: series %v, want %v\n%s", endpoint, tc.query, got, tc.want, a.body) + } + } + } + } + for _, endpoint := range []string{"query_range", "query"} { + a := lrAsk(t, p, endpoint, `sort(label_replace(sum by (app) (count_over_time({app="x"}[5m])), "d", "$1", "app", "(unclosed"))`) + if a.code != http.StatusBadRequest || !strings.Contains(a.body, "parse error : invalid regex in label_replace:") { + t.Errorf("%s: HTTP %d, want Loki's parse error\n%s", endpoint, a.code, a.body) + } + } +} + +// The registry row's query, as curl, Grafana Explore and Logs Drilldown send it +// (Drilldown range answers pass through the series-limit rewriter after the +// transform), with Loki's categorize-labels encoding flag. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestLabelReplaceRowQueryForEveryClient(t *testing.T) { + p := newTestProxy(t, lrBackend(t, []map[string]string{{"app": "api-gateway"}, {"app": "web"}}).URL) + q := `label_replace(sum by (app) (rate({app="api-gateway"}[5m])), "app_short", "$1", "app", "(.*)-.*")` + for client, headers := range instantClients { + for _, endpoint := range []string{"query_range", "query"} { + a := lrAskAs(t, p, endpoint, q, headers) + if a.code != 200 || len(a.series) != 2 { + t.Errorf("%s %s: HTTP %d, %d series, want 2\n%s", client, endpoint, a.code, len(a.series), a.body) + continue + } + for _, s := range a.series { + if (s.Metric["app"] == "api-gateway") != (s.Metric["app_short"] == "api") { + t.Errorf("%s %s: app_short=%q for app=%q\n%s", client, endpoint, s.Metric["app_short"], s.Metric["app"], a.body) + } + } + } + } +} + +// A bare range aggregation carries the stream identity; the new label rides +// along with it. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestLabelReplaceOverBareRangeAggregation(t *testing.T) { + p := newTestProxy(t, lrBackend(t, []map[string]string{{"app": "api-gateway", "_stream": `{app="api-gateway"}`, "level": "info"}}).URL) + for _, endpoint := range []string{"query_range", "query"} { + a := lrAsk(t, p, endpoint, `label_replace(rate({app="api-gateway"}[5m]), "app_short", "$1", "app", "(.*)-.*")`) + if a.code != 200 || len(a.series) != 1 || a.series[0].Metric["app_short"] != "api" || a.series[0].Metric["app"] != "api-gateway" { + t.Errorf("%s: HTTP %d, want app_short=api next to app\n%s", endpoint, a.code, a.body) + } + } +} + +// An invalid regexp is Loki's parse error on both endpoints, nested or not. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestLabelReplaceInvalidRegexp(t *testing.T) { + p := newTestProxy(t, lrBackend(t, []map[string]string{{"app": "a"}}).URL) + agg := `sum by (app) (count_over_time({app="a"}[5m]))` + for _, q := range []string{ + `label_replace(` + agg + `, "d", "$1", "app", "[invalid")`, + `label_replace(label_replace(` + agg + `, "d", "$1", "app", "(.*)"), "e", "$1", "d", "(unclosed")`, + `label_replace(` + agg + ` * 2, "d", "$1", "app", "*bad")`, + } { + for _, endpoint := range []string{"query_range", "query"} { + a := lrAsk(t, p, endpoint, q) + if a.code != http.StatusBadRequest || !strings.Contains(a.body, "parse error : invalid regex in label_replace: error parsing regexp:") { + t.Errorf("%s %s: HTTP %d, want Loki's 400 parse error\n%s", endpoint, q, a.code, a.body) + } + } + } +} + +// A call that does not hold the four string arguments of Loki's grammar, or a +// string whose escapes are invalid, is a 400 parse error on both endpoints: it +// is never taken for a log query or answered with untransformed series. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestLabelReplaceMalformedArguments(t *testing.T) { + p := newTestProxy(t, lrBackend(t, []map[string]string{{"app": "a"}}).URL) + agg := `sum by (app) (count_over_time({app="a"}[5m]))` + // The messages are the ones Loki v3.7.7's parser raises for these queries. + for _, tc := range []struct{ query, want string }{ + {`label_replace(` + agg + `, "d", "$1", "app", "(.+):\d+")`, "parse error at line 1, col 80: invalid char escape"}, + {`label_replace(` + agg + `, "d", "$1", "app")`, "parse error at line 1, col 78: syntax error: unexpected ), expecting ,"}, + {`label_replace(` + agg + `, "d", "$1")`, "parse error at line 1, col 71: syntax error: unexpected ), expecting ,"}, + {`label_replace(` + agg + `, "d", "$1", "app", "x", "y")`, "parse error at line 1, col 83: syntax error: unexpected ,, expecting )"}, + {`label_replace(` + agg + `, d, "$1", "app", "x")`, "parse error at line 1, col 62: syntax error: unexpected IDENTIFIER, expecting STRING"}, + {`label_replace(` + agg + `, "d", "$1", "app", 5)`, "parse error at line 1, col 80: syntax error: unexpected NUMBER, expecting STRING"}, + {`label_replace(` + agg + ` "d", "$1", "app", "x")`, "parse error at line 1, col 61: syntax error: unexpected STRING"}, + } { + for _, endpoint := range []string{"query_range", "query"} { + a := lrAsk(t, p, endpoint, tc.query) + if a.code != http.StatusBadRequest || !strings.Contains(a.body, `"message":"`+tc.want+`"`) { + t.Errorf("%s %s: HTTP %d, want a 400 with %q\n%s", endpoint, tc.query, a.code, tc.want, a.body) + } + } + } +} + +// Loki sorts the result after the transform (pkg/logql/engine.go JoinSampleVector: +// sort.Sort(result) for a range, sort.Slice(vec, labels.Compare) for an instant +// query), so the order of the backend's series does not survive it. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestLabelReplaceSortsTheResult(t *testing.T) { + agg := `sum by (app) (count_over_time({app="x"}[5m]))` + lr := func(inner, dst, repl, src, re string) string { + return fmt.Sprintf(`label_replace(%s, %q, %q, %q, %q)`, inner, dst, repl, src, re) + } + for _, tc := range []struct { + name string + sets []map[string]string + query string + want []string + }{ + {"input in reverse order", []map[string]string{{"app": "web"}, {"app": "api-gateway"}}, lr(agg, "app_short", "$1", "app", "(.*)-.*"), + []string{`{app="api-gateway",app_short="api"}`, `{app="web"}`}}, + {"the transform reorders the label sets", []map[string]string{{"app": "a"}, {"app": "b"}}, + lr(lr(agg, "a_new", "2", "app", "a"), "a_new", "1", "app", "b"), + []string{`{a_new="1",app="b"}`, `{a_new="2",app="a"}`}}, + {"a shorter prefix first", []map[string]string{{"app": "b", "k": "v"}, {"app": "a"}, {"app": "b"}}, lr(agg, "none", "", "app", "zz"), + []string{`{app="a"}`, `{app="b"}`, `{app="b",k="v"}`}}, + } { + p := newTestProxy(t, lrBackend(t, tc.sets).URL) + for _, endpoint := range []string{"query_range", "query"} { + a := lrAsk(t, p, endpoint, tc.query) + var got []string + for _, s := range a.series { + got = append(got, lrKey(s.Metric)) + } + if fmt.Sprint(got) != fmt.Sprint(tc.want) { + t.Errorf("%s %s: order %v, want %v\n%s", tc.name, endpoint, got, tc.want, a.body) + } + } + } + // sort and sort_desc order by value: that order is kept + if !queryIsSortable(`label_replace(sort(sum(x)), "a", "b", "c", "d")`) || queryIsSortable(`label_replace(sum by (resort) (x), "a", "b", "c", "d")`) { + t.Error("queryIsSortable does not tell sort(...) from a name that ends in it") + } +} + +// compareLabelSets is Prometheus' labels.Compare. +func TestCompareLabelSets(t *testing.T) { + for _, tc := range []struct { + a, b map[string]string + want int + }{ + {map[string]string{"a": "1"}, map[string]string{"a": "1"}, 0}, + {map[string]string{"a": "1"}, map[string]string{"a": "2"}, -1}, + {map[string]string{"a": "1"}, map[string]string{"a": "1", "b": "2"}, -1}, + {map[string]string{"a": "1", "c": "1"}, map[string]string{"a": "1", "b": "2"}, 1}, + {map[string]string{"b": "1"}, map[string]string{"a": "9"}, 1}, + {map[string]string{}, map[string]string{"a": "9"}, -1}, + } { + if got := compareLabelSets(tc.a, tc.b); (got > 0) != (tc.want > 0) || (got < 0) != (tc.want < 0) { + t.Errorf("compare(%v, %v) = %d, want sign %d", tc.a, tc.b, got, tc.want) + } + } +} + +// label_replace over `sum without`: the without marker no longer swallows the +// label_replace marker that follows it. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestLabelReplaceOverSumWithout(t *testing.T) { + p := newTestProxy(t, lrBackend(t, []map[string]string{ + {"_stream": `{app="api-gateway"}`, "level": "info"}, {"_stream": `{app="web"}`, "level": "info"}, + }).URL) + for _, endpoint := range []string{"query_range", "query"} { + a := lrAsk(t, p, endpoint, `label_replace(sum without (level) (count_over_time({app=~".+"}[5m])), "app_short", "$1", "app", "(.*)-.*")`) + if a.code != 200 || len(a.series) != 2 { + t.Fatalf("%s: HTTP %d, %d series, want 2\n%s", endpoint, a.code, len(a.series), a.body) + } + for _, s := range a.series { + if _, kept := s.Metric["level"]; kept { + t.Errorf("%s: level kept by without: %v", endpoint, s.Metric) + } + if (s.Metric["app"] == "api-gateway") != (s.Metric["app_short"] == "api") { + t.Errorf("%s: app_short=%q for app=%q, want api for api-gateway only", endpoint, s.Metric["app_short"], s.Metric["app"]) + } + } + } +} + +// The differential test: generated label sets and generated transforms, the +// proxy's answer on both endpoints against the reference applied to the +// untransformed answer of the same backend. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestLabelReplaceDifferential(t *testing.T) { + values := []string{"api-gateway", "web", "x", "api-1.2", "", "żółć-9", "a|b", "10.20", "ab"} + regexes := []string{`(.*)-.*`, `(.*)`, `.*`, `.+`, `api`, `(?P[a-z]+)-(?P.*)`, `([^-]+)`, `a|b`, `(\d+)\.(\d+)`, `.+-(.*)`, ``, `(a)(b)?`, `web|x`, `(?i)API-.*`, `[a-z]+`, `(.)(.)`} + repls := []string{`$1`, `${1}`, `$2`, `${head}`, `$1-$2`, ``, `static`, `$$1`, `$1x`, `${1}x`, `$none`, `$0`, `<${tail}>`} + names := []string{"a", "b", "c", "new", "d.e"} + rng := rand.New(rand.NewSource(7)) + for round := 0; round < 40; round++ { + var sets []map[string]string + seen := map[string]bool{} + for i, n := 0, 1+rng.Intn(5); i < n; i++ { + m := map[string]string{} + for _, k := range []string{"a", "b", "c"} { + if rng.Intn(4) > 0 { + m[k] = values[rng.Intn(len(values))] + } + } + if key := lrKey(m); !seen[key] { + seen[key] = true + sets = append(sets, m) + } + } + p := newTestProxy(t, lrBackend(t, sets).URL) + type step struct{ dst, repl, src, re string } + steps := make([]step, 1+rng.Intn(2)) + for i := range steps { + steps[i] = step{names[rng.Intn(len(names))], repls[rng.Intn(len(repls))], names[rng.Intn(len(names))], regexes[rng.Intn(len(regexes))]} + } + base := `sum by (a, b, c) (count_over_time({app="x"}[5m]))` + q := base + for _, s := range steps { + q = fmt.Sprintf(`label_replace(%s, %s, %s, %s, %s)`, q, strconv.Quote(s.dst), strconv.Quote(s.repl), strconv.Quote(s.src), strconv.Quote(s.re)) + } + for _, endpoint := range []string{"query_range", "query"} { + instant := endpoint == "query" + baseAns := lrAsk(t, p, endpoint, base) + if baseAns.code != 200 || len(baseAns.series) != len(sets) { + t.Fatalf("round %d %s: the untransformed answer has %d series (HTTP %d), want %d\n%s", round, endpoint, len(baseAns.series), baseAns.code, len(sets), baseAns.body) + } + want := lrReferenceJoin(baseAns.series, instant, func(m map[string]string) map[string]string { + for _, s := range steps { + m = lrReference(m, s.dst, s.repl, s.src, s.re) + } + return m + }) + got := lrAsk(t, p, endpoint, q) + if got.code != 200 { + t.Fatalf("round %d %s %s: HTTP %d\n%s", round, endpoint, q, got.code, got.body) + } + if g, w := lrGot(got, instant), want; fmt.Sprint(g) != fmt.Sprint(w) { + t.Fatalf("round %d %s %s over %v\n got %v\nwant %v", round, endpoint, q, sets, g, w) + } + } + } +} diff --git a/internal/proxy/metric_agg.go b/internal/proxy/metric_agg.go index b0de00b52..8a1e1674e 100644 --- a/internal/proxy/metric_agg.go +++ b/internal/proxy/metric_agg.go @@ -203,8 +203,11 @@ func preserveMetricStreamIdentity(originalLogQL, translatedLogsQL string, withou if strings.Contains(translatedLogsQL, "| stats by (") { return translatedLogsQL } - if len(withoutLabels) > 0 || isBareMetricFunctionQuery(strings.TrimSpace(originalLogQL)) { - return addStatsByStreamClause(translatedLogsQL, logqlpkg.ParserCaptureLabels(originalLogQL)) + // A label_replace or label_join around the metric does not change which + // series it yields: the identity is that of the expression under it. + metric := stripLabelTransforms(originalLogQL) + if len(withoutLabels) > 0 || isBareMetricFunctionQuery(strings.TrimSpace(metric)) { + return addStatsByStreamClause(translatedLogsQL, logqlpkg.ParserCaptureLabels(metric)) } return translatedLogsQL } @@ -262,6 +265,12 @@ func (p *Proxy) handleInstantMetricPostAggregation(w http.ResponseWriter, r *htt return } translatedInner, withoutLabels := translator.ParseWithoutMarker(translatedInner) + translatedInner, transforms, regexErr := parseLabelTransformMarkers(translatedInner) + if regexErr != "" { + p.writeError(w, http.StatusBadRequest, regexErr) + p.metrics.RecordRequest("query", unwrapRecordedStatus(r.Context(), http.StatusBadRequest), time.Since(start)) + return + } translatedInner = preserveMetricStreamIdentity(postAgg.inner, translatedInner, withoutLabels) r = p.withRequestScope(r) @@ -300,6 +309,9 @@ func (p *Proxy) handleInstantMetricPostAggregation(w http.ResponseWriter, r *htt if len(withoutLabels) > 0 { bw.body = applyWithoutGrouping(bw.body, withoutLabels, withoutAggregationOp(originalQuery)) } + if sc.code < http.StatusBadRequest { + bw.body = transforms.apply(bw.body) + } if sc.code >= http.StatusBadRequest { copyHeaders(w.Header(), bw.Header()) @@ -335,6 +347,12 @@ func (p *Proxy) handleRangeMetricPostAggregation(w http.ResponseWriter, r *http. return } translatedInner, withoutLabels := translator.ParseWithoutMarker(translatedInner) + translatedInner, transforms, regexErr := parseLabelTransformMarkers(translatedInner) + if regexErr != "" { + p.writeError(w, http.StatusBadRequest, regexErr) + p.metrics.RecordRequest("query_range", unwrapRecordedStatus(r.Context(), http.StatusBadRequest), time.Since(start)) + return + } translatedInner = preserveMetricStreamIdentity(postAgg.inner, translatedInner, withoutLabels) r = p.withRequestScope(r) @@ -378,6 +396,9 @@ func (p *Proxy) handleRangeMetricPostAggregation(w http.ResponseWriter, r *http. if len(withoutLabels) > 0 { bw.body = applyWithoutGrouping(bw.body, withoutLabels, withoutAggregationOp(originalQuery)) } + if sc.code < http.StatusBadRequest { + bw.body = transforms.apply(bw.body) + } if sc.code >= http.StatusBadRequest { copyHeaders(w.Header(), bw.Header()) diff --git a/internal/proxy/ordered_json_metric.go b/internal/proxy/ordered_json_metric.go index 3791c3fda..ed60425ef 100644 --- a/internal/proxy/ordered_json_metric.go +++ b/internal/proxy/ordered_json_metric.go @@ -98,10 +98,47 @@ type orderedJSONMetricPlan struct { parser string } +// handleOrderedJSONMetricUnderTransform answers a label_replace or label_join +// around a metric that the ordered JSON evaluator owns: the evaluator answers +// the expression under the transforms, as Loki evaluates it, and the transforms +// are applied to what it answered. A query it does not own, or whose transform +// arguments are invalid, is left to the general path (which reports them). +func (p *Proxy) handleOrderedJSONMetricUnderTransform(w http.ResponseWriter, r *http.Request, requestStart time.Time, query, inner string, isRange bool) bool { + if _, ok := compileOrderedJSONMetric(inner); !ok { + return false + } + translated, err := p.translateQueryWithContext(r.Context(), query) + if err != nil { + return false + } + _, transforms, invalid := parseLabelTransformMarkers(translated) + if invalid != "" || (len(transforms.replaces) == 0 && transforms.join == nil) { + return false + } + buffered := &bufferedResponseWriter{header: make(http.Header)} + if !p.handleOrderedJSONMetric(buffered, cloneMetricQueryRequest(r, inner), requestStart, inner, isRange) { + return false + } + body := buffered.body + if buffered.code == 0 || buffered.code == http.StatusOK { + body = sortSeriesByLabels(transforms.apply(body), !isRange && queryIsSortable(query)) + } + copyBackendHeaders(w.Header(), buffered.Header()) + w.Header().Set("Content-Type", "application/json") + if buffered.code != 0 { + w.WriteHeader(buffered.code) + } + _, _ = w.Write(body) + return true +} + // handleOrderedJSONMetric uses the existing metric accumulators, but executes // parser-dependent stages before reducing rows to samples. VictoriaLogs does not // attach Loki's parser error labels, so pushing those filters into VL loses rows. func (p *Proxy) handleOrderedJSONMetric(w http.ResponseWriter, r *http.Request, requestStart time.Time, query string, isRange bool) bool { + if inner := stripLabelTransforms(query); inner != query { + return p.handleOrderedJSONMetricUnderTransform(w, r, requestStart, query, inner, isRange) + } plan, ok := compileOrderedJSONMetric(query) if !ok { return false diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 71c6076d0..7b8efa4b6 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -2362,7 +2362,7 @@ func (p *Proxy) handleQueryRange(w http.ResponseWriter, r *http.Request) { } } logsqlQuery = preserveMetricStreamIdentity(logqlQuery, logsqlQuery, withoutLabels) - if isBareMetricFunctionQuery(strings.TrimSpace(logqlQuery)) && !isStatsQuery(logsqlQuery) { + if isBareMetricFunctionQuery(strings.TrimSpace(stripLabelTransforms(logqlQuery))) && !isStatsQuery(logsqlQuery) { p.writeError(w, http.StatusBadRequest, "unsupported metric query: range aggregations require compatible unwrap or translator support") p.metrics.RecordRequest("query_range", http.StatusBadRequest, time.Since(start)) return @@ -2384,7 +2384,18 @@ func (p *Proxy) handleQueryRange(w http.ResponseWriter, r *http.Request) { } // Route using the original LogQL AST — more reliable than re-parsing translated markers. - parsedForRouting, _ := logqlpkg.Parse(logqlQuery) + // A label_replace or label_join around a binary expression routes by that + // expression: its operands are evaluated, then the transform is applied below. + routedQuery := logqlQuery + if len(labelReplaceSpecs) > 0 || labelJoinSpec != nil { + routedQuery = stripLabelTransforms(logqlQuery) + } + if routedQuery != logqlQuery { + // From the request's own query: logqlQuery may have lost a parser stage + // that one operand needs (preferWorkingParser). + r = cloneMetricQueryRequest(r, stripLabelTransforms(binaryRequestQuery(r))) + } + parsedForRouting, _ := logqlpkg.Parse(routedQuery) if binOp, ok := parsedForRouting.(*logqlpkg.BinOpExpr); ok { // Binary metric expression: sum(rate(...)) / sum(rate(...)) // translateBinOpSide handles scalar literals (e.g. * 100) without translation. @@ -2426,6 +2437,9 @@ func (p *Proxy) handleQueryRange(w http.ResponseWriter, r *http.Request) { if labelJoinSpec != nil { cacheOut = applyLabelJoin(cacheOut, *labelJoinSpec) } + if len(labelReplaceSpecs) > 0 || labelJoinSpec != nil { + cacheOut = sortSeriesByLabels(cacheOut, false) + } copyBackendHeaders(w.Header(), capture.Header()) if w.Header().Get("Content-Type") == "" { w.Header().Set("Content-Type", "application/json") @@ -2631,7 +2645,7 @@ func (p *Proxy) handleQuery(w http.ResponseWriter, r *http.Request) { } } logsqlQuery = preserveMetricStreamIdentity(logqlQuery, logsqlQuery, withoutLabels) - if isBareMetricFunctionQuery(strings.TrimSpace(logqlQuery)) && !isStatsQuery(logsqlQuery) { + if isBareMetricFunctionQuery(strings.TrimSpace(stripLabelTransforms(logqlQuery))) && !isStatsQuery(logsqlQuery) { p.writeError(w, http.StatusBadRequest, "unsupported metric query: range aggregations require compatible unwrap or translator support") p.metrics.RecordRequest("query", http.StatusBadRequest, time.Since(start)) return @@ -2654,6 +2668,11 @@ func (p *Proxy) handleQuery(w http.ResponseWriter, r *http.Request) { if len(labelReplaceSpecs) > 0 || labelJoinSpec != nil { routedQuery = stripLabelTransforms(logqlQuery) } + if routedQuery != logqlQuery { + // From the request's own query: logqlQuery may have lost a parser stage + // that one operand needs (preferWorkingParser). + r = cloneMetricQueryRequest(r, stripLabelTransforms(binaryRequestQuery(r))) + } parsedForRoutingQ, _ := logqlpkg.Parse(routedQuery) if binOp, ok := parsedForRoutingQ.(*logqlpkg.BinOpExpr); ok { leftLogsql, leftErr := p.translateBinOpSide(r.Context(), binOp.Left) @@ -2663,9 +2682,6 @@ func (p *Proxy) handleQuery(w http.ResponseWriter, r *http.Request) { } else if rightErr != nil { p.writeError(sc, http.StatusBadRequest, rightErr.Error()) } else { - if routedQuery != logqlQuery { - r = cloneMetricQueryRequest(r, routedQuery) - } p.proxyBinaryMetricQueryVM(sc, r, binOp.Op, leftLogsql, rightLogsql, binOpExprToVMInfo(binOp)) } } else if isStatsQuery(logsqlQuery) { @@ -2693,6 +2709,9 @@ func (p *Proxy) handleQuery(w http.ResponseWriter, r *http.Request) { if labelJoinSpec != nil { result = applyLabelJoin(result, *labelJoinSpec) } + if len(labelReplaceSpecs) > 0 || labelJoinSpec != nil { + result = sortSeriesByLabels(result, queryIsSortable(logqlQuery)) + } w.Header().Set("Content-Type", "application/json") w.Write(result) } diff --git a/internal/proxy/vector_matching.go b/internal/proxy/vector_matching.go index c3b2ee72a..77fd1d81c 100644 --- a/internal/proxy/vector_matching.go +++ b/internal/proxy/vector_matching.go @@ -659,58 +659,217 @@ func applyGroupNormalization(body []byte) []byte { return out } -// applyLabelReplace applies label_replace semantics to all series in a matrix response. -// For each series: new_value = regexp.ReplaceAll(src_label_value, replacement). -// If new_value is non-empty, dst_label is set; otherwise dst_label is deleted. -func applyLabelReplace(body []byte, spec translator.LabelReplaceSpec) []byte { - re, err := regexp.Compile("^(?:" + spec.Regex + ")$") - if err != nil { +// transformSeriesLabels rewrites the label set of every series of a matrix or +// a vector answer with fn, keeping the rest of the answer as it is. Series +// that end with one label set are not an error in Loki: an instant vector +// keeps each sample, a range matrix files their points under one series, those +// of each step in turn (pkg/logql/engine.go JoinSampleVector, vectorsToSeries). +func transformSeriesLabels(body []byte, fn func(map[string]string) map[string]string) []byte { + var top, data map[string]json.RawMessage + if json.Unmarshal(body, &top) != nil || json.Unmarshal(top["data"], &data) != nil { return body } - var resp matrixResponseRW - if err := json.Unmarshal(body, &resp); err != nil || resp.Data.ResultType != "matrix" { + var resultType string + var series []map[string]json.RawMessage + _ = json.Unmarshal(data["resultType"], &resultType) + if (resultType != "matrix" && resultType != "vector") || json.Unmarshal(data["result"], &series) != nil { return body } - for i, s := range resp.Data.Result { - src := s.Metric[spec.SrcLabel] - // Prometheus semantics: if regex matches, apply replacement; if not, series unchanged. - if re.MatchString(src) { - newVal := re.ReplaceAllString(src, spec.Replacement) - if newVal != "" { - resp.Data.Result[i].Metric[spec.DstLabel] = newVal - } else { - delete(resp.Data.Result[i].Metric, spec.DstLabel) + merged := make(map[string]int, len(series)) + out := series[:0] + for _, s := range series { + var metric map[string]string + _ = json.Unmarshal(s["metric"], &metric) + if metric == nil { + metric = map[string]string{} + } + metric = fn(metric) + s["metric"], _ = json.Marshal(metric) + if resultType == "matrix" { + key := binaryLabelKey(metric) + if at, dup := merged[key]; dup { + var more, have []json.RawMessage + _ = json.Unmarshal(s["values"], &more) + _ = json.Unmarshal(out[at]["values"], &have) + all := append(have, more...) + sort.SliceStable(all, func(i, j int) bool { return pointSeconds(all[i]) < pointSeconds(all[j]) }) + out[at]["values"], _ = json.Marshal(all) + continue } + merged[key] = len(out) } + out = append(out, s) } - out, err := json.Marshal(resp) + data["result"], _ = json.Marshal(out) + top["data"], _ = json.Marshal(data) + b, err := json.Marshal(top) if err != nil { return body } - return out + return b } -// applyLabelJoin applies label_join semantics to all series in a matrix response. -// dst_label is set to the src_labels values joined by separator. -func applyLabelJoin(body []byte, spec translator.LabelJoinSpec) []byte { - var resp matrixResponseRW - if err := json.Unmarshal(body, &resp); err != nil || resp.Data.ResultType != "matrix" { +// sortSeriesByLabels orders the series of a matrix or vector answer as Loki +// does after a label_replace: a range result by label set (pkg/logql/engine.go +// JoinSampleVector, sort.Sort(result)), an instant vector the same way unless +// the query sorts by value (sort, sort_desc: evaluator.go Sortable), in which +// case sortable is set and the order is kept. +func sortSeriesByLabels(body []byte, sortable bool) []byte { + var top, data map[string]json.RawMessage + if json.Unmarshal(body, &top) != nil || json.Unmarshal(top["data"], &data) != nil { return body } - for i, s := range resp.Data.Result { + var resultType string + var series []struct { + raw json.RawMessage + metric map[string]string + } + var raws []json.RawMessage + _ = json.Unmarshal(data["resultType"], &resultType) + if (resultType != "matrix" && resultType != "vector") || (resultType == "vector" && sortable) || json.Unmarshal(data["result"], &raws) != nil { + return body + } + for _, raw := range raws { + var m struct { + Metric map[string]string `json:"metric"` + } + _ = json.Unmarshal(raw, &m) + series = append(series, struct { + raw json.RawMessage + metric map[string]string + }{raw, m.Metric}) + } + sort.SliceStable(series, func(i, j int) bool { return compareLabelSets(series[i].metric, series[j].metric) < 0 }) + for i := range series { + raws[i] = series[i].raw + } + data["result"], _ = json.Marshal(raws) + top["data"], _ = json.Marshal(data) + if b, err := json.Marshal(top); err == nil { + return b + } + return body +} + +var sortCallRE = regexp.MustCompile(`(^|[^\w])sort(_desc)?\s*\(`) + +// queryIsSortable reports whether a query orders by value (evaluator.go +// Sortable: a sort or sort_desc anywhere in it). +func queryIsSortable(logql string) bool { return sortCallRE.MatchString(logql) } + +// compareLabelSets is Prometheus' labels.Compare: label by label in name +// order, the name first and then the value, a shorter prefix first. +func compareLabelSets(a, b map[string]string) int { + names := func(m map[string]string) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out + } + na, nb := names(a), names(b) + for i := 0; i < len(na) && i < len(nb); i++ { + if na[i] != nb[i] { + return strings.Compare(na[i], nb[i]) + } + if c := strings.Compare(a[na[i]], b[nb[i]]); c != 0 { + return c + } + } + return len(na) - len(nb) +} + +// pointSeconds is the time of a sample point `[time, "value"]`. +func pointSeconds(point json.RawMessage) float64 { + var p []any + if json.Unmarshal(point, &p) != nil || len(p) == 0 { + return 0 + } + t, _ := sampleTimeSeconds(p[0]) + return t +} + +// applyLabelReplace applies label_replace to every series of a matrix or vector +// answer as Loki's LabelReplaceEvaluator does (pkg/logql/evaluator.go): the +// regexp is anchored, a series it does not match is left as it is, and the +// expanded replacement ($1, ${1}, $name) replaces the destination label, which +// an empty expansion removes. +func applyLabelReplace(body []byte, spec translator.LabelReplaceSpec) []byte { + re, err := regexp.Compile("^(?:" + spec.Regex + ")$") + if err != nil { + return body + } + return transformSeriesLabels(body, func(in map[string]string) map[string]string { + src := in[spec.SrcLabel] + idx := re.FindStringSubmatchIndex(src) + if idx == nil { + return in + } + res := re.ExpandString(nil, spec.Replacement, src, idx) + out := make(map[string]string, len(in)+1) + for k, v := range in { + out[k] = v + } + delete(out, spec.DstLabel) + if len(res) > 0 { + out[spec.DstLabel] = string(res) + } + return out + }) +} + +// applyLabelJoin applies label_join to every series of a matrix or vector +// answer: the destination label is the values of the source labels, the +// absent ones skipped, joined by the separator. Loki v3.7.7 has no label_join; +// accepting it is the open gap semantics/proxy-accepts-operators-not-in-loki. +func applyLabelJoin(body []byte, spec translator.LabelJoinSpec) []byte { + return transformSeriesLabels(body, func(in map[string]string) map[string]string { parts := make([]string, 0, len(spec.SrcLabels)) for _, src := range spec.SrcLabels { - if v := s.Metric[src]; v != "" { + if v := in[src]; v != "" { parts = append(parts, v) } } - resp.Data.Result[i].Metric[spec.DstLabel] = strings.Join(parts, spec.Separator) + out := make(map[string]string, len(in)+1) + for k, v := range in { + out[k] = v + } + out[spec.DstLabel] = strings.Join(parts, spec.Separator) + return out + }) +} + +// labelTransforms are the label_replace and label_join calls of a query, in +// the order they apply: the innermost first. +type labelTransforms struct { + replaces []translator.LabelReplaceSpec + join *translator.LabelJoinSpec +} + +// parseLabelTransformMarkers strips the label_replace and label_join markers +// the translator left in a query. It returns Loki's parse error text when a +// label_replace regexp does not compile. +func parseLabelTransformMarkers(logsql string) (string, labelTransforms, string) { + logsql, replaces := translator.ParseAllLabelReplaceMarkers(logsql) + logsql, join := translator.ParseLabelJoinMarker(logsql) + for _, spec := range replaces { + if _, err := regexp.Compile("^(?:" + spec.Regex + ")$"); err != nil { + return logsql, labelTransforms{}, fmt.Sprintf("parse error : invalid regex in label_replace: %v", err) + } } - out, err := json.Marshal(resp) - if err != nil { - return body + return logsql, labelTransforms{replaces: replaces, join: join}, "" +} + +// apply runs the transforms over a matrix or vector answer. +func (t labelTransforms) apply(body []byte) []byte { + for _, spec := range t.replaces { + body = applyLabelReplace(body, spec) } - return out + if t.join != nil { + body = applyLabelJoin(body, *t.join) + } + return body } // sampleTimeSeconds reads a sample time of a decoded response: a JSON number, or diff --git a/internal/translator/coverage_test.go b/internal/translator/coverage_test.go index 46e4dd860..9e4ab7bba 100644 --- a/internal/translator/coverage_test.go +++ b/internal/translator/coverage_test.go @@ -184,3 +184,15 @@ func TestCoverage_AddByClause_DeduplicatesTranslatedLabels(t *testing.T) { t.Fatalf("addByClause returned %q, want %q", got, want) } } + +// A marker appended after the without marker ends its label list and stays on +// the query, so the label_replace, label_join and group markers of a +// `label_replace(sum without (pod) (...), ...)` still reach their parsers. +func TestParseWithoutMarkerBeforeOtherMarkers(t *testing.T) { + for _, tail := range []string{"__lvp_lr:YQBiAGMAZA==__", "__lvp_lj:YQBiAGM=__", "__lvp_group__"} { + clean, labels := ParseWithoutMarker(`app:="api"` + WithoutMarkerSuffix + `pod,node` + tail) + if clean != `app:="api"`+tail || len(labels) != 2 || labels[0] != "pod" || labels[1] != "node" { + t.Errorf("tail %q: clean=%q labels=%v, want the marker kept and the labels pod,node", tail, clean, labels) + } + } +} diff --git a/internal/translator/translator.go b/internal/translator/translator.go index 384bce345..4edf95ac0 100644 --- a/internal/translator/translator.go +++ b/internal/translator/translator.go @@ -525,6 +525,12 @@ func ParseWithoutMarker(query string) (cleanQuery string, excludeLabels []string } cleanQuery = query[:idx] labelStr := query[idx+len(WithoutMarkerSuffix):] + // A marker appended after this one (label_replace, label_join, group) + // ends the label list and stays on the query for its own parser. + if end := strings.Index(labelStr, "__lvp_"); end >= 0 { + cleanQuery += labelStr[end:] + labelStr = labelStr[:end] + } for _, l := range strings.Split(labelStr, ",") { l = strings.TrimSpace(l) if l != "" { @@ -1802,6 +1808,29 @@ func parseAllStringArgs(s string) []string { return args } +// parseUnquotedStringArgs reads the comma-separated string arguments that +// follow the first argument of label_replace or label_join, unquoted as the +// LogQL lexer does: a double-quoted string has its escapes resolved, a +// backtick one is raw (pkg/logql/syntax/lex.go). It stops at ')' or at the +// first argument that is not a valid string. +func parseUnquotedStringArgs(s string) []string { + var args []string + for { + s = strings.TrimSpace(s) + quoted, err := strconv.QuotedPrefix(s) + if err != nil { + break + } + arg, err := strconv.Unquote(quoted) + if err != nil { + break + } + args = append(args, arg) + s = strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(s[len(quoted):]), ",")) + } + return args +} + // tryTranslateLabelReplace handles label_replace(v, "dst", "repl", "src", "regex"). // It translates the inner expression and embeds a marker for proxy post-processing. func tryTranslateLabelReplace(logql string, labelFn LabelTranslateFunc) (string, bool) { @@ -1813,7 +1842,7 @@ func tryTranslateLabelReplace(logql string, labelFn LabelTranslateFunc) (string, if !ok { return "", false } - args := parseAllStringArgs(rest) + args := parseUnquotedStringArgs(rest) if len(args) != 4 { return "", false } @@ -1838,7 +1867,7 @@ func tryTranslateLabelJoin(logql string, labelFn LabelTranslateFunc) (string, bo if !ok { return "", false } - args := parseAllStringArgs(rest) + args := parseUnquotedStringArgs(rest) if len(args) < 3 { // dst + sep + at least one src return "", false } diff --git a/internal/translator/translator_test.go b/internal/translator/translator_test.go index 7e19eb15f..bc786e878 100644 --- a/internal/translator/translator_test.go +++ b/internal/translator/translator_test.go @@ -871,6 +871,33 @@ func TestLabelReplaceTranslation(t *testing.T) { } } +// The string arguments of label_replace are LogQL strings: a double-quoted one +// is unescaped (pkg/logql/syntax/lex.go), so "\\." is the regexp \. and not a +// backslash followed by any character; a backtick one is taken as it is. +func TestLabelReplaceStringArgumentsAreUnquoted(t *testing.T) { + for _, tc := range []struct{ name, in, dst, repl, src, regex string }{ + {"escaped dot", `label_replace(rate({app="api"}[5m]), "d", "$1", "s", "(\\w+)\\.(\\w+)")`, "d", "$1", "s", `(\w+)\.(\w+)`}, + {"escaped quote", `label_replace(rate({app="api"}[5m]), "d", "a\"b", "s", "x")`, "d", `a"b`, "s", "x"}, + {"backtick is raw", "label_replace(rate({app=\"api\"}[5m]), `d`, `$1`, `s`, `(\\w+)\\.`)", "d", "$1", "s", `(\w+)\.`}, + {"escaped newline", `label_replace(rate({app="api"}[5m]), "d", "a\nb", "s", "x")`, "d", "a\nb", "s", "x"}, + {"comma and paren in a string", `label_replace(rate({app="api"}[5m]), "d", "a,b)", "s", "(x),(y)")`, "d", "a,b)", "s", "(x),(y)"}, + } { + t.Run(tc.name, func(t *testing.T) { + result, err := TranslateLogQL(tc.in) + if err != nil { + t.Fatal(err) + } + _, spec := ParseLabelReplaceMarker(result) + if spec == nil { + t.Fatalf("no marker in %q", result) + } + if spec.DstLabel != tc.dst || spec.Replacement != tc.repl || spec.SrcLabel != tc.src || spec.Regex != tc.regex { + t.Fatalf("spec %+v, want dst=%q repl=%q src=%q regex=%q", *spec, tc.dst, tc.repl, tc.src, tc.regex) + } + }) + } +} + func TestParseAllLabelReplaceMarkers(t *testing.T) { t.Run("single marker", func(t *testing.T) { q := `label_replace(rate({app="api"}[5m]), "host", "$1", "instance", "(.*):.+")` diff --git a/test/e2e-compat/label_replace_new_label_test.go b/test/e2e-compat/label_replace_new_label_test.go new file mode 100644 index 000000000..b3ab40cb7 --- /dev/null +++ b/test/e2e-compat/label_replace_new_label_test.go @@ -0,0 +1,242 @@ +//go:build e2e + +package e2e_compat + +import ( + "fmt" + "net/http" + "net/url" + "sort" + "strings" + "testing" + "time" +) + +// label_replace sets, rewrites or removes one label of every series its +// expression yields (pkg/logql/evaluator.go LabelReplaceEvaluator.Next): the +// regexp is anchored, a series it does not match is left alone, an empty +// replacement removes the destination, and series that end with one label set +// are not an error (pkg/logql/engine.go JoinSampleVector files their points +// under one series on a range query, keeps each sample on an instant query). +// +// The proxy's instant answers lacked the destination label (the transform only +// ran on range matrices), an escaped regexp such as `\.` never matched, and a +// label_replace around a binary expression on query_range sent the proxy's +// internal binary marker to VictoriaLogs. +// +// Every shape is asked of Loki and of the proxy on query_range and on query. +// Both must answer the same status; when they answer, the same result type, the +// same label sets (and, on query, the same number of samples per label set). The +// values and the points per step are compared by the other registry items. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestCompat_LabelReplaceNewLabel(t *testing.T) { + ensureDataIngested(t) + waitForLokiMetricData(t) + + // The structured metadata of the fixture streams that Loki adds to a bare + // range function's series (see the bare field below). + structuredMetadataLabels := map[string]bool{ + "cloud_provider": true, "cloud_region": true, "deployment_name": true, "deployment_version": true, + "host_name": true, "k8s_node_name": true, "queue_name": true, "request_id": true, "span_id": true, + "telemetry_sdk_language": true, "telemetry_sdk_name": true, "telemetry_sdk_version": true, + "trace_id": true, "user_agent": true, "worker_id": true, + } + + sel := `{env="production"}` + count := `count_over_time(` + sel + `[5m])` + rate := `rate(` + sel + `[5m])` + agg := `sum by (app) (` + count + `)` + lr := func(inner, dst, repl, src, re string) string { + return fmt.Sprintf(`label_replace(%s, %q, %q, %q, %q)`, inner, dst, repl, src, re) + } + shapes := []struct { + name string + query string + // points compares the points per series of a range answer as well: + // where two series end with one label set, Loki files the points of + // both under one series. + points bool + // bare marks a range function with no aggregation: Loki's series carry + // the entry's structured metadata and the proxy's keep the stream + // (registry: metric-identity/structured-metadata-splits-series, an + // owner-accepted deviation), so those labels are left out of both + // sides. The destination label, the stream labels and detected_level + // are still compared. + bare bool + }{ + // the request of the registry row, its bare form, and the clusters' forms + {"row", `label_replace(sum by (app) (rate({app="api-gateway"}[5m])), "app_short", "$1", "app", "(.*)-.*")`, false, false}, + {"bare rate", `label_replace(rate({app="api-gateway"}[5m]), "app_short", "$1", "app", "(.*)-.*")`, false, false}, + {"bare count_over_time", lr(count, "app_short", "$1", "app", "(.*)-.*"), false, true}, + {"bytes_over_time", lr(`bytes_over_time(`+sel+`[5m])`, "buzz", "$2", "app", "(.*)-(.*)"), false, true}, + {"nested", lr(lr(agg, "service", "$1", "app", "(.*)"), "short", "$1", "service", "^([^-]+)"), false, false}, + {"nested, the outer reads the inner's label", lr(lr(agg, "a1", "$1", "app", "(.*)-.*"), "a2", "x$1", "a1", "(.*)"), false, false}, + {"nested, three deep", lr(lr(lr(agg, "a", "$1", "app", "(.*)-.*"), "b", "$1$1", "a", "(.*)"), "c", "${1}!", "b", "(.*)"), false, false}, + // regexp and replacement rules + {"anchored: a prefix does not match", lr(agg, "d", "x", "app", "api"), false, false}, + {"anchored: the whole value", lr(agg, "d", "x", "app", "api-gateway"), false, false}, + {"no match keeps the series", lr(agg, "new_label", "default", "app", "^nonexistent$"), false, false}, + {"escaped regexp", lr(agg, "d", "$1", "app", `([a-z]+)\-(\w+)`), false, false}, + {"escaped dot never matches a hyphen", lr(agg, "d", "$1", "app", `(\w+)\.(\w+)`), false, false}, + {"named group", lr(agg, "d", "${head}", "app", `(?P[a-z]+)-.*`), false, false}, + {"bare group name swallows the suffix", lr(agg, "d", "$1x", "app", "(.*)-.*"), false, false}, + {"two groups", lr(agg, "d", "$2/$1", "app", "([a-z]+)-([a-z]+)"), false, false}, + {"constant replacement", lr(agg, "d", "const", "app", ".*"), false, false}, + {"missing source, empty-matching regexp", lr(agg, "d", "dflt", "nope", "(.*)"), false, false}, + {"missing source, non-empty regexp", lr(agg, "d", "dflt", "nope", ".+"), false, false}, + {"empty replacement removes the destination", lr(agg, "app", "", "app", "api-gateway"), false, false}, + {"destination equal to the source", lr(agg, "app", "$1", "app", "(.*)-.*"), false, false}, + {"two series with one label set", lr(agg, "app", "all", "app", ".+"), true, false}, + {"all labels removed", lr(agg, "app", "", "app", ".*"), true, false}, + {"label names are not validated", lr(agg, "1bad", "x", "app", ".*"), false, false}, + // around and with other expressions + {"over a binary expression", lr(agg+` * 2`, "app_short", "$1", "app", "(.*)-.*"), false, false}, + {"over a binary of two aggregations", lr(agg+` / `+agg, "app_short", "$1", "app", "(.*)-.*"), false, false}, + {"over a rate aggregation", lr(`sum by (app) (`+rate+`)`, "app_short", "$1", "app", "(.*)-.*"), false, false}, + {"over a parser", lr(`sum by (app) (count_over_time(`+sel+` | json | __error__="" [5m]))`, "app_short", "$1", "app", "(.*)-.*"), false, false}, + {"under sort", `sort(` + lr(agg, "app_short", "$1", "app", "(.*)-.*") + `)`, false, false}, + {"under topk", `topk(1, ` + lr(agg, "app_short", "$1", "app", "(.*)-.*") + `)`, false, false}, + // an invalid regexp is Loki's parse error + {"invalid regexp", lr(agg, "new", "$1", "app", "[invalid"), false, false}, + {"invalid regexp, nested", lr(lr(agg, "d", "$1", "app", "(.*)"), "e", "$1", "d", "(unclosed"), false, false}, + {"invalid regexp under topk", `topk(2, ` + lr(agg, "new", "$1", "app", "*bad") + `)`, false, false}, + // a call the grammar rejects + {"invalid escape in a string", `label_replace(` + agg + `, "d", "$1", "app", "(.+):\d+")`, false, false}, + {"three string arguments", `label_replace(` + agg + `, "d", "$1", "app")`, false, false}, + {"five string arguments", `label_replace(` + agg + `, "d", "$1", "app", "x", "y")`, false, false}, + } + + // labelCounts reads a decoded answer as {label set: samples}: an instant + // vector's samples per label set, a range series' points when points is + // set and 1 otherwise (the points per step are other registry items'). + labelCounts := func(payload map[string]interface{}, instant, points, bare bool) map[string]int { + out := map[string]int{} + data, _ := payload["data"].(map[string]interface{}) + result, _ := data["result"].([]interface{}) + for _, item := range result { + series, _ := item.(map[string]interface{}) + metric, _ := series["metric"].(map[string]interface{}) + if bare { + kept := map[string]interface{}{} + for name, value := range metric { + if !structuredMetadataLabels[name] { + kept[name] = value + } + } + metric = kept + } + key := canonicalMetricKey(metric) + if instant { + out[key]++ + continue + } + values, _ := series["values"].([]interface{}) + if points { + out[key] += len(values) + } else { + out[key] = 1 + } + } + return out + } + resultType := func(payload map[string]interface{}) string { + data, _ := payload["data"].(map[string]interface{}) + rt, _ := data["resultType"].(string) + return rt + } + render := func(m map[string]int) string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, fmt.Sprintf("%s x%d", k, m[k])) + } + sort.Strings(keys) + return strings.Join(keys, "\n ") + } + + for _, shape := range shapes { + for _, endpoint := range []string{"query_range", "query"} { + t.Run(shape.name+"/"+endpoint, func(t *testing.T) { + instant := endpoint == "query" + at := time.Now().UTC().Truncate(time.Minute) + params := url.Values{"query": {shape.query}} + if instant { + params.Set("time", fmt.Sprintf("%d", at.UnixNano())) + } else { + params.Set("start", fmt.Sprintf("%d", at.Add(-15*time.Minute).UnixNano())) + params.Set("end", fmt.Sprintf("%d", at.UnixNano())) + params.Set("step", "60") + } + headers := map[string]string{"X-Scope-OrgID": "0", "X-Loki-Response-Encoding-Flags": "categorize-labels"} + full := "/loki/api/v1/" + endpoint + "?" + params.Encode() + lokiStatus, lokiBody, lokiPayload := doJSONGET(t, lokiURL+full, headers) + proxyStatus, proxyBody, proxyPayload := doJSONGET(t, proxyURL+full, headers) + if lokiStatus != proxyStatus { + t.Fatalf("HTTP status loki=%d proxy=%d\nloki: %s\nproxy: %s", lokiStatus, proxyStatus, lokiBody, proxyBody) + } + if lokiStatus != http.StatusOK { + if lokiStatus == http.StatusBadRequest && !strings.Contains(proxyBody, "parse error") { + t.Errorf("the proxy's 400 is not Loki's parse error\nloki: %s\nproxy: %s", lokiBody, proxyBody) + } + if strings.Contains(shape.name, "invalid regexp") && !strings.Contains(proxyBody, "invalid regex in label_replace") { + t.Errorf("the proxy's error is not Loki's invalid regex error\nloki: %s\nproxy: %s", lokiBody, proxyBody) + } + return + } + if lt, pt := resultType(lokiPayload), resultType(proxyPayload); lt != pt { + t.Fatalf("resultType loki=%q proxy=%q\nloki: %s\nproxy: %s", lt, pt, lokiBody, proxyBody) + } + lokiSeries, proxySeries := labelCounts(lokiPayload, instant, shape.points, shape.bare), labelCounts(proxyPayload, instant, shape.points, shape.bare) + if len(lokiSeries) == 0 { + t.Skipf("Loki answered no series for %s; nothing to compare", shape.query) + } + if render(lokiSeries) != render(proxySeries) { + t.Errorf("label sets differ\nloki:\n %s\nproxy:\n %s", render(lokiSeries), render(proxySeries)) + } + }) + } + } +} + +// Two series that a label_replace gives one label set are ranked as two +// samples by the vector aggregation above it, and filed under one series only +// at the end (pkg/logql/evaluator.go VectorAggEvaluator.Next, pkg/logql/engine.go +// vectorsToSeries). The proxy merges them before the ranking +// (semantics/label-replace-duplicates-before-ranking, open). +// conformance: semantics/label-replace-duplicates-before-ranking, loki_api_v1_query_range +func TestCompat_LabelReplaceDuplicatesBeforeRanking(t *testing.T) { + ensureDataIngested(t) + waitForLokiMetricData(t) + + at := time.Now().UTC().Truncate(time.Minute) + for _, query := range []string{ + `topk(1, label_replace(sum by (app) (count_over_time({env="production"}[5m])), "app", "all", "app", ".+"))`, + `bottomk(1, label_replace(sum by (app) (count_over_time({env="production"}[5m])), "app", "all", "app", ".+"))`, + } { + params := url.Values{ + "query": {query}, + "start": {fmt.Sprintf("%d", at.Add(-15*time.Minute).UnixNano())}, + "end": {fmt.Sprintf("%d", at.UnixNano())}, + "step": {"60"}, + } + full := "/loki/api/v1/query_range?" + params.Encode() + lokiStatus, lokiBody, lokiPayload := doJSONGET(t, lokiURL+full, nil) + proxyStatus, proxyBody, proxyPayload := doJSONGET(t, proxyURL+full, nil) + if lokiStatus != http.StatusOK || proxyStatus != http.StatusOK { + t.Fatalf("%s: HTTP loki=%d proxy=%d\nloki: %s\nproxy: %s", query, lokiStatus, proxyStatus, lokiBody, proxyBody) + } + count := func(payload map[string]interface{}) int { + data, _ := payload["data"].(map[string]interface{}) + result, _ := data["result"].([]interface{}) + points := 0 + for _, item := range result { + series, _ := item.(map[string]interface{}) + values, _ := series["values"].([]interface{}) + points += len(values) + } + return points + } + if l, p := count(lokiPayload), count(proxyPayload); l != p { + t.Errorf("%s: points loki=%d proxy=%d\nloki: %s\nproxy: %s", query, l, p, lokiBody, proxyBody) + } + } +}