From 0c852a1fb38f0a0ddb4db06918fc44ca14539143 Mon Sep 17 00:00:00 2001 From: szibis Date: Mon, 12 Oct 2026 12:27:11 +0200 Subject: [PATCH 01/18] fix: an instant query keeps its failure status and always answers a resultType A query handleQuery post-processes (without, label_replace, label_join) was buffered and a failure of it was written with the status dropped, so an error reached the client as a 200 with no data and no resultType; query_range already sent the failing status, the instant path now does too. A label_replace or label_join around a binary expression sent the translator's internal binary marker to VictoriaLogs as a query; the expression under the transform now routes like any binary expression. Also on the instant endpoint: a scalar literal answers a scalar as Loki's evalLiteral does, vector() stamps its sample in seconds, and a log query with every tenant filtered out is rejected with Loki's 400 instead of an empty streams answer. --- CHANGELOG.md | 29 ++ .../instant-result-type-missing.yaml | 59 ++- internal/proxy/coverage_gaps_test.go | 12 +- internal/proxy/instant_result_type_test.go | 449 ++++++++++++++++++ internal/proxy/metric_agg.go | 29 +- internal/proxy/multitenant.go | 5 + internal/proxy/proxy.go | 26 +- internal/proxy/range_metric_compat.go | 16 +- test/e2e-compat/instant_result_type_test.go | 160 +++++++ 9 files changed, 761 insertions(+), 24 deletions(-) create mode 100644 internal/proxy/instant_result_type_test.go create mode 100644 test/e2e-compat/instant_result_type_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index feca92cbd..7cbda1928 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- **An instant query is answered with its failure status and a resultType, as + Loki does, not with a 200 that has neither (#TBD).** `stdvar without (app) + (...)` and `label_replace(avg by (app) (a + b / c) * 2, ...)` came back as + HTTP 200 with an error body and no `resultType`, which Grafana and every client + decoding by type (`pkg/loghttp/query.go` `QueryResponse.UnmarshalJSON` fails on + a missing or unknown type) cannot read, where Loki answers `vector` + (`pkg/logql/engine.go` `evalSample`, `JoinSampleVector`; `pkg/querier/queryrange/marshal.go` + `ResultToResponse`). Two causes. A query the proxy post-processes (`without`, + `label_replace`, `label_join`) was buffered, and a failure of it (a + VictoriaLogs error, a series limit, a rejected query) was written with the + status dropped, so every such error reached the client as a 200; query_range + already sent the failing status, the instant path now does too. And a + `label_replace` or `label_join` around a binary expression sent the + translator's internal binary marker to VictoriaLogs as a LogsQL query; the + expression under the transform now routes like any binary expression, its + operands run as instant queries and combine, and the transform applies to the + result. Also on the instant endpoint: a scalar literal (`1`, `-1`) answers a + `scalar` as Loki's `evalLiteral` does, where it was rejected as a log query, + also under a multi-tenant `X-Scope-OrgID`; `vector(1)` and `vector(1) + + vector(2)` stamp their sample in seconds like every other sample + (`pkg/util/marshal/query.go` `NewVector`) where they carried nanoseconds; and + a log query sent with every tenant filtered out by `__tenant_id__` is rejected + with Loki's 400 for a log query as an instant query, not answered with an + empty `streams`. An aggregation over a binary expression (`sum by (job) (a / + b)`) is still not evaluated as Loki does; where the proxy rejects it, it now + does so with an error status, no longer with a typeless 200. + ## [2.11.2] - 2026-10-12 ### CI diff --git a/conformance/registry/cases/semantics/instant-result-type-missing.yaml b/conformance/registry/cases/semantics/instant-result-type-missing.yaml index e880cf8e1..853e9a57e 100644 --- a/conformance/registry/cases/semantics/instant-result-type-missing.yaml +++ b/conformance/registry/cases/semantics/instant-result-type-missing.yaml @@ -1,13 +1,50 @@ id: semantics/instant-result-type-missing -title: Some instant binary/label_replace answers carry no resultType (open) +title: An instant query answers its failure status and a resultType, like Loki (fixed) proves: - loki_api_v1_query consumer: grafana-datasource why: | - Loki answers every instant metric query with `resultType: vector`. For - `label_replace(avg by (app) (sum by (app) (rate(...)) + sum by (app) - (rate(...))), ...)` and `stdvar without (app) (...)` the proxy's 200 answer - has no resultType, which Grafana and API clients cannot decode. + Loki answers every metric instant query with `resultType: vector`, a scalar + literal with `scalar`, and rejects a log query with 400 (pkg/logql/engine.go + evalSample ends in JoinSampleVector, which returns the vector for an instant + query; evalLiteral returns a promql.Scalar; evalVector a one-sample vector; + pkg/querier/queryrange/marshal.go ResultToResponse labels promql.Vector + loghttp.ResultTypeVector and promql.Scalar loghttp.ResultTypeScalar; + pkg/querier/http.go InstantQueryHandler fails a log selector with + ErrUnsupportedSyntaxForInstantQuery, HTTP 400). Its own decoder, and the + Grafana Loki datasource, read the answer by that type + (pkg/loghttp/query.go QueryResponse.UnmarshalJSON: a missing or unknown + resultType is an error). + + The proxy's 200 answer for `label_replace(avg by (app) (sum by (app) (rate(...)) + + sum by (app) (rate(...))) ...)` and `stdvar without (app) (...)` had no + resultType (parity run of 2026-10-05, clusters 307bbdd1, f2066991, ce65447e: + both sides 200, "vector vs None"). Two causes, both reproduced against a + simulated VictoriaLogs (internal/proxy/instant_result_type_test.go): + - A query the proxy post-processes (`without`, `label_replace`, + `label_join`) is buffered by handleQuery, and the buffered answer was + written with the failing status dropped: any VictoriaLogs error, series + limit or rejection became a 200 whose body is the error, with no data. The + range handler already sent the failing status; the instant handler now + does too. + - A `label_replace` or `label_join` around a binary expression translated to + `__binary__::|||` plus the transform marker. With the + marker stripped, the AST routing saw an opaque expression, found a stats + pipe in the text and sent the binary marker to VictoriaLogs as a query. The + expression under the transform (stripLabelTransforms) now routes like any + binary expression. + Found on the way, same endpoint, same symptom class: a scalar literal (`1`) + was rejected as a log query (Loki: scalar) and a scalar under a multi-tenant + X-Scope-OrgID merged to an empty result; vector() carried nanosecond sample + times; a log query with every tenant filtered out answered an empty `streams` + (Loki: 400). + + Not fixed here, registered separately: an aggregation over a binary + expression (`sum by (job) (a / b)`) is translated to its first operand only, + so the value is wrong when it is answered and the shape of cluster f2066991 + (`| json | unwrap duration(latency)` over `/ count_over_time`) is rejected + with a 400; and the same `__binary__` marker still reaches + stats_query_range on query_range for a label_replace around a binary. request: method: GET path: /loki/api/v1/query @@ -22,13 +59,15 @@ compare: expect: | vector. gap: - status: open + status: fixed + fixed_by: "#TBD" impact: grafana-datasource effort: S - area: 'internal/proxy instant response assembly (binary/label_replace/without paths)' - loki: 'pkg/logqlmodel result types; pkg/util/marshal query response' - proxy: 'resultType missing' - planned_test: 'TestCompat_InstantResultTypeAlwaysSet (test/e2e-compat, group semantics)' + area: 'internal/proxy handleQuery (captured post-processing status, label transform routing, constants), multitenant.go' + loki: 'pkg/logql/engine.go evalSample/evalLiteral/evalVector, JoinSampleVector; pkg/querier/queryrange/marshal.go ResultToResponse; pkg/querier/http.go InstantQueryHandler; pkg/loghttp/query.go UnmarshalJSON' + proxy: 'the failing status of a post-processed instant query is kept; a label transform over a binary expression evaluates the expression; literals answer scalar' + planned_test: 'TestCompat_InstantResultTypeAlwaysSet (test/e2e-compat, group semantics); TestInstantQueryAlwaysHasLokiResultType, TestInstantQueryPostProcessedBackendFailureKeepsStatus, TestInstantConstantsCarrySecondsTimestamps, TestInstantQueryMultiTenantResultTypes and TestInstantLabelTransformOfBinaryEvaluatesTheBinary (internal/proxy)' + note: 'The e2e is written and not yet run against the compose stack. Cluster f2066991 stays claimed here until the aggregation-over-binary row is registered: its queries no longer answer a typeless 200 but are still rejected (400) or answered from their first operand only. Cluster ce65447e is owned by semantics/stddev-stdvar-grouped-values; its typeless answer is fixed here, its values are not.' clusters: - 307bbdd1 - f2066991 diff --git a/internal/proxy/coverage_gaps_test.go b/internal/proxy/coverage_gaps_test.go index f57f75606..8750a8c1f 100644 --- a/internal/proxy/coverage_gaps_test.go +++ b/internal/proxy/coverage_gaps_test.go @@ -278,7 +278,7 @@ func TestParseTimestampToUnix(t *testing.T) { } } -func TestEvaluateConstantInstantVectorQuery(t *testing.T) { +func TestEvaluateConstantInstantQuery(t *testing.T) { tests := []struct { name string query string @@ -287,18 +287,18 @@ func TestEvaluateConstantInstantVectorQuery(t *testing.T) { wantTS float64 expectEval bool }{ - {name: "vector literal", query: `vector(1)`, timeParam: "4", wantValue: "1", wantTS: 4_000_000_000, expectEval: true}, - {name: "binary add", query: `vector(1)+vector(1)`, timeParam: "4", wantValue: "2", wantTS: 4_000_000_000, expectEval: true}, - {name: "binary divide", query: `vector(9) / vector(3)`, timeParam: "4", wantValue: "3", wantTS: 4_000_000_000, expectEval: true}, + {name: "vector literal", query: `vector(1)`, timeParam: "4", wantValue: "1", wantTS: 4, expectEval: true}, + {name: "binary add", query: `vector(1)+vector(1)`, timeParam: "4", wantValue: "2", wantTS: 4, expectEval: true}, + {name: "binary divide", query: `vector(9) / vector(3)`, timeParam: "4", wantValue: "3", wantTS: 4, expectEval: true}, {name: "divide by zero", query: `vector(1)/vector(0)`, timeParam: "4", expectEval: false}, {name: "non vector", query: `{app="api"}`, timeParam: "4", expectEval: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - body, ok := evaluateConstantInstantVectorQuery(tt.query, tt.timeParam) + body, ok := evaluateConstantInstantQuery(tt.query, tt.timeParam) if ok != tt.expectEval { - t.Fatalf("evaluateConstantInstantVectorQuery(%q) ok=%v, want %v", tt.query, ok, tt.expectEval) + t.Fatalf("evaluateConstantInstantQuery(%q) ok=%v, want %v", tt.query, ok, tt.expectEval) } if !tt.expectEval { return diff --git a/internal/proxy/instant_result_type_test.go b/internal/proxy/instant_result_type_test.go new file mode 100644 index 000000000..1510ce11b --- /dev/null +++ b/internal/proxy/instant_result_type_test.go @@ -0,0 +1,449 @@ +package proxy + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "testing" + "time" +) + +// An instant query answers with the result type of its expression, never +// without one. Loki v3.7.7: +// +// - every metric expression is a vector: pkg/logql/engine.go evalSample ends +// in JoinSampleVector (instant: `return vec, nil`), and vector(n) is a +// one-sample vector with no labels (evalVector); pkg/querier/queryrange +// /marshal.go ResultToResponse labels promql.Vector loghttp.ResultTypeVector; +// - a scalar literal, or a binary operation of two literals which the parser +// folds into one (syntax/ast.go mustNewBinOpExpr), is a scalar (evalLiteral); +// - a log selector is no instant query: pkg/querier/http.go +// InstantQueryHandler fails it with ErrUnsupportedSyntaxForInstantQuery +// (HTTP 400, pkg/util/server/error.go), so an instant answer is never +// `streams`; +// - a request wholly before max_query_lookback is answered with an empty +// vector, whatever the expression (queryrange/codec.go NewEmptyResponse, +// LokiInstantRequest). +// +// The sample time of a vector or scalar is seconds (model.Time, millisecond +// precision), not nanoseconds (pkg/util/marshal/query.go NewVector, NewScalar). + +type instantBackendMode string + +const ( + instantBackendData instantBackendMode = "data" + instantBackendEmpty instantBackendMode = "empty" + instantBackendNoType instantBackendMode = "no-result-type" + instantBackendBad400 instantBackendMode = "backend-400" + instantBackendDown500 instantBackendMode = "backend-500" +) + +// instantBackend is a VictoriaLogs that answers every stats_query and every raw +// query in one mode: with series, with none, without the resultType field, +// or with an error status. +func instantBackend(t *testing.T, mode instantBackendMode) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = r.ParseForm() + if q := r.FormValue("query"); strings.Contains(q, "__binary__") || strings.Contains(q, "|||") { + // VictoriaLogs cannot parse an internal marker: a marker that + // reaches it is a proxy bug, answered as VictoriaLogs would. + t.Errorf("an internal marker reached VictoriaLogs: %s", q) + http.Error(w, "cannot parse query: unexpected token", http.StatusBadRequest) + return + } + switch r.URL.Path { + case "/select/logsql/stats_query": + switch mode { + case instantBackendEmpty: + _, _ = w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[]}}`)) + case instantBackendNoType: + _, _ = w.Write([]byte(`{"status":"success","data":{"result":[{"metric":{"app":"a"},"value":[1704067200,"5"]}]}}`)) + case instantBackendBad400: + http.Error(w, "cannot parse stats", http.StatusBadRequest) + case instantBackendDown500: + http.Error(w, "storage unavailable", http.StatusInternalServerError) + default: + _, _ = w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[` + + `{"metric":{"app":"a","_stream":"{app=\"a\"}","level":"info"},"value":[1704067200,"5"]},` + + `{"metric":{"app":"b","_stream":"{app=\"b\"}","level":"info"},"value":[1704067200,"7"]}]}}`)) + } + case "/select/logsql/query": + switch mode { + case instantBackendBad400: + http.Error(w, "cannot parse query", http.StatusBadRequest) + case instantBackendDown500: + http.Error(w, "storage unavailable", http.StatusInternalServerError) + case instantBackendEmpty: + default: + a, b := "level=info latency=12 bytes=5", "level=info latency=30 bytes=9" + if strings.Contains(r.FormValue("query"), "unpack_json") { + a, b = `{"level":"info","latency":12}`, `{"level":"info","latency":30}` + } + _, _ = w.Write([]byte(`{"_time":"2024-01-01T00:00:00Z","_msg":` + strconv.Quote(a) + `,"_stream":"{app=\"a\"}","app":"a"}` + "\n" + + `{"_time":"2024-01-01T00:00:10Z","_msg":` + strconv.Quote(b) + `,"_stream":"{app=\"b\"}","app":"b"}` + "\n")) + } + case "/health": + _, _ = w.Write([]byte("ok")) + default: + _, _ = w.Write([]byte(`{"values":[]}`)) + } + })) + t.Cleanup(srv.Close) + return srv +} + +type instantShape struct { + name string + query string + // want is the resultType of a successful answer; "error" is a rejected + // query (HTTP 400) on every backend. + want string +} + +// instantRejectedShapes are the shapes the proxy does not evaluate yet, an +// aggregation over a binary operation: it may answer with an error status, but +// never with a 200 that has no resultType. +var instantRejectedShapes = map[string]bool{ + "parity f2066991 unwrap ratio": true, + "parity f2066991 offset": true, +} + +const instantNS = `{namespace="prod"}` + +func instantShapes() []instantShape { + rate := `rate(` + instantNS + `[1m])` + count := `count_over_time(` + instantNS + `[1m])` + return []instantShape{ + // native stats + {"range aggregation", count, "vector"}, + {"range aggregation line filter", `count_over_time(` + instantNS + ` |~ ".+bar" [1m])`, "vector"}, + {"parenthesized log range", `count_over_time((` + instantNS + ` |~ ".+bar")[1m])`, "vector"}, + {"bytes_over_time", `bytes_over_time(` + instantNS + `[5m])`, "vector"}, + {"sum", `sum(` + count + `)`, "vector"}, + {"sum by", `sum by (app) (` + rate + `)`, "vector"}, + {"avg by", `avg by (app) (` + count + `)`, "vector"}, + {"max", `max(` + count + `)`, "vector"}, + {"count by", `count by (app) (` + count + `)`, "vector"}, + {"stddev", `stddev(` + count + `)`, "vector"}, + {"stdvar by", `stdvar by (app) (` + count + `)`, "vector"}, + {"topk", `topk(2, sum by (app) (` + count + `))`, "vector"}, + {"bottomk", `bottomk(1, sum by (app) (` + count + `))`, "vector"}, + {"sort", `sort(sum by (app) (` + count + `))`, "vector"}, + {"quantile_over_time", `quantile_over_time(0.9, ` + instantNS + ` | logfmt | unwrap latency [5m])`, "vector"}, + {"sum over unwrap", `sum(sum_over_time(` + instantNS + ` | logfmt | unwrap latency [5m]))`, "vector"}, + {"bare parser", `count_over_time(` + instantNS + ` | logfmt [5m])`, "vector"}, + {"absent_over_time", `absent_over_time(` + instantNS + `[5m])`, "vector"}, + {"offset", `sum(count_over_time(` + instantNS + `[1m] offset 5m))`, "vector"}, + // without grouping (post-processed) + {"stdvar without", `stdvar without (app) (` + count + `)`, "vector"}, + {"stdvar without line filter", `stdvar without (app) (count_over_time((` + instantNS + ` |~ ".+bar")[1m]))`, "vector"}, + {"sum without", `sum without (app) (` + count + `)`, "vector"}, + {"avg without", `avg without (app) (` + count + `)`, "vector"}, + {"sum without nothing", `sum without () (` + count + `)`, "vector"}, + // label_replace and label_join + {"label_replace", `label_replace(sum by (app) (` + count + `), "x", "$1", "app", "(.*)")`, "vector"}, + {"label_replace line filter", `label_replace(bytes_over_time((` + instantNS + ` |= "baz" |~ "blip" != "flip" !~ "flap")[5m]), "buzz", "$2", "bar", "(.*):(.*)")`, "vector"}, + {"label_replace nested", `label_replace(label_replace(sum by (app) (` + count + `), "x", "$1", "app", "(.*)"), "y", "$1", "x", "(.*)")`, "vector"}, + {"label_join", `label_join(sum by (app, level) (` + count + `), "z", "-", "app", "level")`, "vector"}, + {"label_replace of avg without", `label_replace(avg without (app) (` + count + `), "x", "$1", "level", "(.*)")`, "vector"}, + // binary operations + {"vector add", `sum by (app) (` + rate + `) + sum by (app) (` + rate + `)`, "vector"}, + {"vector divide scalar", `sum by (app) (` + rate + `) / 60`, "vector"}, + {"scalar multiply vector", `2 * sum by (app) (` + count + `)`, "vector"}, + {"comparison filter", `sum by (app) (` + count + `) > 5`, "vector"}, + {"comparison bool", `sum by (app) (` + count + `) > bool 5`, "vector"}, + {"or vector zero", `sum(` + count + `) or vector(0)`, "vector"}, + {"and", `sum by (app) (` + count + `) and sum by (app) (` + rate + `)`, "vector"}, + {"on matching", `sum by (app) (` + count + `) / on (app) sum by (app) (` + rate + `)`, "vector"}, + {"nested binary", `sum(` + count + `) + sum(` + count + `) / sum(` + count + `)`, "vector"}, + {"label_replace of binary", `label_replace(sum by (app) (` + rate + `) + sum by (app) (` + rate + `), "x", "$1", "app", "(.*)")`, "vector"}, + {"label_replace of binary scalar", `label_replace(sum by (app) (` + rate + `) * 2, "x", "$1", "app", "(.*)")`, "vector"}, + {"label_replace of avg over nested binary", `label_replace(avg by (app) (sum by (app) (` + rate + `) + sum by (app) (` + rate + `) / sum by (app) (` + rate + `)) * 2, "new", "$1", "app", "f(.*)")`, "vector"}, + {"label_replace of unwrap binary", `label_replace(sum by (job) (sum_over_time(` + instantNS + ` | logfmt | unwrap latency | __error__!~".*" [5m]) / count_over_time(` + instantNS + ` | logfmt | label_format foo=bar [5m])), "foo", "$1", "service", "(.*):.*")`, "vector"}, + // the queries of the parity run that answered 200 without a resultType + // (clusters 307bbdd1, f2066991, ce65447e), verbatim + {"parity 307bbdd1 avg binary", `label_replace( avg by (app) ( sum by (app) (rate({namespace="prod"} |~".+bar" [1m])) + sum by (app) (rate({namespace="prod"} |~".+bar" [1m])) / sum by (app) (rate({namespace="prod"} |~".+bar" [1m])) ) * 2, "new", "$1", "app", "f(.*)" )`, "vector"}, + {"parity 307bbdd1 bytes_over_time", `label_replace( bytes_over_time(({namespace="prod"} |= "baz" |~ "blip" != "flip" !~ "flap")[5m]), "buzz", "$2", "bar", "(.*):(.*)" )`, "vector"}, + {"parity f2066991 unwrap ratio", `label_replace( sum by (job) ( sum_over_time( {service_name="api-gateway"} |= "level=error" | json | avg=5 and bar<25ms | unwrap duration(latency) | __error__!~".*" [5m] ) / count_over_time({service_name="api-gateway"} | logfmt | label_format foo=bar[5m]) ), "foo", "$1", "service", "(.*):.*" )`, "vector"}, + {"parity f2066991 offset", `label_replace( sum by (job) ( sum_over_time( {service_name="api-gateway"} |= "level=error" | json | avg=5 and bar<25ms | unwrap duration(latency) | __error__!~".*" [5m] offset 1h ) / count_over_time({service_name="api-gateway"} | logfmt | label_format foo=bar[5m] offset 1h) ), "foo", "$1", "service", "(.*):.*" )`, "vector"}, + {"parity ce65447e stdvar without", `stdvar without (app) (count_over_time(({namespace="prod"} |~".+bar")[1m]))`, "vector"}, + // constants + {"vector()", `vector(1)`, "vector"}, + {"vector binary", `vector(1) + vector(2)`, "vector"}, + {"vector compared", `vector(1) > bool 0`, "vector"}, + {"scalar literal", `1`, "scalar"}, + {"negative literal", `-1`, "scalar"}, + {"scalar binary", `1 + 1`, "scalar"}, + {"scalar nested binary", `2 * (3 + 4)`, "scalar"}, + // a log query is no instant query + {"log selector", instantNS, "error"}, + {"log selector with filter", instantNS + ` |= "x" | json`, "error"}, + } +} + +type instantAnswer struct { + code int + body []byte + resp struct { + Status string `json:"status"` + ErrorType string `json:"errorType"` + Data *struct { + ResultType string `json:"resultType"` + Result []json.RawMessage `json:"result"` + } `json:"data"` + } + raw map[string]json.RawMessage +} + +func askInstant(t *testing.T, p *Proxy, q, orgID string, extra url.Values) instantAnswer { + t.Helper() + params := url.Values{"query": {q}, "time": {"1704067200"}} + for k, v := range extra { + params[k] = v + } + req := httptest.NewRequest("GET", "/loki/api/v1/query?"+params.Encode(), nil) + req.Header.Set("X-Scope-OrgID", orgID) + req.Header.Set("X-Loki-Response-Encoding-Flags", "categorize-labels") + rec := httptest.NewRecorder() + p.handleQuery(rec, req) + a := instantAnswer{code: rec.Code, body: rec.Body.Bytes()} + if err := json.Unmarshal(a.body, &a.resp); err != nil { + t.Fatalf("%s: body is not JSON (HTTP %d): %v\n%s", q, a.code, err, a.body) + } + _ = json.Unmarshal(a.body, &a.raw) + return a +} + +// requireInstantAnswer holds an answer to Loki's contract: a 2xx carries +// status success, the resultType want and a result of that type's shape; a +// failure is an error status with an error body and no data. The reverse of +// either, a 200 that holds an error or a success without a type, is what a +// client cannot decode. +func requireInstantAnswer(t *testing.T, label string, a instantAnswer, want string, mayFail bool) { + t.Helper() + if a.code >= 400 { + if !mayFail { + t.Errorf("%s: HTTP %d, want a %s answer\n%s", label, a.code, want, a.body) + } + if a.resp.Status != "error" { + t.Errorf("%s: HTTP %d with status %q, want error\n%s", label, a.code, a.resp.Status, a.body) + } + if a.resp.Data != nil { + t.Errorf("%s: an error answer holds data\n%s", label, a.body) + } + return + } + if want == "error" { + t.Errorf("%s: HTTP %d, want Loki's 400 for a log query as an instant query\n%s", label, a.code, a.body) + return + } + if a.resp.Status != "success" || a.resp.Data == nil { + t.Errorf("%s: HTTP %d holds status %q and no data, want a %s answer (an error under a 200)\n%s", label, a.code, a.resp.Status, want, a.body) + return + } + if got := a.resp.Data.ResultType; got != want { + t.Errorf("%s: resultType %q, want %q\n%s", label, got, want, a.body) + return + } + requireInstantResultShape(t, label, a, want) +} + +// requireInstantResultShape checks the result against loghttp's types: a vector +// sample is {metric, value:[seconds, "v"]}, a scalar is [seconds, "v"]. +func requireInstantResultShape(t *testing.T, label string, a instantAnswer, want string) { + t.Helper() + checkPoint := func(raw json.RawMessage) { + var point []any + if err := json.Unmarshal(raw, &point); err != nil || len(point) != 2 { + t.Errorf("%s: sample point %s is not [time, value]\n%s", label, raw, a.body) + return + } + ts, ok := point[0].(float64) + if !ok || ts < 1e9 || ts > 1e11 { + t.Errorf("%s: sample time %v is not seconds since the epoch\n%s", label, point[0], a.body) + } + if _, ok := point[1].(string); !ok { + t.Errorf("%s: sample value %v is not a string\n%s", label, point[1], a.body) + } + } + switch want { + case "scalar": + var data struct { + Result json.RawMessage `json:"result"` + } + _ = json.Unmarshal(a.raw["data"], &data) + checkPoint(data.Result) + case "vector": + for _, item := range a.resp.Data.Result { + var sample struct { + Metric map[string]string `json:"metric"` + Value json.RawMessage `json:"value"` + Values json.RawMessage `json:"values"` + } + if err := json.Unmarshal(item, &sample); err != nil || sample.Metric == nil || sample.Value == nil || sample.Values != nil { + t.Errorf("%s: %s is not a vector sample {metric, value}\n%s", label, item, a.body) + continue + } + checkPoint(sample.Value) + } + } +} + +// conformance: semantics/instant-result-type-missing, loki_api_v1_query +func TestInstantQueryAlwaysHasLokiResultType(t *testing.T) { + modes := []instantBackendMode{instantBackendData, instantBackendEmpty, instantBackendNoType, instantBackendBad400, instantBackendDown500} + for _, mode := range modes { + t.Run(string(mode), func(t *testing.T) { + p := newTestProxy(t, instantBackend(t, mode).URL) + for _, shape := range instantShapes() { + t.Run(shape.name, func(t *testing.T) { + a := askInstant(t, p, shape.query, "0", nil) + // A backend that fails fails the query; a literal or vector() + // has no backend to fail. A log selector is rejected whatever + // the backend says. + mayFail := mode == instantBackendBad400 || mode == instantBackendDown500 || shape.want == "error" || instantRejectedShapes[shape.name] + if shape.want == "scalar" || strings.HasPrefix(shape.query, "vector(") { + mayFail = false + } + requireInstantAnswer(t, string(mode)+" "+shape.query, a, shape.want, mayFail) + if shape.want == "error" && a.code != http.StatusBadRequest { + t.Errorf("%s: HTTP %d, want 400", shape.query, a.code) + } + }) + } + }) + } +} + +// A backend failure under a post-processed query (without, label_replace, +// label_join) keeps the failing status: the buffered answer used to be sent +// with 200, an error body under a success status. +// conformance: semantics/instant-result-type-missing, loki_api_v1_query +func TestInstantQueryPostProcessedBackendFailureKeepsStatus(t *testing.T) { + for _, tc := range []struct { + mode instantBackendMode + code int + }{{instantBackendBad400, http.StatusBadRequest}, {instantBackendDown500, http.StatusInternalServerError}} { + p := newTestProxy(t, instantBackend(t, tc.mode).URL) + for _, q := range []string{ + `stdvar without (app) (count_over_time(` + instantNS + `[1m]))`, + `label_replace(sum by (app) (count_over_time(` + instantNS + `[1m])), "x", "$1", "app", "(.*)")`, + `label_join(sum by (app, level) (count_over_time(` + instantNS + `[1m])), "z", "-", "app", "level")`, + } { + a := askInstant(t, p, q, "0", nil) + if a.code != tc.code || a.resp.Status != "error" { + t.Errorf("%s with %s: HTTP %d status %q, want HTTP %d with status error\n%s", q, tc.mode, a.code, a.resp.Status, tc.code, a.body) + } + } + } +} + +// Sample times of vector() are seconds, like every other sample, and the +// scalar of a literal follows pkg/util/marshal NewScalar. +// conformance: semantics/instant-result-type-missing, loki_api_v1_query +func TestInstantConstantsCarrySecondsTimestamps(t *testing.T) { + p := newTestProxy(t, instantBackend(t, instantBackendData).URL) + for _, tc := range []struct{ query, time string }{ + {`vector(1)`, "1704067200"}, + {`vector(1)`, "1704067200000000000"}, + {`vector(1)`, "2024-01-01T00:00:00Z"}, + {`vector(1) + vector(2)`, "1704067200.5"}, + {`1`, "1704067200"}, + {`1 + 1`, "1704067200000000000"}, + } { + a := askInstant(t, p, tc.query, "0", url.Values{"time": {tc.time}}) + want := "vector" + if !strings.Contains(tc.query, "vector") { + want = "scalar" + } + requireInstantAnswer(t, tc.query+" at "+tc.time, a, want, false) + } + // without a time, the evaluation time is now + req := httptest.NewRequest("GET", "/loki/api/v1/query?query="+url.QueryEscape(`1`), nil) + req.Header.Set("X-Scope-OrgID", "0") + rec := httptest.NewRecorder() + p.handleQuery(rec, req) + var got struct { + Data struct { + ResultType string `json:"resultType"` + Result []any `json:"result"` + } `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil || got.Data.ResultType != "scalar" || len(got.Data.Result) != 2 { + t.Fatalf("scalar literal without time: HTTP %d %s", rec.Code, rec.Body.String()) + } + if ts, _ := got.Data.Result[0].(float64); time.Since(time.Unix(int64(ts), 0)) > time.Minute { + t.Fatalf("scalar literal without time is stamped %v, want now", got.Data.Result[0]) + } +} + +// conformance: semantics/instant-result-type-missing, loki_api_v1_query +func TestInstantQueryMultiTenantResultTypes(t *testing.T) { + p := newTestProxy(t, instantBackend(t, instantBackendData).URL) + for _, shape := range instantShapes() { + t.Run(shape.name, func(t *testing.T) { + a := askInstant(t, p, shape.query, "a|b", nil) + requireInstantAnswer(t, "a|b "+shape.query, a, shape.want, shape.want == "error" || instantRejectedShapes[shape.name]) + if shape.want == "error" && a.code != http.StatusBadRequest { + t.Errorf("a|b %s: HTTP %d, want 400", shape.query, a.code) + } + }) + } + // Every tenant filtered out by __tenant_id__ leaves nothing to ask; a log + // query is still no instant query, whatever tenants are left. + a := askInstant(t, p, `{__tenant_id__="nobody"}`, "a|b", nil) + requireInstantAnswer(t, "a|b nobody", a, "error", true) + if a.code != http.StatusBadRequest { + t.Errorf("log query for no tenant: HTTP %d, want 400\n%s", a.code, a.body) + } +} + +// A label transform around a binary expression evaluates the expression: its +// operands run as instant queries and combine, and the transform applies to +// the result. The marker the translator builds for the expression used to be +// sent to VictoriaLogs as a query. +// conformance: semantics/instant-result-type-missing, loki_api_v1_query +func TestInstantLabelTransformOfBinaryEvaluatesTheBinary(t *testing.T) { + p := newTestProxy(t, instantBackend(t, instantBackendData).URL) + count := `count_over_time(` + instantNS + `[1m])` + for _, q := range []string{ + `label_replace(sum by (app) (` + count + `) * 2, "x", "$1", "app", "(.*)")`, + `label_replace(2 * sum by (app) (` + count + `), "x", "$1", "app", "(.*)")`, + `label_replace(label_replace(sum by (app) (` + count + `) * 2, "x", "$1", "app", "(.*)"), "y", "$1", "app", "(.*)")`, + `label_join(sum by (app) (` + count + `) * 2, "z", "-", "app", "app")`, + } { + a := askInstant(t, p, q, "0", nil) + requireInstantAnswer(t, q, a, "vector", false) + got := map[string]string{} + for _, item := range a.resp.Data.Result { + var sample struct { + Metric map[string]string `json:"metric"` + Value []any `json:"value"` + } + _ = json.Unmarshal(item, &sample) + got[sample.Metric["app"]], _ = sample.Value[1].(string) + } + if got["a"] != "10" || got["b"] != "14" || len(got) != 2 { + t.Errorf("%s: values by app %v, want a=10 and b=14 (the backend's 5 and 7, doubled)\n%s", q, got, a.body) + } + } +} + +func TestStripLabelTransforms(t *testing.T) { + for _, tc := range []struct{ in, want string }{ + {`rate({a="b"}[1m])`, `rate({a="b"}[1m])`}, + {`label_replace(rate({a="b"}[1m]), "x", "$1", "a", "(.*)")`, `rate({a="b"}[1m])`}, + {`label_join(sum by (a, b) (rate({a="b"}[1m])), "z", "-", "a", "b")`, `sum by (a, b) (rate({a="b"}[1m]))`}, + {`label_join(label_replace( x + y , "x", "$1", "a", "(.*)"), "z", "-", "a")`, `x + y`}, + {`label_replace(rate({a="b"}[1m])`, `label_replace(rate({a="b"}[1m])`}, // no argument list: left as it is + {`label_replace(`, `label_replace(`}, + } { + if got := stripLabelTransforms(tc.in); got != tc.want { + t.Errorf("stripLabelTransforms(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} diff --git a/internal/proxy/metric_agg.go b/internal/proxy/metric_agg.go index a3177e046..565f993c6 100644 --- a/internal/proxy/metric_agg.go +++ b/internal/proxy/metric_agg.go @@ -44,7 +44,11 @@ func unwrapEmptySumWithout(query string) (logqlpkg.Expr, int) { return expr, count } -func evaluateConstantInstantVectorQuery(expr, timeParam string) ([]byte, bool) { +// evaluateConstantInstantQuery answers an instant query of constants, which has +// no data behind it. vector(n), and a binary operation of two of them, is a +// vector of one sample with no labels (Loki pkg/logql/engine.go evalVector); a +// literal, or a binary operation of literals, is a scalar (evalLiteral). +func evaluateConstantInstantQuery(expr, timeParam string) ([]byte, bool) { expr = strings.TrimSpace(expr) if expr == "" { return nil, false @@ -74,11 +78,30 @@ func evaluateConstantInstantVectorQuery(expr, timeParam string) ([]byte, bool) { return buildConstantVectorResponse(timeParam, value), true } + if parsed, err := logqlpkg.Parse(expr); err == nil { + if value, ok := binaryScalarValue(parsed, 0); ok { + result, _ := json.Marshal(map[string]interface{}{ + "status": "success", + "data": map[string]interface{}{ + "resultType": "scalar", + "result": []interface{}{constantInstantSeconds(timeParam), strconv.FormatFloat(value, 'f', -1, 64)}, + }, + }) + return result, true + } + } + return nil, false } +// constantInstantSeconds is the evaluation time as a sample time: seconds with +// millisecond precision (Loki pkg/util/marshal NewVector and NewScalar), not +// the nanoseconds the request carries. +func constantInstantSeconds(timeParam string) float64 { + return float64(parseInstantVectorTime(timeParam)/int64(time.Millisecond)) / 1e3 +} + func buildConstantVectorResponse(timeParam string, value float64) []byte { - ts := parseInstantVectorTime(timeParam) result, _ := json.Marshal(map[string]interface{}{ "status": "success", "data": map[string]interface{}{ @@ -86,7 +109,7 @@ func buildConstantVectorResponse(timeParam string, value float64) []byte { "result": []map[string]interface{}{ { "metric": map[string]string{}, - "value": []interface{}{ts, strconv.FormatFloat(value, 'f', -1, 64)}, + "value": []interface{}{constantInstantSeconds(timeParam), strconv.FormatFloat(value, 'f', -1, 64)}, }, }, "stats": map[string]interface{}{}, diff --git a/internal/proxy/multitenant.go b/internal/proxy/multitenant.go index 84b422628..c04260cfe 100644 --- a/internal/proxy/multitenant.go +++ b/internal/proxy/multitenant.go @@ -33,6 +33,11 @@ func (p *Proxy) handleMultiTenantFanout(w http.ResponseWriter, r *http.Request, return true } if len(filteredTenants) == 0 { + if endpoint == "query" && !logQLProducesSamples(r.FormValue("query")) { + // No tenant left to ask is no reason to answer `streams`: a log query is no instant query. + p.writeError(w, http.StatusBadRequest, lokiInstantLogQueryError) + return true + } p.writeJSON(w, emptyMultiTenantResponse(endpoint)) return true } diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 82fc238ca..71c6076d0 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -2494,6 +2494,10 @@ func (p *Proxy) detectedLevelCacheKey() string { return "dl1-stored" } +// lokiInstantLogQueryError is Loki's rejection of a log query sent to the +// instant query endpoint (pkg/logqlmodel/error.go ErrUnsupportedSyntaxForInstantQuery). +const lokiInstantLogQueryError = "log queries are not supported as an instant query type, please change your query to a range query type" + // handleQuery translates Loki instant queries. // //nolint:gocyclo // dispatches across cache, multi-tenant fanout, stats vs logs, binary metric, and streaming modes; branching is inherent to Loki instant query parity. @@ -2506,7 +2510,7 @@ func (p *Proxy) handleQuery(w http.ResponseWriter, r *http.Request) { } p.log.Debug("query request", "logql", redactQuery(logqlQuery, p.debugLogRawQueries)) - if body, ok := evaluateConstantInstantVectorQuery(logqlQuery, r.FormValue("time")); ok { + if body, ok := evaluateConstantInstantQuery(logqlQuery, r.FormValue("time")); ok { w.Header().Set("Content-Type", "application/json") _, _ = w.Write(body) elapsed := time.Since(start) @@ -2644,7 +2648,13 @@ func (p *Proxy) handleQuery(w http.ResponseWriter, r *http.Request) { } // Route using the original LogQL AST — more reliable than re-parsing translated markers. - parsedForRoutingQ, _ := logqlpkg.Parse(logqlQuery) + // A label_replace or label_join around a binary expression routes by that + // expression: its operands are evaluated, then the transform is applied below. + routedQuery := logqlQuery + if len(labelReplaceSpecs) > 0 || labelJoinSpec != nil { + routedQuery = stripLabelTransforms(logqlQuery) + } + parsedForRoutingQ, _ := logqlpkg.Parse(routedQuery) if binOp, ok := parsedForRoutingQ.(*logqlpkg.BinOpExpr); ok { leftLogsql, leftErr := p.translateBinOpSide(r.Context(), binOp.Left) rightLogsql, rightErr := p.translateBinOpSide(r.Context(), binOp.Right) @@ -2653,15 +2663,23 @@ func (p *Proxy) handleQuery(w http.ResponseWriter, r *http.Request) { } else if rightErr != nil { p.writeError(sc, http.StatusBadRequest, rightErr.Error()) } else { + if routedQuery != logqlQuery { + r = cloneMetricQueryRequest(r, routedQuery) + } p.proxyBinaryMetricQueryVM(sc, r, binOp.Op, leftLogsql, rightLogsql, binOpExprToVMInfo(binOp)) } } else if isStatsQuery(logsqlQuery) { p.proxyStatsQuery(sc, r, logsqlQuery) } else { - p.writeError(sc, http.StatusBadRequest, "log queries are not supported as an instant query type, please change your query to a range query type") + p.writeError(sc, http.StatusBadRequest, lokiInstantLogQueryError) } - if bw != nil && needsCapture { + if bw != nil && needsCapture && sc.code != http.StatusOK { + // A failed answer is no sample to post-process: send it with its own + // status, as query_range does. Sent with 200 it is an error body under + // a success status, with no resultType for the client to decode. + p.writeError(w, sc.code, p.redactBackendError(bw.body)) + } else if bw != nil && needsCapture { result := bw.body if len(withoutLabels) > 0 { result = applyWithoutGrouping(result, withoutLabels, withoutAggregationOp(logqlQuery)) diff --git a/internal/proxy/range_metric_compat.go b/internal/proxy/range_metric_compat.go index 6f2a2810d..ca97299fe 100644 --- a/internal/proxy/range_metric_compat.go +++ b/internal/proxy/range_metric_compat.go @@ -130,9 +130,23 @@ func parseSingleFieldCountSpec(logsqlQuery string) (statsCompatSpec, bool) { // parseOriginalRangeMetricSpec to reach the actual metric function even when // the query is wrapped in one or more label_replace() calls. func stripOuterLabelReplace(logql string) string { + return stripOuterCalls(logql, "label_replace(") +} + +// stripLabelTransforms is stripOuterLabelReplace for label_join as well: the +// metric expression under the label transforms of a query. +func stripLabelTransforms(logql string) string { + return stripOuterCalls(logql, "label_replace(", "label_join(") +} + +func stripOuterCalls(logql string, prefixes ...string) string { for { logql = strings.TrimSpace(logql) - if !strings.HasPrefix(logql, "label_replace(") { + wrapped := false + for _, prefix := range prefixes { + wrapped = wrapped || strings.HasPrefix(logql, prefix) + } + if !wrapped { break } idx := strings.Index(logql, "(") diff --git a/test/e2e-compat/instant_result_type_test.go b/test/e2e-compat/instant_result_type_test.go new file mode 100644 index 000000000..bbf69c7c4 --- /dev/null +++ b/test/e2e-compat/instant_result_type_test.go @@ -0,0 +1,160 @@ +//go:build e2e + +package e2e_compat + +import ( + "fmt" + "net/http" + "net/url" + "strconv" + "testing" + "time" +) + +// Loki answers every metric instant query with `resultType: vector` and a +// scalar literal with `scalar` (pkg/logql/engine.go evalSample, evalLiteral; +// pkg/querier/queryrange/marshal.go ResultToResponse), and its own decoder +// fails on a missing or unknown type (pkg/loghttp/query.go UnmarshalJSON). The +// Grafana Loki datasource parses the answer by that type. The proxy answered +// `stdvar without (...)` and a `label_replace` around a binary expression with +// a 200 and no resultType: the failure of the post-processed query was sent +// with its status dropped, and the translator's binary marker was sent to +// VictoriaLogs as a query (semantics/instant-result-type-missing). +// +// Each shape is asked of Loki and of the proxy. Both must answer, and answer +// the same status and the same resultType; the series of the answers are +// compared by the other registry items, not here. +// conformance: semantics/instant-result-type-missing, loki_api_v1_query +func TestCompat_InstantResultTypeAlwaysSet(t *testing.T) { + ensureDataIngested(t) + waitForLokiMetricData(t) + + sel := `{app="api-gateway"}` + count := `count_over_time(` + sel + `[1m])` + rate := `rate(` + sel + `[1m])` + shapes := []struct { + name string + query string + }{ + // the request of the registry row and the queries of its clusters + {"row stdvar without", `stdvar without (app) (count_over_time({namespace="prod"}[1m]))`}, + {"cluster ce65447e", `stdvar without (app) (count_over_time(({namespace="prod"} |~".+bar")[1m]))`}, + {"cluster 307bbdd1 avg binary", `label_replace( avg by (app) ( sum by (app) (rate({namespace="prod"} |~".+bar" [1m])) + sum by (app) (rate({namespace="prod"} |~".+bar" [1m])) / sum by (app) (rate({namespace="prod"} |~".+bar" [1m])) ) * 2, "new", "$1", "app", "f(.*)" )`}, + {"cluster 307bbdd1 bytes_over_time", `label_replace( bytes_over_time(({namespace="prod"} |= "baz" |~ "blip" != "flip" !~ "flap")[5m]), "buzz", "$2", "bar", "(.*):(.*)" )`}, + // the same shapes over data + {"stdvar without", `stdvar without (app) (` + count + `)`}, + {"stddev without", `stddev without (app) (` + count + `)`}, + {"sum without", `sum without (app) (` + count + `)`}, + {"avg without", `avg without (app) (` + count + `)`}, + {"label_replace of avg binary", `label_replace(avg by (app) (sum by (app) (` + rate + `) + sum by (app) (` + rate + `)), "x", "$1", "app", "(.*)")`}, + {"label_replace of binary", `label_replace(sum by (app) (` + rate + `) + sum by (app) (` + rate + `), "x", "$1", "app", "(.*)")`}, + {"label_replace of binary scalar", `label_replace(sum by (app) (` + rate + `) * 2, "x", "$1", "app", "(.*)")`}, + {"label_replace of nested binary scalar", `label_replace(avg by (app) (sum by (app) (` + rate + `) + sum by (app) (` + rate + `) / sum by (app) (` + rate + `)) * 2, "new", "$1", "app", "f(.*)")`}, + {"label_join of binary", `label_join(sum by (app) (` + count + `) * 2, "z", "-", "app", "app")`}, + {"label_replace", `label_replace(sum by (app) (` + count + `), "x", "$1", "app", "(.*)")`}, + {"label_replace of bytes_over_time", `label_replace(bytes_over_time(` + sel + `[5m]), "buzz", "$2", "app", "(.*)-(.*)")`}, + // the other shapes + {"range aggregation", count}, + {"sum by", `sum by (app) (` + rate + `)`}, + {"binary", `sum by (app) (` + rate + `) / sum by (app) (` + count + `)`}, + {"binary scalar", `sum(` + count + `) * 60`}, + {"comparison", `sum by (app) (` + count + `) > 0`}, + {"or vector", `sum(` + count + `) or vector(0)`}, + {"topk", `topk(2, sum by (app) (` + count + `))`}, + {"sort", `sort(sum by (app) (` + count + `))`}, + {"absent_over_time", `absent_over_time({app="no-such-app-for-resulttype"}[5m])`}, + {"vector()", `vector(1)`}, + {"vector binary", `vector(1) + vector(2)`}, + {"scalar literal", `1`}, + {"negative literal", `-1`}, + {"scalar binary", `1 + 1`}, + {"log selector", sel}, + } + + ask := func(t *testing.T, base, query, orgID string, at time.Time) (int, string, map[string]interface{}) { + params := url.Values{"query": {query}, "time": {strconv.FormatInt(at.UnixNano(), 10)}} + return doJSONGET(t, base+"/loki/api/v1/query?"+params.Encode(), map[string]string{ + "X-Scope-OrgID": orgID, + "X-Loki-Response-Encoding-Flags": "categorize-labels", + }) + } + resultType := func(payload map[string]interface{}) string { + data, _ := payload["data"].(map[string]interface{}) + rt, _ := data["resultType"].(string) + return rt + } + check := func(t *testing.T, query, orgID string) { + t.Helper() + at := time.Now() + lokiStatus, lokiBody, lokiPayload := ask(t, lokiURL, query, orgID, at) + proxyStatus, proxyBody, proxyPayload := ask(t, proxyURL, query, orgID, at) + if proxyStatus == http.StatusOK && resultType(proxyPayload) == "" { + t.Errorf("%s: the proxy answers 200 without a resultType: %s", query, proxyBody) + } + if lokiStatus != proxyStatus { + t.Errorf("%s: HTTP status loki=%d proxy=%d\nloki: %s\nproxy: %s", query, lokiStatus, proxyStatus, lokiBody, proxyBody) + return + } + if lokiStatus != http.StatusOK { + return // both reject the query alike + } + if lt, pt := resultType(lokiPayload), resultType(proxyPayload); lt == "" || lt != pt { + t.Errorf("%s: resultType loki=%q proxy=%q\nloki: %s\nproxy: %s", query, lt, pt, lokiBody, proxyBody) + } + } + + for _, shape := range shapes { + t.Run(shape.name, func(t *testing.T) { check(t, shape.query, "0") }) + } + + // Several tenants merge their answers: a vector stays a vector, a log + // query stays rejected, and a constant needs no tenant. + t.Run("multi-tenant", func(t *testing.T) { + for _, query := range []string{ + `sum by (app) (` + count + `)`, + `label_replace(sum by (app) (` + rate + `) * 2, "x", "$1", "app", "(.*)")`, + `stdvar without (app) (` + count + `)`, + `vector(1)`, + `1`, + `1 + 1`, + sel, + } { + check(t, query, "0|fake") + } + }) + + // A vector sample is stamped in seconds (pkg/util/marshal NewVector), not + // in the nanoseconds of the request. + t.Run("sample time is seconds", func(t *testing.T) { + at := time.Now() + for name, base := range map[string]string{"loki": lokiURL, "proxy": proxyURL} { + _, body, payload := ask(t, base, `vector(1)`, "0", at) + data, _ := payload["data"].(map[string]interface{}) + result, _ := data["result"].([]interface{}) + if len(result) != 1 { + t.Fatalf("%s: vector(1) holds %d samples: %s", name, len(result), body) + } + sample, _ := result[0].(map[string]interface{}) + value, _ := sample["value"].([]interface{}) + ts, _ := value[0].(float64) + if diff := ts - float64(at.UnixMilli())/1e3; diff < -1 || diff > 1 { + t.Errorf("%s: vector(1) is stamped %v, want the evaluation time %s in seconds", name, value[0], fmt.Sprint(float64(at.UnixMilli())/1e3)) + } + } + }) + + // An aggregation over a binary expression (`sum by (job) (a / b)`) is not + // evaluated by the proxy yet (semantics/aggregation-over-binary-dropped); it + // may be rejected, but never with a 200 that has no resultType. + t.Run("aggregation over binary is never typeless", func(t *testing.T) { + for _, query := range []string{ + `label_replace( sum by (job) ( sum_over_time( {service_name="api-gateway"} |= "level=error" | json | avg=5 and bar<25ms | unwrap duration(latency) | __error__!~".*" [5m] ) / count_over_time({service_name="api-gateway"} | logfmt | label_format foo=bar[5m]) ), "foo", "$1", "service", "(.*):.*" )`, + `label_replace( sum by (job) ( sum_over_time( {service_name="api-gateway"} |= "level=error" | json | avg=5 and bar<25ms | unwrap duration(latency) | __error__!~".*" [5m] offset 1h ) / count_over_time({service_name="api-gateway"} | logfmt | label_format foo=bar[5m] offset 1h) ), "foo", "$1", "service", "(.*):.*" )`, + } { + status, body, payload := ask(t, proxyURL, query, "0", time.Now()) + if status == http.StatusOK && resultType(payload) == "" { + t.Errorf("%s: the proxy answers 200 without a resultType: %s", query, body) + } + } + }) +} From c437a21dcbe82cf147d797f7c4a3069d23d76364 Mon Sep 17 00:00:00 2001 From: szibis Date: Mon, 12 Oct 2026 12:32:30 +0200 Subject: [PATCH 02/18] test: instant result types per client, and the multi-tenant e2e asks the proxy alone The instant result-type table runs as a bare client, Grafana Explore and Logs Drilldown, which take their own failure and series-limit branches. The multi-tenant e2e checks the proxy against the type Loki gives the expression, as the compose Loki is not configured for multi-tenant queries. --- internal/proxy/instant_result_type_test.go | 46 +++++++++++++++------ test/e2e-compat/instant_result_type_test.go | 31 +++++++++----- 2 files changed, 54 insertions(+), 23 deletions(-) diff --git a/internal/proxy/instant_result_type_test.go b/internal/proxy/instant_result_type_test.go index 1510ce11b..91a72e907 100644 --- a/internal/proxy/instant_result_type_test.go +++ b/internal/proxy/instant_result_type_test.go @@ -199,7 +199,20 @@ type instantAnswer struct { raw map[string]json.RawMessage } +// instantClients are the header sets of the clients that read an instant +// answer: a bare client, Grafana's Explore and Logs Drilldown. Grafana-sourced +// requests take their own failure and series-limit branches. +var instantClients = map[string]map[string]string{ + "curl": nil, + "explore": {"User-Agent": "Grafana/13.0.1", "X-Grafana-Org-Id": "1", "X-Query-Tags": "Source=grafana"}, + "drilldown": {"User-Agent": "Grafana/13.0.1", "X-Grafana-Org-Id": "1", "X-Query-Tags": "Source=grafana-lokiexplore-app"}, +} + func askInstant(t *testing.T, p *Proxy, q, orgID string, extra url.Values) instantAnswer { + return askInstantAs(t, p, q, orgID, extra, nil) +} + +func askInstantAs(t *testing.T, p *Proxy, q, orgID string, extra url.Values, headers map[string]string) instantAnswer { t.Helper() params := url.Values{"query": {q}, "time": {"1704067200"}} for k, v := range extra { @@ -208,6 +221,9 @@ func askInstant(t *testing.T, p *Proxy, q, orgID string, extra url.Values) insta req := httptest.NewRequest("GET", "/loki/api/v1/query?"+params.Encode(), nil) req.Header.Set("X-Scope-OrgID", orgID) req.Header.Set("X-Loki-Response-Encoding-Flags", "categorize-labels") + for k, v := range headers { + req.Header.Set(k, v) + } rec := httptest.NewRecorder() p.handleQuery(rec, req) a := instantAnswer{code: rec.Code, body: rec.Body.Bytes()} @@ -299,19 +315,23 @@ func TestInstantQueryAlwaysHasLokiResultType(t *testing.T) { for _, mode := range modes { t.Run(string(mode), func(t *testing.T) { p := newTestProxy(t, instantBackend(t, mode).URL) - for _, shape := range instantShapes() { - t.Run(shape.name, func(t *testing.T) { - a := askInstant(t, p, shape.query, "0", nil) - // A backend that fails fails the query; a literal or vector() - // has no backend to fail. A log selector is rejected whatever - // the backend says. - mayFail := mode == instantBackendBad400 || mode == instantBackendDown500 || shape.want == "error" || instantRejectedShapes[shape.name] - if shape.want == "scalar" || strings.HasPrefix(shape.query, "vector(") { - mayFail = false - } - requireInstantAnswer(t, string(mode)+" "+shape.query, a, shape.want, mayFail) - if shape.want == "error" && a.code != http.StatusBadRequest { - t.Errorf("%s: HTTP %d, want 400", shape.query, a.code) + for client, headers := range instantClients { + t.Run(client, func(t *testing.T) { + for _, shape := range instantShapes() { + t.Run(shape.name, func(t *testing.T) { + a := askInstantAs(t, p, shape.query, "0", nil, headers) + // A backend that fails fails the query; a literal or vector() + // has no backend to fail. A log selector is rejected whatever + // the backend says. + mayFail := mode == instantBackendBad400 || mode == instantBackendDown500 || shape.want == "error" || instantRejectedShapes[shape.name] + if shape.want == "scalar" || strings.HasPrefix(shape.query, "vector(") { + mayFail = false + } + requireInstantAnswer(t, string(mode)+" "+client+" "+shape.query, a, shape.want, mayFail) + if shape.want == "error" && a.code != http.StatusBadRequest { + t.Errorf("%s: HTTP %d, want 400", shape.query, a.code) + } + }) } }) } diff --git a/test/e2e-compat/instant_result_type_test.go b/test/e2e-compat/instant_result_type_test.go index bbf69c7c4..cec1e21c9 100644 --- a/test/e2e-compat/instant_result_type_test.go +++ b/test/e2e-compat/instant_result_type_test.go @@ -108,18 +108,29 @@ func TestCompat_InstantResultTypeAlwaysSet(t *testing.T) { } // Several tenants merge their answers: a vector stays a vector, a log - // query stays rejected, and a constant needs no tenant. + // query stays rejected, and a constant needs no tenant. The compose Loki is + // not configured for multi-tenant queries, so this asks the proxy alone for + // the type Loki gives the expression. t.Run("multi-tenant", func(t *testing.T) { - for _, query := range []string{ - `sum by (app) (` + count + `)`, - `label_replace(sum by (app) (` + rate + `) * 2, "x", "$1", "app", "(.*)")`, - `stdvar without (app) (` + count + `)`, - `vector(1)`, - `1`, - `1 + 1`, - sel, + for query, want := range map[string]string{ + `sum by (app) (` + count + `)`: "vector", + `label_replace(sum by (app) (` + rate + `) * 2, "x", "$1", "app", "(.*)")`: "vector", + `stdvar without (app) (` + count + `)`: "vector", + `vector(1)`: "vector", + `1`: "scalar", + `1 + 1`: "scalar", + sel: "", } { - check(t, query, "0|fake") + status, body, payload := ask(t, proxyURL, query, "0|fake", time.Now()) + if want == "" { + if status != http.StatusBadRequest { + t.Errorf("%s: HTTP %d, want Loki's 400 for a log query as an instant query: %s", query, status, body) + } + continue + } + if status != http.StatusOK || resultType(payload) != want { + t.Errorf("%s: HTTP %d resultType %q, want 200 %s: %s", query, status, resultType(payload), want, body) + } } }) From d6ac3f81fa0e73974a827f7c287e4af9ca217ae5 Mon Sep 17 00:00:00 2001 From: szibis Date: Mon, 12 Oct 2026 12:56:57 +0200 Subject: [PATCH 03/18] fix: label transforms split their first argument outside strings, and instant constants follow Loki label_replace and label_join split the expression they wrap at the first comma outside strings and brackets, in the translator and in the instant routing, so a parenthesis or comma in a line filter or a matcher no longer sends the query to the log path or the binary marker to VictoriaLogs. vector(n) with a scalar, and a scalar with vector(n), are evaluated as constants through one path, a literal division by zero is NaN as in Loki's MergeBinOp, and with every tenant filtered out an unparseable query gets the parse error. Cluster f2066991 is no longer claimed by the instant result-type row; a stub row owns it. --- CHANGELOG.md | 10 ++-- .../aggregation-over-binary-dropped.yaml | 37 +++++++++++++ .../instant-result-type-missing.yaml | 3 +- internal/logql/canonical.go | 44 ++++++--------- internal/proxy/binary_scalar.go | 50 +++++++++++++---- internal/proxy/coverage_gaps_test.go | 6 ++- internal/proxy/instant_result_type_test.go | 30 +++++++++++ internal/proxy/metric_agg.go | 53 +++++-------------- internal/proxy/multitenant.go | 14 +++-- internal/proxy/range_metric_compat.go | 38 +++---------- internal/translator/first_argument_test.go | 26 +++++++++ internal/translator/translator.go | 43 +++++++++++---- 12 files changed, 227 insertions(+), 127 deletions(-) create mode 100644 conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml create mode 100644 internal/translator/first_argument_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cbda1928..5019854a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,9 +27,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 operands run as instant queries and combine, and the transform applies to the result. Also on the instant endpoint: a scalar literal (`1`, `-1`) answers a `scalar` as Loki's `evalLiteral` does, where it was rejected as a log query, - also under a multi-tenant `X-Scope-OrgID`; `vector(1)` and `vector(1) + - vector(2)` stamp their sample in seconds like every other sample - (`pkg/util/marshal/query.go` `NewVector`) where they carried nanoseconds; and + also under a multi-tenant `X-Scope-OrgID`; `vector(1)`, `vector(1) + 2` and + `vector(1) + vector(2)` are evaluated as constants (a literal division by zero + is NaN, as in Loki's `MergeBinOp`) and stamp their sample in seconds like + every other sample (`pkg/util/marshal/query.go` `NewVector`) where they + carried nanoseconds; the first argument of a `label_replace` or `label_join` + is split outside strings and brackets, so a `)` or `,` in a line filter or a + matcher no longer breaks the translation; and a log query sent with every tenant filtered out by `__tenant_id__` is rejected with Loki's 400 for a log query as an instant query, not answered with an empty `streams`. An aggregation over a binary expression (`sum by (job) (a / diff --git a/conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml b/conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml new file mode 100644 index 000000000..96e7d7fe0 --- /dev/null +++ b/conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml @@ -0,0 +1,37 @@ +id: semantics/aggregation-over-binary-dropped +title: An aggregation over a binary expression is translated to its first operand (open) +proves: + - loki_api_v1_query +consumer: explore +why: | + `sum by (job) (rate(a[1m]) / count_over_time(a[1m]))` is a vector aggregation + over a binary operation on Loki (pkg/logql/evaluator.go VectorAggEvaluator + over a binary evaluator). The translator keeps the first operand only: the + right operand and the operator are dropped (`sum(rate(a[1m]) / 2)` loses the + `/ 2`), so an answered query has wrong values, and the shape of parity + cluster f2066991 (`| json | unwrap duration(latency)` over `/ + count_over_time(...)` under label_replace) is rejected with 400 ("log queries + are not supported as an instant query type") where Loki answers a vector. +request: + method: GET + path: /loki/api/v1/query + params: + query: 'sum by (job) (rate({app="api-gateway"}[1m]) / count_over_time({app="api-gateway"}[1m]))' + headers: + X-Scope-OrgID: "0" + X-Loki-Response-Encoding-Flags: categorize-labels +compare: + mode: observed + fields: [series, values] + expect: | + Loki's vector: the quotient per stream, summed by job. +gap: + status: open + impact: explore-visible + effort: M + area: 'internal/translator extractOuterAggregation and internal/proxy handleQuery routing of an aggregation over a binary expression' + loki: 'pkg/logql/evaluator.go VectorAggEvaluator over a binary evaluator' + proxy: 'first operand only, or a 400' + planned_test: 'TestCompat_AggregationOverBinary (test/e2e-compat, group semantics)' + clusters: + - f2066991 diff --git a/conformance/registry/cases/semantics/instant-result-type-missing.yaml b/conformance/registry/cases/semantics/instant-result-type-missing.yaml index 853e9a57e..b8f2d1496 100644 --- a/conformance/registry/cases/semantics/instant-result-type-missing.yaml +++ b/conformance/registry/cases/semantics/instant-result-type-missing.yaml @@ -67,7 +67,6 @@ gap: loki: 'pkg/logql/engine.go evalSample/evalLiteral/evalVector, JoinSampleVector; pkg/querier/queryrange/marshal.go ResultToResponse; pkg/querier/http.go InstantQueryHandler; pkg/loghttp/query.go UnmarshalJSON' proxy: 'the failing status of a post-processed instant query is kept; a label transform over a binary expression evaluates the expression; literals answer scalar' planned_test: 'TestCompat_InstantResultTypeAlwaysSet (test/e2e-compat, group semantics); TestInstantQueryAlwaysHasLokiResultType, TestInstantQueryPostProcessedBackendFailureKeepsStatus, TestInstantConstantsCarrySecondsTimestamps, TestInstantQueryMultiTenantResultTypes and TestInstantLabelTransformOfBinaryEvaluatesTheBinary (internal/proxy)' - note: 'The e2e is written and not yet run against the compose stack. Cluster f2066991 stays claimed here until the aggregation-over-binary row is registered: its queries no longer answer a typeless 200 but are still rejected (400) or answered from their first operand only. Cluster ce65447e is owned by semantics/stddev-stdvar-grouped-values; its typeless answer is fixed here, its values are not.' + note: 'The e2e is written and not yet run against the compose stack. Cluster f2066991 is not claimed here: its queries no longer answer a typeless 200 but the proxy still answers 400 where Loki answers a vector; it belongs to semantics/aggregation-over-binary-dropped. Cluster ce65447e is owned by semantics/stddev-stdvar-grouped-values; its typeless answer is fixed here, its values are not.' clusters: - 307bbdd1 - - f2066991 diff --git a/internal/logql/canonical.go b/internal/logql/canonical.go index 5ede677f5..bd1e76134 100644 --- a/internal/logql/canonical.go +++ b/internal/logql/canonical.go @@ -1,6 +1,10 @@ package logql -import "strings" +import ( + "strings" + + "github.com/ReliablyObserve/Loki-VL-proxy/internal/translator" +) // CanonicalizeLogRanges rewrites the alternative forms Loki accepts for a log // expression into the plain `{selector} pipeline [range]` form the translator @@ -95,32 +99,14 @@ func skipSpace(q string, i int) int { return i } -// firstArgumentLen is the length of the first argument of a call, given the -// text after its '(': up to the first comma or unmatched ')' outside strings. -func firstArgumentLen(s string) int { - depth := 0 - var quote byte - for i := 0; i < len(s); i++ { - c := s[i] - switch { - case quote != 0: - if c == '\\' && quote == '"' { - i++ - } else if c == quote { - quote = 0 - } - case c == '"' || c == '`': - quote = c - case c == '(' || c == '[' || c == '{': - depth++ - case c == ')' || c == ']' || c == '}': - if depth == 0 { - return i - } - depth-- - case c == ',' && depth == 0: - return i - } - } - return len(s) +// FirstArgument is the first argument of a call, given the text after its '(', +// and whether a comma follows it (a call with further arguments). Strings in +// double quotes or backticks and nested brackets are skipped. +func FirstArgument(s string) (string, bool) { + n := firstArgumentLen(s) + return s[:n], n < len(s) && s[n] == ',' } + +// firstArgumentLen is the length of the first argument of a call, given the +// text after its '(': see translator.FirstArgumentLen. +func firstArgumentLen(s string) int { return translator.FirstArgumentLen(s) } diff --git a/internal/proxy/binary_scalar.go b/internal/proxy/binary_scalar.go index d0df43ee0..2fe4d07b1 100644 --- a/internal/proxy/binary_scalar.go +++ b/internal/proxy/binary_scalar.go @@ -3,33 +3,63 @@ package proxy import ( "encoding/json" "fmt" + "math" "net/http" "strconv" + "strings" logqlpkg "github.com/ReliablyObserve/Loki-VL-proxy/internal/logql" ) +// binaryScalarValue is the value of a scalar expression: literals and +// arithmetic of them, which Loki folds into one literal. func binaryScalarValue(expr logqlpkg.Expr, depth int) (float64, bool) { + value, isVector, ok := binaryConstantValue(expr, depth) + return value, ok && !isVector +} + +// binaryConstantValue folds an expression of literals and vector(n) into its +// value, and says whether any operand was a vector(n), making the result a +// one-sample vector with no labels instead of a scalar. A division by zero is +// NaN, as in Loki's MergeBinOp (pkg/logql/syntax/ast.go). A comparison with a +// vector operand needs the bool modifier and the generic binary path. +func binaryConstantValue(expr logqlpkg.Expr, depth int) (value float64, isVector, ok bool) { if depth >= 64 { - return 0, false + return 0, false, false } switch e := expr.(type) { case *logqlpkg.LiteralExpr: - return e.Value, true + return e.Value, false, true + case *logqlpkg.OpaqueMetricExpr: + if matches := vectorLiteralRE.FindStringSubmatch(e.Raw); len(matches) == 2 { + if value, err := strconv.ParseFloat(strings.TrimSpace(matches[1]), 64); err == nil { + return value, true, true + } + } case *logqlpkg.BinOpExpr: if e.VectorMatching != nil || e.Op == "and" || e.Op == "or" || e.Op == "unless" { - return 0, false + return 0, false, false + } + left, leftVector, leftOK := binaryConstantValue(e.Left, depth+1) + right, rightVector, rightOK := binaryConstantValue(e.Right, depth+1) + if !leftOK || !rightOK || ((leftVector || rightVector) && isComparisonOp(e.Op)) { + return 0, false, false } - left, leftOK := binaryScalarValue(e.Left, depth+1) - right, rightOK := binaryScalarValue(e.Right, depth+1) - if !leftOK || !rightOK { - return 0, false + if e.Op == "/" && right == 0 { + return math.NaN(), leftVector || rightVector, true } value, _ := binarySampleValue(left, right, e.Op, true) - return value, true - default: - return 0, false + return value, leftVector || rightVector, true + } + return 0, false, false +} + +func isComparisonOp(op string) bool { + switch op { + case "==", "!=", ">", "<", ">=", "<=": + return true } + return false } func binaryConstantVectorResponse(r *http.Request, value float64, resultType string) ([]byte, error) { diff --git a/internal/proxy/coverage_gaps_test.go b/internal/proxy/coverage_gaps_test.go index 8750a8c1f..2adcde093 100644 --- a/internal/proxy/coverage_gaps_test.go +++ b/internal/proxy/coverage_gaps_test.go @@ -290,7 +290,11 @@ func TestEvaluateConstantInstantQuery(t *testing.T) { {name: "vector literal", query: `vector(1)`, timeParam: "4", wantValue: "1", wantTS: 4, expectEval: true}, {name: "binary add", query: `vector(1)+vector(1)`, timeParam: "4", wantValue: "2", wantTS: 4, expectEval: true}, {name: "binary divide", query: `vector(9) / vector(3)`, timeParam: "4", wantValue: "3", wantTS: 4, expectEval: true}, - {name: "divide by zero", query: `vector(1)/vector(0)`, timeParam: "4", expectEval: false}, + {name: "divide by zero is NaN like Loki", query: `vector(1)/vector(0)`, timeParam: "4", wantValue: "NaN", wantTS: 4, expectEval: true}, + {name: "vector plus scalar", query: `vector(1) + 2`, timeParam: "4", wantValue: "3", wantTS: 4, expectEval: true}, + {name: "scalar times vector", query: `3 * vector(2)`, timeParam: "4", wantValue: "6", wantTS: 4, expectEval: true}, + {name: "vector divided by scalar zero", query: `vector(1) / 0`, timeParam: "4", wantValue: "NaN", wantTS: 4, expectEval: true}, + {name: "vector comparison needs the generic path", query: `vector(1) > 0`, timeParam: "4", expectEval: false}, {name: "non vector", query: `{app="api"}`, timeParam: "4", expectEval: false}, } diff --git a/internal/proxy/instant_result_type_test.go b/internal/proxy/instant_result_type_test.go index 91a72e907..a1ee6f693 100644 --- a/internal/proxy/instant_result_type_test.go +++ b/internal/proxy/instant_result_type_test.go @@ -171,10 +171,18 @@ func instantShapes() []instantShape { {"parity f2066991 unwrap ratio", `label_replace( sum by (job) ( sum_over_time( {service_name="api-gateway"} |= "level=error" | json | avg=5 and bar<25ms | unwrap duration(latency) | __error__!~".*" [5m] ) / count_over_time({service_name="api-gateway"} | logfmt | label_format foo=bar[5m]) ), "foo", "$1", "service", "(.*):.*" )`, "vector"}, {"parity f2066991 offset", `label_replace( sum by (job) ( sum_over_time( {service_name="api-gateway"} |= "level=error" | json | avg=5 and bar<25ms | unwrap duration(latency) | __error__!~".*" [5m] offset 1h ) / count_over_time({service_name="api-gateway"} | logfmt | label_format foo=bar[5m] offset 1h) ), "foo", "$1", "service", "(.*):.*" )`, "vector"}, {"parity ce65447e stdvar without", `stdvar without (app) (count_over_time(({namespace="prod"} |~".+bar")[1m]))`, "vector"}, + // a quote or bracket in the expression under a label transform + {"label_replace of binary with a quoted paren", `label_replace(rate(` + instantNS + ` |= ")" [5m]) + rate({b="z"}[5m]), "d", "$1", "app", "(.*)")`, "vector"}, + {"label_replace of binary with a regex paren", `label_replace(rate({a=~"x\\(y"}[5m]) + rate({b="z"}[5m]), "d", "$1", "app", "(.*)")`, "vector"}, + {"label_replace of binary with a backtick", "label_replace(rate(" + instantNS + " |= `,)` [5m]) + rate({b=\"z\"}[5m]), \"d\", \"$1\", \"app\", \"(.*)\")", "vector"}, // constants {"vector()", `vector(1)`, "vector"}, {"vector binary", `vector(1) + vector(2)`, "vector"}, {"vector compared", `vector(1) > bool 0`, "vector"}, + {"vector plus scalar", `vector(1) + 2`, "vector"}, + {"scalar plus vector", `1 + vector(2)`, "vector"}, + {"vector divided by zero", `vector(1) / 0`, "vector"}, + {"scalar divided by zero", `1 / 0`, "scalar"}, {"scalar literal", `1`, "scalar"}, {"negative literal", `-1`, "scalar"}, {"scalar binary", `1 + 1`, "scalar"}, @@ -460,6 +468,12 @@ func TestStripLabelTransforms(t *testing.T) { {`label_join(sum by (a, b) (rate({a="b"}[1m])), "z", "-", "a", "b")`, `sum by (a, b) (rate({a="b"}[1m]))`}, {`label_join(label_replace( x + y , "x", "$1", "a", "(.*)"), "z", "-", "a")`, `x + y`}, {`label_replace(rate({a="b"}[1m])`, `label_replace(rate({a="b"}[1m])`}, // no argument list: left as it is + // strings and brackets hide their commas and parentheses + {`label_replace(rate({a=") , ("}[5m]), "d","$1","s","(.*)")`, `rate({a=") , ("}[5m])`}, + {`label_replace(rate({a="x"} |= ")" [5m]) + rate({b="z"}[5m]), "d","$1","s","(.*)")`, `rate({a="x"} |= ")" [5m]) + rate({b="z"}[5m])`}, + {`label_replace(rate({a=~"x(y"}[5m]) + rate({b="z"}[5m]), "d","$1","s","(.*)")`, `rate({a=~"x(y"}[5m]) + rate({b="z"}[5m])`}, + {"label_replace(rate({a=\"x\"} |= `,)(` [5m]) + rate({b=\"z\"}[5m]), \"d\",\"$1\",\"s\",\"(.*)\")", "rate({a=\"x\"} |= `,)(` [5m]) + rate({b=\"z\"}[5m])"}, + {`label_join(label_replace(rate({a="\",)"}[5m]), "d","$1","s","(.*)"), "z", "-", "a")`, `rate({a="\",)"}[5m])`}, {`label_replace(`, `label_replace(`}, } { if got := stripLabelTransforms(tc.in); got != tc.want { @@ -467,3 +481,19 @@ func TestStripLabelTransforms(t *testing.T) { } } } + +// With every tenant filtered out, an unparseable query is a parse error, not a +// log query. +// conformance: semantics/instant-result-type-missing, loki_api_v1_query +func TestInstantQueryMultiTenantNoTenantUnparseable(t *testing.T) { + p := newTestProxy(t, instantBackend(t, instantBackendData).URL) + req := httptest.NewRequest("GET", "/loki/api/v1/query?"+url.Values{"query": {`{__tenant_id__="nobody"} | json | (`}, "time": {"1704067200"}}.Encode(), nil) + req.Header.Set("X-Scope-OrgID", "a|b") + rec := httptest.NewRecorder() + if !p.handleMultiTenantFanout(rec, req, "query") { + t.Fatal("the fanout did not answer") + } + if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "parse error") || strings.Contains(rec.Body.String(), "instant query type") { + t.Fatalf("HTTP %d %s, want 400 with a parse error", rec.Code, rec.Body.String()) + } +} diff --git a/internal/proxy/metric_agg.go b/internal/proxy/metric_agg.go index 565f993c6..b0de00b52 100644 --- a/internal/proxy/metric_agg.go +++ b/internal/proxy/metric_agg.go @@ -17,7 +17,6 @@ import ( var ( vectorLiteralRE = regexp.MustCompile(`^\s*vector\(\s*([^)]+?)\s*\)\s*$`) - vectorBinaryRE = regexp.MustCompile(`^\s*vector\(\s*([^)]+?)\s*\)\s*([+\-*/])\s*vector\(\s*([^)]+?)\s*\)\s*$`) ) // unwrapEmptySumWithout preserves every series for an explicit sum without(). @@ -49,49 +48,25 @@ func unwrapEmptySumWithout(query string) (logqlpkg.Expr, int) { // vector of one sample with no labels (Loki pkg/logql/engine.go evalVector); a // literal, or a binary operation of literals, is a scalar (evalLiteral). func evaluateConstantInstantQuery(expr, timeParam string) ([]byte, bool) { - expr = strings.TrimSpace(expr) - if expr == "" { + parsed, err := logqlpkg.Parse(strings.TrimSpace(expr)) + if err != nil { return nil, false } - - if matches := vectorLiteralRE.FindStringSubmatch(expr); len(matches) == 2 { - value, err := strconv.ParseFloat(strings.TrimSpace(matches[1]), 64) - if err != nil { - return nil, false - } - return buildConstantVectorResponse(timeParam, value), true + value, isVector, ok := binaryConstantValue(parsed, 0) + if !ok { + return nil, false } - - if matches := vectorBinaryRE.FindStringSubmatch(expr); len(matches) == 4 { - left, err := strconv.ParseFloat(strings.TrimSpace(matches[1]), 64) - if err != nil { - return nil, false - } - right, err := strconv.ParseFloat(strings.TrimSpace(matches[3]), 64) - if err != nil { - return nil, false - } - value, ok := applyConstantBinaryOp(left, right, matches[2]) - if !ok { - return nil, false - } + if isVector { return buildConstantVectorResponse(timeParam, value), true } - - if parsed, err := logqlpkg.Parse(expr); err == nil { - if value, ok := binaryScalarValue(parsed, 0); ok { - result, _ := json.Marshal(map[string]interface{}{ - "status": "success", - "data": map[string]interface{}{ - "resultType": "scalar", - "result": []interface{}{constantInstantSeconds(timeParam), strconv.FormatFloat(value, 'f', -1, 64)}, - }, - }) - return result, true - } - } - - return nil, false + result, _ := json.Marshal(map[string]interface{}{ + "status": "success", + "data": map[string]interface{}{ + "resultType": "scalar", + "result": []interface{}{constantInstantSeconds(timeParam), strconv.FormatFloat(value, 'f', -1, 64)}, + }, + }) + return result, true } // constantInstantSeconds is the evaluation time as a sample time: seconds with diff --git a/internal/proxy/multitenant.go b/internal/proxy/multitenant.go index c04260cfe..435e191d7 100644 --- a/internal/proxy/multitenant.go +++ b/internal/proxy/multitenant.go @@ -12,6 +12,8 @@ import ( "strings" "sync" "time" + + logqlpkg "github.com/ReliablyObserve/Loki-VL-proxy/internal/logql" ) func (p *Proxy) handleMultiTenantFanout(w http.ResponseWriter, r *http.Request, endpoint string) bool { @@ -33,10 +35,16 @@ func (p *Proxy) handleMultiTenantFanout(w http.ResponseWriter, r *http.Request, return true } if len(filteredTenants) == 0 { - if endpoint == "query" && !logQLProducesSamples(r.FormValue("query")) { + if endpoint == "query" { // No tenant left to ask is no reason to answer `streams`: a log query is no instant query. - p.writeError(w, http.StatusBadRequest, lokiInstantLogQueryError) - return true + if _, err := logqlpkg.Parse(r.FormValue("query")); err != nil { + p.writeError(w, http.StatusBadRequest, fmt.Sprintf("parse error : %v", err)) + return true + } + if !logQLProducesSamples(r.FormValue("query")) { + p.writeError(w, http.StatusBadRequest, lokiInstantLogQueryError) + return true + } } p.writeJSON(w, emptyMultiTenantResponse(endpoint)) return true diff --git a/internal/proxy/range_metric_compat.go b/internal/proxy/range_metric_compat.go index ca97299fe..f04a5eea1 100644 --- a/internal/proxy/range_metric_compat.go +++ b/internal/proxy/range_metric_compat.go @@ -142,41 +142,19 @@ func stripLabelTransforms(logql string) string { func stripOuterCalls(logql string, prefixes ...string) string { for { logql = strings.TrimSpace(logql) - wrapped := false + inner, ok := "", false for _, prefix := range prefixes { - wrapped = wrapped || strings.HasPrefix(logql, prefix) - } - if !wrapped { - break - } - idx := strings.Index(logql, "(") - if idx < 0 { - break - } - depth := 0 - commaAt := -1 - for i := idx + 1; i < len(logql); i++ { - c := logql[i] - switch { - case c == '(': - depth++ - case c == ')': - if depth == 0 { - goto done - } - depth-- - case c == ',' && depth == 0: - commaAt = i - goto done + if strings.HasPrefix(logql, prefix) { + // The first argument, up to the comma outside strings and brackets. + inner, ok = logqlpkg.FirstArgument(logql[len(prefix):]) + break } } - done: - if commaAt < 0 { - break + if !ok { + return logql } - logql = strings.TrimSpace(logql[idx+1 : commaAt]) + logql = inner } - return logql } func parseOriginalRangeMetricSpec(logql string) (originalRangeMetricSpec, bool) { diff --git a/internal/translator/first_argument_test.go b/internal/translator/first_argument_test.go new file mode 100644 index 000000000..b541b6a7c --- /dev/null +++ b/internal/translator/first_argument_test.go @@ -0,0 +1,26 @@ +package translator + +import "testing" + +func TestFirstArgumentLenSkipsStringsAndBrackets(t *testing.T) { + for _, tc := range []struct{ in, want string }{ + {`a, b)`, `a`}, + {`rate({a="x"}[5m]), "d")`, `rate({a="x"}[5m])`}, + {`rate({a=") , ("}[5m]), "d")`, `rate({a=") , ("}[5m])`}, + {`rate({a="x"} |= ")" [5m]) + rate({b="z"}[5m]), "d")`, `rate({a="x"} |= ")" [5m]) + rate({b="z"}[5m])`}, + {"rate({a=\"x\"} |= `,)(` [5m]), \"d\")", "rate({a=\"x\"} |= `,)(` [5m])"}, + {`rate({a="\",)"}[5m]), "d")`, `rate({a="\",)"}[5m])`}, + {`sum by (a, b) (x))`, `sum by (a, b) (x)`}, + {`unclosed(`, `unclosed(`}, + } { + if got := tc.in[:FirstArgumentLen(tc.in)]; got != tc.want { + t.Errorf("FirstArgumentLen(%q) takes %q, want %q", tc.in, got, tc.want) + } + } + if first, rest, ok := splitFuncFirstArg(`rate({a=") , ("}[5m]), "d", "e")`); !ok || first != `rate({a=") , ("}[5m])` || rest != `"d", "e")` { + t.Errorf("splitFuncFirstArg = %q, %q, %v", first, rest, ok) + } + if _, _, ok := splitFuncFirstArg(`unclosed(`); ok { + t.Error("splitFuncFirstArg accepted an unclosed call") + } +} diff --git a/internal/translator/translator.go b/internal/translator/translator.go index 8d0baaeb1..ca551a0f5 100644 --- a/internal/translator/translator.go +++ b/internal/translator/translator.go @@ -1740,23 +1740,46 @@ func convertGoTemplate(tmpl string) string { // returning (firstArg, rest, ok). rest is the text after the first comma at depth 0. // Input starts right after the opening '(' of the outer function. func splitFuncFirstArg(s string) (first, rest string, ok bool) { + n := FirstArgumentLen(s) + if n == len(s) { + return "", "", false + } + if s[n] == ',' { + return strings.TrimSpace(s[:n]), strings.TrimSpace(s[n+1:]), true + } + return strings.TrimSpace(s[:n]), "", true +} + +// FirstArgumentLen is the length of the first argument of a call, given the +// text after its '(': up to the first comma or unmatched ')' outside strings +// (double-quoted with escapes, or backtick-quoted) and nested brackets. It is +// len(s) when the call is not closed. +func FirstArgumentLen(s string) int { depth := 0 - for i, c := range s { - switch c { - case '(': + var quote byte + for i := 0; i < len(s); i++ { + c := s[i] + switch { + case quote != 0: + if c == '\\' && quote == '"' { + i++ + } else if c == quote { + quote = 0 + } + case c == '"' || c == '`': + quote = c + case c == '(' || c == '[' || c == '{': depth++ - case ')': + case c == ')' || c == ']' || c == '}': if depth == 0 { - return strings.TrimSpace(s[:i]), "", true + return i } depth-- - case ',': - if depth == 0 { - return strings.TrimSpace(s[:i]), strings.TrimSpace(s[i+1:]), true - } + case c == ',' && depth == 0: + return i } } - return "", "", false + return len(s) } // parseAllStringArgs parses all quoted string arguments (double- or back-tick-quoted) From d42e815cfa817ff801ec7b69d8df20d0fab34e2c Mon Sep 17 00:00:00 2001 From: szibis Date: Mon, 12 Oct 2026 12:56:58 +0200 Subject: [PATCH 04/18] docs: regenerate the conformance wiring --- conformance/registry/generated/wiring.json | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/conformance/registry/generated/wiring.json b/conformance/registry/generated/wiring.json index 84533f221..111432b84 100644 --- a/conformance/registry/generated/wiring.json +++ b/conformance/registry/generated/wiring.json @@ -181,12 +181,14 @@ "test/e2e-compat/metadata_scan_admission_test.go" ], "loki_api_v1_query": [ + "internal/proxy/instant_result_type_test.go", "internal/proxy/parenthesized_range_test.go", "internal/proxy/series_limit_routes_test.go", "internal/proxy/tenant_query_limits_test.go", "internal/proxy/tumbling_axis_parity_test.go", "internal/proxy/unwrap_conversion_error_test.go", "test/e2e-compat/extracted_suffix_collision_test.go", + "test/e2e-compat/instant_result_type_test.go", "test/e2e-compat/parenthesized_range_or_filters_test.go", "test/e2e-compat/parser_capture_identity_test.go", "test/e2e-compat/plain_name_after_parser_test.go", @@ -373,6 +375,10 @@ "test/e2e-compat/extracted_metric_pushdown_test.go", "test/e2e-compat/extracted_suffix_collision_test.go" ], + "semantics/instant-result-type-missing": [ + "internal/proxy/instant_result_type_test.go", + "test/e2e-compat/instant_result_type_test.go" + ], "semantics/json-extracted-metric-pushdown": [ "internal/proxy/ordered_json_extracted_differential_test.go", "internal/proxy/ordered_json_extracted_pushdown_test.go", @@ -541,7 +547,7 @@ "test/e2e-compat/unwrap_ungrouped_sum_test.go" ] }, - "registry_items": 299, + "registry_items": 300, "routes_without_registry_item": [], "test_to_items": { "cmd/proxy/main_test.go": [ @@ -632,6 +638,10 @@ "internal/proxy/gaps_test.go": [ "loki_api_v1_index_stats" ], + "internal/proxy/instant_result_type_test.go": [ + "loki_api_v1_query", + "semantics/instant-result-type-missing" + ], "internal/proxy/instant_unwrap_sum_test.go": [ "profiles/sanitized-json-key-grouping" ], @@ -944,6 +954,10 @@ "semantics/sliding-window-recurrence-exact", "series-limits-and-partial-results" ], + "test/e2e-compat/instant_result_type_test.go": [ + "loki_api_v1_query", + "semantics/instant-result-type-missing" + ], "test/e2e-compat/json_filter_pushdown_test.go": [ "parser-error-and-label-collision", "semantics/json-filter-pushdown-translated-label", @@ -1064,7 +1078,7 @@ "profiles/structured-metadata-keys-per-profile" ] }, - "tests_with_markers": 96, + "tests_with_markers": 98, "unknown_ids": {}, "unwired_items": [ "api_prom_label", @@ -1131,6 +1145,7 @@ "quality/labels-backfill-into-cached-bucket", "quality/stats-volume-freshness", "semantics/absent-over-time-semantics", + "semantics/aggregation-over-binary-dropped", "semantics/binary-ignoring-matching", "semantics/count-values-unsupported", "semantics/detected-fields-extracted-suffix", @@ -1139,7 +1154,6 @@ "semantics/extracted-derived-labels-raw-evaluator", "semantics/extracted-ungrouped-metric-raw-rows", "semantics/first-last-over-time-translation", - "semantics/instant-result-type-missing", "semantics/ip-filter-precision", "semantics/label-filter-expression-syntax", "semantics/label-replace-new-label", @@ -1172,5 +1186,5 @@ "versions/capabilities-floor-vs-matrix", "versions/rejected-stats-query-wording-vl-1.47" ], - "wired_items": 106 + "wired_items": 107 } From ccee2363b542617e619fa4ee55b277f280d59ce7 Mon Sep 17 00:00:00 2001 From: szibis Date: Mon, 12 Oct 2026 13:04:32 +0200 Subject: [PATCH 05/18] chore(conformance): record #735 as the fix of instant-result-type-missing --- CHANGELOG.md | 2 +- .../instant-result-type-missing.yaml | 4 +- conformance/reports/parity-gaps.md | 59 ++++++++++--------- 3 files changed, 34 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5019854a8..403645ce7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - **An instant query is answered with its failure status and a resultType, as - Loki does, not with a 200 that has neither (#TBD).** `stdvar without (app) + Loki does, not with a 200 that has neither (#735).** `stdvar without (app) (...)` and `label_replace(avg by (app) (a + b / c) * 2, ...)` came back as HTTP 200 with an error body and no `resultType`, which Grafana and every client decoding by type (`pkg/loghttp/query.go` `QueryResponse.UnmarshalJSON` fails on diff --git a/conformance/registry/cases/semantics/instant-result-type-missing.yaml b/conformance/registry/cases/semantics/instant-result-type-missing.yaml index b8f2d1496..d8ff70cd2 100644 --- a/conformance/registry/cases/semantics/instant-result-type-missing.yaml +++ b/conformance/registry/cases/semantics/instant-result-type-missing.yaml @@ -60,8 +60,10 @@ compare: vector. gap: status: fixed - fixed_by: "#TBD" + fixed_by: "#735" impact: grafana-datasource + visual: none + visual_reason: 'instant queries only; the visual harness captures Explore range panels (queryType range), so the fix is proven by the instant resultType table and the e2e against Loki' effort: S area: 'internal/proxy handleQuery (captured post-processing status, label transform routing, constants), multitenant.go' loki: 'pkg/logql/engine.go evalSample/evalLiteral/evalVector, JoinSampleVector; pkg/querier/queryrange/marshal.go ResultToResponse; pkg/querier/http.go InstantQueryHandler; pkg/loghttp/query.go UnmarshalJSON' diff --git a/conformance/reports/parity-gaps.md b/conformance/reports/parity-gaps.md index 47d3231db..558e2db2a 100644 --- a/conformance/reports/parity-gaps.md +++ b/conformance/reports/parity-gaps.md @@ -36,35 +36,35 @@ Last run: 2026-10-05T20:16:15Z on `fcaccfcc`, window [1791220800, 1791224400]. | 15 | `semantics/stddev-stdvar-grouped-values` — stddev and stdvar by a label compute different values (open) | open | explore-visible | 7 | 5 | S | internal/translator stddev/stdvar outer aggregation (by-group fold) | pkg/logql/evaluator.go VectorAggEvaluator stddev/stdvar | TestCompat_StddevStdvarBy (test/e2e-compat, group semantics) | | 16 | `semantics/label-replace-new-label` — label_replace does not add a new destination label (open) | open | explore-visible | 5 | 1 | S | internal/proxy label_replace post-processing | pkg/logql/evaluator.go:387, :1253 label_replace evaluator | TestCompat_LabelReplaceNewLabel (test/e2e-compat, group semantics) | | 17 | `quality/label-values-chunk-window` — Loki lists label values of streams whose chunks overlap the window; the proxy only of streams with lines in it (open) | open | explore-visible | 2 | 2 | M | internal/proxy label values window (chunk-overlap semantics) | pkg/querier/querier.go Label (index lookups by chunk overlap) | TestCompat_LabelValuesWindowEdge (test/e2e-compat, group core) | -| 18 | `semantics/pattern-line-filter` — An empty pattern line filter (`\|> ""`, `!> ""`) selects the opposite lines (open) | open | explore-visible | 2 | 2 | S | internal/translator pattern line filter | pkg/logql/syntax/lex.go:26,29 (NPA, PIPE_PATTERN); pkg/logql/log/filter.go pattern filter | TestCompat_PatternLineFilter (test/e2e-compat, group semantics) | -| 19 | `semantics/log-entries-differ-same-count` — A log query returns the same number of lines with different entries (open, cause not isolated) | open | explore-visible | 1 | 1 | S | to be isolated (log query paging / numeric label filters) | pkg/logql/log/label_filter.go numeric filters; pkg/iter entry ordering | TestCompat_NumericLabelFilterLines (test/e2e-compat, group semantics) | -| 20 | `profiles/label-filter-regexp-anchoring` — A label-filter regexp after a parser is not anchored the way Loki's is (open) | open | explore-visible | — | — | S | internal/translator label-filter regexp emission (anchor ^(?:re)$); external PR #572 open | pkg/logql/log/label_filter.go (regexp matcher on the whole value) | TestCompat_LabelFilterRegexpAnchoring (test/e2e-compat, group semantics) | -| 21 | `profiles/parsed-key-colliding-with-structured-metadata` — A parsed key named like structured metadata loses the metadata value in VictoriaLogs (open) | open | explore-visible | — | — | L | internal/translator json / logfmt unpack (original stored value) and internal/proxy row classification | pkg/logql/log/parser.go:152 HasInCategory(key, StructuredMetadataLabel) then duplicateSuffix | TestCompat_ParsedKeyAgainstStructuredMetadata (test/e2e-compat, group otel-edge) | -| 22 | `quality/label-cache-aged-labels-kept` — Labels that age out of the window are never dropped from the merged label cache (open) | open | explore-visible | — | — | S | internal/proxy/label_metadata.go:1259 mergeLabelsIntoCache | pkg/querier/querier.go Label (window-bounded) | TestE2E_LabelsAgeOutOfWindow (test/e2e-compat, group core) | -| 23 | `quality/labels-backfill-into-cached-bucket` — Rows backfilled into a past bucket the label inventory already cached with data appear in /labels only after its revalidation delay (open) | open | explore-visible | — | — | M | internal/proxy/metadata_inventory.go (bucket revalidation delay) | pkg/querier/querier.go Label (index read per request) | TestCompat_LabelsBackfillIntoCachedBucket (test/e2e-compat, group core) | -| 24 | `semantics/drop-keep-plain-name-after-parser` — A log query's drop, keep or label_format source of a stream label's plain name after a parser acts on the stored field; Loki acts on the ... | open | explore-visible | — | — | S | internal/proxy stage-field exposure (drop and keep of a name a parser also reads) | pkg/logql/log/labels.go LabelsBuilder.Del / getWithCategory; pkg/logql/log/drop_labels.go, keep_labels.go | TestCompat_DropKeepPlainNameAfterParser (test/e2e-compat, group core) | -| 25 | `semantics/extracted-ungrouped-metric-raw-rows` — An ungrouped range metric over a `\| json` pipeline filtered on name_extracted still reads raw rows and fails at the row limit (open) | open | explore-visible | — | — | L | internal/proxy bare-parser metric evaluator (fetchBareParserMetricSeries) and the ordered JSON pushdown (without grouping) | pkg/logql range aggregation keeps every label of the sample | TestCompat_ExtractedUngroupedMetricAtSixHours (test/e2e-compat, group core) | -| 26 | `semantics/repeated-label-filter-replaced-across-label-format` — A second filter on the same label replaces the first even with a label_format between them (open) | open | explore-visible | — | — | S | internal/translator labelFilterLatest (latest filter per field replaces the earlier one in place) | pkg/logql/log/pipeline.go (stages run in order); label_filter.go | TestCompat_RepeatedLabelFilterAcrossLabelFormat (test/e2e-compat, group core) | -| 27 | `semantics/two-parsers-keep-one` — A query with `\| json` and `\| logfmt` runs the one parser the data answers to (open) | open | explore-visible | — | — | M | internal/proxy/query_translation.go preferWorkingParser (probe-based removal of a parser stage) | pkg/logql/log/pipeline.go: every stage runs on every line; parser.go skips a key ParserHint.Extracted | TestCompat_TwoParsersOnMixedFormats (test/e2e-compat, group semantics) | -| 28 | `semantics/unwrap-duration-seconds` — '`unwrap duration_seconds(label)` answers no data where Loki converts it like duration() (open)' | open | explore-visible | — | — | S | internal/translator extractUnwrapField and unwrapGateField (duration_seconds conversion) | pkg/logql/syntax/extractor.go:63-71 | TestCompat_UnwrapDurationSeconds (test/e2e-compat, group core) | -| 29 | `semantics/unwrap-first-last-under-aggregation` — first_over_time and last_over_time with unwrap under sum fail with a VictoriaLogs 422 (open) | open | explore-visible | — | — | M | internal/translator first_over_time / last_over_time under a vector aggregation | pkg/logql/range_vector.go first/last | an e2e over sum(first_over_time\|last_over_time(... \| unwrap ...)) | -| 30 | `semantics/unwrap-over-time-range-windows` — sum, avg, max and min over time with unwrap answer the wrong window on query_range (open) | open | explore-visible | — | — | M | internal/proxy range_metric_compat.go shouldUseManualRangeMetricCompat (unwrap sum/avg/min/max) and metric_binary.go statsRateRangeEquals... | pkg/logql/range_vector.go:102-189 batchRangeVectorIterator (window (t-range, t]); range_vector.go:371-420 sumOverTime, avgOverTime, maxOv... | TestCompat_UnwrapOverTimeRangeWindows (test/e2e-compat, group core) over sum/avg/max/min x tumbling/sliding x logfmt/json | -| 31 | `semantics/unwrap-per-series-identity-raw-rows` — A function taken per series (avg, max, min over time, rate_counter) under sum or sum by uses the stream as the series (open) | open | explore-visible | — | — | L | internal/translator unwrapInnerGrouping (sum by over a per-series function) and the raw evaluators series key (collectRangeMetricSamples) | pkg/logql/evaluator.go:435-529 VectorAggEvaluator.Next; pkg/logql/log/metrics_extraction.go:150-178 labelSampleExtractor (series = stream... | TestCompat_UnwrapPerSeriesIdentity (test/e2e-compat, group core) over sum and sum by x avg/max/min/rate_counter, logfmt and json | -| 32 | `semantics/unwrap-quantile-response-limit` — quantile_over_time over a JSON unwrap fails at the manual metric response limit from 6 hours (open) | open | explore-visible | — | — | L | internal/proxy raw-row quantile evaluator (manual metric response cap) | pkg/logql/range_vector.go quantile_over_time | bench/ab set unwrap-conversion-error E2 at 6h/24h/7d | -| 33 | `semantics/unwrap-range-last-point` — A grouped unwrap range metric misses the sample at `end` (open) | open | explore-visible | — | — | S | internal/proxy range-metric unwrap path (end bucket) | pkg/logql/range_vector.go step iteration start..end inclusive | TestCompat_UnwrapRangeEndSample (test/e2e-compat, group semantics) | -| 34 | `semantics/unwrap-sum-without-collapses` — sum without over an unwrap range function returns a single series (open) | open | explore-visible | — | — | M | internal/translator `without` grouping over unwrap range functions | pkg/logql/evaluator.go newVectorAggEvaluator (without) | an e2e over sum without (...) with unwrap on query_range and query | -| 35 | `drilldown/detected-field-values` — detected_field values differ: none for service_name, other values for numeric and id fields (open) | open | drilldown-visible | 33 | 3 | M | internal/proxy detected_field values (sampling window and stream-label fields) | pkg/querier detected field values (v3.7.7): values of the sampled lines | TestCompat_DetectedFieldValues (test/e2e-compat, group drilldown) | -| 36 | `semantics/detected-fields-extracted-suffix` — detected_fields omits Loki's level_extracted and version_extracted (open; service fields are owner-kept) | open | drilldown-visible | 15 | 2 | M | internal/proxy/label_handlers.go handleDetectedFields (parsed fields named like a stream label) | pkg/logql/log/parser.go:25 duplicateSuffix; detected fields come from the parsed labels of sampled entries | TestCompat_DetectedFieldsExtractedSuffix (test/e2e-compat, group drilldown) | -| 37 | `profiles/json-number-label-text` — JSON numbers become label values in a different text form ("14.0" vs "14") (open) | open | drilldown-visible | 12 | 6 | M | internal/proxy JSON number rendering of unpacked values (VictoriaLogs unpack_json normalises) | pkg/logql/log/parser.go readValue (raw number bytes) | TestCompat_JSONNumberLabelText (test/e2e-compat, group drilldown) | -| 38 | `severity/detected-level-filter-json-level` — `\| detected_level=` filters select different lines than Loki (JSON-level lines, stream level precedence) (open) | open | drilldown-visible | 12 | 8 | M | internal/translator/detected_level_logsql.go (filter predicate) vs internal/proxy/detected_level.go (response derivation) | pkg/distributor field detection (v3.7.7) stores detected_level as structured metadata; filter reads it | TestCompat_DetectedLevelFilterJSONLevel (test/e2e-compat, group drilldown) | -| 39 | `limits/drilldown-7d-breakdown-response-limit` — Metric queries fail on the 64 MiB buffered-response budget where Loki answers (open) | open | drilldown-visible | 8 | 4 | M | internal/proxy/range_metric_compat.go:767, execution_limits.go:11 (raw-row metric paths buffer rows; push down or stream) | pkg/logql engine series limit (max_query_series) with partial result for Drilldown | TestCompat_MetricBufferedBudget1h + TestE2E_Drilldown7dHighCardinalityBreakdown (test/e2e-compat, groups semantics and drilldown) | -| 40 | `quality/index-stats-streams-differ` — index/stats stream counts differ from Loki in both directions (open) | open | drilldown-visible | 6 | 2 | S | internal/proxy/label_handlers.go handleIndexStats (stream counting per window) | pkg/storage index stats (streams from the index for the window) | bench/parity rerun with the /series cross-check, then TestCompat_IndexStatsStreams (test/e2e-compat, group drilldown) | -| 41 | `drilldown/detected-fields-type-inference` — detected_fields infers a different type for some fields (open) | open | drilldown-visible | — | — | S | internal/proxy detected_fields type inference | pkg/querier detected fields type detection (v3.7.7): int before duration before string | TestCompat_DetectedFieldsTypes (test/e2e-compat, group drilldown) | -| 42 | `profiles/existence-filter-on-sanitized-key` — An existence filter on a sanitized key reads the stored dotted field, not the JSON line key (open) | open | drilldown-visible | — | — | M | internal/translator (label-filter key variants, parsed_key_variants.go); Drilldown single-field fast paths | pkg/logql/log/parser.go:141 sanitizeLabelKey on every extracted key; label filter reads the parsed label | TestCompat_ExistenceFilterSanitizedKey (test/e2e-compat, group semantics) | -| 43 | `quality/detected-metadata-proxy-cache` — detected_labels and detected_fields are served from proxy caches; Loki answers each request fresh (open) | open | drilldown-visible | — | — | S | internal/proxy/cache_keys.go:48,130 (detected_* cache TTL) + recent-tail freshness bypass | pkg/querier detected labels/fields: no results cache in v3.7.7 | TestE2E_DetectedMetadataFreshness (test/e2e-compat, group drilldown) | -| 44 | `quality/stats-volume-freshness` — index/stats and volume answers ending now are cached; Loki does not cache inside max_stats_cache_freshness (10m) (open) | open | drilldown-visible | — | — | S | internal/proxy/volume.go handleVolume, label_handlers.go handleIndexStats (cache freshness) | pkg/validation/limits.go:413-414 max_stats_cache_freshness default 10m | TestE2E_StatsVolumeFreshness (test/e2e-compat, group drilldown) | -| 45 | `semantics/parse-error-text-parity` — Invalid queries fail on both sides, with different error text or status (open) | open | grafana-datasource | 210 | 129 | L | internal/logql parser error messages (Loki's goyacc messages and positions) + VictoriaLogs error mapping | pkg/logql/syntax/parser.go ParseError (line, col) and syntax.y | TestCompat_ParseErrorText corpus (test/e2e-compat, bad_request_status_parity_test.go) | -| 46 | `semantics/instant-result-type-missing` — Some instant binary/label_replace answers carry no resultType (open) | open | grafana-datasource | 4 | 2 | S | internal/proxy instant response assembly (binary/label_replace/without paths) | pkg/logqlmodel result types; pkg/util/marshal query response | TestCompat_InstantResultTypeAlwaysSet (test/e2e-compat, group semantics) | +| 18 | `semantics/aggregation-over-binary-dropped` — An aggregation over a binary expression is translated to its first operand (open) | open | explore-visible | 2 | 1 | M | internal/translator extractOuterAggregation and internal/proxy handleQuery routing of an aggregation over a binary expression | pkg/logql/evaluator.go VectorAggEvaluator over a binary evaluator | TestCompat_AggregationOverBinary (test/e2e-compat, group semantics) | +| 19 | `semantics/pattern-line-filter` — An empty pattern line filter (`\|> ""`, `!> ""`) selects the opposite lines (open) | open | explore-visible | 2 | 2 | S | internal/translator pattern line filter | pkg/logql/syntax/lex.go:26,29 (NPA, PIPE_PATTERN); pkg/logql/log/filter.go pattern filter | TestCompat_PatternLineFilter (test/e2e-compat, group semantics) | +| 20 | `semantics/log-entries-differ-same-count` — A log query returns the same number of lines with different entries (open, cause not isolated) | open | explore-visible | 1 | 1 | S | to be isolated (log query paging / numeric label filters) | pkg/logql/log/label_filter.go numeric filters; pkg/iter entry ordering | TestCompat_NumericLabelFilterLines (test/e2e-compat, group semantics) | +| 21 | `profiles/label-filter-regexp-anchoring` — A label-filter regexp after a parser is not anchored the way Loki's is (open) | open | explore-visible | — | — | S | internal/translator label-filter regexp emission (anchor ^(?:re)$); external PR #572 open | pkg/logql/log/label_filter.go (regexp matcher on the whole value) | TestCompat_LabelFilterRegexpAnchoring (test/e2e-compat, group semantics) | +| 22 | `profiles/parsed-key-colliding-with-structured-metadata` — A parsed key named like structured metadata loses the metadata value in VictoriaLogs (open) | open | explore-visible | — | — | L | internal/translator json / logfmt unpack (original stored value) and internal/proxy row classification | pkg/logql/log/parser.go:152 HasInCategory(key, StructuredMetadataLabel) then duplicateSuffix | TestCompat_ParsedKeyAgainstStructuredMetadata (test/e2e-compat, group otel-edge) | +| 23 | `quality/label-cache-aged-labels-kept` — Labels that age out of the window are never dropped from the merged label cache (open) | open | explore-visible | — | — | S | internal/proxy/label_metadata.go:1259 mergeLabelsIntoCache | pkg/querier/querier.go Label (window-bounded) | TestE2E_LabelsAgeOutOfWindow (test/e2e-compat, group core) | +| 24 | `quality/labels-backfill-into-cached-bucket` — Rows backfilled into a past bucket the label inventory already cached with data appear in /labels only after its revalidation delay (open) | open | explore-visible | — | — | M | internal/proxy/metadata_inventory.go (bucket revalidation delay) | pkg/querier/querier.go Label (index read per request) | TestCompat_LabelsBackfillIntoCachedBucket (test/e2e-compat, group core) | +| 25 | `semantics/drop-keep-plain-name-after-parser` — A log query's drop, keep or label_format source of a stream label's plain name after a parser acts on the stored field; Loki acts on the ... | open | explore-visible | — | — | S | internal/proxy stage-field exposure (drop and keep of a name a parser also reads) | pkg/logql/log/labels.go LabelsBuilder.Del / getWithCategory; pkg/logql/log/drop_labels.go, keep_labels.go | TestCompat_DropKeepPlainNameAfterParser (test/e2e-compat, group core) | +| 26 | `semantics/extracted-ungrouped-metric-raw-rows` — An ungrouped range metric over a `\| json` pipeline filtered on name_extracted still reads raw rows and fails at the row limit (open) | open | explore-visible | — | — | L | internal/proxy bare-parser metric evaluator (fetchBareParserMetricSeries) and the ordered JSON pushdown (without grouping) | pkg/logql range aggregation keeps every label of the sample | TestCompat_ExtractedUngroupedMetricAtSixHours (test/e2e-compat, group core) | +| 27 | `semantics/repeated-label-filter-replaced-across-label-format` — A second filter on the same label replaces the first even with a label_format between them (open) | open | explore-visible | — | — | S | internal/translator labelFilterLatest (latest filter per field replaces the earlier one in place) | pkg/logql/log/pipeline.go (stages run in order); label_filter.go | TestCompat_RepeatedLabelFilterAcrossLabelFormat (test/e2e-compat, group core) | +| 28 | `semantics/two-parsers-keep-one` — A query with `\| json` and `\| logfmt` runs the one parser the data answers to (open) | open | explore-visible | — | — | M | internal/proxy/query_translation.go preferWorkingParser (probe-based removal of a parser stage) | pkg/logql/log/pipeline.go: every stage runs on every line; parser.go skips a key ParserHint.Extracted | TestCompat_TwoParsersOnMixedFormats (test/e2e-compat, group semantics) | +| 29 | `semantics/unwrap-duration-seconds` — '`unwrap duration_seconds(label)` answers no data where Loki converts it like duration() (open)' | open | explore-visible | — | — | S | internal/translator extractUnwrapField and unwrapGateField (duration_seconds conversion) | pkg/logql/syntax/extractor.go:63-71 | TestCompat_UnwrapDurationSeconds (test/e2e-compat, group core) | +| 30 | `semantics/unwrap-first-last-under-aggregation` — first_over_time and last_over_time with unwrap under sum fail with a VictoriaLogs 422 (open) | open | explore-visible | — | — | M | internal/translator first_over_time / last_over_time under a vector aggregation | pkg/logql/range_vector.go first/last | an e2e over sum(first_over_time\|last_over_time(... \| unwrap ...)) | +| 31 | `semantics/unwrap-over-time-range-windows` — sum, avg, max and min over time with unwrap answer the wrong window on query_range (open) | open | explore-visible | — | — | M | internal/proxy range_metric_compat.go shouldUseManualRangeMetricCompat (unwrap sum/avg/min/max) and metric_binary.go statsRateRangeEquals... | pkg/logql/range_vector.go:102-189 batchRangeVectorIterator (window (t-range, t]); range_vector.go:371-420 sumOverTime, avgOverTime, maxOv... | TestCompat_UnwrapOverTimeRangeWindows (test/e2e-compat, group core) over sum/avg/max/min x tumbling/sliding x logfmt/json | +| 32 | `semantics/unwrap-per-series-identity-raw-rows` — A function taken per series (avg, max, min over time, rate_counter) under sum or sum by uses the stream as the series (open) | open | explore-visible | — | — | L | internal/translator unwrapInnerGrouping (sum by over a per-series function) and the raw evaluators series key (collectRangeMetricSamples) | pkg/logql/evaluator.go:435-529 VectorAggEvaluator.Next; pkg/logql/log/metrics_extraction.go:150-178 labelSampleExtractor (series = stream... | TestCompat_UnwrapPerSeriesIdentity (test/e2e-compat, group core) over sum and sum by x avg/max/min/rate_counter, logfmt and json | +| 33 | `semantics/unwrap-quantile-response-limit` — quantile_over_time over a JSON unwrap fails at the manual metric response limit from 6 hours (open) | open | explore-visible | — | — | L | internal/proxy raw-row quantile evaluator (manual metric response cap) | pkg/logql/range_vector.go quantile_over_time | bench/ab set unwrap-conversion-error E2 at 6h/24h/7d | +| 34 | `semantics/unwrap-range-last-point` — A grouped unwrap range metric misses the sample at `end` (open) | open | explore-visible | — | — | S | internal/proxy range-metric unwrap path (end bucket) | pkg/logql/range_vector.go step iteration start..end inclusive | TestCompat_UnwrapRangeEndSample (test/e2e-compat, group semantics) | +| 35 | `semantics/unwrap-sum-without-collapses` — sum without over an unwrap range function returns a single series (open) | open | explore-visible | — | — | M | internal/translator `without` grouping over unwrap range functions | pkg/logql/evaluator.go newVectorAggEvaluator (without) | an e2e over sum without (...) with unwrap on query_range and query | +| 36 | `drilldown/detected-field-values` — detected_field values differ: none for service_name, other values for numeric and id fields (open) | open | drilldown-visible | 33 | 3 | M | internal/proxy detected_field values (sampling window and stream-label fields) | pkg/querier detected field values (v3.7.7): values of the sampled lines | TestCompat_DetectedFieldValues (test/e2e-compat, group drilldown) | +| 37 | `semantics/detected-fields-extracted-suffix` — detected_fields omits Loki's level_extracted and version_extracted (open; service fields are owner-kept) | open | drilldown-visible | 15 | 2 | M | internal/proxy/label_handlers.go handleDetectedFields (parsed fields named like a stream label) | pkg/logql/log/parser.go:25 duplicateSuffix; detected fields come from the parsed labels of sampled entries | TestCompat_DetectedFieldsExtractedSuffix (test/e2e-compat, group drilldown) | +| 38 | `profiles/json-number-label-text` — JSON numbers become label values in a different text form ("14.0" vs "14") (open) | open | drilldown-visible | 12 | 6 | M | internal/proxy JSON number rendering of unpacked values (VictoriaLogs unpack_json normalises) | pkg/logql/log/parser.go readValue (raw number bytes) | TestCompat_JSONNumberLabelText (test/e2e-compat, group drilldown) | +| 39 | `severity/detected-level-filter-json-level` — `\| detected_level=` filters select different lines than Loki (JSON-level lines, stream level precedence) (open) | open | drilldown-visible | 12 | 8 | M | internal/translator/detected_level_logsql.go (filter predicate) vs internal/proxy/detected_level.go (response derivation) | pkg/distributor field detection (v3.7.7) stores detected_level as structured metadata; filter reads it | TestCompat_DetectedLevelFilterJSONLevel (test/e2e-compat, group drilldown) | +| 40 | `limits/drilldown-7d-breakdown-response-limit` — Metric queries fail on the 64 MiB buffered-response budget where Loki answers (open) | open | drilldown-visible | 8 | 4 | M | internal/proxy/range_metric_compat.go:767, execution_limits.go:11 (raw-row metric paths buffer rows; push down or stream) | pkg/logql engine series limit (max_query_series) with partial result for Drilldown | TestCompat_MetricBufferedBudget1h + TestE2E_Drilldown7dHighCardinalityBreakdown (test/e2e-compat, groups semantics and drilldown) | +| 41 | `quality/index-stats-streams-differ` — index/stats stream counts differ from Loki in both directions (open) | open | drilldown-visible | 6 | 2 | S | internal/proxy/label_handlers.go handleIndexStats (stream counting per window) | pkg/storage index stats (streams from the index for the window) | bench/parity rerun with the /series cross-check, then TestCompat_IndexStatsStreams (test/e2e-compat, group drilldown) | +| 42 | `drilldown/detected-fields-type-inference` — detected_fields infers a different type for some fields (open) | open | drilldown-visible | — | — | S | internal/proxy detected_fields type inference | pkg/querier detected fields type detection (v3.7.7): int before duration before string | TestCompat_DetectedFieldsTypes (test/e2e-compat, group drilldown) | +| 43 | `profiles/existence-filter-on-sanitized-key` — An existence filter on a sanitized key reads the stored dotted field, not the JSON line key (open) | open | drilldown-visible | — | — | M | internal/translator (label-filter key variants, parsed_key_variants.go); Drilldown single-field fast paths | pkg/logql/log/parser.go:141 sanitizeLabelKey on every extracted key; label filter reads the parsed label | TestCompat_ExistenceFilterSanitizedKey (test/e2e-compat, group semantics) | +| 44 | `quality/detected-metadata-proxy-cache` — detected_labels and detected_fields are served from proxy caches; Loki answers each request fresh (open) | open | drilldown-visible | — | — | S | internal/proxy/cache_keys.go:48,130 (detected_* cache TTL) + recent-tail freshness bypass | pkg/querier detected labels/fields: no results cache in v3.7.7 | TestE2E_DetectedMetadataFreshness (test/e2e-compat, group drilldown) | +| 45 | `quality/stats-volume-freshness` — index/stats and volume answers ending now are cached; Loki does not cache inside max_stats_cache_freshness (10m) (open) | open | drilldown-visible | — | — | S | internal/proxy/volume.go handleVolume, label_handlers.go handleIndexStats (cache freshness) | pkg/validation/limits.go:413-414 max_stats_cache_freshness default 10m | TestE2E_StatsVolumeFreshness (test/e2e-compat, group drilldown) | +| 46 | `semantics/parse-error-text-parity` — Invalid queries fail on both sides, with different error text or status (open) | open | grafana-datasource | 210 | 129 | L | internal/logql parser error messages (Loki's goyacc messages and positions) + VictoriaLogs error mapping | pkg/logql/syntax/parser.go ParseError (line, col) and syntax.y | TestCompat_ParseErrorText corpus (test/e2e-compat, bad_request_status_parity_test.go) | | 47 | `semantics/proxy-rejects-valid-logql` — The proxy rejects LogQL Loki accepts: comments, negative and compound offsets, filters after a range, logfmt flags, ip() label filters (o... | open | api-only | 79 | 48 | M | internal/logql parser (lexer: comments, signed durations, flags) + translator | pkg/logql/syntax/lex.go:149 comments, :90-91 OpStrict/OpKeepEmpty; syntax.y offsetExpr; pkg/logql/log/ip.go | TestCompat_LogQLSurfaceAccepted (test/e2e-compat, group semantics), one subtest per family | | 48 | `semantics/proxy-accepts-invalid-logql` — Queries Loki rejects with 400 are answered by the proxy (open) | open | api-only | 18 | 14 | S | internal/logql parser validation | pkg/logql/syntax/ast.go:1446,1579 grouping not allowed; duration parsing | TestCompat_InvalidLogQLRejected (test/e2e-compat, bad_request_status_parity_test.go) | | 49 | `parsed-identity/regexp-partial-match-series-count` — A partially-matching \| regexp returns about 3x Loki's series count | open | api-only | 2 | 2 | M | internal/proxy regexp capture series identity (non-matching lines) | pkg/logql/log/parser.go RegexpParser (no capture label for a non-matching line) | TestCompat_RegexpPartialMatchSeries (test/e2e-compat, group semantics) | @@ -114,4 +114,5 @@ Last run: 2026-10-05T20:16:15Z on `fcaccfcc`, window [1791220800, 1791224400]. | `semantics/unwrap-rate-ungrouped-sum-per-stream` — An ungrouped sum over rate with unwrap is one series on query_range, like Loki (fixed) | #716 | 0 | 0 | | `severity/otlp-severity-before-vl-1.50` — OTLP rows stored by VictoriaLogs before v1.50 (one `severity` field, no severity_number) get detected_level "Unspecified" / "Info2" (fixed) | #666 | 0 | 0 | | `versions/rejected-stats-query-wording-vl-1.47` — The proxy did not recognise VictoriaLogs v1.47 parse errors, so a rejected query was not Loki's 400 (fixed) | #659 | 0 | 0 | +| `semantics/instant-result-type-missing` — An instant query answers its failure status and a resultType, like Loki (fixed) | #735 | 1 | 2 | | `versions/capabilities-floor-vs-matrix` — logsql.Capabilities treated VictoriaLogs below v1.40 as unsupported while the matrix claimed v1.30 (fixed) | #659 | 0 | 0 | From 5dab7521db362b53533834466e64a7a41a494a00 Mon Sep 17 00:00:00 2001 From: szibis Date: Mon, 12 Oct 2026 13:47:42 +0200 Subject: [PATCH 06/18] fix: a label transform is stripped only when the call is the whole expression label_replace(a, ...) / b is a binary expression, not a transform of a: the instant routing stripped the call by its prefix and answered a alone. The wrapper is now removed only when its closing parenthesis ends the expression. --- internal/proxy/instant_result_type_test.go | 53 ++++++++++++++++++++++ internal/proxy/range_metric_compat.go | 12 ++++- 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/internal/proxy/instant_result_type_test.go b/internal/proxy/instant_result_type_test.go index a1ee6f693..228982fb0 100644 --- a/internal/proxy/instant_result_type_test.go +++ b/internal/proxy/instant_result_type_test.go @@ -497,3 +497,56 @@ func TestInstantQueryMultiTenantNoTenantUnparseable(t *testing.T) { t.Fatalf("HTTP %d %s, want 400 with a parse error", rec.Code, rec.Body.String()) } } + +// A label transform is stripped only when the call is the whole expression: +// with an operator or another call around it, the expression is not a transform. +func TestStripLabelTransformsWholeCallOnly(t *testing.T) { + for _, in := range []string{ + `label_replace(a, "d","$1","s","(.*)") / b`, + `label_replace(a, "d","$1","s","(.*)") + label_replace(b, "d","$1","s","(.*)")`, + `sum(label_replace(a, "d","$1","s","(.*)"))`, + `label_join(a, "d","-","s") * 2`, + `label_replace(a, "d","$1","s","(.*)") / label_replace(b, "x",")","s",".*")`, + } { + if got := stripLabelTransforms(in); got != in { + t.Errorf("stripLabelTransforms(%q) = %q, want it unchanged", in, got) + } + } + for _, tc := range []struct{ in, want string }{ + {`label_replace(a, "d","$1","s","(.*)")`, `a`}, + {` label_replace(a, "d","$1","s","(.*)") `, `a`}, + {`label_replace(label_join(a + b, "z","-","s"), "d","$1","s","(.*)")`, `a + b`}, + } { + if got := stripLabelTransforms(tc.in); got != tc.want { + t.Errorf("stripLabelTransforms(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +// A label_replace as an operand of a binary expression leaves the binary +// expression to evaluate: the answer is not the operand alone. +// conformance: semantics/instant-result-type-missing, loki_api_v1_query +func TestInstantBinaryOfLabelReplaceEvaluatesTheBinary(t *testing.T) { + p := newTestProxy(t, instantBackend(t, instantBackendData).URL) + lr := `label_replace(sum by (app) (count_over_time(` + instantNS + `[1m])), "x", "$1", "app", "(.*)")` + for q, want := range map[string]map[string]string{ + lr + ` / 2`: {"a": "2.5", "b": "3.5"}, + lr + ` + ` + lr: {"a": "10", "b": "14"}, + `2 * ` + lr: {"a": "10", "b": "14"}, + } { + a := askInstant(t, p, q, "0", nil) + requireInstantAnswer(t, q, a, "vector", false) + got := map[string]string{} + for _, item := range a.resp.Data.Result { + var sample struct { + Metric map[string]string `json:"metric"` + Value []any `json:"value"` + } + _ = json.Unmarshal(item, &sample) + got[sample.Metric["app"]], _ = sample.Value[1].(string) + } + if len(got) != len(want) || got["a"] != want["a"] || got["b"] != want["b"] { + t.Errorf("%s: values by app %v, want %v\n%s", q, got, want, a.body) + } + } +} diff --git a/internal/proxy/range_metric_compat.go b/internal/proxy/range_metric_compat.go index f04a5eea1..fd542616f 100644 --- a/internal/proxy/range_metric_compat.go +++ b/internal/proxy/range_metric_compat.go @@ -145,8 +145,16 @@ func stripOuterCalls(logql string, prefixes ...string) string { inner, ok := "", false for _, prefix := range prefixes { if strings.HasPrefix(logql, prefix) { - // The first argument, up to the comma outside strings and brackets. - inner, ok = logqlpkg.FirstArgument(logql[len(prefix):]) + // The first argument, up to the comma outside strings and brackets, + // of a call that is the whole expression: `label_replace(a, ...) / b` + // is a binary, not a transform. + args := logql[len(prefix):] + inner, ok = logqlpkg.FirstArgument(args) + end := len(inner) + for end < len(args) && args[end] == ',' { + end += 1 + translator.FirstArgumentLen(args[end+1:]) + } + ok = ok && end < len(args) && strings.TrimSpace(args[end+1:]) == "" break } } From 76f3a2156370657e5fd79228a6ebdf148ec128a5 Mon Sep 17 00:00:00 2001 From: szibis Date: Mon, 12 Oct 2026 13:29:04 +0200 Subject: [PATCH 07/18] fix: label_replace adds its destination label to instant answers and reads escaped regexps The transform ran on range matrices only, so an instant vector came back without the new label. It now rewrites the labels of a vector and of a matrix alike, keeps the rest of the answer, expands the replacement as Loki does and files the points of series that end with one label set under one series on a range query, as JoinSampleVector does. The string arguments are unquoted like the LogQL lexer does, a call without the four string arguments is a 400 parse error, topk/sort/stddev over a label_replace no longer send the translator's marker to VictoriaLogs, a label_replace around a binary expression on query_range routes by the expression under it, and the without marker no longer swallows the markers that follow it. --- CHANGELOG.md | 26 + .../semantics/label-replace-new-label.yaml | 70 ++- internal/logql/semantic.go | 44 ++ .../proxy/label_replace_semantics_test.go | 595 ++++++++++++++++++ internal/proxy/metric_agg.go | 18 + internal/proxy/proxy.go | 11 +- internal/proxy/vector_matching.go | 149 ++++- internal/translator/coverage_test.go | 12 + internal/translator/translator.go | 33 +- internal/translator/translator_test.go | 27 + .../label_replace_new_label_test.go | 173 +++++ 11 files changed, 1113 insertions(+), 45 deletions(-) create mode 100644 internal/proxy/label_replace_semantics_test.go create mode 100644 test/e2e-compat/label_replace_new_label_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 403645ce7..692205d2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`label_replace` adds its destination label to instant answers, reads an + escaped regexp, and runs under `topk`, `sort` and a binary expression (#TBD).** + `label_replace(rate({app="api-gateway"}[5m]), "app_short", "$1", "app", + "(.*)-.*")` came back from `/loki/api/v1/query` without `app_short` on every + series (also through nested `label_replace`), where Loki adds it + (`pkg/logql/evaluator.go` `LabelReplaceEvaluator.Next`): the proxy applied the + transform to range matrices only. It now rewrites the labels of a vector and + of a matrix alike, keeps the rest of the answer, and expands the replacement + the way Loki does (`$1`, `${1}`, `$name`, `$$`; an empty expansion removes the + destination), with the anchored regexp. Two series that end with one label + set stay two samples on an instant query and become one series on a range + query, as `JoinSampleVector` does, and no error is raised, as in Loki. Also + fixed: the string arguments are unquoted like the LogQL lexer does, so a + regexp written `"(\\w+)\\.(\\w+)"` matches (the backslashes used to be kept, + and the regexp never matched) and an escaped quote no longer ends the + argument; `topk`, `bottomk`, `sort` and `stddev` over a `label_replace` sent + the translator's marker to VictoriaLogs and now rank what the transform + produced; and a `label_replace` around a binary expression on `query_range` + sent the binary marker to VictoriaLogs, as the instant path used to. An + invalid regexp under those wrappers is Loki's `parse error : invalid regex in + label_replace` 400. `label_replace(sum without (level) (...), ...)` kept the + excluded label and lost the transform, because the `without` marker swallowed + the `label_replace` marker after it; and a call without the four string + arguments of Loki's grammar, or with an invalid escape in one, was taken for a + log query (an empty `streams` answer on `query_range`) and is now a 400 parse + error on both endpoints. - **An instant query is answered with its failure status and a resultType, as Loki does, not with a 200 that has neither (#735).** `stdvar without (app) (...)` and `label_replace(avg by (app) (a + b / c) * 2, ...)` came back as diff --git a/conformance/registry/cases/semantics/label-replace-new-label.yaml b/conformance/registry/cases/semantics/label-replace-new-label.yaml index e8a04867c..e1325bb6e 100644 --- a/conformance/registry/cases/semantics/label-replace-new-label.yaml +++ b/conformance/registry/cases/semantics/label-replace-new-label.yaml @@ -1,20 +1,67 @@ id: semantics/label-replace-new-label -title: label_replace does not add a new destination label (open) +title: label_replace adds its destination label to every series, instant and range (fixed) proves: + - loki_api_v1_query - loki_api_v1_query_range consumer: explore why: | `label_replace(rate({app="api-gateway"}[5m]), "app_short", "$1", "app", "(.*)-.*")` adds `app_short` to every series on Loki; the proxy's series - lack the new label (also through nested label_replace). + lacked the new label (also through nested label_replace), in the instant + answers of the parity run (cluster a9d4f85c, endpoint query). The + query_range form of the row's query already carried the label on the base + commit (checked against a simulated VictoriaLogs); the instant form did not, + so the request below is the instant one. + + Loki v3.7.7: pkg/logql/syntax/ast.go mustNewLabelReplaceExpr compiles the + regexp as `^(?:regex)$` (an invalid one is the parse error "invalid regex in + label_replace: ..."); pkg/logql/evaluator.go LabelReplaceEvaluator.Next reads + the source label (empty when absent), leaves a series the regexp does not + match as it is, and otherwise expands the replacement with + Regexp.ExpandString ($1, ${1}, $name, ${name}, $$), deleting the destination + and setting it only when the expansion is not empty. It validates no label + name and raises no error for two series that end with one label set; pkg/ + logql/engine.go JoinSampleVector then keeps both samples of an instant vector + and files the points of a range query's duplicates under one series, those of + each step in turn. label_join is no Loki v3.7.7 function (the lexer has + label_replace only); the proxy keeps accepting it. + + Causes, reproduced against a simulated VictoriaLogs + (internal/proxy/label_replace_semantics_test.go): + - The proxy applied the transform to a range matrix only (applyLabelReplace, + applyLabelJoin decoded `resultType: matrix`), so an instant vector was + returned untouched. It also re-encoded only the series, dropping the rest + of the answer, and it expanded the replacement with ReplaceAllString. + - The translator took the string arguments of label_replace raw: `"\\."` was + the regexp `\\.` (a backslash and any character), never matching, and an + escaped quote ended the argument. They are now unquoted as the LogQL lexer + does (double-quoted: escapes resolved; backtick: raw). + - topk, bottomk, sort and stddev over a label_replace sent the translator's + marker to VictoriaLogs as part of the LogsQL query; the transform now runs on + what VictoriaLogs answered, before the ranking. + - A label_replace around a binary expression on query_range sent the binary + marker to VictoriaLogs (the instant path was fixed by the instant-result-type + gap); the range path now routes by the expression under the transform too. + + - `label_replace(sum without (level) (...))` lost its transform and kept the + excluded label: the without marker's label list swallowed the label_replace + marker that follows it. + - A call without Loki's four string arguments, or with an invalid escape in a + string, was taken for a log query (200 with no streams on query_range, a + wrong 400 on query); it is now Loki's 400 parse error (the position and + wording of the message follow the proxy's other grammar errors). + + Not changed (separate shapes, to register apart): an aggregation over a + label_replace (`sum by (app_short) (label_replace(...))`) is evaluated by + VictoriaLogs on a label it does not have; a label_replace around topk, sort, + group or vector answers empty on query_range and 400 on query; a + label_replace as an operand of a binary expression drops the operation; + label_join, which Loki v3.7.7 does not have, is accepted. request: method: GET - path: /loki/api/v1/query_range + path: /loki/api/v1/query params: query: 'label_replace(sum by (app) (rate({app="api-gateway"}[5m])), "app_short", "$1", "app", "(.*)-.*")' - start: 'now-1h' - end: 'now' - step: '60' headers: X-Scope-OrgID: "0" X-Loki-Response-Encoding-Flags: categorize-labels @@ -24,12 +71,13 @@ compare: expect: | app_short on every series. gap: - status: open + status: fixed + fixed_by: "#TBD" impact: explore-visible effort: S - area: 'internal/proxy label_replace post-processing' - loki: 'pkg/logql/evaluator.go:387, :1253 label_replace evaluator' - proxy: 'destination label missing' - planned_test: 'TestCompat_LabelReplaceNewLabel (test/e2e-compat, group semantics)' + area: 'internal/proxy vector_matching.go (transformSeriesLabels, applyLabelReplace, applyLabelJoin), metric_agg.go (post-aggregation markers), internal/translator parseUnquotedStringArgs' + loki: 'pkg/logql/syntax/ast.go mustNewLabelReplaceExpr; pkg/logql/evaluator.go:1249-1315 LabelReplaceEvaluator; pkg/logql/engine.go:530-600 JoinSampleVector, vectorsToSeries' + proxy: 'the transform runs on the series VictoriaLogs aggregated, matrix and vector alike' + planned_test: 'TestCompat_LabelReplaceNewLabel (test/e2e-compat, group semantics); TestLabelReplaceEdgeTable, TestLabelReplaceOnInstantAndRange, TestLabelReplaceDifferential, TestLabelReplaceUnderPostAggregation, TestLabelReplaceInvalidRegexp (internal/proxy); TestLabelReplaceStringArgumentsAreUnquoted (internal/translator)' clusters: - a9d4f85c diff --git a/internal/logql/semantic.go b/internal/logql/semantic.go index 01e69a639..1c48c0e53 100644 --- a/internal/logql/semantic.go +++ b/internal/logql/semantic.go @@ -42,6 +42,9 @@ func validateSemantics(expr Expr, raw string) string { if errors.As(err, &lokiErr) { return lokiErr.Msg } + if err := labelReplaceArgumentsError(x.Raw); err != nil { + return err.Error() + } if inner != nil { return validateSemantics(inner, raw) } @@ -74,6 +77,47 @@ func opaqueFirstArgument(call string) (Expr, error) { return inner, nil } +// labelReplaceArgumentsError reports the parse error Loki raises for a +// label_replace call that does not hold exactly four string arguments after its +// expression (pkg/logql/syntax/syntax.y labelReplaceExpr), a string whose +// escapes are invalid (the LogQL lexer unquotes it) included. It is nil for +// any other call. +func labelReplaceArgumentsError(call string) error { + p := &parser{sc: newScanner(call)} + p.advance() + if p.cur.Typ != TokIdent || p.cur.Val != "label_replace" { + return nil + } + if p.advance(); p.cur.Typ != TokLParen { + return nil + } + p.advance() + if _, err := p.parseExpr(); err != nil { + return nil // reported by opaqueFirstArgument + } + for i := 0; i < 4; i++ { + if p.cur.Typ != TokComma { + return p.syntaxErrorAt("COMMA") + } + p.advance() + switch p.cur.Typ { + case TokString, TokRawString: + case TokError: + if p.cur.Val == "invalid quoted string" { + return newParseError(1, len(p.sc.src[:p.sc.pos])+1, "invalid syntax") + } + return p.syntaxErrorAt("STRING") + default: + return p.syntaxErrorAt("STRING") + } + p.advance() + } + if p.cur.Typ != TokRParen { + return p.syntaxErrorAt(")") + } + return nil +} + // validateLogQuerySemantics validates a log query. lineIndependent is true // inside range aggregations whose result does not depend on the log line // (every op except bytes_rate/bytes_over_time): there Loki's removeLineformat diff --git a/internal/proxy/label_replace_semantics_test.go b/internal/proxy/label_replace_semantics_test.go new file mode 100644 index 000000000..d573a141c --- /dev/null +++ b/internal/proxy/label_replace_semantics_test.go @@ -0,0 +1,595 @@ +package proxy + +import ( + "encoding/json" + "fmt" + "math/rand" + "net/http" + "net/http/httptest" + "net/url" + "regexp" + "sort" + "strconv" + "strings" + "testing" + + "github.com/ReliablyObserve/Loki-VL-proxy/internal/translator" +) + +// label_replace adds, rewrites or removes one label of every series its +// expression yields. Loki v3.7.7: +// +// - pkg/logql/syntax/ast.go mustNewLabelReplaceExpr compiles the regexp as +// `^(?:regex)$`, so it must match the whole source value; an invalid one is +// the parse error "invalid regex in label_replace: ..."; +// - pkg/logql/evaluator.go LabelReplaceEvaluator.Next reads the source label +// (empty when absent), leaves the series as it is when the regexp does not +// match, otherwise expands the replacement with Regexp.ExpandString ($1, +// ${1}, $name, ${name}, $$) and deletes the destination label before +// setting it when the expansion is not empty. It checks no label name and +// raises no error for two series that end with one label set; +// - pkg/logql/engine.go JoinSampleVector keeps those series apart on an +// instant query (one sample each) and, on a range query, files their +// points under one series (vectorsToSeries keys by label hash, appending +// the points of each step in turn). +// +// The proxy applies the transform to the series VictoriaLogs aggregated, so an +// instant vector and a range matrix go through the same function. + +// lrSeries is one series of a decoded metric answer: labels and points, a +// matrix series' values or an instant vector's single value. +type lrSeries struct { + Metric map[string]string `json:"metric"` + Value []any `json:"value,omitempty"` + Values [][]any `json:"values,omitempty"` +} + +type lrAnswer struct { + code int + resultType string + series []lrSeries + body string +} + +func decodeLR(t *testing.T, code int, body []byte) lrAnswer { + t.Helper() + var resp struct { + Status string `json:"status"` + Data struct { + ResultType string `json:"resultType"` + Result []lrSeries `json:"result"` + } `json:"data"` + } + if err := json.Unmarshal(body, &resp); err != nil { + t.Fatalf("answer is not JSON (HTTP %d): %v\n%s", code, err, body) + } + return lrAnswer{code: code, resultType: resp.Data.ResultType, series: resp.Data.Result, body: string(body)} +} + +// lrKey is a canonical rendering of a label set. +func lrKey(m map[string]string) string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + var b strings.Builder + b.WriteByte('{') + for i, k := range keys { + if i > 0 { + b.WriteByte(',') + } + b.WriteString(k + "=" + strconv.Quote(m[k])) + } + b.WriteByte('}') + return b.String() +} + +// lrBackend is a VictoriaLogs that answers every stats query with series +// carrying exactly the given label sets, two points each on a range query. +func lrBackend(t *testing.T, sets []map[string]string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = r.ParseForm() + if q := r.FormValue("query"); strings.Contains(q, "__lvp_lr:") || strings.Contains(q, "__lvp_lj:") || strings.Contains(q, "__binary__") || strings.Contains(q, "|||") { + t.Errorf("an internal marker reached VictoriaLogs: %s", q) + } + var out []string + switch r.URL.Path { + case "/select/logsql/stats_query_range": + for i, m := range sets { + mb, _ := json.Marshal(m) + out = append(out, fmt.Sprintf(`{"metric":%s,"values":[[1704067200,"%d"],[1704067260,"%d"]]}`, mb, i+1, i+2)) + } + _, _ = fmt.Fprintf(w, `{"status":"success","data":{"resultType":"matrix","result":[%s]}}`, strings.Join(out, ",")) + case "/select/logsql/stats_query": + for i, m := range sets { + mb, _ := json.Marshal(m) + out = append(out, fmt.Sprintf(`{"metric":%s,"value":[1704067200,"%d"]}`, mb, i+1)) + } + _, _ = fmt.Fprintf(w, `{"status":"success","data":{"resultType":"vector","result":[%s]}}`, strings.Join(out, ",")) + default: + _, _ = w.Write([]byte(`{"status":"success","data":{"result":[]}}`)) + } + })) + t.Cleanup(srv.Close) + return srv +} + +func lrAsk(t *testing.T, p *Proxy, endpoint, q string) lrAnswer { + t.Helper() + return lrAskAs(t, p, endpoint, q, nil) +} + +func lrAskAs(t *testing.T, p *Proxy, endpoint, q string, headers map[string]string) lrAnswer { + t.Helper() + v := url.Values{"query": {q}, "start": {"1704067200"}, "end": {"1704067260"}, "step": {"60"}, "time": {"1704067260"}} + req := httptest.NewRequest("GET", "/loki/api/v1/"+endpoint+"?"+v.Encode(), nil) + req.Header.Set("X-Scope-OrgID", "0") + req.Header.Set("X-Loki-Response-Encoding-Flags", "categorize-labels") + for k, val := range headers { + req.Header.Set(k, val) + } + rec := httptest.NewRecorder() + if endpoint == "query" { + p.handleQuery(rec, req) + } else { + p.handleQueryRange(rec, req) + } + return decodeLR(t, rec.Code, rec.Body.Bytes()) +} + +// lrReference is Loki's label_replace written apart from the proxy: the +// evaluator step (LabelReplaceEvaluator.Next) on one label set. +func lrReference(in map[string]string, dst, repl, src, regex string) map[string]string { + re := regexp.MustCompile("^(?:" + regex + ")$") + out := make(map[string]string, len(in)+1) + for k, v := range in { + out[k] = v + } + s := in[src] + idx := re.FindStringSubmatchIndex(s) + if idx == nil { + return out + } + res := re.ExpandString(nil, repl, s, idx) + delete(out, dst) + if len(res) > 0 { + out[dst] = string(res) + } + return out +} + +// lrReferenceJoin is JoinSampleVector for the series of a transformed answer: +// an instant vector keeps every sample, a range matrix files the points of the +// series that now share a label set under one series, step by step. +func lrReferenceJoin(base []lrSeries, instant bool, f func(map[string]string) map[string]string) map[string][]string { + out := map[string][]string{} + if instant { + for _, s := range base { + k := lrKey(f(s.Metric)) + out[k] = append(out[k], fmt.Sprint(s.Value)) + } + for k := range out { + sort.Strings(out[k]) + } + return out + } + steps := map[string]bool{} + for _, s := range base { + for _, v := range s.Values { + steps[fmt.Sprint(v[0])] = true + } + } + order := make([]string, 0, len(steps)) + for ts := range steps { + order = append(order, ts) + } + sort.Slice(order, func(i, j int) bool { + a, _ := strconv.ParseFloat(order[i], 64) + b, _ := strconv.ParseFloat(order[j], 64) + return a < b + }) + for _, ts := range order { + for _, s := range base { + for _, v := range s.Values { + if fmt.Sprint(v[0]) == ts { + k := lrKey(f(s.Metric)) + out[k] = append(out[k], fmt.Sprint(v)) + } + } + } + } + return out +} + +func lrGot(a lrAnswer, instant bool) map[string][]string { + out := map[string][]string{} + for _, s := range a.series { + k := lrKey(s.Metric) + if instant { + out[k] = append(out[k], fmt.Sprint(s.Value)) + continue + } + for _, v := range s.Values { + out[k] = append(out[k], fmt.Sprint(v)) + } + } + if instant { + for k := range out { + sort.Strings(out[k]) + } + } + return out +} + +// The edge table: every rule of the evaluator step, on both answer shapes. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestLabelReplaceEdgeTable(t *testing.T) { + type tc struct { + name string + in map[string]string + dst, repl, src, r string + want map[string]string + } + cases := []tc{ + {"adds the destination", map[string]string{"app": "api-gateway"}, "app_short", "$1", "app", "(.*)-.*", map[string]string{"app": "api-gateway", "app_short": "api"}}, + {"regexp is anchored: a prefix does not match", map[string]string{"app": "api-gateway"}, "d", "x", "app", "api", map[string]string{"app": "api-gateway"}}, + {"regexp is anchored: a suffix does not match", map[string]string{"app": "api-gateway"}, "d", "x", "app", "gateway", map[string]string{"app": "api-gateway"}}, + {"regexp is anchored: the whole value matches", map[string]string{"app": "api"}, "d", "x", "app", "api", map[string]string{"app": "api", "d": "x"}}, + {"alternation is anchored as a whole", map[string]string{"app": "bc"}, "d", "x", "app", "a|bc", map[string]string{"app": "bc", "d": "x"}}, + {"alternation does not leak the anchors", map[string]string{"app": "abc"}, "d", "x", "app", "a|bc", map[string]string{"app": "abc"}}, + {"no match keeps the series and an existing destination", map[string]string{"app": "web", "d": "old"}, "d", "$1", "app", "(.*)-.*", map[string]string{"app": "web", "d": "old"}}, + {"empty replacement removes the destination", map[string]string{"app": "web", "d": "old"}, "d", "", "app", ".*", map[string]string{"app": "web"}}, + {"empty replacement without a destination changes nothing", map[string]string{"app": "web"}, "d", "", "app", ".*", map[string]string{"app": "web"}}, + {"an empty expansion removes the destination", map[string]string{"app": "web", "d": "old"}, "d", "$1", "app", "web()", map[string]string{"app": "web"}}, + {"braced group", map[string]string{"app": "api-gateway"}, "d", "${1}_x", "app", "(.*)-.*", map[string]string{"app": "api-gateway", "d": "api_x"}}, + {"bare group name swallows the suffix", map[string]string{"app": "api-gateway"}, "d", "$1x", "app", "(.*)-.*", map[string]string{"app": "api-gateway"}}, + {"named group", map[string]string{"app": "api-gateway"}, "d", "${head}", "app", "(?P[a-z]+)-.*", map[string]string{"app": "api-gateway", "d": "api"}}, + {"bare named group", map[string]string{"app": "api-gateway"}, "d", "$head.", "app", "(?P[a-z]+)-.*", map[string]string{"app": "api-gateway", "d": "api."}}, + {"two groups", map[string]string{"app": "api-gateway"}, "d", "$2/$1", "app", "([a-z]+)-([a-z]+)", map[string]string{"app": "api-gateway", "d": "gateway/api"}}, + {"unknown group expands to nothing", map[string]string{"app": "api", "d": "old"}, "d", "$2", "app", "(.*)", map[string]string{"app": "api"}}, + {"dollar escape", map[string]string{"app": "api"}, "d", "$$1", "app", "(.*)", map[string]string{"app": "api", "d": "$1"}}, + {"constant replacement", map[string]string{"app": "api"}, "d", "const", "app", ".*", map[string]string{"app": "api", "d": "const"}}, + {"missing source is empty: a regexp that matches empty applies", map[string]string{"app": "api"}, "d", "dflt", "nope", "(.*)", map[string]string{"app": "api", "d": "dflt"}}, + {"missing source is empty: a regexp that needs a character does not", map[string]string{"app": "api"}, "d", "dflt", "nope", ".+", map[string]string{"app": "api"}}, + {"empty regexp matches an empty source", map[string]string{"app": "api"}, "d", "dflt", "nope", "", map[string]string{"app": "api", "d": "dflt"}}, + {"destination equal to the source", map[string]string{"app": "api-gateway"}, "app", "$1", "app", "(.*)-.*", map[string]string{"app": "api"}}, + {"destination equal to the source, removed", map[string]string{"app": "api", "b": "1"}, "app", "", "app", ".*", map[string]string{"b": "1"}}, + {"dot does not match a newline", map[string]string{"app": "a\nb"}, "d", "x", "app", ".*", map[string]string{"app": "a\nb"}}, + {"flag lets dot match a newline", map[string]string{"app": "a\nb"}, "d", "x", "app", "(?s).*", map[string]string{"app": "a\nb", "d": "x"}}, + {"label names are not validated", map[string]string{"app": "api"}, "not a label", "x", "app", ".*", map[string]string{"app": "api", "not a label": "x"}}, + {"dotted destination", map[string]string{"app": "api"}, "service.name", "$0", "app", ".*", map[string]string{"app": "api", "service.name": "api"}}, + {"group zero is the whole match", map[string]string{"app": "api"}, "d", "<$0>", "app", "a(p)i", map[string]string{"app": "api", "d": ""}}, + {"unicode value", map[string]string{"app": "żółć-1"}, "d", "$1", "app", "(.*)-.*", map[string]string{"app": "żółć-1", "d": "żółć"}}, + } + for _, c := range cases { + spec := translator.LabelReplaceSpec{DstLabel: c.dst, Replacement: c.repl, SrcLabel: c.src, Regex: c.r} + for _, shape := range []string{"matrix", "vector"} { + t.Run(c.name+"/"+shape, func(t *testing.T) { + mb, _ := json.Marshal(c.in) + var body string + if shape == "matrix" { + body = fmt.Sprintf(`{"status":"success","data":{"resultType":"matrix","result":[{"metric":%s,"values":[[1704067200,"1"]]}],"stats":{"x":1}}}`, mb) + } else { + body = fmt.Sprintf(`{"status":"success","data":{"resultType":"vector","result":[{"metric":%s,"value":[1704067200,"1"]}],"stats":{"x":1}}}`, mb) + } + out := applyLabelReplace([]byte(body), spec) + a := decodeLR(t, 200, out) + if len(a.series) != 1 || lrKey(a.series[0].Metric) != lrKey(c.want) { + t.Fatalf("labels %s, want %s\n%s", lrKey(a.series[0].Metric), lrKey(c.want), out) + } + if a.resultType != shape { + t.Errorf("resultType %q, want %q", a.resultType, shape) + } + if !strings.Contains(string(out), `"stats":{"x":1}`) { + t.Errorf("the rest of the answer is not kept: %s", out) + } + if ref := lrReference(c.in, c.dst, c.repl, c.src, c.r); lrKey(ref) != lrKey(c.want) { + t.Fatalf("test table disagrees with the reference: %s vs %s", lrKey(ref), lrKey(c.want)) + } + }) + } + } +} + +// Two series that end with one label set: no error (the evaluator checks +// nothing); an instant vector holds both samples, a range matrix one series +// with the points of each step in turn. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestLabelReplaceDuplicateLabelSets(t *testing.T) { + spec := translator.LabelReplaceSpec{DstLabel: "app", Replacement: "all", SrcLabel: "app", Regex: ".*"} + matrix := `{"status":"success","data":{"resultType":"matrix","result":[` + + `{"metric":{"app":"a"},"values":[[1,"1"],[2,"2"]]},{"metric":{"app":"b"},"values":[[1,"10"],[2,"20"]]},{"metric":{"app":"c","k":"v"},"values":[[1,"7"]]}]}}` + got := decodeLR(t, 200, applyLabelReplace([]byte(matrix), spec)) + if len(got.series) != 2 { + t.Fatalf("range: %d series, want the two duplicates filed under one\n%s", len(got.series), got.body) + } + for _, s := range got.series { + if s.Metric["app"] == "all" && len(s.Metric) == 1 { + if v := fmt.Sprint(s.Values); v != "[[1 1] [1 10] [2 2] [2 20]]" { + t.Errorf("merged points %s, want the points of each step in turn", v) + } + } + } + vector := `{"status":"success","data":{"resultType":"vector","result":[` + + `{"metric":{"app":"a"},"value":[1,"1"]},{"metric":{"app":"b"},"value":[1,"10"]}]}}` + got = decodeLR(t, 200, applyLabelReplace([]byte(vector), spec)) + if len(got.series) != 2 || lrKey(got.series[0].Metric) != lrKey(got.series[1].Metric) { + t.Fatalf("instant: want both samples under the shared label set\n%s", got.body) + } +} + +// label_join is no Loki v3.7.7 function (the lexer has label_replace only); +// the proxy keeps accepting it and applies it to both answer shapes. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestLabelJoinInstantAndRange(t *testing.T) { + spec := translator.LabelJoinSpec{DstLabel: "j", Separator: "-", SrcLabels: []string{"a", "b"}} + for _, body := range []string{ + `{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"a":"x","b":"y"},"values":[[1,"1"]]}]}}`, + `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"a":"x","b":"y"},"value":[1,"1"]}]}}`, + } { + a := decodeLR(t, 200, applyLabelJoin([]byte(body), spec)) + if a.series[0].Metric["j"] != "x-y" { + t.Errorf("%s: j=%q, want x-y", a.resultType, a.series[0].Metric["j"]) + } + } +} + +// The registry shape and the shapes around it, on both endpoints. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestLabelReplaceOnInstantAndRange(t *testing.T) { + sets := []map[string]string{{"app": "api-gateway"}, {"app": "web"}} + p := newTestProxy(t, lrBackend(t, sets).URL) + count := `count_over_time({app="api-gateway"}[5m])` + agg := `sum by (app) (` + count + `)` + lr := func(inner, dst, repl, src, re string) string { + return fmt.Sprintf(`label_replace(%s, %q, %q, %q, %q)`, inner, dst, repl, src, re) + } + cases := []struct { + name string + query string + want []map[string]string + }{ + {"over an aggregation", lr(agg, "app_short", "$1", "app", "(.*)-.*"), []map[string]string{{"app": "api-gateway", "app_short": "api"}, {"app": "web"}}}, + {"nested", lr(lr(agg, "a1", "$1", "app", "(.*)-.*"), "a2", "x$1", "a1", "(.*)"), []map[string]string{{"app": "api-gateway", "a1": "api", "a2": "xapi"}, {"app": "web", "a2": "x"}}}, + {"nested, the outer reads what the inner removed", lr(lr(agg, "app", "", "app", "web"), "w", "gone", "app", ""), []map[string]string{{"app": "api-gateway"}, {"w": "gone"}}}, + {"escaped regexp", lr(agg, "d", "$1", "app", `([a-z]+)\-(\w+)`), []map[string]string{{"app": "api-gateway", "d": "api"}, {"app": "web"}}}, + {"escaped dot", lr(agg, "d", "$1", "app", `(\w+)\.(\w+)`), []map[string]string{{"app": "api-gateway"}, {"app": "web"}}}, + {"over a binary expression", lr(agg+` * 2`, "app_short", "$1", "app", "(.*)-.*"), []map[string]string{{"app": "api-gateway", "app_short": "api"}, {"app": "web"}}}, + {"over a binary of two aggregations", lr(agg+` / `+agg, "app_short", "$1", "app", "(.*)-.*"), []map[string]string{{"app": "api-gateway", "app_short": "api"}, {"app": "web"}}}, + } + for _, c := range cases { + for _, endpoint := range []string{"query_range", "query"} { + t.Run(c.name+"/"+endpoint, func(t *testing.T) { + a := lrAsk(t, p, endpoint, c.query) + if a.code != 200 { + t.Fatalf("HTTP %d: %s", a.code, a.body) + } + want := "matrix" + if endpoint == "query" { + want = "vector" + } + if a.resultType != want { + t.Fatalf("resultType %q, want %q\n%s", a.resultType, want, a.body) + } + got, exp := map[string]bool{}, map[string]bool{} + for _, s := range a.series { + got[lrKey(s.Metric)] = true + } + for _, m := range c.want { + exp[lrKey(m)] = true + } + if fmt.Sprint(got) != fmt.Sprint(exp) { + t.Fatalf("label sets %v, want %v\n%s", got, exp, a.body) + } + }) + } + } +} + +// topk, bottomk, sort and stddev rank or fold the series label_replace +// produced: the transform runs on what VictoriaLogs answered, then the +// ranking. The translator's marker never reaches VictoriaLogs. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestLabelReplaceUnderPostAggregation(t *testing.T) { + sets := []map[string]string{{"app": "api-gateway"}, {"app": "web"}} + p := newTestProxy(t, lrBackend(t, sets).URL) + lr := `label_replace(sum by (app) (count_over_time({app="x"}[5m])), "app_short", "$1", "app", "(.*)-.*")` + for _, tc := range []struct { + query string + want []string + }{ + {`topk(2, ` + lr + `)`, []string{`{app="api-gateway",app_short="api"}`, `{app="web"}`}}, + {`bottomk(1, ` + lr + `)`, []string{`{app="api-gateway",app_short="api"}`}}, + {`sort(` + lr + `)`, []string{`{app="api-gateway",app_short="api"}`, `{app="web"}`}}, + {`sort_desc(` + lr + `)`, []string{`{app="api-gateway",app_short="api"}`, `{app="web"}`}}, + } { + for _, endpoint := range []string{"query_range", "query"} { + if endpoint == "query_range" && (strings.HasPrefix(tc.query, "topk") || strings.HasPrefix(tc.query, "bottomk")) { + continue // ranks per step from window statistics, which this backend does not emulate + } + a := lrAsk(t, p, endpoint, tc.query) + got := map[string]bool{} + for _, s := range a.series { + got[lrKey(s.Metric)] = true + } + if a.code != 200 || len(got) != len(tc.want) { + t.Errorf("%s %s: HTTP %d, series %v, want %v\n%s", endpoint, tc.query, a.code, got, tc.want, a.body) + continue + } + for _, w := range tc.want { + if !got[w] { + t.Errorf("%s %s: series %v, want %v\n%s", endpoint, tc.query, got, tc.want, a.body) + } + } + } + } + for _, endpoint := range []string{"query_range", "query"} { + a := lrAsk(t, p, endpoint, `sort(label_replace(sum by (app) (count_over_time({app="x"}[5m])), "d", "$1", "app", "(unclosed"))`) + if a.code != http.StatusBadRequest || !strings.Contains(a.body, "parse error : invalid regex in label_replace:") { + t.Errorf("%s: HTTP %d, want Loki's parse error\n%s", endpoint, a.code, a.body) + } + } +} + +// The registry row's query, as curl, Grafana Explore and Logs Drilldown send it +// (Drilldown range answers pass through the series-limit rewriter after the +// transform), with Loki's categorize-labels encoding flag. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestLabelReplaceRowQueryForEveryClient(t *testing.T) { + p := newTestProxy(t, lrBackend(t, []map[string]string{{"app": "api-gateway"}, {"app": "web"}}).URL) + q := `label_replace(sum by (app) (rate({app="api-gateway"}[5m])), "app_short", "$1", "app", "(.*)-.*")` + for client, headers := range instantClients { + for _, endpoint := range []string{"query_range", "query"} { + a := lrAskAs(t, p, endpoint, q, headers) + if a.code != 200 || len(a.series) != 2 { + t.Errorf("%s %s: HTTP %d, %d series, want 2\n%s", client, endpoint, a.code, len(a.series), a.body) + continue + } + for _, s := range a.series { + if (s.Metric["app"] == "api-gateway") != (s.Metric["app_short"] == "api") { + t.Errorf("%s %s: app_short=%q for app=%q\n%s", client, endpoint, s.Metric["app_short"], s.Metric["app"], a.body) + } + } + } + } +} + +// A bare range aggregation carries the stream identity; the new label rides +// along with it. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestLabelReplaceOverBareRangeAggregation(t *testing.T) { + p := newTestProxy(t, lrBackend(t, []map[string]string{{"app": "api-gateway", "_stream": `{app="api-gateway"}`, "level": "info"}}).URL) + for _, endpoint := range []string{"query_range", "query"} { + a := lrAsk(t, p, endpoint, `label_replace(rate({app="api-gateway"}[5m]), "app_short", "$1", "app", "(.*)-.*")`) + if a.code != 200 || len(a.series) != 1 || a.series[0].Metric["app_short"] != "api" || a.series[0].Metric["app"] != "api-gateway" { + t.Errorf("%s: HTTP %d, want app_short=api next to app\n%s", endpoint, a.code, a.body) + } + } +} + +// An invalid regexp is Loki's parse error on both endpoints, nested or not. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestLabelReplaceInvalidRegexp(t *testing.T) { + p := newTestProxy(t, lrBackend(t, []map[string]string{{"app": "a"}}).URL) + agg := `sum by (app) (count_over_time({app="a"}[5m]))` + for _, q := range []string{ + `label_replace(` + agg + `, "d", "$1", "app", "[invalid")`, + `label_replace(label_replace(` + agg + `, "d", "$1", "app", "(.*)"), "e", "$1", "d", "(unclosed")`, + `label_replace(` + agg + ` * 2, "d", "$1", "app", "*bad")`, + } { + for _, endpoint := range []string{"query_range", "query"} { + a := lrAsk(t, p, endpoint, q) + if a.code != http.StatusBadRequest || !strings.Contains(a.body, "parse error : invalid regex in label_replace: error parsing regexp:") { + t.Errorf("%s %s: HTTP %d, want Loki's 400 parse error\n%s", endpoint, q, a.code, a.body) + } + } + } +} + +// A call that does not hold the four string arguments of Loki's grammar, or a +// string whose escapes are invalid, is a 400 parse error on both endpoints: it +// is never taken for a log query or answered with untransformed series. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestLabelReplaceMalformedArguments(t *testing.T) { + p := newTestProxy(t, lrBackend(t, []map[string]string{{"app": "a"}}).URL) + agg := `sum by (app) (count_over_time({app="a"}[5m]))` + for _, q := range []string{ + `label_replace(` + agg + `, "d", "$1", "app", "(.+):\d+")`, + `label_replace(` + agg + `, "d", "$1", "app")`, + `label_replace(` + agg + `, "d", "$1")`, + `label_replace(` + agg + `, "d", "$1", "app", "x", "y")`, + `label_replace(` + agg + `, d, "$1", "app", "x")`, + `label_replace(` + agg + `, "d", "$1", "app", 5)`, + } { + for _, endpoint := range []string{"query_range", "query"} { + a := lrAsk(t, p, endpoint, q) + if a.code != http.StatusBadRequest || !strings.Contains(a.body, "parse error at line 1, col ") || strings.Contains(a.body, "log queries") { + t.Errorf("%s %s: HTTP %d, want a 400 parse error\n%s", endpoint, q, a.code, a.body) + } + } + } +} + +// label_replace over `sum without`: the without marker no longer swallows the +// label_replace marker that follows it. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestLabelReplaceOverSumWithout(t *testing.T) { + p := newTestProxy(t, lrBackend(t, []map[string]string{ + {"_stream": `{app="api-gateway"}`, "level": "info"}, {"_stream": `{app="web"}`, "level": "info"}, + }).URL) + for _, endpoint := range []string{"query_range", "query"} { + a := lrAsk(t, p, endpoint, `label_replace(sum without (level) (count_over_time({app=~".+"}[5m])), "app_short", "$1", "app", "(.*)-.*")`) + if a.code != 200 || len(a.series) != 2 { + t.Fatalf("%s: HTTP %d, %d series, want 2\n%s", endpoint, a.code, len(a.series), a.body) + } + for _, s := range a.series { + if _, kept := s.Metric["level"]; kept { + t.Errorf("%s: level kept by without: %v", endpoint, s.Metric) + } + if (s.Metric["app"] == "api-gateway") != (s.Metric["app_short"] == "api") { + t.Errorf("%s: app_short=%q for app=%q, want api for api-gateway only", endpoint, s.Metric["app_short"], s.Metric["app"]) + } + } + } +} + +// The differential test: generated label sets and generated transforms, the +// proxy's answer on both endpoints against the reference applied to the +// untransformed answer of the same backend. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestLabelReplaceDifferential(t *testing.T) { + values := []string{"api-gateway", "web", "x", "api-1.2", "", "żółć-9", "a|b", "10.20", "ab"} + regexes := []string{`(.*)-.*`, `(.*)`, `.*`, `.+`, `api`, `(?P[a-z]+)-(?P.*)`, `([^-]+)`, `a|b`, `(\d+)\.(\d+)`, `.+-(.*)`, ``, `(a)(b)?`, `web|x`, `(?i)API-.*`, `[a-z]+`, `(.)(.)`} + repls := []string{`$1`, `${1}`, `$2`, `${head}`, `$1-$2`, ``, `static`, `$$1`, `$1x`, `${1}x`, `$none`, `$0`, `<${tail}>`} + names := []string{"a", "b", "c", "new", "d.e"} + rng := rand.New(rand.NewSource(7)) + for round := 0; round < 40; round++ { + var sets []map[string]string + seen := map[string]bool{} + for i, n := 0, 1+rng.Intn(5); i < n; i++ { + m := map[string]string{} + for _, k := range []string{"a", "b", "c"} { + if rng.Intn(4) > 0 { + m[k] = values[rng.Intn(len(values))] + } + } + if key := lrKey(m); !seen[key] { + seen[key] = true + sets = append(sets, m) + } + } + p := newTestProxy(t, lrBackend(t, sets).URL) + type step struct{ dst, repl, src, re string } + steps := make([]step, 1+rng.Intn(2)) + for i := range steps { + steps[i] = step{names[rng.Intn(len(names))], repls[rng.Intn(len(repls))], names[rng.Intn(len(names))], regexes[rng.Intn(len(regexes))]} + } + base := `sum by (a, b, c) (count_over_time({app="x"}[5m]))` + q := base + for _, s := range steps { + q = fmt.Sprintf(`label_replace(%s, %s, %s, %s, %s)`, q, strconv.Quote(s.dst), strconv.Quote(s.repl), strconv.Quote(s.src), strconv.Quote(s.re)) + } + for _, endpoint := range []string{"query_range", "query"} { + instant := endpoint == "query" + baseAns := lrAsk(t, p, endpoint, base) + if baseAns.code != 200 || len(baseAns.series) != len(sets) { + t.Fatalf("round %d %s: the untransformed answer has %d series (HTTP %d), want %d\n%s", round, endpoint, len(baseAns.series), baseAns.code, len(sets), baseAns.body) + } + want := lrReferenceJoin(baseAns.series, instant, func(m map[string]string) map[string]string { + for _, s := range steps { + m = lrReference(m, s.dst, s.repl, s.src, s.re) + } + return m + }) + got := lrAsk(t, p, endpoint, q) + if got.code != 200 { + t.Fatalf("round %d %s %s: HTTP %d\n%s", round, endpoint, q, got.code, got.body) + } + if g, w := lrGot(got, instant), want; fmt.Sprint(g) != fmt.Sprint(w) { + t.Fatalf("round %d %s %s over %v\n got %v\nwant %v", round, endpoint, q, sets, g, w) + } + } + } +} diff --git a/internal/proxy/metric_agg.go b/internal/proxy/metric_agg.go index b0de00b52..1ddf3f5b7 100644 --- a/internal/proxy/metric_agg.go +++ b/internal/proxy/metric_agg.go @@ -262,6 +262,12 @@ func (p *Proxy) handleInstantMetricPostAggregation(w http.ResponseWriter, r *htt return } translatedInner, withoutLabels := translator.ParseWithoutMarker(translatedInner) + translatedInner, transforms, regexErr := parseLabelTransformMarkers(translatedInner) + if regexErr != "" { + p.writeError(w, http.StatusBadRequest, regexErr) + p.metrics.RecordRequest("query", unwrapRecordedStatus(r.Context(), http.StatusBadRequest), time.Since(start)) + return + } translatedInner = preserveMetricStreamIdentity(postAgg.inner, translatedInner, withoutLabels) r = p.withRequestScope(r) @@ -300,6 +306,9 @@ func (p *Proxy) handleInstantMetricPostAggregation(w http.ResponseWriter, r *htt if len(withoutLabels) > 0 { bw.body = applyWithoutGrouping(bw.body, withoutLabels, withoutAggregationOp(originalQuery)) } + if sc.code < http.StatusBadRequest { + bw.body = transforms.apply(bw.body) + } if sc.code >= http.StatusBadRequest { copyHeaders(w.Header(), bw.Header()) @@ -335,6 +344,12 @@ func (p *Proxy) handleRangeMetricPostAggregation(w http.ResponseWriter, r *http. return } translatedInner, withoutLabels := translator.ParseWithoutMarker(translatedInner) + translatedInner, transforms, regexErr := parseLabelTransformMarkers(translatedInner) + if regexErr != "" { + p.writeError(w, http.StatusBadRequest, regexErr) + p.metrics.RecordRequest("query_range", unwrapRecordedStatus(r.Context(), http.StatusBadRequest), time.Since(start)) + return + } translatedInner = preserveMetricStreamIdentity(postAgg.inner, translatedInner, withoutLabels) r = p.withRequestScope(r) @@ -378,6 +393,9 @@ func (p *Proxy) handleRangeMetricPostAggregation(w http.ResponseWriter, r *http. if len(withoutLabels) > 0 { bw.body = applyWithoutGrouping(bw.body, withoutLabels, withoutAggregationOp(originalQuery)) } + if sc.code < http.StatusBadRequest { + bw.body = transforms.apply(bw.body) + } if sc.code >= http.StatusBadRequest { copyHeaders(w.Header(), bw.Header()) diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 71c6076d0..2bbcff4f8 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -2384,7 +2384,13 @@ func (p *Proxy) handleQueryRange(w http.ResponseWriter, r *http.Request) { } // Route using the original LogQL AST — more reliable than re-parsing translated markers. - parsedForRouting, _ := logqlpkg.Parse(logqlQuery) + // A label_replace or label_join around a binary expression routes by that + // expression: its operands are evaluated, then the transform is applied below. + routedQuery := logqlQuery + if len(labelReplaceSpecs) > 0 || labelJoinSpec != nil { + routedQuery = stripLabelTransforms(logqlQuery) + } + parsedForRouting, _ := logqlpkg.Parse(routedQuery) if binOp, ok := parsedForRouting.(*logqlpkg.BinOpExpr); ok { // Binary metric expression: sum(rate(...)) / sum(rate(...)) // translateBinOpSide handles scalar literals (e.g. * 100) without translation. @@ -2395,6 +2401,9 @@ func (p *Proxy) handleQueryRange(w http.ResponseWriter, r *http.Request) { } else if rightErr != nil { p.writeError(sc, http.StatusBadRequest, rightErr.Error()) } else { + if routedQuery != logqlQuery { + r = cloneMetricQueryRequest(r, routedQuery) + } p.proxyBinaryMetricQueryRangeVM(sc, r, binOp.Op, leftLogsql, rightLogsql, binOpExprToVMInfo(binOp)) } } else if isStatsQuery(logsqlQuery) { diff --git a/internal/proxy/vector_matching.go b/internal/proxy/vector_matching.go index c3b2ee72a..98608820d 100644 --- a/internal/proxy/vector_matching.go +++ b/internal/proxy/vector_matching.go @@ -659,58 +659,145 @@ func applyGroupNormalization(body []byte) []byte { return out } -// applyLabelReplace applies label_replace semantics to all series in a matrix response. -// For each series: new_value = regexp.ReplaceAll(src_label_value, replacement). -// If new_value is non-empty, dst_label is set; otherwise dst_label is deleted. -func applyLabelReplace(body []byte, spec translator.LabelReplaceSpec) []byte { - re, err := regexp.Compile("^(?:" + spec.Regex + ")$") - if err != nil { +// transformSeriesLabels rewrites the label set of every series of a matrix or +// a vector answer with fn, keeping the rest of the answer as it is. Series +// that end with one label set are not an error in Loki: an instant vector +// keeps each sample, a range matrix files their points under one series, those +// of each step in turn (pkg/logql/engine.go JoinSampleVector, vectorsToSeries). +func transformSeriesLabels(body []byte, fn func(map[string]string) map[string]string) []byte { + var top, data map[string]json.RawMessage + if json.Unmarshal(body, &top) != nil || json.Unmarshal(top["data"], &data) != nil { return body } - var resp matrixResponseRW - if err := json.Unmarshal(body, &resp); err != nil || resp.Data.ResultType != "matrix" { + var resultType string + var series []map[string]json.RawMessage + _ = json.Unmarshal(data["resultType"], &resultType) + if (resultType != "matrix" && resultType != "vector") || json.Unmarshal(data["result"], &series) != nil { return body } - for i, s := range resp.Data.Result { - src := s.Metric[spec.SrcLabel] - // Prometheus semantics: if regex matches, apply replacement; if not, series unchanged. - if re.MatchString(src) { - newVal := re.ReplaceAllString(src, spec.Replacement) - if newVal != "" { - resp.Data.Result[i].Metric[spec.DstLabel] = newVal - } else { - delete(resp.Data.Result[i].Metric, spec.DstLabel) + merged := make(map[string]int, len(series)) + out := series[:0] + for _, s := range series { + var metric map[string]string + _ = json.Unmarshal(s["metric"], &metric) + if metric == nil { + metric = map[string]string{} + } + metric = fn(metric) + s["metric"], _ = json.Marshal(metric) + if resultType == "matrix" { + key := binaryLabelKey(metric) + if at, dup := merged[key]; dup { + var more, have []json.RawMessage + _ = json.Unmarshal(s["values"], &more) + _ = json.Unmarshal(out[at]["values"], &have) + all := append(have, more...) + sort.SliceStable(all, func(i, j int) bool { return pointSeconds(all[i]) < pointSeconds(all[j]) }) + out[at]["values"], _ = json.Marshal(all) + continue } + merged[key] = len(out) } + out = append(out, s) } - out, err := json.Marshal(resp) + data["result"], _ = json.Marshal(out) + top["data"], _ = json.Marshal(data) + b, err := json.Marshal(top) if err != nil { return body } - return out + return b } -// applyLabelJoin applies label_join semantics to all series in a matrix response. -// dst_label is set to the src_labels values joined by separator. -func applyLabelJoin(body []byte, spec translator.LabelJoinSpec) []byte { - var resp matrixResponseRW - if err := json.Unmarshal(body, &resp); err != nil || resp.Data.ResultType != "matrix" { +// pointSeconds is the time of a sample point `[time, "value"]`. +func pointSeconds(point json.RawMessage) float64 { + var p []any + if json.Unmarshal(point, &p) != nil || len(p) == 0 { + return 0 + } + t, _ := sampleTimeSeconds(p[0]) + return t +} + +// applyLabelReplace applies label_replace to every series of a matrix or vector +// answer as Loki's LabelReplaceEvaluator does (pkg/logql/evaluator.go): the +// regexp is anchored, a series it does not match is left as it is, and the +// expanded replacement ($1, ${1}, $name) replaces the destination label, which +// an empty expansion removes. +func applyLabelReplace(body []byte, spec translator.LabelReplaceSpec) []byte { + re, err := regexp.Compile("^(?:" + spec.Regex + ")$") + if err != nil { return body } - for i, s := range resp.Data.Result { + return transformSeriesLabels(body, func(in map[string]string) map[string]string { + src := in[spec.SrcLabel] + idx := re.FindStringSubmatchIndex(src) + if idx == nil { + return in + } + res := re.ExpandString(nil, spec.Replacement, src, idx) + out := make(map[string]string, len(in)+1) + for k, v := range in { + out[k] = v + } + delete(out, spec.DstLabel) + if len(res) > 0 { + out[spec.DstLabel] = string(res) + } + return out + }) +} + +// applyLabelJoin applies label_join to every series of a matrix or vector +// answer: the destination label is the values of the source labels, the +// absent ones skipped, joined by the separator. +func applyLabelJoin(body []byte, spec translator.LabelJoinSpec) []byte { + return transformSeriesLabels(body, func(in map[string]string) map[string]string { parts := make([]string, 0, len(spec.SrcLabels)) for _, src := range spec.SrcLabels { - if v := s.Metric[src]; v != "" { + if v := in[src]; v != "" { parts = append(parts, v) } } - resp.Data.Result[i].Metric[spec.DstLabel] = strings.Join(parts, spec.Separator) + out := make(map[string]string, len(in)+1) + for k, v := range in { + out[k] = v + } + out[spec.DstLabel] = strings.Join(parts, spec.Separator) + return out + }) +} + +// labelTransforms are the label_replace and label_join calls of a query, in +// the order they apply: the innermost first. +type labelTransforms struct { + replaces []translator.LabelReplaceSpec + join *translator.LabelJoinSpec +} + +// parseLabelTransformMarkers strips the label_replace and label_join markers +// the translator left in a query. It returns Loki's parse error text when a +// label_replace regexp does not compile. +func parseLabelTransformMarkers(logsql string) (string, labelTransforms, string) { + logsql, replaces := translator.ParseAllLabelReplaceMarkers(logsql) + logsql, join := translator.ParseLabelJoinMarker(logsql) + for _, spec := range replaces { + if _, err := regexp.Compile("^(?:" + spec.Regex + ")$"); err != nil { + return logsql, labelTransforms{}, fmt.Sprintf("parse error : invalid regex in label_replace: %v", err) + } } - out, err := json.Marshal(resp) - if err != nil { - return body + return logsql, labelTransforms{replaces: replaces, join: join}, "" +} + +// apply runs the transforms over a matrix or vector answer. +func (t labelTransforms) apply(body []byte) []byte { + for _, spec := range t.replaces { + body = applyLabelReplace(body, spec) } - return out + if t.join != nil { + body = applyLabelJoin(body, *t.join) + } + return body } // sampleTimeSeconds reads a sample time of a decoded response: a JSON number, or diff --git a/internal/translator/coverage_test.go b/internal/translator/coverage_test.go index 46e4dd860..9e4ab7bba 100644 --- a/internal/translator/coverage_test.go +++ b/internal/translator/coverage_test.go @@ -184,3 +184,15 @@ func TestCoverage_AddByClause_DeduplicatesTranslatedLabels(t *testing.T) { t.Fatalf("addByClause returned %q, want %q", got, want) } } + +// A marker appended after the without marker ends its label list and stays on +// the query, so the label_replace, label_join and group markers of a +// `label_replace(sum without (pod) (...), ...)` still reach their parsers. +func TestParseWithoutMarkerBeforeOtherMarkers(t *testing.T) { + for _, tail := range []string{"__lvp_lr:YQBiAGMAZA==__", "__lvp_lj:YQBiAGM=__", "__lvp_group__"} { + clean, labels := ParseWithoutMarker(`app:="api"` + WithoutMarkerSuffix + `pod,node` + tail) + if clean != `app:="api"`+tail || len(labels) != 2 || labels[0] != "pod" || labels[1] != "node" { + t.Errorf("tail %q: clean=%q labels=%v, want the marker kept and the labels pod,node", tail, clean, labels) + } + } +} diff --git a/internal/translator/translator.go b/internal/translator/translator.go index ca551a0f5..08d3f2dbe 100644 --- a/internal/translator/translator.go +++ b/internal/translator/translator.go @@ -525,6 +525,12 @@ func ParseWithoutMarker(query string) (cleanQuery string, excludeLabels []string } cleanQuery = query[:idx] labelStr := query[idx+len(WithoutMarkerSuffix):] + // A marker appended after this one (label_replace, label_join, group) + // ends the label list and stays on the query for its own parser. + if end := strings.Index(labelStr, "__lvp_"); end >= 0 { + cleanQuery += labelStr[end:] + labelStr = labelStr[:end] + } for _, l := range strings.Split(labelStr, ",") { l = strings.TrimSpace(l) if l != "" { @@ -1808,6 +1814,29 @@ func parseAllStringArgs(s string) []string { return args } +// parseUnquotedStringArgs reads the comma-separated string arguments that +// follow the first argument of label_replace or label_join, unquoted as the +// LogQL lexer does: a double-quoted string has its escapes resolved, a +// backtick one is raw (pkg/logql/syntax/lex.go). It stops at ')' or at the +// first argument that is not a valid string. +func parseUnquotedStringArgs(s string) []string { + var args []string + for { + s = strings.TrimSpace(s) + quoted, err := strconv.QuotedPrefix(s) + if err != nil { + break + } + arg, err := strconv.Unquote(quoted) + if err != nil { + break + } + args = append(args, arg) + s = strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(s[len(quoted):]), ",")) + } + return args +} + // tryTranslateLabelReplace handles label_replace(v, "dst", "repl", "src", "regex"). // It translates the inner expression and embeds a marker for proxy post-processing. func tryTranslateLabelReplace(logql string, labelFn LabelTranslateFunc) (string, bool) { @@ -1819,7 +1848,7 @@ func tryTranslateLabelReplace(logql string, labelFn LabelTranslateFunc) (string, if !ok { return "", false } - args := parseAllStringArgs(rest) + args := parseUnquotedStringArgs(rest) if len(args) != 4 { return "", false } @@ -1844,7 +1873,7 @@ func tryTranslateLabelJoin(logql string, labelFn LabelTranslateFunc) (string, bo if !ok { return "", false } - args := parseAllStringArgs(rest) + args := parseUnquotedStringArgs(rest) if len(args) < 3 { // dst + sep + at least one src return "", false } diff --git a/internal/translator/translator_test.go b/internal/translator/translator_test.go index 7e19eb15f..bc786e878 100644 --- a/internal/translator/translator_test.go +++ b/internal/translator/translator_test.go @@ -871,6 +871,33 @@ func TestLabelReplaceTranslation(t *testing.T) { } } +// The string arguments of label_replace are LogQL strings: a double-quoted one +// is unescaped (pkg/logql/syntax/lex.go), so "\\." is the regexp \. and not a +// backslash followed by any character; a backtick one is taken as it is. +func TestLabelReplaceStringArgumentsAreUnquoted(t *testing.T) { + for _, tc := range []struct{ name, in, dst, repl, src, regex string }{ + {"escaped dot", `label_replace(rate({app="api"}[5m]), "d", "$1", "s", "(\\w+)\\.(\\w+)")`, "d", "$1", "s", `(\w+)\.(\w+)`}, + {"escaped quote", `label_replace(rate({app="api"}[5m]), "d", "a\"b", "s", "x")`, "d", `a"b`, "s", "x"}, + {"backtick is raw", "label_replace(rate({app=\"api\"}[5m]), `d`, `$1`, `s`, `(\\w+)\\.`)", "d", "$1", "s", `(\w+)\.`}, + {"escaped newline", `label_replace(rate({app="api"}[5m]), "d", "a\nb", "s", "x")`, "d", "a\nb", "s", "x"}, + {"comma and paren in a string", `label_replace(rate({app="api"}[5m]), "d", "a,b)", "s", "(x),(y)")`, "d", "a,b)", "s", "(x),(y)"}, + } { + t.Run(tc.name, func(t *testing.T) { + result, err := TranslateLogQL(tc.in) + if err != nil { + t.Fatal(err) + } + _, spec := ParseLabelReplaceMarker(result) + if spec == nil { + t.Fatalf("no marker in %q", result) + } + if spec.DstLabel != tc.dst || spec.Replacement != tc.repl || spec.SrcLabel != tc.src || spec.Regex != tc.regex { + t.Fatalf("spec %+v, want dst=%q repl=%q src=%q regex=%q", *spec, tc.dst, tc.repl, tc.src, tc.regex) + } + }) + } +} + func TestParseAllLabelReplaceMarkers(t *testing.T) { t.Run("single marker", func(t *testing.T) { q := `label_replace(rate({app="api"}[5m]), "host", "$1", "instance", "(.*):.+")` diff --git a/test/e2e-compat/label_replace_new_label_test.go b/test/e2e-compat/label_replace_new_label_test.go new file mode 100644 index 000000000..0116a0623 --- /dev/null +++ b/test/e2e-compat/label_replace_new_label_test.go @@ -0,0 +1,173 @@ +//go:build e2e + +package e2e_compat + +import ( + "fmt" + "net/http" + "net/url" + "sort" + "strings" + "testing" + "time" +) + +// label_replace sets, rewrites or removes one label of every series its +// expression yields (pkg/logql/evaluator.go LabelReplaceEvaluator.Next): the +// regexp is anchored, a series it does not match is left alone, an empty +// replacement removes the destination, and series that end with one label set +// are not an error (pkg/logql/engine.go JoinSampleVector files their points +// under one series on a range query, keeps each sample on an instant query). +// +// The proxy's instant answers lacked the destination label (the transform only +// ran on range matrices), an escaped regexp such as `\.` never matched, and a +// label_replace around a binary expression on query_range sent the proxy's +// internal binary marker to VictoriaLogs. +// +// Every shape is asked of Loki and of the proxy on query_range and on query. +// Both must answer the same status; when they answer, the same result type, the +// same label sets (and, on query, the same number of samples per label set). The +// values and the points per step are compared by the other registry items. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestCompat_LabelReplaceNewLabel(t *testing.T) { + ensureDataIngested(t) + waitForLokiMetricData(t) + + sel := `{env="production"}` + count := `count_over_time(` + sel + `[5m])` + rate := `rate(` + sel + `[5m])` + agg := `sum by (app) (` + count + `)` + lr := func(inner, dst, repl, src, re string) string { + return fmt.Sprintf(`label_replace(%s, %q, %q, %q, %q)`, inner, dst, repl, src, re) + } + shapes := []struct { + name string + query string + // points compares the points per series of a range answer as well: + // where two series end with one label set, Loki files the points of + // both under one series. + points bool + }{ + // the request of the registry row, its bare form, and the clusters' forms + {"row", `label_replace(sum by (app) (rate({app="api-gateway"}[5m])), "app_short", "$1", "app", "(.*)-.*")`, false}, + {"bare rate", `label_replace(rate({app="api-gateway"}[5m]), "app_short", "$1", "app", "(.*)-.*")`, false}, + {"bare count_over_time", lr(count, "app_short", "$1", "app", "(.*)-.*"), false}, + {"bytes_over_time", lr(`bytes_over_time(`+sel+`[5m])`, "buzz", "$2", "app", "(.*)-(.*)"), false}, + {"nested", lr(lr(agg, "service", "$1", "app", "(.*)"), "short", "$1", "service", "^([^-]+)"), false}, + {"nested, the outer reads the inner's label", lr(lr(agg, "a1", "$1", "app", "(.*)-.*"), "a2", "x$1", "a1", "(.*)"), false}, + {"nested, three deep", lr(lr(lr(agg, "a", "$1", "app", "(.*)-.*"), "b", "$1$1", "a", "(.*)"), "c", "${1}!", "b", "(.*)"), false}, + // regexp and replacement rules + {"anchored: a prefix does not match", lr(agg, "d", "x", "app", "api"), false}, + {"anchored: the whole value", lr(agg, "d", "x", "app", "api-gateway"), false}, + {"no match keeps the series", lr(agg, "new_label", "default", "app", "^nonexistent$"), false}, + {"escaped regexp", lr(agg, "d", "$1", "app", `([a-z]+)\-(\w+)`), false}, + {"escaped dot never matches a hyphen", lr(agg, "d", "$1", "app", `(\w+)\.(\w+)`), false}, + {"named group", lr(agg, "d", "${head}", "app", `(?P[a-z]+)-.*`), false}, + {"bare group name swallows the suffix", lr(agg, "d", "$1x", "app", "(.*)-.*"), false}, + {"two groups", lr(agg, "d", "$2/$1", "app", "([a-z]+)-([a-z]+)"), false}, + {"constant replacement", lr(agg, "d", "const", "app", ".*"), false}, + {"missing source, empty-matching regexp", lr(agg, "d", "dflt", "nope", "(.*)"), false}, + {"missing source, non-empty regexp", lr(agg, "d", "dflt", "nope", ".+"), false}, + {"empty replacement removes the destination", lr(agg, "app", "", "app", "api-gateway"), false}, + {"destination equal to the source", lr(agg, "app", "$1", "app", "(.*)-.*"), false}, + {"two series with one label set", lr(agg, "app", "all", "app", ".+"), true}, + {"all labels removed", lr(agg, "app", "", "app", ".*"), true}, + {"label names are not validated", lr(agg, "1bad", "x", "app", ".*"), false}, + // around and with other expressions + {"over a binary expression", lr(agg+` * 2`, "app_short", "$1", "app", "(.*)-.*"), false}, + {"over a binary of two aggregations", lr(agg+` / `+agg, "app_short", "$1", "app", "(.*)-.*"), false}, + {"over a rate aggregation", lr(`sum by (app) (`+rate+`)`, "app_short", "$1", "app", "(.*)-.*"), false}, + {"over a parser", lr(`sum by (app) (count_over_time(`+sel+` | json [5m]))`, "app_short", "$1", "app", "(.*)-.*"), false}, + {"under sort", `sort(` + lr(agg, "app_short", "$1", "app", "(.*)-.*") + `)`, false}, + {"under topk", `topk(2, ` + lr(agg, "app_short", "$1", "app", "(.*)-.*") + `)`, false}, + // an invalid regexp is Loki's parse error + {"invalid regexp", lr(agg, "new", "$1", "app", "[invalid"), false}, + {"invalid regexp, nested", lr(lr(agg, "d", "$1", "app", "(.*)"), "e", "$1", "d", "(unclosed"), false}, + {"invalid regexp under topk", `topk(2, ` + lr(agg, "new", "$1", "app", "*bad") + `)`, false}, + // a call the grammar rejects + {"invalid escape in a string", `label_replace(` + agg + `, "d", "$1", "app", "(.+):\d+")`, false}, + {"three string arguments", `label_replace(` + agg + `, "d", "$1", "app")`, false}, + {"five string arguments", `label_replace(` + agg + `, "d", "$1", "app", "x", "y")`, false}, + } + + // labelCounts reads a decoded answer as {label set: samples}: an instant + // vector's samples per label set, a range series' points when points is + // set and 1 otherwise (the points per step are other registry items'). + labelCounts := func(payload map[string]interface{}, instant, points bool) map[string]int { + out := map[string]int{} + data, _ := payload["data"].(map[string]interface{}) + result, _ := data["result"].([]interface{}) + for _, item := range result { + series, _ := item.(map[string]interface{}) + metric, _ := series["metric"].(map[string]interface{}) + key := canonicalMetricKey(metric) + if instant { + out[key]++ + continue + } + values, _ := series["values"].([]interface{}) + if points { + out[key] += len(values) + } else { + out[key] = 1 + } + } + return out + } + resultType := func(payload map[string]interface{}) string { + data, _ := payload["data"].(map[string]interface{}) + rt, _ := data["resultType"].(string) + return rt + } + render := func(m map[string]int) string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, fmt.Sprintf("%s x%d", k, m[k])) + } + sort.Strings(keys) + return strings.Join(keys, "\n ") + } + + for _, shape := range shapes { + for _, endpoint := range []string{"query_range", "query"} { + t.Run(shape.name+"/"+endpoint, func(t *testing.T) { + instant := endpoint == "query" + at := time.Now().UTC().Truncate(time.Minute) + params := url.Values{"query": {shape.query}} + if instant { + params.Set("time", fmt.Sprintf("%d", at.UnixNano())) + } else { + params.Set("start", fmt.Sprintf("%d", at.Add(-15*time.Minute).UnixNano())) + params.Set("end", fmt.Sprintf("%d", at.UnixNano())) + params.Set("step", "60") + } + headers := map[string]string{"X-Scope-OrgID": "0", "X-Loki-Response-Encoding-Flags": "categorize-labels"} + full := "/loki/api/v1/" + endpoint + "?" + params.Encode() + lokiStatus, lokiBody, lokiPayload := doJSONGET(t, lokiURL+full, headers) + proxyStatus, proxyBody, proxyPayload := doJSONGET(t, proxyURL+full, headers) + if lokiStatus != proxyStatus { + t.Fatalf("HTTP status loki=%d proxy=%d\nloki: %s\nproxy: %s", lokiStatus, proxyStatus, lokiBody, proxyBody) + } + if lokiStatus != http.StatusOK { + if lokiStatus == http.StatusBadRequest && !strings.Contains(proxyBody, "parse error") { + t.Errorf("the proxy's 400 is not Loki's parse error\nloki: %s\nproxy: %s", lokiBody, proxyBody) + } + if strings.Contains(shape.name, "invalid regexp") && !strings.Contains(proxyBody, "invalid regex in label_replace") { + t.Errorf("the proxy's error is not Loki's invalid regex error\nloki: %s\nproxy: %s", lokiBody, proxyBody) + } + return + } + if lt, pt := resultType(lokiPayload), resultType(proxyPayload); lt != pt { + t.Fatalf("resultType loki=%q proxy=%q\nloki: %s\nproxy: %s", lt, pt, lokiBody, proxyBody) + } + lokiSeries, proxySeries := labelCounts(lokiPayload, instant, shape.points), labelCounts(proxyPayload, instant, shape.points) + if len(lokiSeries) == 0 { + t.Skipf("Loki answered no series for %s; nothing to compare", shape.query) + } + if render(lokiSeries) != render(proxySeries) { + t.Errorf("label sets differ\nloki:\n %s\nproxy:\n %s", render(lokiSeries), render(proxySeries)) + } + }) + } + } +} From 7d31aac59300e936157a27b46877d7ec944124b3 Mon Sep 17 00:00:00 2001 From: szibis Date: Mon, 12 Oct 2026 13:30:20 +0200 Subject: [PATCH 08/18] chore(conformance): record #744 as the fix of label-replace-new-label --- CHANGELOG.md | 2 +- .../semantics/label-replace-new-label.yaml | 4 +- conformance/registry/generated/wiring.json | 23 +++- conformance/reports/parity-gaps.md | 126 +++++++++--------- 4 files changed, 87 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 692205d2d..a3c5385a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - **`label_replace` adds its destination label to instant answers, reads an - escaped regexp, and runs under `topk`, `sort` and a binary expression (#TBD).** + escaped regexp, and runs under `topk`, `sort` and a binary expression (#744).** `label_replace(rate({app="api-gateway"}[5m]), "app_short", "$1", "app", "(.*)-.*")` came back from `/loki/api/v1/query` without `app_short` on every series (also through nested `label_replace`), where Loki adds it diff --git a/conformance/registry/cases/semantics/label-replace-new-label.yaml b/conformance/registry/cases/semantics/label-replace-new-label.yaml index e1325bb6e..f9cc63d5f 100644 --- a/conformance/registry/cases/semantics/label-replace-new-label.yaml +++ b/conformance/registry/cases/semantics/label-replace-new-label.yaml @@ -72,8 +72,10 @@ compare: app_short on every series. gap: status: fixed - fixed_by: "#TBD" + fixed_by: "#744" impact: explore-visible + visual: none + visual_reason: 'the visible difference is on instant queries; the visual harness captures Explore range panels (queryType range), where label_replace already matched, so the fix is proven by the instant tests and the e2e against Loki' effort: S area: 'internal/proxy vector_matching.go (transformSeriesLabels, applyLabelReplace, applyLabelJoin), metric_agg.go (post-aggregation markers), internal/translator parseUnquotedStringArgs' loki: 'pkg/logql/syntax/ast.go mustNewLabelReplaceExpr; pkg/logql/evaluator.go:1249-1315 LabelReplaceEvaluator; pkg/logql/engine.go:530-600 JoinSampleVector, vectorsToSeries' diff --git a/conformance/registry/generated/wiring.json b/conformance/registry/generated/wiring.json index 111432b84..3fb801208 100644 --- a/conformance/registry/generated/wiring.json +++ b/conformance/registry/generated/wiring.json @@ -182,6 +182,7 @@ ], "loki_api_v1_query": [ "internal/proxy/instant_result_type_test.go", + "internal/proxy/label_replace_semantics_test.go", "internal/proxy/parenthesized_range_test.go", "internal/proxy/series_limit_routes_test.go", "internal/proxy/tenant_query_limits_test.go", @@ -189,6 +190,7 @@ "internal/proxy/unwrap_conversion_error_test.go", "test/e2e-compat/extracted_suffix_collision_test.go", "test/e2e-compat/instant_result_type_test.go", + "test/e2e-compat/label_replace_new_label_test.go", "test/e2e-compat/parenthesized_range_or_filters_test.go", "test/e2e-compat/parser_capture_identity_test.go", "test/e2e-compat/plain_name_after_parser_test.go", @@ -201,6 +203,7 @@ "internal/proxy/drilldown_breakdown_exact_test.go", "internal/proxy/drilldown_breakdown_matrix_test.go", "internal/proxy/extracted_suffix_collision_test.go", + "internal/proxy/label_replace_semantics_test.go", "internal/proxy/line_fields_without_parser_test.go", "internal/proxy/metadata_query_validation_test.go", "internal/proxy/parenthesized_range_test.go", @@ -215,6 +218,7 @@ "test/e2e-compat/drilldown_field_breakdown_test.go", "test/e2e-compat/extracted_suffix_collision_test.go", "test/e2e-compat/heavy_query_protection_test.go", + "test/e2e-compat/label_replace_new_label_test.go", "test/e2e-compat/line_fields_parity_test.go", "test/e2e-compat/parenthesized_range_or_filters_test.go", "test/e2e-compat/plain_name_after_parser_test.go", @@ -421,6 +425,10 @@ "internal/proxy/label_values_cap_inventory_test.go", "internal/proxy/metadata_inventory_test.go" ], + "semantics/label-replace-new-label": [ + "internal/proxy/label_replace_semantics_test.go", + "test/e2e-compat/label_replace_new_label_test.go" + ], "semantics/labels-end-inclusive-like-loki": [ "internal/proxy/labels_inclusive_end_test.go", "test/e2e-compat/labels_zero_width_test.go" @@ -652,6 +660,11 @@ "loki_api_v1_series", "semantics/metadata-answers-include-last-24h-like-loki" ], + "internal/proxy/label_replace_semantics_test.go": [ + "loki_api_v1_query", + "loki_api_v1_query_range", + "semantics/label-replace-new-label" + ], "internal/proxy/label_surface_test.go": [ "limits/label-values-not-truncated-by-default", "loki_api_v1_label_name_values", @@ -969,6 +982,11 @@ "test/e2e-compat/label_freshness_e2e_test.go": [ "semantics/metadata-answers-include-last-24h-like-loki" ], + "test/e2e-compat/label_replace_new_label_test.go": [ + "loki_api_v1_query", + "loki_api_v1_query_range", + "semantics/label-replace-new-label" + ], "test/e2e-compat/labels_zero_width_test.go": [ "semantics/labels-end-inclusive-like-loki" ], @@ -1078,7 +1096,7 @@ "profiles/structured-metadata-keys-per-profile" ] }, - "tests_with_markers": 98, + "tests_with_markers": 100, "unknown_ids": {}, "unwired_items": [ "api_prom_label", @@ -1156,7 +1174,6 @@ "semantics/first-last-over-time-translation", "semantics/ip-filter-precision", "semantics/label-filter-expression-syntax", - "semantics/label-replace-new-label", "semantics/log-entries-differ-same-count", "semantics/logfmt-existence-filter-ungrouped-sum", "semantics/non-sum-vector-aggregation-unreduced", @@ -1186,5 +1203,5 @@ "versions/capabilities-floor-vs-matrix", "versions/rejected-stats-query-wording-vl-1.47" ], - "wired_items": 107 + "wired_items": 108 } diff --git a/conformance/reports/parity-gaps.md b/conformance/reports/parity-gaps.md index 558e2db2a..e5add94b3 100644 --- a/conformance/reports/parity-gaps.md +++ b/conformance/reports/parity-gaps.md @@ -34,69 +34,68 @@ Last run: 2026-10-05T20:16:15Z on `fcaccfcc`, window [1791220800, 1791224400]. | 13 | `semantics/parse-error-labels-on-log-lines` — A line a parser rejects carries __error__ and __error_details__ in Loki's log answer (open) | open | explore-visible | 10 | 2 | M | internal/proxy log-row assembly (stage-field exposure) + translator parse-error predicate | pkg/logql/log/parser.go:97 addErrLabel(errJSON, err) -> :777-787 SetErr/SetErrorDetails; pkg/logql/log/error.go:5 errJSON | TestCompat_ParseErrorLabelsOnLogLines (test/e2e-compat, group semantics) | | 14 | `semantics/absent-over-time-semantics` — absent_over_time answers when data exists and stays empty when it is absent, after a parser (open) | open | explore-visible | 8 | 6 | S | internal/translator absent_over_time (parser pipelines) | pkg/logql/evaluator.go:647-664, :1320 absentLabels | TestCompat_AbsentOverTimeWithParser (test/e2e-compat, group semantics) | | 15 | `semantics/stddev-stdvar-grouped-values` — stddev and stdvar by a label compute different values (open) | open | explore-visible | 7 | 5 | S | internal/translator stddev/stdvar outer aggregation (by-group fold) | pkg/logql/evaluator.go VectorAggEvaluator stddev/stdvar | TestCompat_StddevStdvarBy (test/e2e-compat, group semantics) | -| 16 | `semantics/label-replace-new-label` — label_replace does not add a new destination label (open) | open | explore-visible | 5 | 1 | S | internal/proxy label_replace post-processing | pkg/logql/evaluator.go:387, :1253 label_replace evaluator | TestCompat_LabelReplaceNewLabel (test/e2e-compat, group semantics) | -| 17 | `quality/label-values-chunk-window` — Loki lists label values of streams whose chunks overlap the window; the proxy only of streams with lines in it (open) | open | explore-visible | 2 | 2 | M | internal/proxy label values window (chunk-overlap semantics) | pkg/querier/querier.go Label (index lookups by chunk overlap) | TestCompat_LabelValuesWindowEdge (test/e2e-compat, group core) | -| 18 | `semantics/aggregation-over-binary-dropped` — An aggregation over a binary expression is translated to its first operand (open) | open | explore-visible | 2 | 1 | M | internal/translator extractOuterAggregation and internal/proxy handleQuery routing of an aggregation over a binary expression | pkg/logql/evaluator.go VectorAggEvaluator over a binary evaluator | TestCompat_AggregationOverBinary (test/e2e-compat, group semantics) | -| 19 | `semantics/pattern-line-filter` — An empty pattern line filter (`\|> ""`, `!> ""`) selects the opposite lines (open) | open | explore-visible | 2 | 2 | S | internal/translator pattern line filter | pkg/logql/syntax/lex.go:26,29 (NPA, PIPE_PATTERN); pkg/logql/log/filter.go pattern filter | TestCompat_PatternLineFilter (test/e2e-compat, group semantics) | -| 20 | `semantics/log-entries-differ-same-count` — A log query returns the same number of lines with different entries (open, cause not isolated) | open | explore-visible | 1 | 1 | S | to be isolated (log query paging / numeric label filters) | pkg/logql/log/label_filter.go numeric filters; pkg/iter entry ordering | TestCompat_NumericLabelFilterLines (test/e2e-compat, group semantics) | -| 21 | `profiles/label-filter-regexp-anchoring` — A label-filter regexp after a parser is not anchored the way Loki's is (open) | open | explore-visible | — | — | S | internal/translator label-filter regexp emission (anchor ^(?:re)$); external PR #572 open | pkg/logql/log/label_filter.go (regexp matcher on the whole value) | TestCompat_LabelFilterRegexpAnchoring (test/e2e-compat, group semantics) | -| 22 | `profiles/parsed-key-colliding-with-structured-metadata` — A parsed key named like structured metadata loses the metadata value in VictoriaLogs (open) | open | explore-visible | — | — | L | internal/translator json / logfmt unpack (original stored value) and internal/proxy row classification | pkg/logql/log/parser.go:152 HasInCategory(key, StructuredMetadataLabel) then duplicateSuffix | TestCompat_ParsedKeyAgainstStructuredMetadata (test/e2e-compat, group otel-edge) | -| 23 | `quality/label-cache-aged-labels-kept` — Labels that age out of the window are never dropped from the merged label cache (open) | open | explore-visible | — | — | S | internal/proxy/label_metadata.go:1259 mergeLabelsIntoCache | pkg/querier/querier.go Label (window-bounded) | TestE2E_LabelsAgeOutOfWindow (test/e2e-compat, group core) | -| 24 | `quality/labels-backfill-into-cached-bucket` — Rows backfilled into a past bucket the label inventory already cached with data appear in /labels only after its revalidation delay (open) | open | explore-visible | — | — | M | internal/proxy/metadata_inventory.go (bucket revalidation delay) | pkg/querier/querier.go Label (index read per request) | TestCompat_LabelsBackfillIntoCachedBucket (test/e2e-compat, group core) | -| 25 | `semantics/drop-keep-plain-name-after-parser` — A log query's drop, keep or label_format source of a stream label's plain name after a parser acts on the stored field; Loki acts on the ... | open | explore-visible | — | — | S | internal/proxy stage-field exposure (drop and keep of a name a parser also reads) | pkg/logql/log/labels.go LabelsBuilder.Del / getWithCategory; pkg/logql/log/drop_labels.go, keep_labels.go | TestCompat_DropKeepPlainNameAfterParser (test/e2e-compat, group core) | -| 26 | `semantics/extracted-ungrouped-metric-raw-rows` — An ungrouped range metric over a `\| json` pipeline filtered on name_extracted still reads raw rows and fails at the row limit (open) | open | explore-visible | — | — | L | internal/proxy bare-parser metric evaluator (fetchBareParserMetricSeries) and the ordered JSON pushdown (without grouping) | pkg/logql range aggregation keeps every label of the sample | TestCompat_ExtractedUngroupedMetricAtSixHours (test/e2e-compat, group core) | -| 27 | `semantics/repeated-label-filter-replaced-across-label-format` — A second filter on the same label replaces the first even with a label_format between them (open) | open | explore-visible | — | — | S | internal/translator labelFilterLatest (latest filter per field replaces the earlier one in place) | pkg/logql/log/pipeline.go (stages run in order); label_filter.go | TestCompat_RepeatedLabelFilterAcrossLabelFormat (test/e2e-compat, group core) | -| 28 | `semantics/two-parsers-keep-one` — A query with `\| json` and `\| logfmt` runs the one parser the data answers to (open) | open | explore-visible | — | — | M | internal/proxy/query_translation.go preferWorkingParser (probe-based removal of a parser stage) | pkg/logql/log/pipeline.go: every stage runs on every line; parser.go skips a key ParserHint.Extracted | TestCompat_TwoParsersOnMixedFormats (test/e2e-compat, group semantics) | -| 29 | `semantics/unwrap-duration-seconds` — '`unwrap duration_seconds(label)` answers no data where Loki converts it like duration() (open)' | open | explore-visible | — | — | S | internal/translator extractUnwrapField and unwrapGateField (duration_seconds conversion) | pkg/logql/syntax/extractor.go:63-71 | TestCompat_UnwrapDurationSeconds (test/e2e-compat, group core) | -| 30 | `semantics/unwrap-first-last-under-aggregation` — first_over_time and last_over_time with unwrap under sum fail with a VictoriaLogs 422 (open) | open | explore-visible | — | — | M | internal/translator first_over_time / last_over_time under a vector aggregation | pkg/logql/range_vector.go first/last | an e2e over sum(first_over_time\|last_over_time(... \| unwrap ...)) | -| 31 | `semantics/unwrap-over-time-range-windows` — sum, avg, max and min over time with unwrap answer the wrong window on query_range (open) | open | explore-visible | — | — | M | internal/proxy range_metric_compat.go shouldUseManualRangeMetricCompat (unwrap sum/avg/min/max) and metric_binary.go statsRateRangeEquals... | pkg/logql/range_vector.go:102-189 batchRangeVectorIterator (window (t-range, t]); range_vector.go:371-420 sumOverTime, avgOverTime, maxOv... | TestCompat_UnwrapOverTimeRangeWindows (test/e2e-compat, group core) over sum/avg/max/min x tumbling/sliding x logfmt/json | -| 32 | `semantics/unwrap-per-series-identity-raw-rows` — A function taken per series (avg, max, min over time, rate_counter) under sum or sum by uses the stream as the series (open) | open | explore-visible | — | — | L | internal/translator unwrapInnerGrouping (sum by over a per-series function) and the raw evaluators series key (collectRangeMetricSamples) | pkg/logql/evaluator.go:435-529 VectorAggEvaluator.Next; pkg/logql/log/metrics_extraction.go:150-178 labelSampleExtractor (series = stream... | TestCompat_UnwrapPerSeriesIdentity (test/e2e-compat, group core) over sum and sum by x avg/max/min/rate_counter, logfmt and json | -| 33 | `semantics/unwrap-quantile-response-limit` — quantile_over_time over a JSON unwrap fails at the manual metric response limit from 6 hours (open) | open | explore-visible | — | — | L | internal/proxy raw-row quantile evaluator (manual metric response cap) | pkg/logql/range_vector.go quantile_over_time | bench/ab set unwrap-conversion-error E2 at 6h/24h/7d | -| 34 | `semantics/unwrap-range-last-point` — A grouped unwrap range metric misses the sample at `end` (open) | open | explore-visible | — | — | S | internal/proxy range-metric unwrap path (end bucket) | pkg/logql/range_vector.go step iteration start..end inclusive | TestCompat_UnwrapRangeEndSample (test/e2e-compat, group semantics) | -| 35 | `semantics/unwrap-sum-without-collapses` — sum without over an unwrap range function returns a single series (open) | open | explore-visible | — | — | M | internal/translator `without` grouping over unwrap range functions | pkg/logql/evaluator.go newVectorAggEvaluator (without) | an e2e over sum without (...) with unwrap on query_range and query | -| 36 | `drilldown/detected-field-values` — detected_field values differ: none for service_name, other values for numeric and id fields (open) | open | drilldown-visible | 33 | 3 | M | internal/proxy detected_field values (sampling window and stream-label fields) | pkg/querier detected field values (v3.7.7): values of the sampled lines | TestCompat_DetectedFieldValues (test/e2e-compat, group drilldown) | -| 37 | `semantics/detected-fields-extracted-suffix` — detected_fields omits Loki's level_extracted and version_extracted (open; service fields are owner-kept) | open | drilldown-visible | 15 | 2 | M | internal/proxy/label_handlers.go handleDetectedFields (parsed fields named like a stream label) | pkg/logql/log/parser.go:25 duplicateSuffix; detected fields come from the parsed labels of sampled entries | TestCompat_DetectedFieldsExtractedSuffix (test/e2e-compat, group drilldown) | -| 38 | `profiles/json-number-label-text` — JSON numbers become label values in a different text form ("14.0" vs "14") (open) | open | drilldown-visible | 12 | 6 | M | internal/proxy JSON number rendering of unpacked values (VictoriaLogs unpack_json normalises) | pkg/logql/log/parser.go readValue (raw number bytes) | TestCompat_JSONNumberLabelText (test/e2e-compat, group drilldown) | -| 39 | `severity/detected-level-filter-json-level` — `\| detected_level=` filters select different lines than Loki (JSON-level lines, stream level precedence) (open) | open | drilldown-visible | 12 | 8 | M | internal/translator/detected_level_logsql.go (filter predicate) vs internal/proxy/detected_level.go (response derivation) | pkg/distributor field detection (v3.7.7) stores detected_level as structured metadata; filter reads it | TestCompat_DetectedLevelFilterJSONLevel (test/e2e-compat, group drilldown) | -| 40 | `limits/drilldown-7d-breakdown-response-limit` — Metric queries fail on the 64 MiB buffered-response budget where Loki answers (open) | open | drilldown-visible | 8 | 4 | M | internal/proxy/range_metric_compat.go:767, execution_limits.go:11 (raw-row metric paths buffer rows; push down or stream) | pkg/logql engine series limit (max_query_series) with partial result for Drilldown | TestCompat_MetricBufferedBudget1h + TestE2E_Drilldown7dHighCardinalityBreakdown (test/e2e-compat, groups semantics and drilldown) | -| 41 | `quality/index-stats-streams-differ` — index/stats stream counts differ from Loki in both directions (open) | open | drilldown-visible | 6 | 2 | S | internal/proxy/label_handlers.go handleIndexStats (stream counting per window) | pkg/storage index stats (streams from the index for the window) | bench/parity rerun with the /series cross-check, then TestCompat_IndexStatsStreams (test/e2e-compat, group drilldown) | -| 42 | `drilldown/detected-fields-type-inference` — detected_fields infers a different type for some fields (open) | open | drilldown-visible | — | — | S | internal/proxy detected_fields type inference | pkg/querier detected fields type detection (v3.7.7): int before duration before string | TestCompat_DetectedFieldsTypes (test/e2e-compat, group drilldown) | -| 43 | `profiles/existence-filter-on-sanitized-key` — An existence filter on a sanitized key reads the stored dotted field, not the JSON line key (open) | open | drilldown-visible | — | — | M | internal/translator (label-filter key variants, parsed_key_variants.go); Drilldown single-field fast paths | pkg/logql/log/parser.go:141 sanitizeLabelKey on every extracted key; label filter reads the parsed label | TestCompat_ExistenceFilterSanitizedKey (test/e2e-compat, group semantics) | -| 44 | `quality/detected-metadata-proxy-cache` — detected_labels and detected_fields are served from proxy caches; Loki answers each request fresh (open) | open | drilldown-visible | — | — | S | internal/proxy/cache_keys.go:48,130 (detected_* cache TTL) + recent-tail freshness bypass | pkg/querier detected labels/fields: no results cache in v3.7.7 | TestE2E_DetectedMetadataFreshness (test/e2e-compat, group drilldown) | -| 45 | `quality/stats-volume-freshness` — index/stats and volume answers ending now are cached; Loki does not cache inside max_stats_cache_freshness (10m) (open) | open | drilldown-visible | — | — | S | internal/proxy/volume.go handleVolume, label_handlers.go handleIndexStats (cache freshness) | pkg/validation/limits.go:413-414 max_stats_cache_freshness default 10m | TestE2E_StatsVolumeFreshness (test/e2e-compat, group drilldown) | -| 46 | `semantics/parse-error-text-parity` — Invalid queries fail on both sides, with different error text or status (open) | open | grafana-datasource | 210 | 129 | L | internal/logql parser error messages (Loki's goyacc messages and positions) + VictoriaLogs error mapping | pkg/logql/syntax/parser.go ParseError (line, col) and syntax.y | TestCompat_ParseErrorText corpus (test/e2e-compat, bad_request_status_parity_test.go) | -| 47 | `semantics/proxy-rejects-valid-logql` — The proxy rejects LogQL Loki accepts: comments, negative and compound offsets, filters after a range, logfmt flags, ip() label filters (o... | open | api-only | 79 | 48 | M | internal/logql parser (lexer: comments, signed durations, flags) + translator | pkg/logql/syntax/lex.go:149 comments, :90-91 OpStrict/OpKeepEmpty; syntax.y offsetExpr; pkg/logql/log/ip.go | TestCompat_LogQLSurfaceAccepted (test/e2e-compat, group semantics), one subtest per family | -| 48 | `semantics/proxy-accepts-invalid-logql` — Queries Loki rejects with 400 are answered by the proxy (open) | open | api-only | 18 | 14 | S | internal/logql parser validation | pkg/logql/syntax/ast.go:1446,1579 grouping not allowed; duration parsing | TestCompat_InvalidLogQLRejected (test/e2e-compat, bad_request_status_parity_test.go) | -| 49 | `parsed-identity/regexp-partial-match-series-count` — A partially-matching \| regexp returns about 3x Loki's series count | open | api-only | 2 | 2 | M | internal/proxy regexp capture series identity (non-matching lines) | pkg/logql/log/parser.go RegexpParser (no capture label for a non-matching line) | TestCompat_RegexpPartialMatchSeries (test/e2e-compat, group semantics) | -| 50 | `profiles/json-extraction-list-missing-key` — A json extraction-list key absent from the line is dropped; Loki keeps it in the metric series (open) | open | api-only | 2 | 2 | S | internal/proxy stage-field exposure (extraction lists, metric path) | pkg/logql/log/parser.go:671-720 JSONExpressionParser.Process | TestCompat_JSONExtractionListMissingKey (test/e2e-compat, group semantics) | -| 51 | `semantics/unpack-non-string-values` — `\| unpack` extracts non-string JSON values that Loki ignores (open) | open | api-only | 2 | 2 | S | internal/translator unpack (string values only) | pkg/logql/log/parser.go:789-820 UnpackParser.unpack | TestCompat_UnpackStringValuesOnly (test/e2e-compat, group semantics) | -| 52 | `semantics/extracted-derived-labels-raw-evaluator` — service_name_extracted and detected_level_extracted metrics keep the raw-row evaluator (open) | open | edge | — | — | S | internal/proxy orderedJSONPushdownLabelOK (service_name, detected_level) | pkg/logql/log/parser.go duplicateSuffix; service_name set by discover_service_name at ingest | TestCompat_ExtractedLabelMetricPushdown rows for service_name_extracted (test/e2e-compat, group core) | -| 53 | `semantics/extracted-metric-known-differences` — Two name_extracted differences against Loki remain in the metric pushdown and the raw evaluator (open) | open | edge | — | — | M | internal/proxy raw evaluator parsed-label naming (processWithStreamLabels) and translator.ExtractedLabelPipes | pkg/logql/log/parser.go duplicateSuffix and ParserHint.Extracted (a key an earlier extraction produced is skipped) | TestCompat_ExtractedBothKeysInLineOrder (test/e2e-compat, group core) | -| 54 | `semantics/unwrap-conversion-error-parser-divergence` — Where VictoriaLogs' logfmt parser reads another value than Loki's, an unwrap misses Loki's SampleExtractionErr or drops samples Loki keep... | open | edge | — | — | L | internal/translator logfmt unpacking (unpack_logfmt keeps the last value); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:379-438 LogfmtParser.Process; pkg/logql/log/logfmt/decode.go | TestUnwrapConversionErrorDifferential counts the first case; TestUnwrapLokiLabels pins the decoder; TestCompat_UnwrapConversionErrorParit... | -| 55 | `semantics/unwrap-conversion-error-postfilter-forms` — An unwrap whose post filter keeps only the error, compares a label, or combines filters with and/or answers 200 where Loki may fail with ... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorPostFilters; the translator's post-filter placement | pkg/logql/log/label_filter.go:90-100 BinaryLabelFilter.Process, 306-340 NumericLabelFilter.Process, 418-424 labelValue | TestUnwrapErrorProbePlans and TestUnwrapConversionErrorDifferential (internal/proxy) assert the detection stays off for these forms; flip... | -| 56 | `semantics/unwrap-conversion-error-stored-value-overwritten` — An unwrap over a label stored as structured metadata and also parsed from the line misses Loki's SampleExtractionErr (open) | open | edge | — | — | M | internal/translator parsed-key collision with stored fields (unpack keep_original_fields); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:151,809 duplicateSuffix | TestUnwrapConversionErrorDifferential counts the first case; TestCompat_UnwrapConversionErrorParity asserts the 200 side | -| 57 | `semantics/unwrap-conversion-error-undetected-shapes` — A few query shapes answer 200 where Loki fails with SampleExtractionErr, because no VictoriaLogs query of theirs carries the conversion c... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorProbes (opaque expressions), unwrapCountingQuery (pipes after stats), unwrapLokiLabe... | pkg/logql/evaluator.go:726-733 | TestCompat_UnwrapConversionErrorParity open/* asserts Loki's 400 and skips naming this case (absent_over_time, label_replace); TestUnwrap... | -| 58 | `semantics/unwrap-gate-parsefloat-divergences` — The unwrap sample gate drops a few values Loki's conversions accept or reject (open) | open | edge | — | — | S | internal/translator/unwrap_gate.go (UnwrapNumberPattern, UnwrapDurationPattern, UnwrapBytesPattern) | pkg/logql/log/metrics_extraction.go:316-338 convertFloat, convertDuration, convertBytes | TestUnwrapNumberPatternMatchesParseFloat (internal/translator) pins the known differences | -| 59 | `versions/stats-bucket-label-v1.45` — Stats-bucket metrics put a line on a bucket edge one bucket late on VictoriaLogs before v1.45 (open) | open | edge | — | — | M | stats-bucket relabel (internal/proxy range_metric_compat.go setSlidingStatsRangeParams / slidingStatsBucket; sliding_window_stats.go) | pkg/logql range vector (T-range, T] | TestRangeMetricCompatibilityJSONFilterPushdown on VictoriaLogs v1.40-v1.44 (version matrix); fix by relabelling edge lines or declining t... | -| 60 | `metric-identity/structured-metadata-splits-series` — Loki splits a stream's metric series per structured-metadata set; the proxy keeps the stream (owner-accepted deviation) | owner-kept | explore-visible | 173 | 37 | L | internal/translator series identity (stats by stream); VictoriaLogs cannot group by a row's remaining fields | pkg/logql/log/metrics_extraction.go:102-120 builder.Add(StructuredMetadataLabel) -> GroupedLabels | differential run cluster count (bench/parity) | -| 61 | `parsed-identity/dynamic-key-parsers` — `\| json` and `\| logfmt` metric series keep the stream identity by default (owner-accepted deviation, opt-in flag) | owner-kept | explore-visible | 95 | 29 | L | internal/translator parser series identity; -exact-parser-series-identity | pkg/logql/log/parser.go label extraction into the sample labels | differential run cluster count (bench/parity) | -| 62 | `profiles/label-filter-on-line-field` — A label filter on a line key no earlier stage exposes matches like Loki | owner-kept | explore-visible | — | — | M | internal/proxy label-filter refill (-label-filter-refill-max-pages); metric queries on keys no stage exposes | pkg/logql/log/label_filter.go (filter reads only labels the pipeline set) | TestCompat_LabelFilterOnLineField (exists, profile tests) | -| 63 | `profiles/detected-fields-no-extracted-suffix` — detected_fields lists service / service.name without Loki's _extracted suffix (owner decision) | owner-kept | drilldown-visible | — | — | S | internal/proxy/label_handlers.go handleDetectedFields | pkg/logql/log/parser.go:25 duplicateSuffix | bench/parity diff rule (documented facet, service fields only) | -| 64 | `profiles/json-line-stored-without-line` — A JSON line pushed without _msg is stored without its line (documented) | documented | explore-visible | — | — | L | client push configuration (disable_message_parsing=1); docs/KNOWN_ISSUES.md | pkg/distributor push keeps the line verbatim | cases/profiles/stage-field-exposure with disable_message_parsing pushes | -| 65 | `drilldown/detected-labels-sampled-cardinality` — detected_labels cardinality is sampled from the window's last minutes; Loki counts every stream its ingesters hold (documented) | documented | drilldown-visible | 20 | 2 | M | internal/proxy/patterns.go handleDetectedLabels | pkg/ingester/instance.go LabelsWithValues | bench/parity diff rule (documented facet) | -| 66 | `profiles/volume-detected-level-extension` — index/volume answers targetLabels=detected_level; Loki returns no volume for it (documented proxy extension) | documented | drilldown-visible | 17 | 2 | S | internal/proxy/volume.go | pkg/storage volume index (stream labels only) | differential run row volume by detected_level | -| 67 | `drilldown/patterns-mined-differently` — Patterns come from different miners on each side (documented) | documented | drilldown-visible | 15 | 3 | L | internal/proxy/patterns.go | pkg/pattern (ingester-side mining) | bench/visual history-dependent comparison (exists) | -| 68 | `profiles/volume-bytes-accounting` — Volume values count line bytes; Loki counts chunk bytes with structured metadata (documented) | documented | drilldown-visible | 4 | 3 | L | internal/proxy/volume.go; docs/KNOWN_ISSUES.md | pkg/storage volume index (chunk bytes incl. structured metadata) | bench/parity diff rule (documented facet) | -| 69 | `limits/drilldown-breakdown-keeps-busiest` — Over max_query_series a Drilldown breakdown keeps the busiest series; Loki keeps the first it meets (documented) | documented | drilldown-visible | 1 | 1 | S | internal/proxy series limit ranking | pkg/logql engine series limit | cases/limits/drilldown-partial-with-warning (exists) | -| 70 | `quality/index-stats-bytes-and-chunks` — index/stats bytes and chunks have no VictoriaLogs equivalent (documented) | documented | drilldown-visible | — | — | L | internal/proxy/label_handlers.go handleIndexStats; docs/KNOWN_ISSUES.md | pkg/storage index stats from chunk refs (bytes, chunks) | bench/parity diff rule (documented facet) | -| 71 | `profiles/grafana-stats-error-as-warning` — A failed stats query from Grafana answers 200 with warnings; Loki returns the error (documented) | documented | grafana-datasource | — | — | S | internal/proxy Grafana-sourced stats error handling | pkg/querier/queryrange error propagation (no warnings conversion) | TestCompat_GrafanaStatsErrorStatus (test/e2e-compat, fault injection) | -| 72 | `profiles/metadata-default-lookback` — Label endpoints without start/end default to a different lookback than Loki's 1h (documented) | documented | api-only | — | — | S | internal/config flag default -metadata-default-lookback | pkg/loghttp/params.go defaultSince = 1h | TestCompat_MetadataDefaultLookback (test/e2e-compat, group core) | -| 73 | `profiles/multi-tenant-volume-merge` — Multi-tenant volume returns per-tenant series with __tenant_id__; Loki sums the tenants' volumes (documented) | documented | api-only | — | — | S | internal/proxy/volume.go multi-tenant merge | pkg/querier/queryrange volume merge | TestCompat_MultiTenantVolume (test/e2e-compat, group tail-multitenancy) | -| 74 | `semantics/count-values-unsupported` — count_values() is rejected with 400 where Loki answers (documented) | documented | api-only | — | — | M | internal/translator + proxy-side post-aggregation of a stats matrix | pkg/logql/syntax OpTypeCountValues; vector evaluator | TestCompat_CountValues (test/e2e-compat, group semantics) | -| 75 | `semantics/eager-pipeline-validation` — Invalid pipeline stages fail with 400 regardless of data; Loki answers 200 empty over a range with no data (documented) | documented | edge | — | — | S | internal/translator validation | pkg/logql/log/fmt.go template build at pipeline construction | TestCompat_EagerValidationEmptyRange (test/e2e-compat, group semantics) | -| 76 | `semantics/ip-filter-precision` — ip() filters are approximated with regular expressions (documented) | documented | edge | — | — | M | internal/proxy/postprocess.go:84 ip filtering | pkg/logql/log/ip.go | TestCompat_IPFilterForms (test/e2e-compat, group semantics) | -| 77 | `drilldown/plugin-legend-sync-race` — Logs Drilldown crashes a breakdown page when a query answers before the panel module loads (upstream, not the proxy) | upstream | drilldown-visible | — | — | S | none (grafana-lokiexplore-app) | n/a (client race) | bench/visual capture waits for the panel module (already) | -| 78 | `quality/index-stats-loki-chunk-accounting` — Loki's index/stats counts chunks it holds twice; the proxy's counts equal Loki's own count_over_time (upstream) | upstream | drilldown-visible | — | — | S | none (Loki chunk index accounting) | pkg/storage stats from chunk refs (flushed and retained chunks both counted) | bench/parity cross-check (run.py loki_counts_itself) | +| 16 | `quality/label-values-chunk-window` — Loki lists label values of streams whose chunks overlap the window; the proxy only of streams with lines in it (open) | open | explore-visible | 2 | 2 | M | internal/proxy label values window (chunk-overlap semantics) | pkg/querier/querier.go Label (index lookups by chunk overlap) | TestCompat_LabelValuesWindowEdge (test/e2e-compat, group core) | +| 17 | `semantics/aggregation-over-binary-dropped` — An aggregation over a binary expression is translated to its first operand (open) | open | explore-visible | 2 | 1 | M | internal/translator extractOuterAggregation and internal/proxy handleQuery routing of an aggregation over a binary expression | pkg/logql/evaluator.go VectorAggEvaluator over a binary evaluator | TestCompat_AggregationOverBinary (test/e2e-compat, group semantics) | +| 18 | `semantics/pattern-line-filter` — An empty pattern line filter (`\|> ""`, `!> ""`) selects the opposite lines (open) | open | explore-visible | 2 | 2 | S | internal/translator pattern line filter | pkg/logql/syntax/lex.go:26,29 (NPA, PIPE_PATTERN); pkg/logql/log/filter.go pattern filter | TestCompat_PatternLineFilter (test/e2e-compat, group semantics) | +| 19 | `semantics/log-entries-differ-same-count` — A log query returns the same number of lines with different entries (open, cause not isolated) | open | explore-visible | 1 | 1 | S | to be isolated (log query paging / numeric label filters) | pkg/logql/log/label_filter.go numeric filters; pkg/iter entry ordering | TestCompat_NumericLabelFilterLines (test/e2e-compat, group semantics) | +| 20 | `profiles/label-filter-regexp-anchoring` — A label-filter regexp after a parser is not anchored the way Loki's is (open) | open | explore-visible | — | — | S | internal/translator label-filter regexp emission (anchor ^(?:re)$); external PR #572 open | pkg/logql/log/label_filter.go (regexp matcher on the whole value) | TestCompat_LabelFilterRegexpAnchoring (test/e2e-compat, group semantics) | +| 21 | `profiles/parsed-key-colliding-with-structured-metadata` — A parsed key named like structured metadata loses the metadata value in VictoriaLogs (open) | open | explore-visible | — | — | L | internal/translator json / logfmt unpack (original stored value) and internal/proxy row classification | pkg/logql/log/parser.go:152 HasInCategory(key, StructuredMetadataLabel) then duplicateSuffix | TestCompat_ParsedKeyAgainstStructuredMetadata (test/e2e-compat, group otel-edge) | +| 22 | `quality/label-cache-aged-labels-kept` — Labels that age out of the window are never dropped from the merged label cache (open) | open | explore-visible | — | — | S | internal/proxy/label_metadata.go:1259 mergeLabelsIntoCache | pkg/querier/querier.go Label (window-bounded) | TestE2E_LabelsAgeOutOfWindow (test/e2e-compat, group core) | +| 23 | `quality/labels-backfill-into-cached-bucket` — Rows backfilled into a past bucket the label inventory already cached with data appear in /labels only after its revalidation delay (open) | open | explore-visible | — | — | M | internal/proxy/metadata_inventory.go (bucket revalidation delay) | pkg/querier/querier.go Label (index read per request) | TestCompat_LabelsBackfillIntoCachedBucket (test/e2e-compat, group core) | +| 24 | `semantics/drop-keep-plain-name-after-parser` — A log query's drop, keep or label_format source of a stream label's plain name after a parser acts on the stored field; Loki acts on the ... | open | explore-visible | — | — | S | internal/proxy stage-field exposure (drop and keep of a name a parser also reads) | pkg/logql/log/labels.go LabelsBuilder.Del / getWithCategory; pkg/logql/log/drop_labels.go, keep_labels.go | TestCompat_DropKeepPlainNameAfterParser (test/e2e-compat, group core) | +| 25 | `semantics/extracted-ungrouped-metric-raw-rows` — An ungrouped range metric over a `\| json` pipeline filtered on name_extracted still reads raw rows and fails at the row limit (open) | open | explore-visible | — | — | L | internal/proxy bare-parser metric evaluator (fetchBareParserMetricSeries) and the ordered JSON pushdown (without grouping) | pkg/logql range aggregation keeps every label of the sample | TestCompat_ExtractedUngroupedMetricAtSixHours (test/e2e-compat, group core) | +| 26 | `semantics/repeated-label-filter-replaced-across-label-format` — A second filter on the same label replaces the first even with a label_format between them (open) | open | explore-visible | — | — | S | internal/translator labelFilterLatest (latest filter per field replaces the earlier one in place) | pkg/logql/log/pipeline.go (stages run in order); label_filter.go | TestCompat_RepeatedLabelFilterAcrossLabelFormat (test/e2e-compat, group core) | +| 27 | `semantics/two-parsers-keep-one` — A query with `\| json` and `\| logfmt` runs the one parser the data answers to (open) | open | explore-visible | — | — | M | internal/proxy/query_translation.go preferWorkingParser (probe-based removal of a parser stage) | pkg/logql/log/pipeline.go: every stage runs on every line; parser.go skips a key ParserHint.Extracted | TestCompat_TwoParsersOnMixedFormats (test/e2e-compat, group semantics) | +| 28 | `semantics/unwrap-duration-seconds` — '`unwrap duration_seconds(label)` answers no data where Loki converts it like duration() (open)' | open | explore-visible | — | — | S | internal/translator extractUnwrapField and unwrapGateField (duration_seconds conversion) | pkg/logql/syntax/extractor.go:63-71 | TestCompat_UnwrapDurationSeconds (test/e2e-compat, group core) | +| 29 | `semantics/unwrap-first-last-under-aggregation` — first_over_time and last_over_time with unwrap under sum fail with a VictoriaLogs 422 (open) | open | explore-visible | — | — | M | internal/translator first_over_time / last_over_time under a vector aggregation | pkg/logql/range_vector.go first/last | an e2e over sum(first_over_time\|last_over_time(... \| unwrap ...)) | +| 30 | `semantics/unwrap-over-time-range-windows` — sum, avg, max and min over time with unwrap answer the wrong window on query_range (open) | open | explore-visible | — | — | M | internal/proxy range_metric_compat.go shouldUseManualRangeMetricCompat (unwrap sum/avg/min/max) and metric_binary.go statsRateRangeEquals... | pkg/logql/range_vector.go:102-189 batchRangeVectorIterator (window (t-range, t]); range_vector.go:371-420 sumOverTime, avgOverTime, maxOv... | TestCompat_UnwrapOverTimeRangeWindows (test/e2e-compat, group core) over sum/avg/max/min x tumbling/sliding x logfmt/json | +| 31 | `semantics/unwrap-per-series-identity-raw-rows` — A function taken per series (avg, max, min over time, rate_counter) under sum or sum by uses the stream as the series (open) | open | explore-visible | — | — | L | internal/translator unwrapInnerGrouping (sum by over a per-series function) and the raw evaluators series key (collectRangeMetricSamples) | pkg/logql/evaluator.go:435-529 VectorAggEvaluator.Next; pkg/logql/log/metrics_extraction.go:150-178 labelSampleExtractor (series = stream... | TestCompat_UnwrapPerSeriesIdentity (test/e2e-compat, group core) over sum and sum by x avg/max/min/rate_counter, logfmt and json | +| 32 | `semantics/unwrap-quantile-response-limit` — quantile_over_time over a JSON unwrap fails at the manual metric response limit from 6 hours (open) | open | explore-visible | — | — | L | internal/proxy raw-row quantile evaluator (manual metric response cap) | pkg/logql/range_vector.go quantile_over_time | bench/ab set unwrap-conversion-error E2 at 6h/24h/7d | +| 33 | `semantics/unwrap-range-last-point` — A grouped unwrap range metric misses the sample at `end` (open) | open | explore-visible | — | — | S | internal/proxy range-metric unwrap path (end bucket) | pkg/logql/range_vector.go step iteration start..end inclusive | TestCompat_UnwrapRangeEndSample (test/e2e-compat, group semantics) | +| 34 | `semantics/unwrap-sum-without-collapses` — sum without over an unwrap range function returns a single series (open) | open | explore-visible | — | — | M | internal/translator `without` grouping over unwrap range functions | pkg/logql/evaluator.go newVectorAggEvaluator (without) | an e2e over sum without (...) with unwrap on query_range and query | +| 35 | `drilldown/detected-field-values` — detected_field values differ: none for service_name, other values for numeric and id fields (open) | open | drilldown-visible | 33 | 3 | M | internal/proxy detected_field values (sampling window and stream-label fields) | pkg/querier detected field values (v3.7.7): values of the sampled lines | TestCompat_DetectedFieldValues (test/e2e-compat, group drilldown) | +| 36 | `semantics/detected-fields-extracted-suffix` — detected_fields omits Loki's level_extracted and version_extracted (open; service fields are owner-kept) | open | drilldown-visible | 15 | 2 | M | internal/proxy/label_handlers.go handleDetectedFields (parsed fields named like a stream label) | pkg/logql/log/parser.go:25 duplicateSuffix; detected fields come from the parsed labels of sampled entries | TestCompat_DetectedFieldsExtractedSuffix (test/e2e-compat, group drilldown) | +| 37 | `profiles/json-number-label-text` — JSON numbers become label values in a different text form ("14.0" vs "14") (open) | open | drilldown-visible | 12 | 6 | M | internal/proxy JSON number rendering of unpacked values (VictoriaLogs unpack_json normalises) | pkg/logql/log/parser.go readValue (raw number bytes) | TestCompat_JSONNumberLabelText (test/e2e-compat, group drilldown) | +| 38 | `severity/detected-level-filter-json-level` — `\| detected_level=` filters select different lines than Loki (JSON-level lines, stream level precedence) (open) | open | drilldown-visible | 12 | 8 | M | internal/translator/detected_level_logsql.go (filter predicate) vs internal/proxy/detected_level.go (response derivation) | pkg/distributor field detection (v3.7.7) stores detected_level as structured metadata; filter reads it | TestCompat_DetectedLevelFilterJSONLevel (test/e2e-compat, group drilldown) | +| 39 | `limits/drilldown-7d-breakdown-response-limit` — Metric queries fail on the 64 MiB buffered-response budget where Loki answers (open) | open | drilldown-visible | 8 | 4 | M | internal/proxy/range_metric_compat.go:767, execution_limits.go:11 (raw-row metric paths buffer rows; push down or stream) | pkg/logql engine series limit (max_query_series) with partial result for Drilldown | TestCompat_MetricBufferedBudget1h + TestE2E_Drilldown7dHighCardinalityBreakdown (test/e2e-compat, groups semantics and drilldown) | +| 40 | `quality/index-stats-streams-differ` — index/stats stream counts differ from Loki in both directions (open) | open | drilldown-visible | 6 | 2 | S | internal/proxy/label_handlers.go handleIndexStats (stream counting per window) | pkg/storage index stats (streams from the index for the window) | bench/parity rerun with the /series cross-check, then TestCompat_IndexStatsStreams (test/e2e-compat, group drilldown) | +| 41 | `drilldown/detected-fields-type-inference` — detected_fields infers a different type for some fields (open) | open | drilldown-visible | — | — | S | internal/proxy detected_fields type inference | pkg/querier detected fields type detection (v3.7.7): int before duration before string | TestCompat_DetectedFieldsTypes (test/e2e-compat, group drilldown) | +| 42 | `profiles/existence-filter-on-sanitized-key` — An existence filter on a sanitized key reads the stored dotted field, not the JSON line key (open) | open | drilldown-visible | — | — | M | internal/translator (label-filter key variants, parsed_key_variants.go); Drilldown single-field fast paths | pkg/logql/log/parser.go:141 sanitizeLabelKey on every extracted key; label filter reads the parsed label | TestCompat_ExistenceFilterSanitizedKey (test/e2e-compat, group semantics) | +| 43 | `quality/detected-metadata-proxy-cache` — detected_labels and detected_fields are served from proxy caches; Loki answers each request fresh (open) | open | drilldown-visible | — | — | S | internal/proxy/cache_keys.go:48,130 (detected_* cache TTL) + recent-tail freshness bypass | pkg/querier detected labels/fields: no results cache in v3.7.7 | TestE2E_DetectedMetadataFreshness (test/e2e-compat, group drilldown) | +| 44 | `quality/stats-volume-freshness` — index/stats and volume answers ending now are cached; Loki does not cache inside max_stats_cache_freshness (10m) (open) | open | drilldown-visible | — | — | S | internal/proxy/volume.go handleVolume, label_handlers.go handleIndexStats (cache freshness) | pkg/validation/limits.go:413-414 max_stats_cache_freshness default 10m | TestE2E_StatsVolumeFreshness (test/e2e-compat, group drilldown) | +| 45 | `semantics/parse-error-text-parity` — Invalid queries fail on both sides, with different error text or status (open) | open | grafana-datasource | 210 | 129 | L | internal/logql parser error messages (Loki's goyacc messages and positions) + VictoriaLogs error mapping | pkg/logql/syntax/parser.go ParseError (line, col) and syntax.y | TestCompat_ParseErrorText corpus (test/e2e-compat, bad_request_status_parity_test.go) | +| 46 | `semantics/proxy-rejects-valid-logql` — The proxy rejects LogQL Loki accepts: comments, negative and compound offsets, filters after a range, logfmt flags, ip() label filters (o... | open | api-only | 79 | 48 | M | internal/logql parser (lexer: comments, signed durations, flags) + translator | pkg/logql/syntax/lex.go:149 comments, :90-91 OpStrict/OpKeepEmpty; syntax.y offsetExpr; pkg/logql/log/ip.go | TestCompat_LogQLSurfaceAccepted (test/e2e-compat, group semantics), one subtest per family | +| 47 | `semantics/proxy-accepts-invalid-logql` — Queries Loki rejects with 400 are answered by the proxy (open) | open | api-only | 18 | 14 | S | internal/logql parser validation | pkg/logql/syntax/ast.go:1446,1579 grouping not allowed; duration parsing | TestCompat_InvalidLogQLRejected (test/e2e-compat, bad_request_status_parity_test.go) | +| 48 | `parsed-identity/regexp-partial-match-series-count` — A partially-matching \| regexp returns about 3x Loki's series count | open | api-only | 2 | 2 | M | internal/proxy regexp capture series identity (non-matching lines) | pkg/logql/log/parser.go RegexpParser (no capture label for a non-matching line) | TestCompat_RegexpPartialMatchSeries (test/e2e-compat, group semantics) | +| 49 | `profiles/json-extraction-list-missing-key` — A json extraction-list key absent from the line is dropped; Loki keeps it in the metric series (open) | open | api-only | 2 | 2 | S | internal/proxy stage-field exposure (extraction lists, metric path) | pkg/logql/log/parser.go:671-720 JSONExpressionParser.Process | TestCompat_JSONExtractionListMissingKey (test/e2e-compat, group semantics) | +| 50 | `semantics/unpack-non-string-values` — `\| unpack` extracts non-string JSON values that Loki ignores (open) | open | api-only | 2 | 2 | S | internal/translator unpack (string values only) | pkg/logql/log/parser.go:789-820 UnpackParser.unpack | TestCompat_UnpackStringValuesOnly (test/e2e-compat, group semantics) | +| 51 | `semantics/extracted-derived-labels-raw-evaluator` — service_name_extracted and detected_level_extracted metrics keep the raw-row evaluator (open) | open | edge | — | — | S | internal/proxy orderedJSONPushdownLabelOK (service_name, detected_level) | pkg/logql/log/parser.go duplicateSuffix; service_name set by discover_service_name at ingest | TestCompat_ExtractedLabelMetricPushdown rows for service_name_extracted (test/e2e-compat, group core) | +| 52 | `semantics/extracted-metric-known-differences` — Two name_extracted differences against Loki remain in the metric pushdown and the raw evaluator (open) | open | edge | — | — | M | internal/proxy raw evaluator parsed-label naming (processWithStreamLabels) and translator.ExtractedLabelPipes | pkg/logql/log/parser.go duplicateSuffix and ParserHint.Extracted (a key an earlier extraction produced is skipped) | TestCompat_ExtractedBothKeysInLineOrder (test/e2e-compat, group core) | +| 53 | `semantics/unwrap-conversion-error-parser-divergence` — Where VictoriaLogs' logfmt parser reads another value than Loki's, an unwrap misses Loki's SampleExtractionErr or drops samples Loki keep... | open | edge | — | — | L | internal/translator logfmt unpacking (unpack_logfmt keeps the last value); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:379-438 LogfmtParser.Process; pkg/logql/log/logfmt/decode.go | TestUnwrapConversionErrorDifferential counts the first case; TestUnwrapLokiLabels pins the decoder; TestCompat_UnwrapConversionErrorParit... | +| 54 | `semantics/unwrap-conversion-error-postfilter-forms` — An unwrap whose post filter keeps only the error, compares a label, or combines filters with and/or answers 200 where Loki may fail with ... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorPostFilters; the translator's post-filter placement | pkg/logql/log/label_filter.go:90-100 BinaryLabelFilter.Process, 306-340 NumericLabelFilter.Process, 418-424 labelValue | TestUnwrapErrorProbePlans and TestUnwrapConversionErrorDifferential (internal/proxy) assert the detection stays off for these forms; flip... | +| 55 | `semantics/unwrap-conversion-error-stored-value-overwritten` — An unwrap over a label stored as structured metadata and also parsed from the line misses Loki's SampleExtractionErr (open) | open | edge | — | — | M | internal/translator parsed-key collision with stored fields (unpack keep_original_fields); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:151,809 duplicateSuffix | TestUnwrapConversionErrorDifferential counts the first case; TestCompat_UnwrapConversionErrorParity asserts the 200 side | +| 56 | `semantics/unwrap-conversion-error-undetected-shapes` — A few query shapes answer 200 where Loki fails with SampleExtractionErr, because no VictoriaLogs query of theirs carries the conversion c... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorProbes (opaque expressions), unwrapCountingQuery (pipes after stats), unwrapLokiLabe... | pkg/logql/evaluator.go:726-733 | TestCompat_UnwrapConversionErrorParity open/* asserts Loki's 400 and skips naming this case (absent_over_time, label_replace); TestUnwrap... | +| 57 | `semantics/unwrap-gate-parsefloat-divergences` — The unwrap sample gate drops a few values Loki's conversions accept or reject (open) | open | edge | — | — | S | internal/translator/unwrap_gate.go (UnwrapNumberPattern, UnwrapDurationPattern, UnwrapBytesPattern) | pkg/logql/log/metrics_extraction.go:316-338 convertFloat, convertDuration, convertBytes | TestUnwrapNumberPatternMatchesParseFloat (internal/translator) pins the known differences | +| 58 | `versions/stats-bucket-label-v1.45` — Stats-bucket metrics put a line on a bucket edge one bucket late on VictoriaLogs before v1.45 (open) | open | edge | — | — | M | stats-bucket relabel (internal/proxy range_metric_compat.go setSlidingStatsRangeParams / slidingStatsBucket; sliding_window_stats.go) | pkg/logql range vector (T-range, T] | TestRangeMetricCompatibilityJSONFilterPushdown on VictoriaLogs v1.40-v1.44 (version matrix); fix by relabelling edge lines or declining t... | +| 59 | `metric-identity/structured-metadata-splits-series` — Loki splits a stream's metric series per structured-metadata set; the proxy keeps the stream (owner-accepted deviation) | owner-kept | explore-visible | 173 | 37 | L | internal/translator series identity (stats by stream); VictoriaLogs cannot group by a row's remaining fields | pkg/logql/log/metrics_extraction.go:102-120 builder.Add(StructuredMetadataLabel) -> GroupedLabels | differential run cluster count (bench/parity) | +| 60 | `parsed-identity/dynamic-key-parsers` — `\| json` and `\| logfmt` metric series keep the stream identity by default (owner-accepted deviation, opt-in flag) | owner-kept | explore-visible | 95 | 29 | L | internal/translator parser series identity; -exact-parser-series-identity | pkg/logql/log/parser.go label extraction into the sample labels | differential run cluster count (bench/parity) | +| 61 | `profiles/label-filter-on-line-field` — A label filter on a line key no earlier stage exposes matches like Loki | owner-kept | explore-visible | — | — | M | internal/proxy label-filter refill (-label-filter-refill-max-pages); metric queries on keys no stage exposes | pkg/logql/log/label_filter.go (filter reads only labels the pipeline set) | TestCompat_LabelFilterOnLineField (exists, profile tests) | +| 62 | `profiles/detected-fields-no-extracted-suffix` — detected_fields lists service / service.name without Loki's _extracted suffix (owner decision) | owner-kept | drilldown-visible | — | — | S | internal/proxy/label_handlers.go handleDetectedFields | pkg/logql/log/parser.go:25 duplicateSuffix | bench/parity diff rule (documented facet, service fields only) | +| 63 | `profiles/json-line-stored-without-line` — A JSON line pushed without _msg is stored without its line (documented) | documented | explore-visible | — | — | L | client push configuration (disable_message_parsing=1); docs/KNOWN_ISSUES.md | pkg/distributor push keeps the line verbatim | cases/profiles/stage-field-exposure with disable_message_parsing pushes | +| 64 | `drilldown/detected-labels-sampled-cardinality` — detected_labels cardinality is sampled from the window's last minutes; Loki counts every stream its ingesters hold (documented) | documented | drilldown-visible | 20 | 2 | M | internal/proxy/patterns.go handleDetectedLabels | pkg/ingester/instance.go LabelsWithValues | bench/parity diff rule (documented facet) | +| 65 | `profiles/volume-detected-level-extension` — index/volume answers targetLabels=detected_level; Loki returns no volume for it (documented proxy extension) | documented | drilldown-visible | 17 | 2 | S | internal/proxy/volume.go | pkg/storage volume index (stream labels only) | differential run row volume by detected_level | +| 66 | `drilldown/patterns-mined-differently` — Patterns come from different miners on each side (documented) | documented | drilldown-visible | 15 | 3 | L | internal/proxy/patterns.go | pkg/pattern (ingester-side mining) | bench/visual history-dependent comparison (exists) | +| 67 | `profiles/volume-bytes-accounting` — Volume values count line bytes; Loki counts chunk bytes with structured metadata (documented) | documented | drilldown-visible | 4 | 3 | L | internal/proxy/volume.go; docs/KNOWN_ISSUES.md | pkg/storage volume index (chunk bytes incl. structured metadata) | bench/parity diff rule (documented facet) | +| 68 | `limits/drilldown-breakdown-keeps-busiest` — Over max_query_series a Drilldown breakdown keeps the busiest series; Loki keeps the first it meets (documented) | documented | drilldown-visible | 1 | 1 | S | internal/proxy series limit ranking | pkg/logql engine series limit | cases/limits/drilldown-partial-with-warning (exists) | +| 69 | `quality/index-stats-bytes-and-chunks` — index/stats bytes and chunks have no VictoriaLogs equivalent (documented) | documented | drilldown-visible | — | — | L | internal/proxy/label_handlers.go handleIndexStats; docs/KNOWN_ISSUES.md | pkg/storage index stats from chunk refs (bytes, chunks) | bench/parity diff rule (documented facet) | +| 70 | `profiles/grafana-stats-error-as-warning` — A failed stats query from Grafana answers 200 with warnings; Loki returns the error (documented) | documented | grafana-datasource | — | — | S | internal/proxy Grafana-sourced stats error handling | pkg/querier/queryrange error propagation (no warnings conversion) | TestCompat_GrafanaStatsErrorStatus (test/e2e-compat, fault injection) | +| 71 | `profiles/metadata-default-lookback` — Label endpoints without start/end default to a different lookback than Loki's 1h (documented) | documented | api-only | — | — | S | internal/config flag default -metadata-default-lookback | pkg/loghttp/params.go defaultSince = 1h | TestCompat_MetadataDefaultLookback (test/e2e-compat, group core) | +| 72 | `profiles/multi-tenant-volume-merge` — Multi-tenant volume returns per-tenant series with __tenant_id__; Loki sums the tenants' volumes (documented) | documented | api-only | — | — | S | internal/proxy/volume.go multi-tenant merge | pkg/querier/queryrange volume merge | TestCompat_MultiTenantVolume (test/e2e-compat, group tail-multitenancy) | +| 73 | `semantics/count-values-unsupported` — count_values() is rejected with 400 where Loki answers (documented) | documented | api-only | — | — | M | internal/translator + proxy-side post-aggregation of a stats matrix | pkg/logql/syntax OpTypeCountValues; vector evaluator | TestCompat_CountValues (test/e2e-compat, group semantics) | +| 74 | `semantics/eager-pipeline-validation` — Invalid pipeline stages fail with 400 regardless of data; Loki answers 200 empty over a range with no data (documented) | documented | edge | — | — | S | internal/translator validation | pkg/logql/log/fmt.go template build at pipeline construction | TestCompat_EagerValidationEmptyRange (test/e2e-compat, group semantics) | +| 75 | `semantics/ip-filter-precision` — ip() filters are approximated with regular expressions (documented) | documented | edge | — | — | M | internal/proxy/postprocess.go:84 ip filtering | pkg/logql/log/ip.go | TestCompat_IPFilterForms (test/e2e-compat, group semantics) | +| 76 | `drilldown/plugin-legend-sync-race` — Logs Drilldown crashes a breakdown page when a query answers before the panel module loads (upstream, not the proxy) | upstream | drilldown-visible | — | — | S | none (grafana-lokiexplore-app) | n/a (client race) | bench/visual capture waits for the panel module (already) | +| 77 | `quality/index-stats-loki-chunk-accounting` — Loki's index/stats counts chunks it holds twice; the proxy's counts equal Loki's own count_over_time (upstream) | upstream | drilldown-visible | — | — | S | none (Loki chunk index accounting) | pkg/storage stats from chunk refs (flushed and retained chunks both counted) | bench/parity cross-check (run.py loki_counts_itself) | ## Fixed, awaiting the next run @@ -108,6 +107,7 @@ Last run: 2026-10-05T20:16:15Z on `fcaccfcc`, window [1791220800, 1791224400]. | `semantics/parenthesized-log-range` — A parenthesized log expression under a range function evaluates like the plain form | #669 | 41 | 77 | | `semantics/line-filter-or-alternatives` — Line filters with `or` alternatives match like Loki | #669 | 6 | 42 | | `semantics/plain-name-after-parser-reads-stream-label` — A filter or grouping on a name a parser also reads reads the parsed value; Loki reads the stream label (fixed) | #676 | 4 | 6 | +| `semantics/label-replace-new-label` — label_replace adds its destination label to every series, instant and range (fixed) | #744 | 1 | 5 | | `semantics/json-extracted-metric-pushdown` — A metric filtered or grouped on a name_extracted label after `\| json` is answered from VictoriaLogs stats (fixed) | #690 | 0 | 0 | | `semantics/labels-end-inclusive-like-loki` — /labels and /label/{name}/values include data at the request's end, so a zero-width request (Grafana "Show context") lists the row's labe... | #703 | 0 | 0 | | `semantics/unwrap-conversion-error` — An unwrap over a value its conversion rejects fails like Loki, with a 400 SampleExtractionErr (fixed) | #704 | 0 | 0 | From a52da44ed048e8fd8f040f9feb06a312f7a1108f Mon Sep 17 00:00:00 2001 From: szibis Date: Mon, 12 Oct 2026 13:48:50 +0200 Subject: [PATCH 09/18] fix: label_replace sorts its result like Loki, routes every branch by the expression under it, and reports Loki's argument errors The series of a label_replace answer are ordered by label set after the transform (a range result, and an instant vector unless the query sorts by value). The request is cloned once, above the routing chain, when a label transform wraps the expression. A malformed call reports the token names and the string start column of Loki's parser. label_join is no longer claimed as a feature of the transform. Ranking over duplicate label sets that a label_replace produced is registered as an open row. --- ...bel-replace-duplicates-before-ranking.yaml | 42 +++++++ .../semantics/label-replace-new-label.yaml | 17 +-- conformance/registry/generated/wiring.json | 8 +- conformance/reports/parity-gaps.md | 105 +++++++++--------- internal/logql/semantic.go | 10 +- .../proxy/label_replace_semantics_test.go | 98 +++++++++++----- internal/proxy/proxy.go | 18 ++- internal/proxy/vector_matching.go | 74 +++++++++++- .../label_replace_new_label_test.go | 44 ++++++++ 9 files changed, 320 insertions(+), 96 deletions(-) create mode 100644 conformance/registry/cases/semantics/label-replace-duplicates-before-ranking.yaml diff --git a/conformance/registry/cases/semantics/label-replace-duplicates-before-ranking.yaml b/conformance/registry/cases/semantics/label-replace-duplicates-before-ranking.yaml new file mode 100644 index 000000000..24131593d --- /dev/null +++ b/conformance/registry/cases/semantics/label-replace-duplicates-before-ranking.yaml @@ -0,0 +1,42 @@ +id: semantics/label-replace-duplicates-before-ranking +title: topk, bottomk and stddev over label sets a label_replace made equal rank merged series on query_range (open) +proves: + - loki_api_v1_query_range +consumer: explore +why: | + When a label_replace makes two series share a label set, Loki keeps both + samples at each step until the query ends: the vector aggregation above it + (topk, bottomk, stddev, stdvar) ranks or folds every sample, and only + JoinSampleVector files the survivors under one series (pkg/logql/evaluator.go + VectorAggEvaluator.Next, pkg/logql/engine.go vectorsToSeries). The proxy + transforms the series VictoriaLogs answered and merges the duplicates into one + series, with the points of each step in turn, before topk, bottomk or stddev + see them, so a step holds two points under one series and the ranking works on + that merged series (internal/proxy handleRangeMetricPostAggregation, + transformSeriesLabels). On an instant query the samples stay apart, so the + instant answer ranks as Loki does. +request: + method: GET + path: /loki/api/v1/query_range + params: + query: 'topk(1, label_replace(sum by (app) (count_over_time({env="production"}[5m])), "app", "all", "app", ".+"))' + start: 'now-1h' + end: 'now' + step: '60' + headers: + X-Scope-OrgID: "0" +compare: + mode: observed + fields: [series, values] + expect: | + One series per step holding the larger of the two samples, as Loki ranks + them before it files them. +gap: + status: open + impact: explore-visible + effort: S + area: 'internal/proxy metric_agg.go (handleRangeMetricPostAggregation) after transformSeriesLabels' + loki: 'pkg/logql/evaluator.go VectorAggEvaluator.Next; pkg/logql/engine.go:440-470 vectorsToSeries' + proxy: 'duplicates merged into one series before the ranking' + planned_test: 'TestCompat_LabelReplaceDuplicatesBeforeRanking (test/e2e-compat, group semantics)' + clusters: [] diff --git a/conformance/registry/cases/semantics/label-replace-new-label.yaml b/conformance/registry/cases/semantics/label-replace-new-label.yaml index f9cc63d5f..44f2aaf97 100644 --- a/conformance/registry/cases/semantics/label-replace-new-label.yaml +++ b/conformance/registry/cases/semantics/label-replace-new-label.yaml @@ -23,14 +23,15 @@ why: | name and raises no error for two series that end with one label set; pkg/ logql/engine.go JoinSampleVector then keeps both samples of an instant vector and files the points of a range query's duplicates under one series, those of - each step in turn. label_join is no Loki v3.7.7 function (the lexer has - label_replace only); the proxy keeps accepting it. + each step in turn; both then sort by label set (sort.Sort(result) for a + range, sort.Slice with labels.Compare for an instant vector unless the query + sorts by value). Causes, reproduced against a simulated VictoriaLogs (internal/proxy/label_replace_semantics_test.go): - - The proxy applied the transform to a range matrix only (applyLabelReplace, - applyLabelJoin decoded `resultType: matrix`), so an instant vector was - returned untouched. It also re-encoded only the series, dropping the rest + - The proxy applied the transform to a range matrix only (applyLabelReplace + decoded `resultType: matrix` only), so an instant vector was returned + untouched. It also re-encoded only the series, dropping the rest of the answer, and it expanded the replacement with ReplaceAllString. - The translator took the string arguments of label_replace raw: `"\\."` was the regexp `\\.` (a backslash and any character), never matching, and an @@ -49,14 +50,16 @@ why: | - A call without Loki's four string arguments, or with an invalid escape in a string, was taken for a log query (200 with no streams on query_range, a wrong 400 on query); it is now Loki's 400 parse error (the position and - wording of the message follow the proxy's other grammar errors). + wording of the message are Loki's: checked against its parser). Not changed (separate shapes, to register apart): an aggregation over a label_replace (`sum by (app_short) (label_replace(...))`) is evaluated by VictoriaLogs on a label it does not have; a label_replace around topk, sort, group or vector answers empty on query_range and 400 on query; a label_replace as an operand of a binary expression drops the operation; - label_join, which Loki v3.7.7 does not have, is accepted. + topk, bottomk and stddev on query_range merge the duplicate label sets a + label_replace produced before ranking, where Loki ranks the samples first + (semantics/label-replace-duplicates-before-ranking). request: method: GET path: /loki/api/v1/query diff --git a/conformance/registry/generated/wiring.json b/conformance/registry/generated/wiring.json index 3fb801208..5483dc8a6 100644 --- a/conformance/registry/generated/wiring.json +++ b/conformance/registry/generated/wiring.json @@ -425,6 +425,9 @@ "internal/proxy/label_values_cap_inventory_test.go", "internal/proxy/metadata_inventory_test.go" ], + "semantics/label-replace-duplicates-before-ranking": [ + "test/e2e-compat/label_replace_new_label_test.go" + ], "semantics/label-replace-new-label": [ "internal/proxy/label_replace_semantics_test.go", "test/e2e-compat/label_replace_new_label_test.go" @@ -555,7 +558,7 @@ "test/e2e-compat/unwrap_ungrouped_sum_test.go" ] }, - "registry_items": 300, + "registry_items": 301, "routes_without_registry_item": [], "test_to_items": { "cmd/proxy/main_test.go": [ @@ -985,6 +988,7 @@ "test/e2e-compat/label_replace_new_label_test.go": [ "loki_api_v1_query", "loki_api_v1_query_range", + "semantics/label-replace-duplicates-before-ranking", "semantics/label-replace-new-label" ], "test/e2e-compat/labels_zero_width_test.go": [ @@ -1203,5 +1207,5 @@ "versions/capabilities-floor-vs-matrix", "versions/rejected-stats-query-wording-vl-1.47" ], - "wired_items": 108 + "wired_items": 109 } diff --git a/conformance/reports/parity-gaps.md b/conformance/reports/parity-gaps.md index e5add94b3..ea597c0a0 100644 --- a/conformance/reports/parity-gaps.md +++ b/conformance/reports/parity-gaps.md @@ -44,58 +44,59 @@ Last run: 2026-10-05T20:16:15Z on `fcaccfcc`, window [1791220800, 1791224400]. | 23 | `quality/labels-backfill-into-cached-bucket` — Rows backfilled into a past bucket the label inventory already cached with data appear in /labels only after its revalidation delay (open) | open | explore-visible | — | — | M | internal/proxy/metadata_inventory.go (bucket revalidation delay) | pkg/querier/querier.go Label (index read per request) | TestCompat_LabelsBackfillIntoCachedBucket (test/e2e-compat, group core) | | 24 | `semantics/drop-keep-plain-name-after-parser` — A log query's drop, keep or label_format source of a stream label's plain name after a parser acts on the stored field; Loki acts on the ... | open | explore-visible | — | — | S | internal/proxy stage-field exposure (drop and keep of a name a parser also reads) | pkg/logql/log/labels.go LabelsBuilder.Del / getWithCategory; pkg/logql/log/drop_labels.go, keep_labels.go | TestCompat_DropKeepPlainNameAfterParser (test/e2e-compat, group core) | | 25 | `semantics/extracted-ungrouped-metric-raw-rows` — An ungrouped range metric over a `\| json` pipeline filtered on name_extracted still reads raw rows and fails at the row limit (open) | open | explore-visible | — | — | L | internal/proxy bare-parser metric evaluator (fetchBareParserMetricSeries) and the ordered JSON pushdown (without grouping) | pkg/logql range aggregation keeps every label of the sample | TestCompat_ExtractedUngroupedMetricAtSixHours (test/e2e-compat, group core) | -| 26 | `semantics/repeated-label-filter-replaced-across-label-format` — A second filter on the same label replaces the first even with a label_format between them (open) | open | explore-visible | — | — | S | internal/translator labelFilterLatest (latest filter per field replaces the earlier one in place) | pkg/logql/log/pipeline.go (stages run in order); label_filter.go | TestCompat_RepeatedLabelFilterAcrossLabelFormat (test/e2e-compat, group core) | -| 27 | `semantics/two-parsers-keep-one` — A query with `\| json` and `\| logfmt` runs the one parser the data answers to (open) | open | explore-visible | — | — | M | internal/proxy/query_translation.go preferWorkingParser (probe-based removal of a parser stage) | pkg/logql/log/pipeline.go: every stage runs on every line; parser.go skips a key ParserHint.Extracted | TestCompat_TwoParsersOnMixedFormats (test/e2e-compat, group semantics) | -| 28 | `semantics/unwrap-duration-seconds` — '`unwrap duration_seconds(label)` answers no data where Loki converts it like duration() (open)' | open | explore-visible | — | — | S | internal/translator extractUnwrapField and unwrapGateField (duration_seconds conversion) | pkg/logql/syntax/extractor.go:63-71 | TestCompat_UnwrapDurationSeconds (test/e2e-compat, group core) | -| 29 | `semantics/unwrap-first-last-under-aggregation` — first_over_time and last_over_time with unwrap under sum fail with a VictoriaLogs 422 (open) | open | explore-visible | — | — | M | internal/translator first_over_time / last_over_time under a vector aggregation | pkg/logql/range_vector.go first/last | an e2e over sum(first_over_time\|last_over_time(... \| unwrap ...)) | -| 30 | `semantics/unwrap-over-time-range-windows` — sum, avg, max and min over time with unwrap answer the wrong window on query_range (open) | open | explore-visible | — | — | M | internal/proxy range_metric_compat.go shouldUseManualRangeMetricCompat (unwrap sum/avg/min/max) and metric_binary.go statsRateRangeEquals... | pkg/logql/range_vector.go:102-189 batchRangeVectorIterator (window (t-range, t]); range_vector.go:371-420 sumOverTime, avgOverTime, maxOv... | TestCompat_UnwrapOverTimeRangeWindows (test/e2e-compat, group core) over sum/avg/max/min x tumbling/sliding x logfmt/json | -| 31 | `semantics/unwrap-per-series-identity-raw-rows` — A function taken per series (avg, max, min over time, rate_counter) under sum or sum by uses the stream as the series (open) | open | explore-visible | — | — | L | internal/translator unwrapInnerGrouping (sum by over a per-series function) and the raw evaluators series key (collectRangeMetricSamples) | pkg/logql/evaluator.go:435-529 VectorAggEvaluator.Next; pkg/logql/log/metrics_extraction.go:150-178 labelSampleExtractor (series = stream... | TestCompat_UnwrapPerSeriesIdentity (test/e2e-compat, group core) over sum and sum by x avg/max/min/rate_counter, logfmt and json | -| 32 | `semantics/unwrap-quantile-response-limit` — quantile_over_time over a JSON unwrap fails at the manual metric response limit from 6 hours (open) | open | explore-visible | — | — | L | internal/proxy raw-row quantile evaluator (manual metric response cap) | pkg/logql/range_vector.go quantile_over_time | bench/ab set unwrap-conversion-error E2 at 6h/24h/7d | -| 33 | `semantics/unwrap-range-last-point` — A grouped unwrap range metric misses the sample at `end` (open) | open | explore-visible | — | — | S | internal/proxy range-metric unwrap path (end bucket) | pkg/logql/range_vector.go step iteration start..end inclusive | TestCompat_UnwrapRangeEndSample (test/e2e-compat, group semantics) | -| 34 | `semantics/unwrap-sum-without-collapses` — sum without over an unwrap range function returns a single series (open) | open | explore-visible | — | — | M | internal/translator `without` grouping over unwrap range functions | pkg/logql/evaluator.go newVectorAggEvaluator (without) | an e2e over sum without (...) with unwrap on query_range and query | -| 35 | `drilldown/detected-field-values` — detected_field values differ: none for service_name, other values for numeric and id fields (open) | open | drilldown-visible | 33 | 3 | M | internal/proxy detected_field values (sampling window and stream-label fields) | pkg/querier detected field values (v3.7.7): values of the sampled lines | TestCompat_DetectedFieldValues (test/e2e-compat, group drilldown) | -| 36 | `semantics/detected-fields-extracted-suffix` — detected_fields omits Loki's level_extracted and version_extracted (open; service fields are owner-kept) | open | drilldown-visible | 15 | 2 | M | internal/proxy/label_handlers.go handleDetectedFields (parsed fields named like a stream label) | pkg/logql/log/parser.go:25 duplicateSuffix; detected fields come from the parsed labels of sampled entries | TestCompat_DetectedFieldsExtractedSuffix (test/e2e-compat, group drilldown) | -| 37 | `profiles/json-number-label-text` — JSON numbers become label values in a different text form ("14.0" vs "14") (open) | open | drilldown-visible | 12 | 6 | M | internal/proxy JSON number rendering of unpacked values (VictoriaLogs unpack_json normalises) | pkg/logql/log/parser.go readValue (raw number bytes) | TestCompat_JSONNumberLabelText (test/e2e-compat, group drilldown) | -| 38 | `severity/detected-level-filter-json-level` — `\| detected_level=` filters select different lines than Loki (JSON-level lines, stream level precedence) (open) | open | drilldown-visible | 12 | 8 | M | internal/translator/detected_level_logsql.go (filter predicate) vs internal/proxy/detected_level.go (response derivation) | pkg/distributor field detection (v3.7.7) stores detected_level as structured metadata; filter reads it | TestCompat_DetectedLevelFilterJSONLevel (test/e2e-compat, group drilldown) | -| 39 | `limits/drilldown-7d-breakdown-response-limit` — Metric queries fail on the 64 MiB buffered-response budget where Loki answers (open) | open | drilldown-visible | 8 | 4 | M | internal/proxy/range_metric_compat.go:767, execution_limits.go:11 (raw-row metric paths buffer rows; push down or stream) | pkg/logql engine series limit (max_query_series) with partial result for Drilldown | TestCompat_MetricBufferedBudget1h + TestE2E_Drilldown7dHighCardinalityBreakdown (test/e2e-compat, groups semantics and drilldown) | -| 40 | `quality/index-stats-streams-differ` — index/stats stream counts differ from Loki in both directions (open) | open | drilldown-visible | 6 | 2 | S | internal/proxy/label_handlers.go handleIndexStats (stream counting per window) | pkg/storage index stats (streams from the index for the window) | bench/parity rerun with the /series cross-check, then TestCompat_IndexStatsStreams (test/e2e-compat, group drilldown) | -| 41 | `drilldown/detected-fields-type-inference` — detected_fields infers a different type for some fields (open) | open | drilldown-visible | — | — | S | internal/proxy detected_fields type inference | pkg/querier detected fields type detection (v3.7.7): int before duration before string | TestCompat_DetectedFieldsTypes (test/e2e-compat, group drilldown) | -| 42 | `profiles/existence-filter-on-sanitized-key` — An existence filter on a sanitized key reads the stored dotted field, not the JSON line key (open) | open | drilldown-visible | — | — | M | internal/translator (label-filter key variants, parsed_key_variants.go); Drilldown single-field fast paths | pkg/logql/log/parser.go:141 sanitizeLabelKey on every extracted key; label filter reads the parsed label | TestCompat_ExistenceFilterSanitizedKey (test/e2e-compat, group semantics) | -| 43 | `quality/detected-metadata-proxy-cache` — detected_labels and detected_fields are served from proxy caches; Loki answers each request fresh (open) | open | drilldown-visible | — | — | S | internal/proxy/cache_keys.go:48,130 (detected_* cache TTL) + recent-tail freshness bypass | pkg/querier detected labels/fields: no results cache in v3.7.7 | TestE2E_DetectedMetadataFreshness (test/e2e-compat, group drilldown) | -| 44 | `quality/stats-volume-freshness` — index/stats and volume answers ending now are cached; Loki does not cache inside max_stats_cache_freshness (10m) (open) | open | drilldown-visible | — | — | S | internal/proxy/volume.go handleVolume, label_handlers.go handleIndexStats (cache freshness) | pkg/validation/limits.go:413-414 max_stats_cache_freshness default 10m | TestE2E_StatsVolumeFreshness (test/e2e-compat, group drilldown) | -| 45 | `semantics/parse-error-text-parity` — Invalid queries fail on both sides, with different error text or status (open) | open | grafana-datasource | 210 | 129 | L | internal/logql parser error messages (Loki's goyacc messages and positions) + VictoriaLogs error mapping | pkg/logql/syntax/parser.go ParseError (line, col) and syntax.y | TestCompat_ParseErrorText corpus (test/e2e-compat, bad_request_status_parity_test.go) | -| 46 | `semantics/proxy-rejects-valid-logql` — The proxy rejects LogQL Loki accepts: comments, negative and compound offsets, filters after a range, logfmt flags, ip() label filters (o... | open | api-only | 79 | 48 | M | internal/logql parser (lexer: comments, signed durations, flags) + translator | pkg/logql/syntax/lex.go:149 comments, :90-91 OpStrict/OpKeepEmpty; syntax.y offsetExpr; pkg/logql/log/ip.go | TestCompat_LogQLSurfaceAccepted (test/e2e-compat, group semantics), one subtest per family | -| 47 | `semantics/proxy-accepts-invalid-logql` — Queries Loki rejects with 400 are answered by the proxy (open) | open | api-only | 18 | 14 | S | internal/logql parser validation | pkg/logql/syntax/ast.go:1446,1579 grouping not allowed; duration parsing | TestCompat_InvalidLogQLRejected (test/e2e-compat, bad_request_status_parity_test.go) | -| 48 | `parsed-identity/regexp-partial-match-series-count` — A partially-matching \| regexp returns about 3x Loki's series count | open | api-only | 2 | 2 | M | internal/proxy regexp capture series identity (non-matching lines) | pkg/logql/log/parser.go RegexpParser (no capture label for a non-matching line) | TestCompat_RegexpPartialMatchSeries (test/e2e-compat, group semantics) | -| 49 | `profiles/json-extraction-list-missing-key` — A json extraction-list key absent from the line is dropped; Loki keeps it in the metric series (open) | open | api-only | 2 | 2 | S | internal/proxy stage-field exposure (extraction lists, metric path) | pkg/logql/log/parser.go:671-720 JSONExpressionParser.Process | TestCompat_JSONExtractionListMissingKey (test/e2e-compat, group semantics) | -| 50 | `semantics/unpack-non-string-values` — `\| unpack` extracts non-string JSON values that Loki ignores (open) | open | api-only | 2 | 2 | S | internal/translator unpack (string values only) | pkg/logql/log/parser.go:789-820 UnpackParser.unpack | TestCompat_UnpackStringValuesOnly (test/e2e-compat, group semantics) | -| 51 | `semantics/extracted-derived-labels-raw-evaluator` — service_name_extracted and detected_level_extracted metrics keep the raw-row evaluator (open) | open | edge | — | — | S | internal/proxy orderedJSONPushdownLabelOK (service_name, detected_level) | pkg/logql/log/parser.go duplicateSuffix; service_name set by discover_service_name at ingest | TestCompat_ExtractedLabelMetricPushdown rows for service_name_extracted (test/e2e-compat, group core) | -| 52 | `semantics/extracted-metric-known-differences` — Two name_extracted differences against Loki remain in the metric pushdown and the raw evaluator (open) | open | edge | — | — | M | internal/proxy raw evaluator parsed-label naming (processWithStreamLabels) and translator.ExtractedLabelPipes | pkg/logql/log/parser.go duplicateSuffix and ParserHint.Extracted (a key an earlier extraction produced is skipped) | TestCompat_ExtractedBothKeysInLineOrder (test/e2e-compat, group core) | -| 53 | `semantics/unwrap-conversion-error-parser-divergence` — Where VictoriaLogs' logfmt parser reads another value than Loki's, an unwrap misses Loki's SampleExtractionErr or drops samples Loki keep... | open | edge | — | — | L | internal/translator logfmt unpacking (unpack_logfmt keeps the last value); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:379-438 LogfmtParser.Process; pkg/logql/log/logfmt/decode.go | TestUnwrapConversionErrorDifferential counts the first case; TestUnwrapLokiLabels pins the decoder; TestCompat_UnwrapConversionErrorParit... | -| 54 | `semantics/unwrap-conversion-error-postfilter-forms` — An unwrap whose post filter keeps only the error, compares a label, or combines filters with and/or answers 200 where Loki may fail with ... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorPostFilters; the translator's post-filter placement | pkg/logql/log/label_filter.go:90-100 BinaryLabelFilter.Process, 306-340 NumericLabelFilter.Process, 418-424 labelValue | TestUnwrapErrorProbePlans and TestUnwrapConversionErrorDifferential (internal/proxy) assert the detection stays off for these forms; flip... | -| 55 | `semantics/unwrap-conversion-error-stored-value-overwritten` — An unwrap over a label stored as structured metadata and also parsed from the line misses Loki's SampleExtractionErr (open) | open | edge | — | — | M | internal/translator parsed-key collision with stored fields (unpack keep_original_fields); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:151,809 duplicateSuffix | TestUnwrapConversionErrorDifferential counts the first case; TestCompat_UnwrapConversionErrorParity asserts the 200 side | -| 56 | `semantics/unwrap-conversion-error-undetected-shapes` — A few query shapes answer 200 where Loki fails with SampleExtractionErr, because no VictoriaLogs query of theirs carries the conversion c... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorProbes (opaque expressions), unwrapCountingQuery (pipes after stats), unwrapLokiLabe... | pkg/logql/evaluator.go:726-733 | TestCompat_UnwrapConversionErrorParity open/* asserts Loki's 400 and skips naming this case (absent_over_time, label_replace); TestUnwrap... | -| 57 | `semantics/unwrap-gate-parsefloat-divergences` — The unwrap sample gate drops a few values Loki's conversions accept or reject (open) | open | edge | — | — | S | internal/translator/unwrap_gate.go (UnwrapNumberPattern, UnwrapDurationPattern, UnwrapBytesPattern) | pkg/logql/log/metrics_extraction.go:316-338 convertFloat, convertDuration, convertBytes | TestUnwrapNumberPatternMatchesParseFloat (internal/translator) pins the known differences | -| 58 | `versions/stats-bucket-label-v1.45` — Stats-bucket metrics put a line on a bucket edge one bucket late on VictoriaLogs before v1.45 (open) | open | edge | — | — | M | stats-bucket relabel (internal/proxy range_metric_compat.go setSlidingStatsRangeParams / slidingStatsBucket; sliding_window_stats.go) | pkg/logql range vector (T-range, T] | TestRangeMetricCompatibilityJSONFilterPushdown on VictoriaLogs v1.40-v1.44 (version matrix); fix by relabelling edge lines or declining t... | -| 59 | `metric-identity/structured-metadata-splits-series` — Loki splits a stream's metric series per structured-metadata set; the proxy keeps the stream (owner-accepted deviation) | owner-kept | explore-visible | 173 | 37 | L | internal/translator series identity (stats by stream); VictoriaLogs cannot group by a row's remaining fields | pkg/logql/log/metrics_extraction.go:102-120 builder.Add(StructuredMetadataLabel) -> GroupedLabels | differential run cluster count (bench/parity) | -| 60 | `parsed-identity/dynamic-key-parsers` — `\| json` and `\| logfmt` metric series keep the stream identity by default (owner-accepted deviation, opt-in flag) | owner-kept | explore-visible | 95 | 29 | L | internal/translator parser series identity; -exact-parser-series-identity | pkg/logql/log/parser.go label extraction into the sample labels | differential run cluster count (bench/parity) | -| 61 | `profiles/label-filter-on-line-field` — A label filter on a line key no earlier stage exposes matches like Loki | owner-kept | explore-visible | — | — | M | internal/proxy label-filter refill (-label-filter-refill-max-pages); metric queries on keys no stage exposes | pkg/logql/log/label_filter.go (filter reads only labels the pipeline set) | TestCompat_LabelFilterOnLineField (exists, profile tests) | -| 62 | `profiles/detected-fields-no-extracted-suffix` — detected_fields lists service / service.name without Loki's _extracted suffix (owner decision) | owner-kept | drilldown-visible | — | — | S | internal/proxy/label_handlers.go handleDetectedFields | pkg/logql/log/parser.go:25 duplicateSuffix | bench/parity diff rule (documented facet, service fields only) | -| 63 | `profiles/json-line-stored-without-line` — A JSON line pushed without _msg is stored without its line (documented) | documented | explore-visible | — | — | L | client push configuration (disable_message_parsing=1); docs/KNOWN_ISSUES.md | pkg/distributor push keeps the line verbatim | cases/profiles/stage-field-exposure with disable_message_parsing pushes | -| 64 | `drilldown/detected-labels-sampled-cardinality` — detected_labels cardinality is sampled from the window's last minutes; Loki counts every stream its ingesters hold (documented) | documented | drilldown-visible | 20 | 2 | M | internal/proxy/patterns.go handleDetectedLabels | pkg/ingester/instance.go LabelsWithValues | bench/parity diff rule (documented facet) | -| 65 | `profiles/volume-detected-level-extension` — index/volume answers targetLabels=detected_level; Loki returns no volume for it (documented proxy extension) | documented | drilldown-visible | 17 | 2 | S | internal/proxy/volume.go | pkg/storage volume index (stream labels only) | differential run row volume by detected_level | -| 66 | `drilldown/patterns-mined-differently` — Patterns come from different miners on each side (documented) | documented | drilldown-visible | 15 | 3 | L | internal/proxy/patterns.go | pkg/pattern (ingester-side mining) | bench/visual history-dependent comparison (exists) | -| 67 | `profiles/volume-bytes-accounting` — Volume values count line bytes; Loki counts chunk bytes with structured metadata (documented) | documented | drilldown-visible | 4 | 3 | L | internal/proxy/volume.go; docs/KNOWN_ISSUES.md | pkg/storage volume index (chunk bytes incl. structured metadata) | bench/parity diff rule (documented facet) | -| 68 | `limits/drilldown-breakdown-keeps-busiest` — Over max_query_series a Drilldown breakdown keeps the busiest series; Loki keeps the first it meets (documented) | documented | drilldown-visible | 1 | 1 | S | internal/proxy series limit ranking | pkg/logql engine series limit | cases/limits/drilldown-partial-with-warning (exists) | -| 69 | `quality/index-stats-bytes-and-chunks` — index/stats bytes and chunks have no VictoriaLogs equivalent (documented) | documented | drilldown-visible | — | — | L | internal/proxy/label_handlers.go handleIndexStats; docs/KNOWN_ISSUES.md | pkg/storage index stats from chunk refs (bytes, chunks) | bench/parity diff rule (documented facet) | -| 70 | `profiles/grafana-stats-error-as-warning` — A failed stats query from Grafana answers 200 with warnings; Loki returns the error (documented) | documented | grafana-datasource | — | — | S | internal/proxy Grafana-sourced stats error handling | pkg/querier/queryrange error propagation (no warnings conversion) | TestCompat_GrafanaStatsErrorStatus (test/e2e-compat, fault injection) | -| 71 | `profiles/metadata-default-lookback` — Label endpoints without start/end default to a different lookback than Loki's 1h (documented) | documented | api-only | — | — | S | internal/config flag default -metadata-default-lookback | pkg/loghttp/params.go defaultSince = 1h | TestCompat_MetadataDefaultLookback (test/e2e-compat, group core) | -| 72 | `profiles/multi-tenant-volume-merge` — Multi-tenant volume returns per-tenant series with __tenant_id__; Loki sums the tenants' volumes (documented) | documented | api-only | — | — | S | internal/proxy/volume.go multi-tenant merge | pkg/querier/queryrange volume merge | TestCompat_MultiTenantVolume (test/e2e-compat, group tail-multitenancy) | -| 73 | `semantics/count-values-unsupported` — count_values() is rejected with 400 where Loki answers (documented) | documented | api-only | — | — | M | internal/translator + proxy-side post-aggregation of a stats matrix | pkg/logql/syntax OpTypeCountValues; vector evaluator | TestCompat_CountValues (test/e2e-compat, group semantics) | -| 74 | `semantics/eager-pipeline-validation` — Invalid pipeline stages fail with 400 regardless of data; Loki answers 200 empty over a range with no data (documented) | documented | edge | — | — | S | internal/translator validation | pkg/logql/log/fmt.go template build at pipeline construction | TestCompat_EagerValidationEmptyRange (test/e2e-compat, group semantics) | -| 75 | `semantics/ip-filter-precision` — ip() filters are approximated with regular expressions (documented) | documented | edge | — | — | M | internal/proxy/postprocess.go:84 ip filtering | pkg/logql/log/ip.go | TestCompat_IPFilterForms (test/e2e-compat, group semantics) | -| 76 | `drilldown/plugin-legend-sync-race` — Logs Drilldown crashes a breakdown page when a query answers before the panel module loads (upstream, not the proxy) | upstream | drilldown-visible | — | — | S | none (grafana-lokiexplore-app) | n/a (client race) | bench/visual capture waits for the panel module (already) | -| 77 | `quality/index-stats-loki-chunk-accounting` — Loki's index/stats counts chunks it holds twice; the proxy's counts equal Loki's own count_over_time (upstream) | upstream | drilldown-visible | — | — | S | none (Loki chunk index accounting) | pkg/storage stats from chunk refs (flushed and retained chunks both counted) | bench/parity cross-check (run.py loki_counts_itself) | +| 26 | `semantics/label-replace-duplicates-before-ranking` — topk, bottomk and stddev over label sets a label_replace made equal rank merged series on query_range (open) | open | explore-visible | — | — | S | internal/proxy metric_agg.go (handleRangeMetricPostAggregation) after transformSeriesLabels | pkg/logql/evaluator.go VectorAggEvaluator.Next; pkg/logql/engine.go:440-470 vectorsToSeries | TestCompat_LabelReplaceDuplicatesBeforeRanking (test/e2e-compat, group semantics) | +| 27 | `semantics/repeated-label-filter-replaced-across-label-format` — A second filter on the same label replaces the first even with a label_format between them (open) | open | explore-visible | — | — | S | internal/translator labelFilterLatest (latest filter per field replaces the earlier one in place) | pkg/logql/log/pipeline.go (stages run in order); label_filter.go | TestCompat_RepeatedLabelFilterAcrossLabelFormat (test/e2e-compat, group core) | +| 28 | `semantics/two-parsers-keep-one` — A query with `\| json` and `\| logfmt` runs the one parser the data answers to (open) | open | explore-visible | — | — | M | internal/proxy/query_translation.go preferWorkingParser (probe-based removal of a parser stage) | pkg/logql/log/pipeline.go: every stage runs on every line; parser.go skips a key ParserHint.Extracted | TestCompat_TwoParsersOnMixedFormats (test/e2e-compat, group semantics) | +| 29 | `semantics/unwrap-duration-seconds` — '`unwrap duration_seconds(label)` answers no data where Loki converts it like duration() (open)' | open | explore-visible | — | — | S | internal/translator extractUnwrapField and unwrapGateField (duration_seconds conversion) | pkg/logql/syntax/extractor.go:63-71 | TestCompat_UnwrapDurationSeconds (test/e2e-compat, group core) | +| 30 | `semantics/unwrap-first-last-under-aggregation` — first_over_time and last_over_time with unwrap under sum fail with a VictoriaLogs 422 (open) | open | explore-visible | — | — | M | internal/translator first_over_time / last_over_time under a vector aggregation | pkg/logql/range_vector.go first/last | an e2e over sum(first_over_time\|last_over_time(... \| unwrap ...)) | +| 31 | `semantics/unwrap-over-time-range-windows` — sum, avg, max and min over time with unwrap answer the wrong window on query_range (open) | open | explore-visible | — | — | M | internal/proxy range_metric_compat.go shouldUseManualRangeMetricCompat (unwrap sum/avg/min/max) and metric_binary.go statsRateRangeEquals... | pkg/logql/range_vector.go:102-189 batchRangeVectorIterator (window (t-range, t]); range_vector.go:371-420 sumOverTime, avgOverTime, maxOv... | TestCompat_UnwrapOverTimeRangeWindows (test/e2e-compat, group core) over sum/avg/max/min x tumbling/sliding x logfmt/json | +| 32 | `semantics/unwrap-per-series-identity-raw-rows` — A function taken per series (avg, max, min over time, rate_counter) under sum or sum by uses the stream as the series (open) | open | explore-visible | — | — | L | internal/translator unwrapInnerGrouping (sum by over a per-series function) and the raw evaluators series key (collectRangeMetricSamples) | pkg/logql/evaluator.go:435-529 VectorAggEvaluator.Next; pkg/logql/log/metrics_extraction.go:150-178 labelSampleExtractor (series = stream... | TestCompat_UnwrapPerSeriesIdentity (test/e2e-compat, group core) over sum and sum by x avg/max/min/rate_counter, logfmt and json | +| 33 | `semantics/unwrap-quantile-response-limit` — quantile_over_time over a JSON unwrap fails at the manual metric response limit from 6 hours (open) | open | explore-visible | — | — | L | internal/proxy raw-row quantile evaluator (manual metric response cap) | pkg/logql/range_vector.go quantile_over_time | bench/ab set unwrap-conversion-error E2 at 6h/24h/7d | +| 34 | `semantics/unwrap-range-last-point` — A grouped unwrap range metric misses the sample at `end` (open) | open | explore-visible | — | — | S | internal/proxy range-metric unwrap path (end bucket) | pkg/logql/range_vector.go step iteration start..end inclusive | TestCompat_UnwrapRangeEndSample (test/e2e-compat, group semantics) | +| 35 | `semantics/unwrap-sum-without-collapses` — sum without over an unwrap range function returns a single series (open) | open | explore-visible | — | — | M | internal/translator `without` grouping over unwrap range functions | pkg/logql/evaluator.go newVectorAggEvaluator (without) | an e2e over sum without (...) with unwrap on query_range and query | +| 36 | `drilldown/detected-field-values` — detected_field values differ: none for service_name, other values for numeric and id fields (open) | open | drilldown-visible | 33 | 3 | M | internal/proxy detected_field values (sampling window and stream-label fields) | pkg/querier detected field values (v3.7.7): values of the sampled lines | TestCompat_DetectedFieldValues (test/e2e-compat, group drilldown) | +| 37 | `semantics/detected-fields-extracted-suffix` — detected_fields omits Loki's level_extracted and version_extracted (open; service fields are owner-kept) | open | drilldown-visible | 15 | 2 | M | internal/proxy/label_handlers.go handleDetectedFields (parsed fields named like a stream label) | pkg/logql/log/parser.go:25 duplicateSuffix; detected fields come from the parsed labels of sampled entries | TestCompat_DetectedFieldsExtractedSuffix (test/e2e-compat, group drilldown) | +| 38 | `profiles/json-number-label-text` — JSON numbers become label values in a different text form ("14.0" vs "14") (open) | open | drilldown-visible | 12 | 6 | M | internal/proxy JSON number rendering of unpacked values (VictoriaLogs unpack_json normalises) | pkg/logql/log/parser.go readValue (raw number bytes) | TestCompat_JSONNumberLabelText (test/e2e-compat, group drilldown) | +| 39 | `severity/detected-level-filter-json-level` — `\| detected_level=` filters select different lines than Loki (JSON-level lines, stream level precedence) (open) | open | drilldown-visible | 12 | 8 | M | internal/translator/detected_level_logsql.go (filter predicate) vs internal/proxy/detected_level.go (response derivation) | pkg/distributor field detection (v3.7.7) stores detected_level as structured metadata; filter reads it | TestCompat_DetectedLevelFilterJSONLevel (test/e2e-compat, group drilldown) | +| 40 | `limits/drilldown-7d-breakdown-response-limit` — Metric queries fail on the 64 MiB buffered-response budget where Loki answers (open) | open | drilldown-visible | 8 | 4 | M | internal/proxy/range_metric_compat.go:767, execution_limits.go:11 (raw-row metric paths buffer rows; push down or stream) | pkg/logql engine series limit (max_query_series) with partial result for Drilldown | TestCompat_MetricBufferedBudget1h + TestE2E_Drilldown7dHighCardinalityBreakdown (test/e2e-compat, groups semantics and drilldown) | +| 41 | `quality/index-stats-streams-differ` — index/stats stream counts differ from Loki in both directions (open) | open | drilldown-visible | 6 | 2 | S | internal/proxy/label_handlers.go handleIndexStats (stream counting per window) | pkg/storage index stats (streams from the index for the window) | bench/parity rerun with the /series cross-check, then TestCompat_IndexStatsStreams (test/e2e-compat, group drilldown) | +| 42 | `drilldown/detected-fields-type-inference` — detected_fields infers a different type for some fields (open) | open | drilldown-visible | — | — | S | internal/proxy detected_fields type inference | pkg/querier detected fields type detection (v3.7.7): int before duration before string | TestCompat_DetectedFieldsTypes (test/e2e-compat, group drilldown) | +| 43 | `profiles/existence-filter-on-sanitized-key` — An existence filter on a sanitized key reads the stored dotted field, not the JSON line key (open) | open | drilldown-visible | — | — | M | internal/translator (label-filter key variants, parsed_key_variants.go); Drilldown single-field fast paths | pkg/logql/log/parser.go:141 sanitizeLabelKey on every extracted key; label filter reads the parsed label | TestCompat_ExistenceFilterSanitizedKey (test/e2e-compat, group semantics) | +| 44 | `quality/detected-metadata-proxy-cache` — detected_labels and detected_fields are served from proxy caches; Loki answers each request fresh (open) | open | drilldown-visible | — | — | S | internal/proxy/cache_keys.go:48,130 (detected_* cache TTL) + recent-tail freshness bypass | pkg/querier detected labels/fields: no results cache in v3.7.7 | TestE2E_DetectedMetadataFreshness (test/e2e-compat, group drilldown) | +| 45 | `quality/stats-volume-freshness` — index/stats and volume answers ending now are cached; Loki does not cache inside max_stats_cache_freshness (10m) (open) | open | drilldown-visible | — | — | S | internal/proxy/volume.go handleVolume, label_handlers.go handleIndexStats (cache freshness) | pkg/validation/limits.go:413-414 max_stats_cache_freshness default 10m | TestE2E_StatsVolumeFreshness (test/e2e-compat, group drilldown) | +| 46 | `semantics/parse-error-text-parity` — Invalid queries fail on both sides, with different error text or status (open) | open | grafana-datasource | 210 | 129 | L | internal/logql parser error messages (Loki's goyacc messages and positions) + VictoriaLogs error mapping | pkg/logql/syntax/parser.go ParseError (line, col) and syntax.y | TestCompat_ParseErrorText corpus (test/e2e-compat, bad_request_status_parity_test.go) | +| 47 | `semantics/proxy-rejects-valid-logql` — The proxy rejects LogQL Loki accepts: comments, negative and compound offsets, filters after a range, logfmt flags, ip() label filters (o... | open | api-only | 79 | 48 | M | internal/logql parser (lexer: comments, signed durations, flags) + translator | pkg/logql/syntax/lex.go:149 comments, :90-91 OpStrict/OpKeepEmpty; syntax.y offsetExpr; pkg/logql/log/ip.go | TestCompat_LogQLSurfaceAccepted (test/e2e-compat, group semantics), one subtest per family | +| 48 | `semantics/proxy-accepts-invalid-logql` — Queries Loki rejects with 400 are answered by the proxy (open) | open | api-only | 18 | 14 | S | internal/logql parser validation | pkg/logql/syntax/ast.go:1446,1579 grouping not allowed; duration parsing | TestCompat_InvalidLogQLRejected (test/e2e-compat, bad_request_status_parity_test.go) | +| 49 | `parsed-identity/regexp-partial-match-series-count` — A partially-matching \| regexp returns about 3x Loki's series count | open | api-only | 2 | 2 | M | internal/proxy regexp capture series identity (non-matching lines) | pkg/logql/log/parser.go RegexpParser (no capture label for a non-matching line) | TestCompat_RegexpPartialMatchSeries (test/e2e-compat, group semantics) | +| 50 | `profiles/json-extraction-list-missing-key` — A json extraction-list key absent from the line is dropped; Loki keeps it in the metric series (open) | open | api-only | 2 | 2 | S | internal/proxy stage-field exposure (extraction lists, metric path) | pkg/logql/log/parser.go:671-720 JSONExpressionParser.Process | TestCompat_JSONExtractionListMissingKey (test/e2e-compat, group semantics) | +| 51 | `semantics/unpack-non-string-values` — `\| unpack` extracts non-string JSON values that Loki ignores (open) | open | api-only | 2 | 2 | S | internal/translator unpack (string values only) | pkg/logql/log/parser.go:789-820 UnpackParser.unpack | TestCompat_UnpackStringValuesOnly (test/e2e-compat, group semantics) | +| 52 | `semantics/extracted-derived-labels-raw-evaluator` — service_name_extracted and detected_level_extracted metrics keep the raw-row evaluator (open) | open | edge | — | — | S | internal/proxy orderedJSONPushdownLabelOK (service_name, detected_level) | pkg/logql/log/parser.go duplicateSuffix; service_name set by discover_service_name at ingest | TestCompat_ExtractedLabelMetricPushdown rows for service_name_extracted (test/e2e-compat, group core) | +| 53 | `semantics/extracted-metric-known-differences` — Two name_extracted differences against Loki remain in the metric pushdown and the raw evaluator (open) | open | edge | — | — | M | internal/proxy raw evaluator parsed-label naming (processWithStreamLabels) and translator.ExtractedLabelPipes | pkg/logql/log/parser.go duplicateSuffix and ParserHint.Extracted (a key an earlier extraction produced is skipped) | TestCompat_ExtractedBothKeysInLineOrder (test/e2e-compat, group core) | +| 54 | `semantics/unwrap-conversion-error-parser-divergence` — Where VictoriaLogs' logfmt parser reads another value than Loki's, an unwrap misses Loki's SampleExtractionErr or drops samples Loki keep... | open | edge | — | — | L | internal/translator logfmt unpacking (unpack_logfmt keeps the last value); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:379-438 LogfmtParser.Process; pkg/logql/log/logfmt/decode.go | TestUnwrapConversionErrorDifferential counts the first case; TestUnwrapLokiLabels pins the decoder; TestCompat_UnwrapConversionErrorParit... | +| 55 | `semantics/unwrap-conversion-error-postfilter-forms` — An unwrap whose post filter keeps only the error, compares a label, or combines filters with and/or answers 200 where Loki may fail with ... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorPostFilters; the translator's post-filter placement | pkg/logql/log/label_filter.go:90-100 BinaryLabelFilter.Process, 306-340 NumericLabelFilter.Process, 418-424 labelValue | TestUnwrapErrorProbePlans and TestUnwrapConversionErrorDifferential (internal/proxy) assert the detection stays off for these forms; flip... | +| 56 | `semantics/unwrap-conversion-error-stored-value-overwritten` — An unwrap over a label stored as structured metadata and also parsed from the line misses Loki's SampleExtractionErr (open) | open | edge | — | — | M | internal/translator parsed-key collision with stored fields (unpack keep_original_fields); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:151,809 duplicateSuffix | TestUnwrapConversionErrorDifferential counts the first case; TestCompat_UnwrapConversionErrorParity asserts the 200 side | +| 57 | `semantics/unwrap-conversion-error-undetected-shapes` — A few query shapes answer 200 where Loki fails with SampleExtractionErr, because no VictoriaLogs query of theirs carries the conversion c... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorProbes (opaque expressions), unwrapCountingQuery (pipes after stats), unwrapLokiLabe... | pkg/logql/evaluator.go:726-733 | TestCompat_UnwrapConversionErrorParity open/* asserts Loki's 400 and skips naming this case (absent_over_time, label_replace); TestUnwrap... | +| 58 | `semantics/unwrap-gate-parsefloat-divergences` — The unwrap sample gate drops a few values Loki's conversions accept or reject (open) | open | edge | — | — | S | internal/translator/unwrap_gate.go (UnwrapNumberPattern, UnwrapDurationPattern, UnwrapBytesPattern) | pkg/logql/log/metrics_extraction.go:316-338 convertFloat, convertDuration, convertBytes | TestUnwrapNumberPatternMatchesParseFloat (internal/translator) pins the known differences | +| 59 | `versions/stats-bucket-label-v1.45` — Stats-bucket metrics put a line on a bucket edge one bucket late on VictoriaLogs before v1.45 (open) | open | edge | — | — | M | stats-bucket relabel (internal/proxy range_metric_compat.go setSlidingStatsRangeParams / slidingStatsBucket; sliding_window_stats.go) | pkg/logql range vector (T-range, T] | TestRangeMetricCompatibilityJSONFilterPushdown on VictoriaLogs v1.40-v1.44 (version matrix); fix by relabelling edge lines or declining t... | +| 60 | `metric-identity/structured-metadata-splits-series` — Loki splits a stream's metric series per structured-metadata set; the proxy keeps the stream (owner-accepted deviation) | owner-kept | explore-visible | 173 | 37 | L | internal/translator series identity (stats by stream); VictoriaLogs cannot group by a row's remaining fields | pkg/logql/log/metrics_extraction.go:102-120 builder.Add(StructuredMetadataLabel) -> GroupedLabels | differential run cluster count (bench/parity) | +| 61 | `parsed-identity/dynamic-key-parsers` — `\| json` and `\| logfmt` metric series keep the stream identity by default (owner-accepted deviation, opt-in flag) | owner-kept | explore-visible | 95 | 29 | L | internal/translator parser series identity; -exact-parser-series-identity | pkg/logql/log/parser.go label extraction into the sample labels | differential run cluster count (bench/parity) | +| 62 | `profiles/label-filter-on-line-field` — A label filter on a line key no earlier stage exposes matches like Loki | owner-kept | explore-visible | — | — | M | internal/proxy label-filter refill (-label-filter-refill-max-pages); metric queries on keys no stage exposes | pkg/logql/log/label_filter.go (filter reads only labels the pipeline set) | TestCompat_LabelFilterOnLineField (exists, profile tests) | +| 63 | `profiles/detected-fields-no-extracted-suffix` — detected_fields lists service / service.name without Loki's _extracted suffix (owner decision) | owner-kept | drilldown-visible | — | — | S | internal/proxy/label_handlers.go handleDetectedFields | pkg/logql/log/parser.go:25 duplicateSuffix | bench/parity diff rule (documented facet, service fields only) | +| 64 | `profiles/json-line-stored-without-line` — A JSON line pushed without _msg is stored without its line (documented) | documented | explore-visible | — | — | L | client push configuration (disable_message_parsing=1); docs/KNOWN_ISSUES.md | pkg/distributor push keeps the line verbatim | cases/profiles/stage-field-exposure with disable_message_parsing pushes | +| 65 | `drilldown/detected-labels-sampled-cardinality` — detected_labels cardinality is sampled from the window's last minutes; Loki counts every stream its ingesters hold (documented) | documented | drilldown-visible | 20 | 2 | M | internal/proxy/patterns.go handleDetectedLabels | pkg/ingester/instance.go LabelsWithValues | bench/parity diff rule (documented facet) | +| 66 | `profiles/volume-detected-level-extension` — index/volume answers targetLabels=detected_level; Loki returns no volume for it (documented proxy extension) | documented | drilldown-visible | 17 | 2 | S | internal/proxy/volume.go | pkg/storage volume index (stream labels only) | differential run row volume by detected_level | +| 67 | `drilldown/patterns-mined-differently` — Patterns come from different miners on each side (documented) | documented | drilldown-visible | 15 | 3 | L | internal/proxy/patterns.go | pkg/pattern (ingester-side mining) | bench/visual history-dependent comparison (exists) | +| 68 | `profiles/volume-bytes-accounting` — Volume values count line bytes; Loki counts chunk bytes with structured metadata (documented) | documented | drilldown-visible | 4 | 3 | L | internal/proxy/volume.go; docs/KNOWN_ISSUES.md | pkg/storage volume index (chunk bytes incl. structured metadata) | bench/parity diff rule (documented facet) | +| 69 | `limits/drilldown-breakdown-keeps-busiest` — Over max_query_series a Drilldown breakdown keeps the busiest series; Loki keeps the first it meets (documented) | documented | drilldown-visible | 1 | 1 | S | internal/proxy series limit ranking | pkg/logql engine series limit | cases/limits/drilldown-partial-with-warning (exists) | +| 70 | `quality/index-stats-bytes-and-chunks` — index/stats bytes and chunks have no VictoriaLogs equivalent (documented) | documented | drilldown-visible | — | — | L | internal/proxy/label_handlers.go handleIndexStats; docs/KNOWN_ISSUES.md | pkg/storage index stats from chunk refs (bytes, chunks) | bench/parity diff rule (documented facet) | +| 71 | `profiles/grafana-stats-error-as-warning` — A failed stats query from Grafana answers 200 with warnings; Loki returns the error (documented) | documented | grafana-datasource | — | — | S | internal/proxy Grafana-sourced stats error handling | pkg/querier/queryrange error propagation (no warnings conversion) | TestCompat_GrafanaStatsErrorStatus (test/e2e-compat, fault injection) | +| 72 | `profiles/metadata-default-lookback` — Label endpoints without start/end default to a different lookback than Loki's 1h (documented) | documented | api-only | — | — | S | internal/config flag default -metadata-default-lookback | pkg/loghttp/params.go defaultSince = 1h | TestCompat_MetadataDefaultLookback (test/e2e-compat, group core) | +| 73 | `profiles/multi-tenant-volume-merge` — Multi-tenant volume returns per-tenant series with __tenant_id__; Loki sums the tenants' volumes (documented) | documented | api-only | — | — | S | internal/proxy/volume.go multi-tenant merge | pkg/querier/queryrange volume merge | TestCompat_MultiTenantVolume (test/e2e-compat, group tail-multitenancy) | +| 74 | `semantics/count-values-unsupported` — count_values() is rejected with 400 where Loki answers (documented) | documented | api-only | — | — | M | internal/translator + proxy-side post-aggregation of a stats matrix | pkg/logql/syntax OpTypeCountValues; vector evaluator | TestCompat_CountValues (test/e2e-compat, group semantics) | +| 75 | `semantics/eager-pipeline-validation` — Invalid pipeline stages fail with 400 regardless of data; Loki answers 200 empty over a range with no data (documented) | documented | edge | — | — | S | internal/translator validation | pkg/logql/log/fmt.go template build at pipeline construction | TestCompat_EagerValidationEmptyRange (test/e2e-compat, group semantics) | +| 76 | `semantics/ip-filter-precision` — ip() filters are approximated with regular expressions (documented) | documented | edge | — | — | M | internal/proxy/postprocess.go:84 ip filtering | pkg/logql/log/ip.go | TestCompat_IPFilterForms (test/e2e-compat, group semantics) | +| 77 | `drilldown/plugin-legend-sync-race` — Logs Drilldown crashes a breakdown page when a query answers before the panel module loads (upstream, not the proxy) | upstream | drilldown-visible | — | — | S | none (grafana-lokiexplore-app) | n/a (client race) | bench/visual capture waits for the panel module (already) | +| 78 | `quality/index-stats-loki-chunk-accounting` — Loki's index/stats counts chunks it holds twice; the proxy's counts equal Loki's own count_over_time (upstream) | upstream | drilldown-visible | — | — | S | none (Loki chunk index accounting) | pkg/storage stats from chunk refs (flushed and retained chunks both counted) | bench/parity cross-check (run.py loki_counts_itself) | ## Fixed, awaiting the next run diff --git a/internal/logql/semantic.go b/internal/logql/semantic.go index 1c48c0e53..a4933544e 100644 --- a/internal/logql/semantic.go +++ b/internal/logql/semantic.go @@ -5,6 +5,7 @@ import ( "regexp" "regexp/syntax" "strconv" + "unicode/utf8" ) // errEmptyCompatibleMatchers is Loki's syntax.errAtleastOneEqualityMatcherRequired, @@ -97,14 +98,19 @@ func labelReplaceArgumentsError(call string) error { } for i := 0; i < 4; i++ { if p.cur.Typ != TokComma { - return p.syntaxErrorAt("COMMA") + if i == 0 { + return p.syntaxErrorAt("") // the expression could go on: Loki names nothing + } + return p.syntaxErrorAt(",") } p.advance() switch p.cur.Typ { case TokString, TokRawString: case TokError: if p.cur.Val == "invalid quoted string" { - return newParseError(1, len(p.sc.src[:p.sc.pos])+1, "invalid syntax") + // Loki's lexer reports the start of the string. + before := p.sc.src[:p.pos()] + return newParseError(1, utf8.RuneCountInString(before)+1, "invalid char escape") } return p.syntaxErrorAt("STRING") default: diff --git a/internal/proxy/label_replace_semantics_test.go b/internal/proxy/label_replace_semantics_test.go index d573a141c..abdac9068 100644 --- a/internal/proxy/label_replace_semantics_test.go +++ b/internal/proxy/label_replace_semantics_test.go @@ -320,22 +320,6 @@ func TestLabelReplaceDuplicateLabelSets(t *testing.T) { } } -// label_join is no Loki v3.7.7 function (the lexer has label_replace only); -// the proxy keeps accepting it and applies it to both answer shapes. -// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query -func TestLabelJoinInstantAndRange(t *testing.T) { - spec := translator.LabelJoinSpec{DstLabel: "j", Separator: "-", SrcLabels: []string{"a", "b"}} - for _, body := range []string{ - `{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"a":"x","b":"y"},"values":[[1,"1"]]}]}}`, - `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"a":"x","b":"y"},"value":[1,"1"]}]}}`, - } { - a := decodeLR(t, 200, applyLabelJoin([]byte(body), spec)) - if a.series[0].Metric["j"] != "x-y" { - t.Errorf("%s: j=%q, want x-y", a.resultType, a.series[0].Metric["j"]) - } - } -} - // The registry shape and the shapes around it, on both endpoints. // conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query func TestLabelReplaceOnInstantAndRange(t *testing.T) { @@ -495,19 +479,81 @@ func TestLabelReplaceInvalidRegexp(t *testing.T) { func TestLabelReplaceMalformedArguments(t *testing.T) { p := newTestProxy(t, lrBackend(t, []map[string]string{{"app": "a"}}).URL) agg := `sum by (app) (count_over_time({app="a"}[5m]))` - for _, q := range []string{ - `label_replace(` + agg + `, "d", "$1", "app", "(.+):\d+")`, - `label_replace(` + agg + `, "d", "$1", "app")`, - `label_replace(` + agg + `, "d", "$1")`, - `label_replace(` + agg + `, "d", "$1", "app", "x", "y")`, - `label_replace(` + agg + `, d, "$1", "app", "x")`, - `label_replace(` + agg + `, "d", "$1", "app", 5)`, + // The messages are the ones Loki v3.7.7's parser raises for these queries. + for _, tc := range []struct{ query, want string }{ + {`label_replace(` + agg + `, "d", "$1", "app", "(.+):\d+")`, "parse error at line 1, col 80: invalid char escape"}, + {`label_replace(` + agg + `, "d", "$1", "app")`, "parse error at line 1, col 78: syntax error: unexpected ), expecting ,"}, + {`label_replace(` + agg + `, "d", "$1")`, "parse error at line 1, col 71: syntax error: unexpected ), expecting ,"}, + {`label_replace(` + agg + `, "d", "$1", "app", "x", "y")`, "parse error at line 1, col 83: syntax error: unexpected ,, expecting )"}, + {`label_replace(` + agg + `, d, "$1", "app", "x")`, "parse error at line 1, col 62: syntax error: unexpected IDENTIFIER, expecting STRING"}, + {`label_replace(` + agg + `, "d", "$1", "app", 5)`, "parse error at line 1, col 80: syntax error: unexpected NUMBER, expecting STRING"}, + {`label_replace(` + agg + ` "d", "$1", "app", "x")`, "parse error at line 1, col 61: syntax error: unexpected STRING"}, } { for _, endpoint := range []string{"query_range", "query"} { - a := lrAsk(t, p, endpoint, q) - if a.code != http.StatusBadRequest || !strings.Contains(a.body, "parse error at line 1, col ") || strings.Contains(a.body, "log queries") { - t.Errorf("%s %s: HTTP %d, want a 400 parse error\n%s", endpoint, q, a.code, a.body) + a := lrAsk(t, p, endpoint, tc.query) + if a.code != http.StatusBadRequest || !strings.Contains(a.body, `"message":"`+tc.want+`"`) { + t.Errorf("%s %s: HTTP %d, want a 400 with %q\n%s", endpoint, tc.query, a.code, tc.want, a.body) + } + } + } +} + +// Loki sorts the result after the transform (pkg/logql/engine.go JoinSampleVector: +// sort.Sort(result) for a range, sort.Slice(vec, labels.Compare) for an instant +// query), so the order of the backend's series does not survive it. +// conformance: semantics/label-replace-new-label, loki_api_v1_query_range, loki_api_v1_query +func TestLabelReplaceSortsTheResult(t *testing.T) { + agg := `sum by (app) (count_over_time({app="x"}[5m]))` + lr := func(inner, dst, repl, src, re string) string { + return fmt.Sprintf(`label_replace(%s, %q, %q, %q, %q)`, inner, dst, repl, src, re) + } + for _, tc := range []struct { + name string + sets []map[string]string + query string + want []string + }{ + {"input in reverse order", []map[string]string{{"app": "web"}, {"app": "api-gateway"}}, lr(agg, "app_short", "$1", "app", "(.*)-.*"), + []string{`{app="api-gateway",app_short="api"}`, `{app="web"}`}}, + {"the transform reorders the label sets", []map[string]string{{"app": "a"}, {"app": "b"}}, + lr(lr(agg, "a_new", "2", "app", "a"), "a_new", "1", "app", "b"), + []string{`{a_new="1",app="b"}`, `{a_new="2",app="a"}`}}, + {"a shorter prefix first", []map[string]string{{"app": "b", "k": "v"}, {"app": "a"}, {"app": "b"}}, lr(agg, "none", "", "app", "zz"), + []string{`{app="a"}`, `{app="b"}`, `{app="b",k="v"}`}}, + } { + p := newTestProxy(t, lrBackend(t, tc.sets).URL) + for _, endpoint := range []string{"query_range", "query"} { + a := lrAsk(t, p, endpoint, tc.query) + var got []string + for _, s := range a.series { + got = append(got, lrKey(s.Metric)) } + if fmt.Sprint(got) != fmt.Sprint(tc.want) { + t.Errorf("%s %s: order %v, want %v\n%s", tc.name, endpoint, got, tc.want, a.body) + } + } + } + // sort and sort_desc order by value: that order is kept + if !queryIsSortable(`label_replace(sort(sum(x)), "a", "b", "c", "d")`) || queryIsSortable(`label_replace(sum by (resort) (x), "a", "b", "c", "d")`) { + t.Error("queryIsSortable does not tell sort(...) from a name that ends in it") + } +} + +// compareLabelSets is Prometheus' labels.Compare. +func TestCompareLabelSets(t *testing.T) { + for _, tc := range []struct { + a, b map[string]string + want int + }{ + {map[string]string{"a": "1"}, map[string]string{"a": "1"}, 0}, + {map[string]string{"a": "1"}, map[string]string{"a": "2"}, -1}, + {map[string]string{"a": "1"}, map[string]string{"a": "1", "b": "2"}, -1}, + {map[string]string{"a": "1", "c": "1"}, map[string]string{"a": "1", "b": "2"}, 1}, + {map[string]string{"b": "1"}, map[string]string{"a": "9"}, 1}, + {map[string]string{}, map[string]string{"a": "9"}, -1}, + } { + if got := compareLabelSets(tc.a, tc.b); (got > 0) != (tc.want > 0) || (got < 0) != (tc.want < 0) { + t.Errorf("compare(%v, %v) = %d, want sign %d", tc.a, tc.b, got, tc.want) } } } diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 2bbcff4f8..e17a5b5f8 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -2390,6 +2390,9 @@ func (p *Proxy) handleQueryRange(w http.ResponseWriter, r *http.Request) { if len(labelReplaceSpecs) > 0 || labelJoinSpec != nil { routedQuery = stripLabelTransforms(logqlQuery) } + if routedQuery != logqlQuery { + r = cloneMetricQueryRequest(r, routedQuery) + } parsedForRouting, _ := logqlpkg.Parse(routedQuery) if binOp, ok := parsedForRouting.(*logqlpkg.BinOpExpr); ok { // Binary metric expression: sum(rate(...)) / sum(rate(...)) @@ -2401,9 +2404,6 @@ func (p *Proxy) handleQueryRange(w http.ResponseWriter, r *http.Request) { } else if rightErr != nil { p.writeError(sc, http.StatusBadRequest, rightErr.Error()) } else { - if routedQuery != logqlQuery { - r = cloneMetricQueryRequest(r, routedQuery) - } p.proxyBinaryMetricQueryRangeVM(sc, r, binOp.Op, leftLogsql, rightLogsql, binOpExprToVMInfo(binOp)) } } else if isStatsQuery(logsqlQuery) { @@ -2435,6 +2435,9 @@ func (p *Proxy) handleQueryRange(w http.ResponseWriter, r *http.Request) { if labelJoinSpec != nil { cacheOut = applyLabelJoin(cacheOut, *labelJoinSpec) } + if len(labelReplaceSpecs) > 0 || labelJoinSpec != nil { + cacheOut = sortSeriesByLabels(cacheOut, false) + } copyBackendHeaders(w.Header(), capture.Header()) if w.Header().Get("Content-Type") == "" { w.Header().Set("Content-Type", "application/json") @@ -2663,6 +2666,9 @@ func (p *Proxy) handleQuery(w http.ResponseWriter, r *http.Request) { if len(labelReplaceSpecs) > 0 || labelJoinSpec != nil { routedQuery = stripLabelTransforms(logqlQuery) } + if routedQuery != logqlQuery { + r = cloneMetricQueryRequest(r, routedQuery) + } parsedForRoutingQ, _ := logqlpkg.Parse(routedQuery) if binOp, ok := parsedForRoutingQ.(*logqlpkg.BinOpExpr); ok { leftLogsql, leftErr := p.translateBinOpSide(r.Context(), binOp.Left) @@ -2672,9 +2678,6 @@ func (p *Proxy) handleQuery(w http.ResponseWriter, r *http.Request) { } else if rightErr != nil { p.writeError(sc, http.StatusBadRequest, rightErr.Error()) } else { - if routedQuery != logqlQuery { - r = cloneMetricQueryRequest(r, routedQuery) - } p.proxyBinaryMetricQueryVM(sc, r, binOp.Op, leftLogsql, rightLogsql, binOpExprToVMInfo(binOp)) } } else if isStatsQuery(logsqlQuery) { @@ -2702,6 +2705,9 @@ func (p *Proxy) handleQuery(w http.ResponseWriter, r *http.Request) { if labelJoinSpec != nil { result = applyLabelJoin(result, *labelJoinSpec) } + if len(labelReplaceSpecs) > 0 || labelJoinSpec != nil { + result = sortSeriesByLabels(result, queryIsSortable(logqlQuery)) + } w.Header().Set("Content-Type", "application/json") w.Write(result) } diff --git a/internal/proxy/vector_matching.go b/internal/proxy/vector_matching.go index 98608820d..77fd1d81c 100644 --- a/internal/proxy/vector_matching.go +++ b/internal/proxy/vector_matching.go @@ -709,6 +709,77 @@ func transformSeriesLabels(body []byte, fn func(map[string]string) map[string]st return b } +// sortSeriesByLabels orders the series of a matrix or vector answer as Loki +// does after a label_replace: a range result by label set (pkg/logql/engine.go +// JoinSampleVector, sort.Sort(result)), an instant vector the same way unless +// the query sorts by value (sort, sort_desc: evaluator.go Sortable), in which +// case sortable is set and the order is kept. +func sortSeriesByLabels(body []byte, sortable bool) []byte { + var top, data map[string]json.RawMessage + if json.Unmarshal(body, &top) != nil || json.Unmarshal(top["data"], &data) != nil { + return body + } + var resultType string + var series []struct { + raw json.RawMessage + metric map[string]string + } + var raws []json.RawMessage + _ = json.Unmarshal(data["resultType"], &resultType) + if (resultType != "matrix" && resultType != "vector") || (resultType == "vector" && sortable) || json.Unmarshal(data["result"], &raws) != nil { + return body + } + for _, raw := range raws { + var m struct { + Metric map[string]string `json:"metric"` + } + _ = json.Unmarshal(raw, &m) + series = append(series, struct { + raw json.RawMessage + metric map[string]string + }{raw, m.Metric}) + } + sort.SliceStable(series, func(i, j int) bool { return compareLabelSets(series[i].metric, series[j].metric) < 0 }) + for i := range series { + raws[i] = series[i].raw + } + data["result"], _ = json.Marshal(raws) + top["data"], _ = json.Marshal(data) + if b, err := json.Marshal(top); err == nil { + return b + } + return body +} + +var sortCallRE = regexp.MustCompile(`(^|[^\w])sort(_desc)?\s*\(`) + +// queryIsSortable reports whether a query orders by value (evaluator.go +// Sortable: a sort or sort_desc anywhere in it). +func queryIsSortable(logql string) bool { return sortCallRE.MatchString(logql) } + +// compareLabelSets is Prometheus' labels.Compare: label by label in name +// order, the name first and then the value, a shorter prefix first. +func compareLabelSets(a, b map[string]string) int { + names := func(m map[string]string) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out + } + na, nb := names(a), names(b) + for i := 0; i < len(na) && i < len(nb); i++ { + if na[i] != nb[i] { + return strings.Compare(na[i], nb[i]) + } + if c := strings.Compare(a[na[i]], b[nb[i]]); c != 0 { + return c + } + } + return len(na) - len(nb) +} + // pointSeconds is the time of a sample point `[time, "value"]`. func pointSeconds(point json.RawMessage) float64 { var p []any @@ -750,7 +821,8 @@ func applyLabelReplace(body []byte, spec translator.LabelReplaceSpec) []byte { // applyLabelJoin applies label_join to every series of a matrix or vector // answer: the destination label is the values of the source labels, the -// absent ones skipped, joined by the separator. +// absent ones skipped, joined by the separator. Loki v3.7.7 has no label_join; +// accepting it is the open gap semantics/proxy-accepts-operators-not-in-loki. func applyLabelJoin(body []byte, spec translator.LabelJoinSpec) []byte { return transformSeriesLabels(body, func(in map[string]string) map[string]string { parts := make([]string, 0, len(spec.SrcLabels)) diff --git a/test/e2e-compat/label_replace_new_label_test.go b/test/e2e-compat/label_replace_new_label_test.go index 0116a0623..471da200e 100644 --- a/test/e2e-compat/label_replace_new_label_test.go +++ b/test/e2e-compat/label_replace_new_label_test.go @@ -171,3 +171,47 @@ func TestCompat_LabelReplaceNewLabel(t *testing.T) { } } } + +// Two series that a label_replace gives one label set are ranked as two +// samples by the vector aggregation above it, and filed under one series only +// at the end (pkg/logql/evaluator.go VectorAggEvaluator.Next, pkg/logql/engine.go +// vectorsToSeries). The proxy merges them before the ranking +// (semantics/label-replace-duplicates-before-ranking, open). +// conformance: semantics/label-replace-duplicates-before-ranking, loki_api_v1_query_range +func TestCompat_LabelReplaceDuplicatesBeforeRanking(t *testing.T) { + ensureDataIngested(t) + waitForLokiMetricData(t) + + at := time.Now().UTC().Truncate(time.Minute) + for _, query := range []string{ + `topk(1, label_replace(sum by (app) (count_over_time({env="production"}[5m])), "app", "all", "app", ".+"))`, + `bottomk(1, label_replace(sum by (app) (count_over_time({env="production"}[5m])), "app", "all", "app", ".+"))`, + } { + params := url.Values{ + "query": {query}, + "start": {fmt.Sprintf("%d", at.Add(-15*time.Minute).UnixNano())}, + "end": {fmt.Sprintf("%d", at.UnixNano())}, + "step": {"60"}, + } + full := "/loki/api/v1/query_range?" + params.Encode() + lokiStatus, lokiBody, lokiPayload := doJSONGET(t, lokiURL+full, nil) + proxyStatus, proxyBody, proxyPayload := doJSONGET(t, proxyURL+full, nil) + if lokiStatus != http.StatusOK || proxyStatus != http.StatusOK { + t.Fatalf("%s: HTTP loki=%d proxy=%d\nloki: %s\nproxy: %s", query, lokiStatus, proxyStatus, lokiBody, proxyBody) + } + count := func(payload map[string]interface{}) int { + data, _ := payload["data"].(map[string]interface{}) + result, _ := data["result"].([]interface{}) + points := 0 + for _, item := range result { + series, _ := item.(map[string]interface{}) + values, _ := series["values"].([]interface{}) + points += len(values) + } + return points + } + if l, p := count(lokiPayload), count(proxyPayload); l != p { + t.Errorf("%s: points loki=%d proxy=%d\nloki: %s\nproxy: %s", query, l, p, lokiBody, proxyBody) + } + } +} From 17c6e6bfaf9ad57a1f60420a0f21d874d7edbdf3 Mon Sep 17 00:00:00 2001 From: szibis Date: Mon, 12 Oct 2026 13:59:05 +0200 Subject: [PATCH 10/18] fix: a label transform around a binary clones the request from its own query, so each operand keeps its parser --- internal/proxy/binary_logql.go | 13 +++++++++++++ internal/proxy/proxy.go | 8 ++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/internal/proxy/binary_logql.go b/internal/proxy/binary_logql.go index f5f1578ae..178c05134 100644 --- a/internal/proxy/binary_logql.go +++ b/internal/proxy/binary_logql.go @@ -36,6 +36,19 @@ func nextBinaryEvaluation(r *http.Request) (*http.Request, error) { // Resolve original operands through the normal handlers: VL stats buckets do // not implement Loki's trailing range windows or parser-error semantics. +// binaryRequestQuery is the request's own LogQL with Grafana tokens resolved +// and a uniform offset stripped, as binaryExprForRequest reads it. +func binaryRequestQuery(r *http.Request) string { + query := stripIncompleteUnwrapStubs(r.FormValue("query")) + query = resolveGrafanaRangeTemplateTokens(query, r.FormValue("start"), r.FormValue("end"), r.FormValue("step")) + // The outer handler has already shifted uniform offsets. Mixed offsets + // remain on their individual operands and are applied by each child handler. + if _, stripped, err := extractLogQLOffset(query); err == nil { + query = stripped + } + return query +} + func binaryExprForRequest(r *http.Request) *logqlpkg.BinOpExpr { query := resolveGrafanaRangeTemplateTokens(r.FormValue("query"), r.FormValue("start"), r.FormValue("end"), r.FormValue("step")) // The outer handler has already shifted uniform offsets. Mixed offsets diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index e17a5b5f8..6a82167eb 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -2391,7 +2391,9 @@ func (p *Proxy) handleQueryRange(w http.ResponseWriter, r *http.Request) { routedQuery = stripLabelTransforms(logqlQuery) } if routedQuery != logqlQuery { - r = cloneMetricQueryRequest(r, routedQuery) + // From the request's own query: logqlQuery may have lost a parser stage + // that one operand needs (preferWorkingParser). + r = cloneMetricQueryRequest(r, stripLabelTransforms(binaryRequestQuery(r))) } parsedForRouting, _ := logqlpkg.Parse(routedQuery) if binOp, ok := parsedForRouting.(*logqlpkg.BinOpExpr); ok { @@ -2667,7 +2669,9 @@ func (p *Proxy) handleQuery(w http.ResponseWriter, r *http.Request) { routedQuery = stripLabelTransforms(logqlQuery) } if routedQuery != logqlQuery { - r = cloneMetricQueryRequest(r, routedQuery) + // From the request's own query: logqlQuery may have lost a parser stage + // that one operand needs (preferWorkingParser). + r = cloneMetricQueryRequest(r, stripLabelTransforms(binaryRequestQuery(r))) } parsedForRoutingQ, _ := logqlpkg.Parse(routedQuery) if binOp, ok := parsedForRoutingQ.(*logqlpkg.BinOpExpr); ok { From a76fb3ab20b87e550e3424cd8093e7874e98db43 Mon Sep 17 00:00:00 2001 From: szibis Date: Mon, 12 Oct 2026 13:03:48 +0200 Subject: [PATCH 11/18] fix: an aggregation over a binary expression aggregates the whole binary, like Loki A sum, avg, min, max or count over a binary expression answered only the first operand: the translator's outer-aggregation extraction dropped the operator and the second operand, so `sum by (job) (A / B)`, `sum(A / 2)`, a comparison and on()/ignoring() under an aggregation returned wrong values with a 200, and an instant query with a literal or a nested binary first answered 400 for a log query. Loki joins the operands per series and folds the joined series per step (newBinOpStepEvaluator, vectorBinop, VectorAggEvaluator.Next). The query and query_range routing now send such an aggregation to the binary evaluator: each operand stays a native VictoriaLogs stats query, the proxy joins the aggregated series and folds them with the aggregation's operator. The expression is read from the request so an operand keeps its own parser, a one-sided offset is accepted as for a bare binary, and folded series come in Loki's label order. Registers semantics/aggregation-over-binary-dropped (fixed) and moves the discovered cluster f2066991 to it. --- CHANGELOG.md | 25 + .../aggregation-over-binary-dropped.yaml | 63 ++- internal/proxy/binary_aggregation.go | 104 ++++ .../binary_aggregation_differential_test.go | 449 ++++++++++++++++++ internal/proxy/binary_aggregation_test.go | 257 ++++++++++ internal/proxy/binary_logql.go | 27 +- internal/proxy/proxy.go | 10 + internal/proxy/vector_matching.go | 10 +- .../aggregation_over_binary_test.go | 96 ++++ 9 files changed, 1010 insertions(+), 31 deletions(-) create mode 100644 internal/proxy/binary_aggregation.go create mode 100644 internal/proxy/binary_aggregation_differential_test.go create mode 100644 internal/proxy/binary_aggregation_test.go create mode 100644 test/e2e-compat/aggregation_over_binary_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index a3c5385a9..5e9eb13f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 b)`) is still not evaluated as Loki does; where the proxy rejects it, it now does so with an error status, no longer with a typeless 200. +- **A `sum`, `avg`, `min`, `max` or `count` over a binary expression aggregates + the whole binary, as Loki does, instead of only its first operand.** + `sum by (job) (rate({app="api-gateway"}[1m]) / count_over_time({app="api-gateway"}[1m]))` + answered the aggregated rate alone, `sum(rate({app="x"}[1m]) / 2)` lost the + `/ 2`, a comparison (`max(rate(...) > 0.5)`) filtered nothing, and `on()`, + `ignoring()` and `group_left` under an aggregation were ignored: all with a 200 + and wrong values on query_range. On an instant query, a literal or a nested + binary as the first operand (`sum(2 * rate(...))`) answered 400 "log queries are + not supported as an instant query type". Loki 3.7.7 joins the operands per + series (`pkg/logql/evaluator.go:791` `newBinOpStepEvaluator`, `:955` + `vectorBinop`, `:1096` `resultMetric`; a comparison keeps the left value or, + with `bool`, answers 0 or 1) and then folds the joined series per step by the + aggregation's grouping (`VectorAggEvaluator.Next`, `:435-630`). The proxy sent + only a top-level binary to its binary evaluator; under an aggregation the + translator kept the first operand's stats chain and dropped the rest. Now + query and query_range send an aggregation (by, without or no grouping, nested + aggregations included) over a binary to the binary evaluator: each operand + stays a native VictoriaLogs stats query through its own handler, the proxy + joins the aggregated operand series and folds them per step, with no raw rows + and work bounded by the operands' series. An operand keeps its own parser when + the query names both `| json` and `| logfmt` (the shape the differential + corpus found inside a `label_replace`), an offset on one side only is accepted as for a + bare binary, and the series of an aggregation folded by the proxy come in + Loki's label order (#TBD). + ## [2.11.2] - 2026-10-12 ### CI diff --git a/conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml b/conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml index 96e7d7fe0..f624568f6 100644 --- a/conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml +++ b/conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml @@ -1,37 +1,64 @@ id: semantics/aggregation-over-binary-dropped -title: An aggregation over a binary expression is translated to its first operand (open) +title: A vector aggregation over a binary expression aggregates the whole binary, like Loki (fixed) proves: + - loki_api_v1_query_range - loki_api_v1_query consumer: explore why: | - `sum by (job) (rate(a[1m]) / count_over_time(a[1m]))` is a vector aggregation - over a binary operation on Loki (pkg/logql/evaluator.go VectorAggEvaluator - over a binary evaluator). The translator keeps the first operand only: the - right operand and the operator are dropped (`sum(rate(a[1m]) / 2)` loses the - `/ 2`), so an answered query has wrong values, and the shape of parity - cluster f2066991 (`| json | unwrap duration(latency)` over `/ - count_over_time(...)` under label_replace) is rejected with 400 ("log queries - are not supported as an instant query type") where Loki answers a vector. + `sum by (job) (rate({app="api-gateway"}[1m]) / count_over_time({app="api-gateway"}[1m]))` + and `sum(rate({app="x"}[1m]) / 2)` are evaluated by Loki 3.7.7 as the binary + per series, then the aggregation per step: newBinOpStepEvaluator joins the + operands' samples (pkg/logql/evaluator.go:791 and vectorBinop:955, matching + on every label or on()/ignoring() with group_left/group_right, resultMetric:1096; + a literal on either side through newLiteralStepEvaluator:1130; a comparison + keeps the left value or, with bool, answers 0/1, pkg/logql/syntax/ast.go:1944 + MergeBinOp), and VectorAggEvaluator.Next (pkg/logql/evaluator.go:435-630) + folds the joined series by the grouping. + + The proxy routed only a top-level binary to its binary evaluator. Under an + aggregation the query went to the translator, whose outer-aggregation + extraction (internal/translator/translator.go extractOuterAggregation and + applyOuterAggregation) kept the first operand's stats chain and dropped the + operator and the second operand: `/ count_over_time(...)`, `/ 2`, `> 0.5` and + `/ on (job) group_left ...` all vanished with a 200 and wrong values on + query_range; an instant query whose first operand was a literal (`sum(2 * + rate(...))`) or a nested binary was translated to a line search and answered + 400 "log queries are not supported as an instant query type". + + Fixed: the routing of query_range and query sends a sum, avg, min, max or + count (by, without or no grouping, nested) over a binary to the binary + evaluator (proxyBinaryAggregation): each operand stays a native VictoriaLogs + stats query through its own handler, the evaluator joins the aggregated + operand series, and applyWithoutGrouping folds the joined series per step + with the aggregation's operator. The expression is read from the request, so + an operand keeps its parser when the whole query names `| json` and `| + logfmt` (the discovered cluster below). A mixed offset (one side only) is + accepted as for a bare binary, and the folded series are in Loki's label + order. request: method: GET - path: /loki/api/v1/query + path: /loki/api/v1/query_range params: query: 'sum by (job) (rate({app="api-gateway"}[1m]) / count_over_time({app="api-gateway"}[1m]))' + start: 'now-1h' + end: 'now' + step: '60' headers: X-Scope-OrgID: "0" - X-Loki-Response-Encoding-Flags: categorize-labels compare: mode: observed - fields: [series, values] + fields: [resultType, series, values] expect: | - Loki's vector: the quotient per stream, summed by job. + Loki's series and values: the binary per series, then the aggregation per step. gap: - status: open + status: fixed + fixed_by: "#TBD" impact: explore-visible effort: M - area: 'internal/translator extractOuterAggregation and internal/proxy handleQuery routing of an aggregation over a binary expression' - loki: 'pkg/logql/evaluator.go VectorAggEvaluator over a binary evaluator' - proxy: 'first operand only, or a 400' - planned_test: 'TestCompat_AggregationOverBinary (test/e2e-compat, group semantics)' + area: 'internal/proxy binary_aggregation.go (proxyBinaryAggregation, foldBinaryAggregation); proxy.go query and query_range routing' + loki: 'pkg/logql/evaluator.go:791 newBinOpStepEvaluator, :955 vectorBinop, :1096 resultMetric, :400-630 newVectorAggEvaluator / VectorAggEvaluator.Next; pkg/logql/syntax/ast.go:1944 MergeBinOp' + proxy: 'the operands as native stats queries, the join and the fold over their aggregated series' + planned_test: 'TestCompat_AggregationOverBinary and TestCompat_AggregationOverBinaryMixedParsers (test/e2e-compat, group core); TestAggregationOverBinaryEdgeCases, TestAggregationOverBinaryRange, TestAggregationOverBinaryDifferential, TestAggregationOverBinaryKeepsEachOperandsParser and TestAggregationOverBinaryMixedOffset (internal/proxy)' + note: 'The e2e is written and not yet run against the stack.' clusters: - f2066991 diff --git a/internal/proxy/binary_aggregation.go b/internal/proxy/binary_aggregation.go new file mode 100644 index 000000000..6c427a63c --- /dev/null +++ b/internal/proxy/binary_aggregation.go @@ -0,0 +1,104 @@ +package proxy + +import ( + "encoding/json" + "net/http" + + logqlpkg "github.com/ReliablyObserve/Loki-VL-proxy/internal/logql" +) + +// binaryAggregationChain returns the sum, avg, min, max and count aggregations +// over a binary expression, innermost first, and that expression. Loki +// evaluates the binary per series (pkg/logql/evaluator.go newBinOpStepEvaluator) +// and aggregates its answer per step (VectorAggEvaluator.Next); a binary of two +// literals is a literal, which no aggregation here takes. +func binaryAggregationChain(expr logqlpkg.Expr) ([]*logqlpkg.VectorAggregation, *logqlpkg.BinOpExpr, bool) { + var chain []*logqlpkg.VectorAggregation + for { + agg, ok := expr.(*logqlpkg.VectorAggregation) + if !ok { + break + } + switch agg.Op { + case logqlpkg.VectorSum, logqlpkg.VectorAvg, logqlpkg.VectorMin, logqlpkg.VectorMax, logqlpkg.VectorCount: + default: + return nil, nil, false + } + chain = append([]*logqlpkg.VectorAggregation{agg}, chain...) + expr = agg.Inner + } + binary, ok := expr.(*logqlpkg.BinOpExpr) + if !ok || len(chain) == 0 { + return nil, nil, false + } + _, leftScalar := binaryScalarValue(binary.Left, 0) + _, rightScalar := binaryScalarValue(binary.Right, 0) + return chain, binary, !leftScalar || !rightScalar +} + +// proxyBinaryAggregation answers a vector aggregation over a binary expression +// when routed is one: each operand is its own VictoriaLogs stats query, joined +// by the binary evaluator, and the aggregation folds the joined series per +// step. Both work on aggregated series only. The expression is read from the +// request, as the binary path does: the handler's query may have lost a parser +// stage another operand needed (preferWorkingParser). +func (p *Proxy) proxyBinaryAggregation(w http.ResponseWriter, r *http.Request, routed logqlpkg.Expr, resultType string) bool { + if _, _, ok := binaryAggregationChain(routed); !ok { + return false + } + chain, binary, ok := binaryAggregationChain(binaryRequestExpr(r)) + if !ok { + p.writeError(w, http.StatusBadRequest, "unsupported aggregation over a binary expression") + return true + } + buf := &bufferedResponseWriter{} + p.proxyBinaryLogQL(buf, r, binary, resultType) + body := buf.body + if buf.code == 0 || buf.code == http.StatusOK { + for _, agg := range chain { + body = foldBinaryAggregation(body, agg) + } + } + copyHeaders(w.Header(), buf.Header()) + if buf.code != 0 { + w.WriteHeader(buf.code) + } + _, _ = w.Write(body) // nosemgrep: go.lang.security.audit.xss.no-direct-write-to-responsewriter -- the binary evaluator's own JSON, Content-Type copied above + return true +} + +// foldBinaryAggregation applies one aggregation to a matrix or vector body: +// series that share the labels the grouping keeps become one, valued by the +// operator (VectorAggEvaluator.Next: `by` keeps the listed labels, `without` +// drops them, no grouping keeps none). +func foldBinaryAggregation(body []byte, agg *logqlpkg.VectorAggregation) []byte { + op := string(agg.Op) + if agg.Grouping != nil && agg.Grouping.Without { + return applyWithoutGrouping(body, agg.Grouping.Labels, op) + } + exclude := []string{withoutAllLabels} + if agg.Grouping != nil && len(agg.Grouping.Labels) > 0 { + keep := make(map[string]bool, len(agg.Grouping.Labels)) + for _, label := range agg.Grouping.Labels { + keep[label] = true + } + var resp struct { + Data struct { + Result []struct { + Metric map[string]string `json:"metric"` + } `json:"result"` + } `json:"data"` + } + _ = json.Unmarshal(body, &resp) + exclude = exclude[:0] + for _, series := range resp.Data.Result { + for name := range series.Metric { + if !keep[name] { + keep[name] = true // exclude each name once + exclude = append(exclude, name) + } + } + } + } + return applyWithoutGrouping(body, exclude, op) +} diff --git a/internal/proxy/binary_aggregation_differential_test.go b/internal/proxy/binary_aggregation_differential_test.go new file mode 100644 index 000000000..8d23ffbcd --- /dev/null +++ b/internal/proxy/binary_aggregation_differential_test.go @@ -0,0 +1,449 @@ +package proxy + +import ( + "fmt" + "math" + "math/rand" + "sort" + "strconv" + "strings" + "testing" +) + +// This file pins a vector aggregation over a binary expression to Loki's +// answer, over generated rows, windows and expressions, with a reference +// evaluator written apart from the proxy: Loki v3.7.7 pkg/logql/evaluator.go +// newBinOpStepEvaluator and vectorBinop (per step, the right-hand samples by +// matching signature, one match per left sample, resultMetric for the labels: +// on() keeps the listed labels and ignoring() drops them for one-to-one, +// group_left keeps the left labels; a comparison keeps the left value or, with +// bool, answers 0 or 1), newLiteralStepEvaluator (a literal on either side) and +// VectorAggEvaluator.Next (the aggregation per step over the binary's samples). +// The operands are `sum by (...)` over count_over_time and rate, read on the +// window (t-range, t]; only VictoriaLogs is simulated (vlEmulator). + +type refSample struct { + labels map[string]string + v float64 +} + +// refNode is an operand (fn set), a literal (scalar set) or a binary. +type refNode struct { + fn string // "count", "rate" or "amount" (rate of the unwrapped amount) + by []string + scalar *float64 + + op string + boolean bool + match string // "", "on" or "ignoring" + labels []string + groupLeft bool + l, r *refNode +} + +func (n *refNode) query(window string) string { + switch { + case n.fn != "": + inner := map[string]string{ + "count": `count_over_time({app="uw"}[` + window + `])`, + "rate": `rate({app="uw"}[` + window + `])`, + "amount": `rate({app="uw"} | unwrap amount [` + window + `])`, + }[n.fn] + return "sum by (" + strings.Join(n.by, ", ") + ") (" + inner + ")" + case n.scalar != nil: + return strconv.FormatFloat(*n.scalar, 'f', -1, 64) + } + side := func(c *refNode) string { + if c.op != "" { + return "(" + c.query(window) + ")" + } + return c.query(window) + } + q := side(n.l) + " " + n.op + if n.boolean { + q += " bool" + } + if n.match != "" { + q += " " + n.match + " (" + strings.Join(n.labels, ", ") + ")" + } + if n.groupLeft { + q += " group_left" + } + return q + " " + side(n.r) +} + +func refStreamLabels(stream string) map[string]string { + out := map[string]string{} + for _, m := range emuStreamLabelRE.FindAllStringSubmatch(stream, -1) { + out[m[1]] = m[2] + } + return out +} + +func refValue(op string, a, b float64) (float64, bool) { + switch op { + case "+": + return a + b, true + case "-": + return a - b, true + case "*": + return a * b, true + case "/": + return a / b, true // divisors are never zero here: Loki answers NaN, see the report + case ">": + return a, a > b + case "<": + return a, a < b + case ">=": + return a, a >= b + case "<=": + return a, a <= b + } + panic("reference: " + op) +} + +func refComparison(op string) bool { + return op == ">" || op == "<" || op == ">=" || op == "<=" +} + +// eval returns the vector at t, or the literal. +func (n *refNode) eval(rows []emuRow, t, window int64) ([]refSample, float64, bool) { + if n.scalar != nil { + return nil, *n.scalar, true + } + if n.fn != "" { + type group struct { + labels map[string]string + count, sum float64 + } + groups := map[string]*group{} + for _, row := range rows { + if row.ts <= t-window || row.ts > t { + continue + } + stream := refStreamLabels(row.stream) + labels := map[string]string{} + for _, name := range n.by { + if v := stream[name]; v != "" { + labels[name] = v + } + } + key := binaryLabelKey(labels) + if groups[key] == nil { + groups[key] = &group{labels: labels} + } + groups[key].count++ + groups[key].sum += row.value + } + var out []refSample + seconds := float64(window) / 1e9 + for _, g := range groups { + v := map[string]float64{"count": g.count, "rate": g.count / seconds, "amount": g.sum / seconds}[n.fn] + out = append(out, refSample{g.labels, v}) + } + return out, 0, false + } + lhs, lv, lScalar := n.l.eval(rows, t, window) + rhs, rv, rScalar := n.r.eval(rows, t, window) + var out []refSample + if lScalar || rScalar { + vec := lhs + if lScalar { + vec = rhs + } + for _, s := range vec { + a, b := s.v, rv + if lScalar { + a, b = lv, s.v + } + v, keep := refValue(n.op, a, b) + if refComparison(n.op) { + v = s.v // the vector's value, whichever side the literal is on + if n.boolean { + v, keep = map[bool]float64{true: 1, false: 0}[keep], true + } + } + if keep { + out = append(out, refSample{s.labels, v}) + } + } + return out, 0, false + } + signature := func(labels map[string]string) string { + sig := map[string]string{} + for k, v := range labels { + sig[k] = v + } + switch n.match { + case "on": + sig = map[string]string{} + for _, name := range n.labels { + if v := labels[name]; v != "" { + sig[name] = v + } + } + case "ignoring": + for _, name := range n.labels { + delete(sig, name) + } + } + return binaryLabelKey(sig) + } + right := map[string]refSample{} + for _, s := range rhs { + right[signature(s.labels)] = s + } + for _, s := range lhs { + match, ok := right[signature(s.labels)] + if !ok { + continue + } + labels := s.labels + if !n.groupLeft && n.match == "on" { + labels = map[string]string{} + for _, name := range n.labels { + if v := s.labels[name]; v != "" { + labels[name] = v + } + } + } + v, keep := refValue(n.op, s.v, match.v) + if n.boolean { + v, keep = map[bool]float64{true: 1, false: 0}[keep], true + } + if keep { + out = append(out, refSample{labels, v}) + } + } + return out, 0, false +} + +type refAggregation struct { + op string + without bool + labels []string // nil: no grouping +} + +func (a refAggregation) wrap(q string) string { + switch { + case a.labels == nil: + return a.op + "(" + q + ")" + case a.without: + return a.op + " without (" + strings.Join(a.labels, ", ") + ") (" + q + ")" + } + return a.op + " by (" + strings.Join(a.labels, ", ") + ") (" + q + ")" +} + +func (a refAggregation) apply(in []refSample) []refSample { + type group struct { + labels map[string]string + v float64 + n int + } + groups := map[string]*group{} + var order []string + for _, s := range in { + labels := map[string]string{} + if a.without { + for k, v := range s.labels { + labels[k] = v + } + for _, name := range a.labels { + delete(labels, name) + } + } else { + for _, name := range a.labels { + if v := s.labels[name]; v != "" { + labels[name] = v + } + } + } + key := binaryLabelKey(labels) + g := groups[key] + if g == nil { + groups[key] = &group{labels: labels, v: s.v, n: 1} + order = append(order, key) + continue + } + g.n++ + switch a.op { + case "sum": + g.v += s.v + case "avg": + g.v += (s.v - g.v) / float64(g.n) + case "min": + g.v = math.Min(g.v, s.v) + case "max": + g.v = math.Max(g.v, s.v) + } + } + out := make([]refSample, 0, len(order)) + for _, key := range order { + g := groups[key] + if a.op == "count" { + g.v = float64(g.n) + } + out = append(out, refSample{g.labels, g.v}) + } + return out +} + +// generateBinaryAggregation draws an expression: operands grouped by (job, pod) +// or by (job), joined on every label, on() or ignoring() with group_left, with +// a literal on either side, a comparison with or without bool, and binaries +// nested in binaries; then one or two aggregations over it. A divisor is never +// a value that can be zero. +func generateBinaryAggregation(rnd *rand.Rand) (*refNode, []refAggregation) { + pick := func(xs ...string) string { return xs[rnd.Intn(len(xs))] } + operand := func(by ...string) *refNode { return &refNode{fn: pick("count", "rate", "amount"), by: by} } + divisor := func(by ...string) *refNode { return &refNode{fn: pick("count", "rate"), by: by} } + literal := func(v float64) *refNode { return &refNode{scalar: &v} } + full := []string{"job", "pod"} + var expr *refNode + switch rnd.Intn(7) { + case 0: // metric op metric on every label + op := pick("+", "-", "*", "/") + right := operand(full...) + if op == "/" { + right = divisor(full...) + } + expr = &refNode{op: op, l: operand(full...), r: right} + case 1: // metric op literal, literal op metric + op := pick("+", "-", "*", "/") + v := float64(1 + rnd.Intn(4)) + if rnd.Intn(2) == 0 { + expr = &refNode{op: op, l: operand(full...), r: literal(v)} + } else if op == "/" { + expr = &refNode{op: op, l: literal(v), r: divisor(full...)} + } else { + expr = &refNode{op: op, l: literal(v), r: operand(full...)} + } + case 2: // comparison with a literal + v := []float64{0, 0.025, 0.04, 1.5, 2.5}[rnd.Intn(5)] + if rnd.Intn(2) == 0 { + expr = &refNode{op: pick(">", "<", ">=", "<="), boolean: rnd.Intn(2) == 0, l: operand(full...), r: literal(v)} + } else { + expr = &refNode{op: pick(">", "<", ">=", "<="), boolean: rnd.Intn(2) == 0, l: literal(v), r: operand(full...)} + } + case 3: // comparison of two vectors + expr = &refNode{op: pick(">", "<", ">=", "<="), boolean: rnd.Intn(2) == 0, l: operand(full...), r: operand(full...)} + case 4: // many-to-one + match, labels := "on", []string{"job"} + if rnd.Intn(2) == 0 { + match, labels = "ignoring", []string{"pod"} + } + expr = &refNode{op: pick("/", "*", "-"), match: match, labels: labels, groupLeft: true, l: operand(full...), r: divisor("job")} + case 5: // one-to-one on() the full label set: the answer keeps only those + expr = &refNode{op: pick("+", "/"), match: "on", labels: full, l: operand(full...), r: divisor(full...)} + default: // nested + inner := &refNode{op: pick("+", "-", "*"), l: operand(full...), r: operand(full...)} + if rnd.Intn(2) == 0 { + expr = &refNode{op: pick("/", "*"), l: inner, r: literal(float64(2 + rnd.Intn(3)))} + } else { + expr = &refNode{op: "/", l: inner, r: divisor(full...)} + } + } + grouping := func() refAggregation { + a := refAggregation{op: pick("sum", "avg", "min", "max", "count")} + switch rnd.Intn(4) { + case 0: + a.labels = []string{"job"} + case 1: + a.labels = []string{"pod"} + case 2: + a.without, a.labels = true, []string{"pod"} + } + return a + } + chain := []refAggregation{grouping()} + if rnd.Intn(4) == 0 { + chain = append(chain, refAggregation{op: pick("sum", "max", "count")}) + } + return expr, chain +} + +// TestAggregationOverBinaryDifferential: over generated rows (three streams in +// two jobs, values including zero, lines on window edges), generated windows +// (sliding, tumbling, longer and shorter than the step, instant) and generated +// expressions, the proxy's answer to an aggregation over a binary is Loki's: +// the same series, timestamps and values (to 1e-9 relative: sums of floats in +// another order), on VictoriaLogs v1.52 and v1.40. +// +// conformance: semantics/aggregation-over-binary-dropped, loki_api_v1_query_range, loki_api_v1_query +func TestAggregationOverBinaryDifferential(t *testing.T) { + jobs := map[string]string{"a": "x", "b": "x", "c": "y"} + for _, version := range []string{"v1.52.0", "v1.40.0"} { + for _, instant := range []bool{false, true} { + name := fmt.Sprintf("version=%s/instant=%v", version, instant) + t.Run(name, func(t *testing.T) { + rnd := rand.New(rand.NewSource(int64(len(name))*6151 + 3)) + for i := 0; i < 40; i++ { + s := generateUnwrapScenario(rnd, "rate", "", instant) + if s.step > 0 { // the binary evaluates on the step-aligned axis + s.start -= s.start % int64(s.step) + s.end -= s.end % int64(s.step) + } + for j := range s.rows { + pod := s.rows[j].stream[len(`{app="uw",pod="`) : len(`{app="uw",pod="`)+1] + s.rows[j].stream = fmt.Sprintf(`{app="uw",job="%s",pod="%s"}`, jobs[pod], pod) + if version == "v1.40.0" && s.step > 0 && s.rows[j].ts%int64(s.step) == 0 { + // Before v1.45 VictoriaLogs has no stats_query_range offset: its + // epoch buckets [T, T+step) put a line exactly on T in the next + // tumbling window, not Loki's (T-step, T]. Registered apart. + s.rows[j].ts++ + } + } + expr, chain := generateBinaryAggregation(rnd) + query := expr.query(uwWindowText(s.window)) + for _, agg := range chain { + query = agg.wrap(query) + } + want := map[string]map[int64]float64{} + for _, ts := range s.times() { + samples, _, _ := expr.eval(s.rows, ts, int64(s.window)) + for _, agg := range chain { + samples = agg.apply(samples) + } + for _, sample := range samples { + key := binaryLabelKey(sample.labels) + if want[key] == nil { + want[key] = map[int64]float64{} + } + want[key][ts/1e9] = sample.v + } + } + emu := &vlEmulator{t: t, rows: s.rows, field: "amount", version: version} + got, _ := binaryAggregationAnswer(t, emu, query, s.start, s.end, s.step) + label := fmt.Sprintf("%s #%d %s step=%v window=%v rows=%d", name, i, query, s.step, s.window, len(s.rows)) + compareBinaryAggregation(t, label, got, want) + } + }) + } + } +} + +func compareBinaryAggregation(t *testing.T, label string, got, want map[string]map[int64]float64) { + t.Helper() + keys := func(m map[string]map[int64]float64) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out + } + if strings.Join(keys(got), " ") != strings.Join(keys(want), " ") { + t.Fatalf("%s:\nseries %v, Loki has %v\n got %v\nwant %v", label, keys(got), keys(want), got, want) + } + for key, points := range want { + if len(got[key]) != len(points) { + t.Fatalf("%s: %s has %d points, Loki %d\n got %v\nwant %v", label, key, len(got[key]), len(points), got[key], points) + } + for ts, w := range points { + g, ok := got[key][ts] + if !ok || math.Abs(g-w) > 1e-9*math.Max(1, math.Abs(w)) { + t.Fatalf("%s: %s at %d got %v (present %v), Loki has %v", label, key, ts, g, ok, w) + } + } + } +} diff --git a/internal/proxy/binary_aggregation_test.go b/internal/proxy/binary_aggregation_test.go new file mode 100644 index 000000000..dd8c01fd6 --- /dev/null +++ b/internal/proxy/binary_aggregation_test.go @@ -0,0 +1,257 @@ +package proxy + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "github.com/ReliablyObserve/Loki-VL-proxy/internal/cache" +) + +// binaryAggregationAnswer sends query through the real handler against the +// emulated VictoriaLogs and returns label set -> timestamp (s) -> value. +func binaryAggregationAnswer(t *testing.T, emu *vlEmulator, query string, start, end int64, step time.Duration) (map[string]map[int64]float64, string) { + t.Helper() + backend := httptest.NewServer(emu.handler()) + defer backend.Close() + p, err := New(Config{BackendURL: backend.URL, Cache: cache.New(60*time.Second, 1000), LogLevel: "error", BackendMinVersion: "v1.40.0", BackendVersionCheckTimeout: time.Second}) + if err != nil { + t.Fatal(err) + } + if emu.version != "" { + if err := p.ValidateBackendVersionCompatibility(context.Background()); err != nil { + t.Fatal(err) + } + } + params := url.Values{"query": {query}} + target := "/loki/api/v1/query_range?" + if step == 0 { + target = "/loki/api/v1/query?" + params.Set("time", strconv.FormatInt(end/1e9, 10)) + } else { + params.Set("start", strconv.FormatInt(start/1e9, 10)) + params.Set("end", strconv.FormatInt(end/1e9, 10)) + params.Set("step", strconv.FormatInt(int64(step/time.Second), 10)) + } + res := doCompatProxyRequest(p, target+params.Encode(), map[string]string{"X-Scope-OrgID": "0"}) + if res.Code != http.StatusOK { + t.Fatalf("%s: %d %s", query, res.Code, res.Body) + } + var resp struct { + Data struct { + ResultType string `json:"resultType"` + Result []struct { + Metric map[string]string `json:"metric"` + Values [][]any `json:"values"` + Value []any `json:"value"` + } `json:"result"` + } `json:"data"` + } + if err := json.Unmarshal(res.Body.Bytes(), &resp); err != nil { + t.Fatalf("%s: %v %s", query, err, res.Body) + } + got := map[string]map[int64]float64{} + for i, series := range resp.Data.Result { + if i > 0 && metricKeyStr(resp.Data.Result[i-1].Metric) >= metricKeyStr(series.Metric) { + t.Fatalf("%s: series out of Loki's label order: %s", query, res.Body) + } + key := binaryLabelKey(series.Metric) + if got[key] != nil { + t.Fatalf("%s: series %s twice in %s", query, key, res.Body) + } + got[key] = map[int64]float64{} + points := series.Values + if series.Value != nil { + points = [][]any{series.Value} + } + for _, point := range points { + v, err := strconv.ParseFloat(point[1].(string), 64) + if err != nil { + t.Fatalf("%s: value %v", query, point[1]) + } + got[key][int64(point[0].(float64))] = v + } + } + return got, resp.Data.ResultType +} + +// binaryAggregationRows: three streams in two jobs, inside the minute before +// binaryAggregationAt. count_over_time per (job, pod) is p1=3, p2=1, p3=2 and +// sum_over_time of amount is p1=6, p2=4, p3=12. +const binaryAggregationAt = int64(1_800_000_060) * 1e9 + +func binaryAggregationRows() []emuRow { + row := func(offset int64, stream string, value float64) emuRow { + return emuRow{ts: binaryAggregationAt - offset*1e9, stream: stream, value: value} + } + p1, p2, p3 := `{app="uw",job="a",pod="p1"}`, `{app="uw",job="a",pod="p2"}`, `{app="uw",job="b",pod="p3"}` + return []emuRow{ + row(50, p1, 1), row(30, p1, 2), row(10, p1, 3), + row(40, p2, 4), + row(45, p3, 5), row(5, p3, 7), + } +} + +// TestAggregationOverBinaryEdgeCases: a vector aggregation over a binary +// expression aggregates the binary's answer, per step, like Loki +// (pkg/logql/evaluator.go newBinOpStepEvaluator then VectorAggEvaluator.Next): +// both operands, vector matching on the full label set or on()/ignoring() with +// group_left, a scalar on either side, nested binaries, a comparison filtering +// series or answering 0/1 with bool, then sum, avg, min, max and count grouped +// by labels, without labels or not at all, and an aggregation over that. +// +// conformance: semantics/aggregation-over-binary-dropped, loki_api_v1_query, loki_api_v1_query_range +func TestAggregationOverBinaryEdgeCases(t *testing.T) { + const ( + count = `sum by (job, pod) (count_over_time({app="uw"}[1m]))` + amount = `sum by (job, pod) (sum_over_time({app="uw"} | unwrap amount [1m]))` + perJob = `sum by (job) (count_over_time({app="uw"}[1m]))` + ) + ja, jb, none := `{"job":"a"}`, `{"job":"b"}`, `{}` + for _, tc := range []struct { + name, query string + want map[string]float64 + }{ + {"sum by over metric/metric", `sum by (job) (` + amount + ` / ` + count + `)`, map[string]float64{ja: 6, jb: 6}}, + {"avg by", `avg by (job) (` + amount + ` / ` + count + `)`, map[string]float64{ja: 3, jb: 6}}, + {"min by", `min by (job) (` + amount + ` / ` + count + `)`, map[string]float64{ja: 2, jb: 6}}, + {"max by", `max by (job) (` + amount + ` / ` + count + `)`, map[string]float64{ja: 4, jb: 6}}, + {"count by", `count by (job) (` + amount + ` / ` + count + `)`, map[string]float64{ja: 2, jb: 1}}, + {"ungrouped sum", `sum(` + amount + ` / ` + count + `)`, map[string]float64{none: 12}}, + {"sum by ()", `sum by () (` + amount + ` / ` + count + `)`, map[string]float64{none: 12}}, + {"sum without", `sum without (pod) (` + amount + ` / ` + count + `)`, map[string]float64{ja: 6, jb: 6}}, + {"count without folds once", `count without (pod) (` + amount + ` / ` + count + `)`, map[string]float64{ja: 2, jb: 1}}, + {"avg without", `avg without (pod) (` + amount + ` / ` + count + `)`, map[string]float64{ja: 3, jb: 6}}, + {"metric op scalar", `sum by (job) (` + count + ` * 2)`, map[string]float64{ja: 8, jb: 4}}, + {"scalar op metric", `sum(2 * ` + count + `)`, map[string]float64{none: 12}}, + {"metric divided by scalar", `sum(` + count + ` / 2)`, map[string]float64{none: 3}}, + {"comparison filters", `count(` + count + ` > 1)`, map[string]float64{none: 2}}, + {"comparison keeps the vector value", `sum(` + count + ` > 1)`, map[string]float64{none: 5}}, + {"scalar on the left of a comparison", `sum(1 < ` + count + `)`, map[string]float64{none: 5}}, + {"comparison with bool", `sum(` + count + ` > bool 1)`, map[string]float64{none: 2}}, + {"count over bool counts every series", `count(` + count + ` > bool 1)`, map[string]float64{none: 3}}, + {"vector comparison keeps the left value", `sum by (job) (` + amount + ` > ` + count + `)`, map[string]float64{ja: 10, jb: 12}}, + {"vector comparison with bool", `max by (job) (` + count + ` >= bool ` + amount + `)`, map[string]float64{ja: 0, jb: 0}}, + {"nested binary", `sum by (job) ((` + count + ` + ` + count + `) / 2)`, map[string]float64{ja: 4, jb: 2}}, + {"on group_left", `sum by (job) (` + count + ` / on (job) group_left ` + perJob + `)`, map[string]float64{ja: 1, jb: 1}}, + {"ignoring group_left", `max(` + count + ` / ignoring (pod) group_left ` + perJob + `)`, map[string]float64{none: 1}}, + {"aggregation over the aggregation", `max(sum by (job) (` + amount + ` / ` + count + `))`, map[string]float64{none: 6}}, + {"empty after the filter", `sum(` + count + ` > 100)`, map[string]float64{}}, + } { + t.Run(tc.name, func(t *testing.T) { + for _, version := range []string{"v1.52.0", "v1.40.0"} { + emu := &vlEmulator{t: t, rows: binaryAggregationRows(), field: "amount", version: version} + got, resultType := binaryAggregationAnswer(t, emu, tc.query, binaryAggregationAt, binaryAggregationAt, 0) + if resultType != "vector" { + t.Fatalf("%s %s: resultType %q, want vector", version, tc.query, resultType) + } + flat := map[string]float64{} + for key, points := range got { + for ts, v := range points { + if ts != binaryAggregationAt/1e9 { + t.Fatalf("%s %s: sample at %d, want %d", version, tc.query, ts, binaryAggregationAt/1e9) + } + flat[key] = v + } + } + if !reflect.DeepEqual(flat, tc.want) { + t.Fatalf("%s %s:\n got %v\nwant %v", version, tc.query, flat, tc.want) + } + } + }) + } +} + +// The same on query_range: every step aggregates the binary's samples at that +// step, and a step where the binary has no sample has no point. +// +// conformance: semantics/aggregation-over-binary-dropped, loki_api_v1_query_range +func TestAggregationOverBinaryRange(t *testing.T) { + rows := binaryAggregationRows() + // A second minute: p1 once (amount 10), p3 once (amount 1); p2 is silent. + rows = append(rows, + emuRow{ts: binaryAggregationAt + 20e9, stream: `{app="uw",job="a",pod="p1"}`, value: 10}, + emuRow{ts: binaryAggregationAt + 30e9, stream: `{app="uw",job="b",pod="p3"}`, value: 1}, + ) + // Rates: the ratio of the two is amount per line, as above. A sum_over_time + // range query is registered apart (semantics/unwrap-over-time-range-windows). + const ( + count = `sum by (job, pod) (rate({app="uw"}[1m]))` + amount = `sum by (job, pod) (rate({app="uw"} | unwrap amount [1m]))` + ) + t0, t1 := binaryAggregationAt/1e9, binaryAggregationAt/1e9+60 + for _, tc := range []struct { + query string + want map[string]map[int64]float64 + }{ + {`sum by (job) (` + amount + ` / ` + count + `)`, map[string]map[int64]float64{`{"job":"a"}`: {t0: 6, t1: 10}, `{"job":"b"}`: {t0: 6, t1: 1}}}, + {`count(` + count + ` * 60 > bool 1)`, map[string]map[int64]float64{`{}`: {t0: 3, t1: 2}}}, + {`sum(60 * ` + count + ` > 1)`, map[string]map[int64]float64{`{}`: {t0: 5}}}, + {`avg without (pod) (` + count + ` * 180)`, map[string]map[int64]float64{`{"job":"a"}`: {t0: 6, t1: 3}, `{"job":"b"}`: {t0: 6, t1: 3}}}, + } { + for _, version := range []string{"v1.52.0", "v1.40.0"} { + emu := &vlEmulator{t: t, rows: rows, field: "amount", version: version} + got, resultType := binaryAggregationAnswer(t, emu, tc.query, binaryAggregationAt, binaryAggregationAt+60e9, time.Minute) + if resultType != "matrix" { + t.Fatalf("%s %s: resultType %q, want matrix", version, tc.query, resultType) + } + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("%s %s:\n got %v\nwant %v", version, tc.query, got, tc.want) + } + } + } +} + +// The registry's discovered shape (cluster f2066991, inside its label_replace): +// one operand parses JSON, the other logfmt. Each operand reaches VictoriaLogs +// with its own parser, though the whole query names both (the handler drops +// one of two parsers it finds in a query), and the answer is a sample type on +// both endpoints, not the 400 for a log query or a stream. +// +// conformance: semantics/aggregation-over-binary-dropped, loki_api_v1_query, loki_api_v1_query_range +func TestAggregationOverBinaryKeepsEachOperandsParser(t *testing.T) { + const query = `sum by (job) ( sum_over_time( {app="uw"} |= "level=error" | json | avg=5 and bar<25ms | unwrap duration(latency) | __error__!~".*" [5m] ) / count_over_time({app="uw"} | logfmt | label_format foo=bar[5m]) )` + for _, step := range []time.Duration{0, time.Minute} { + emu := &vlEmulator{t: t, rows: binaryAggregationRows(), field: "amount", version: "v1.52.0"} + _, resultType := binaryAggregationAnswer(t, emu, query, binaryAggregationAt-300e9, binaryAggregationAt, step) + if want := map[bool]string{true: "vector", false: "matrix"}[step == 0]; resultType != want { + t.Fatalf("step=%v: resultType %q, want %q", step, resultType, want) + } + var jsonOperand, logfmtOperand bool + for _, call := range emu.calls { + if call.path == "/select/logsql/query" && strings.Contains(call.query, " / ") { + continue // the parser probe of the whole query string + } + jsonOperand = jsonOperand || (strings.Contains(call.query, "unpack_json") && strings.Contains(call.query, "level=error")) + logfmtOperand = logfmtOperand || (strings.Contains(call.query, "unpack_logfmt") && !strings.Contains(call.query, "level=error")) + } + if !jsonOperand || !logfmtOperand { + t.Fatalf("step=%v: an operand lost its parser (json %v, logfmt %v): %v", step, jsonOperand, logfmtOperand, emu.calls) + } + } +} + +// Loki takes an offset on one side of the binary only (each side is its own +// step evaluator); so does the proxy under an aggregation, as for a bare binary. +// +// conformance: semantics/aggregation-over-binary-dropped, loki_api_v1_query +func TestAggregationOverBinaryMixedOffset(t *testing.T) { + rows := append(binaryAggregationRows(), + emuRow{ts: binaryAggregationAt + 20e9, stream: `{app="uw",job="a",pod="p1"}`, value: 10}, + emuRow{ts: binaryAggregationAt + 30e9, stream: `{app="uw",job="b",pod="p3"}`, value: 1}, + ) + // A minute earlier: p1=3, p2=1, p3=2; now: p1=1, p3=1. + query := `sum(sum by (job, pod) (count_over_time({app="uw"}[1m] offset 1m)) / sum by (job, pod) (count_over_time({app="uw"}[1m])))` + emu := &vlEmulator{t: t, rows: rows, field: "amount", version: "v1.52.0"} + got, _ := binaryAggregationAnswer(t, emu, query, 0, binaryAggregationAt+60e9, 0) + if want := map[string]map[int64]float64{`{}`: {binaryAggregationAt/1e9 + 60: 5}}; !reflect.DeepEqual(got, want) { + t.Fatalf("got %v, want %v", got, want) + } +} diff --git a/internal/proxy/binary_logql.go b/internal/proxy/binary_logql.go index 178c05134..b8580d0ac 100644 --- a/internal/proxy/binary_logql.go +++ b/internal/proxy/binary_logql.go @@ -50,16 +50,7 @@ func binaryRequestQuery(r *http.Request) string { } func binaryExprForRequest(r *http.Request) *logqlpkg.BinOpExpr { - query := resolveGrafanaRangeTemplateTokens(r.FormValue("query"), r.FormValue("start"), r.FormValue("end"), r.FormValue("step")) - // The outer handler has already shifted uniform offsets. Mixed offsets - // remain on their individual operands and are applied by each child handler. - if _, stripped, err := extractLogQLOffset(query); err == nil { - query = stripped - } - expr, err := logqlpkg.Parse(query) - if err != nil { - return nil - } + expr := binaryRequestExpr(r) for { switch x := expr.(type) { case *logqlpkg.BinOpExpr: @@ -72,6 +63,22 @@ func binaryExprForRequest(r *http.Request) *logqlpkg.BinOpExpr { } } +// binaryRequestExpr parses the request's query. A label_replace around it is +// applied to the answer by the handler, so the expression is the one inside. +func binaryRequestExpr(r *http.Request) logqlpkg.Expr { + query := resolveGrafanaRangeTemplateTokens(r.FormValue("query"), r.FormValue("start"), r.FormValue("end"), r.FormValue("step")) + // The outer handler has already shifted uniform offsets. Mixed offsets + // remain on their individual operands and are applied by each child handler. + if _, stripped, err := extractLogQLOffset(query); err == nil { + query = stripped + } + expr, err := logqlpkg.Parse(stripOuterLabelReplace(query)) + if err != nil { + return nil + } + return expr +} + type binaryOperandResponse struct { header http.Header body bytes.Buffer diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 6a82167eb..c1ddaa04b 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -2260,6 +2260,9 @@ func (p *Proxy) handleQueryRange(w http.ResponseWriter, r *http.Request) { isMixedOffset := strings.Contains(offsetErr.Error(), "loki-vl-proxy requires a uniform offset") parsedForOffset, _ := logqlpkg.Parse(logqlQuery) _, isBinaryForOffset := parsedForOffset.(*logqlpkg.BinOpExpr) + if _, _, aggregated := binaryAggregationChain(parsedForOffset); aggregated { + isBinaryForOffset = true // an aggregation over one evaluates its sides the same way + } if !isMixedOffset || !isBinaryForOffset { p.writeError(w, http.StatusBadRequest, offsetErr.Error()) p.metrics.RecordRequest("query_range", http.StatusBadRequest, time.Since(start)) @@ -2408,6 +2411,8 @@ func (p *Proxy) handleQueryRange(w http.ResponseWriter, r *http.Request) { } else { p.proxyBinaryMetricQueryRangeVM(sc, r, binOp.Op, leftLogsql, rightLogsql, binOpExprToVMInfo(binOp)) } + } else if p.proxyBinaryAggregation(sc, r, parsedForRouting, "matrix") { + withoutLabels = nil // folded by the aggregation itself } else if isStatsQuery(logsqlQuery) { p.proxyStatsQueryRange(sc, r, logsqlQuery) } else { @@ -2559,6 +2564,9 @@ func (p *Proxy) handleQuery(w http.ResponseWriter, r *http.Request) { isMixedOffset := strings.Contains(offsetErr.Error(), "loki-vl-proxy requires a uniform offset") parsedForOffset, _ := logqlpkg.Parse(logqlQuery) _, isBinaryForOffset := parsedForOffset.(*logqlpkg.BinOpExpr) + if _, _, aggregated := binaryAggregationChain(parsedForOffset); aggregated { + isBinaryForOffset = true // an aggregation over one evaluates its sides the same way + } if !isMixedOffset || !isBinaryForOffset { p.writeError(w, http.StatusBadRequest, offsetErr.Error()) p.metrics.RecordRequest("query", http.StatusBadRequest, time.Since(start)) @@ -2684,6 +2692,8 @@ func (p *Proxy) handleQuery(w http.ResponseWriter, r *http.Request) { } else { p.proxyBinaryMetricQueryVM(sc, r, binOp.Op, leftLogsql, rightLogsql, binOpExprToVMInfo(binOp)) } + } else if p.proxyBinaryAggregation(sc, r, parsedForRoutingQ, "vector") { + withoutLabels = nil // folded by the aggregation itself } else if isStatsQuery(logsqlQuery) { p.proxyStatsQuery(sc, r, logsqlQuery) } else { diff --git a/internal/proxy/vector_matching.go b/internal/proxy/vector_matching.go index 77fd1d81c..4d2c39e81 100644 --- a/internal/proxy/vector_matching.go +++ b/internal/proxy/vector_matching.go @@ -4,9 +4,11 @@ import ( "context" "encoding/json" "fmt" + "maps" "math" "net/http" "regexp" + "slices" "sort" "strconv" "strings" @@ -186,9 +188,10 @@ func applyWithoutVector(body []byte, exclude map[string]bool, merge withoutMerge } } - // Build result + // Build result, in label order as Loki sorts the vector (pkg/logql/engine.go). result := []map[string]interface{}{} - for _, g := range groups { + for _, key := range slices.Sorted(maps.Keys(groups)) { + g := groups[key] result = append(result, map[string]interface{}{ "metric": g.metric, "value": []interface{}{g.ts, strconv.FormatFloat(g.value, 'f', -1, 64)}, @@ -272,7 +275,8 @@ func applyWithoutMatrix(body []byte, exclude map[string]bool, merge withoutMerge } result := []map[string]interface{}{} - for _, group := range groups { + for _, key := range slices.Sorted(maps.Keys(groups)) { // label order, as Loki's sort.Sort(matrix) + group := groups[key] type sampleTime struct { key string seconds float64 diff --git a/test/e2e-compat/aggregation_over_binary_test.go b/test/e2e-compat/aggregation_over_binary_test.go new file mode 100644 index 000000000..d01794d1c --- /dev/null +++ b/test/e2e-compat/aggregation_over_binary_test.go @@ -0,0 +1,96 @@ +//go:build e2e + +package e2e_compat + +import ( + "net/http" + "net/url" + "strconv" + "strings" + "testing" +) + +// Loki evaluates a vector aggregation over a binary expression by joining the +// operands per series (pkg/logql/evaluator.go newBinOpStepEvaluator and +// vectorBinop: matching on every label or on()/ignoring(), group_left, a +// literal on either side, a comparison filtering or answering 0/1 with bool) +// and aggregating the joined series per step (VectorAggEvaluator.Next). The +// proxy kept only the first operand of the binary under the aggregation, or +// answered 400 on an instant query (semantics/aggregation-over-binary-dropped). + +// TestCompat_AggregationOverBinary: sum, avg, min, max and count, grouped by a +// stream label, by a parsed label, without one and not at all, over binaries of +// two metrics, a metric and a literal, a comparison with and without bool, +// on()/ignoring() with group_left and nested binaries, over the unwrap sum +// fixture's logfmt streams, on every window shape and at an instant: the +// proxy's series, timestamps and values are Loki's. +// conformance: semantics/aggregation-over-binary-dropped, loki_api_v1_query_range, loki_api_v1_query +func TestCompat_AggregationOverBinary(t *testing.T) { + fx := ensureUnwrapSumFixtures(t) + fx.waitForLoki(t) + for _, shape := range unwrapSumShapes { + sel := fx.selLogfmt + amount := "sum by (app, route) (rate(" + sel + " | logfmt | unwrap amount [" + shape.window + "]))" + lines := "sum by (app, route) (count_over_time(" + sel + " | logfmt [" + shape.window + "]))" + perApp := "sum by (app) (count_over_time(" + sel + " [" + shape.window + "]))" + for _, query := range []string{ + "sum by (app) (" + amount + " / " + lines + ")", + "avg by (route) (" + amount + " / " + lines + ")", + "min(" + amount + " / " + lines + ")", + "max without (route) (" + amount + " / " + lines + ")", + "count without (route) (" + amount + " / " + lines + ")", + "sum by (app) (" + lines + " * 2)", + "max(2 * " + lines + ")", + // A rate of the amount is about 0.22 per series on every window + // (a line per 30 s per route, amounts 0.5 to 13): the thresholds + // keep some series and drop others. + "count(" + amount + " > 0.2)", + "sum by (route) (" + amount + " > 0.1)", + "sum(" + amount + " > bool 0.2)", + "count(0.2 < " + amount + ")", + "sum by (app) (" + lines + " > " + amount + ")", + "max by (app) (" + amount + " >= bool " + lines + ")", + "sum by (app) (" + lines + " / on (app) group_left " + perApp + ")", + "max by (route) (" + lines + " / ignoring (route) group_left " + perApp + ")", + "sum by (app) ((" + lines + " + " + lines + ") / 2)", + "max(sum by (app) (" + amount + " / " + lines + "))", + } { + t.Run(shape.name+"/"+query, func(t *testing.T) { + loki := shape.run(t, fx, lokiURL, query) + proxy := shape.run(t, fx, proxyURL, query) + compareUnwrapSumSeries(t, query, loki, proxy) + if shape.step == 0 { + assertInstantVector(t, query, fx) + } + }) + } + } +} + +// TestCompat_AggregationOverBinaryMixedParsers: the registry's discovered shape +// (cluster f2066991, inside its label_replace): the operands parse with +// different parsers, JSON on one side and logfmt on the other, so the whole +// query names both; each operand keeps its own. An instant query only: a +// sum_over_time range query is registered apart +// (semantics/unwrap-over-time-range-windows). +// conformance: semantics/aggregation-over-binary-dropped, loki_api_v1_query +func TestCompat_AggregationOverBinaryMixedParsers(t *testing.T) { + fx := ensureUnwrapSumFixtures(t) + fx.waitForLoki(t) + shape := unwrapSumShape{name: "instant", window: "5m", stop: unwrapSumShapes[len(unwrapSumShapes)-1].stop} + query := "sum by (route) (sum by (route) (sum_over_time(" + fx.selJSON + " | json | unwrap amount [5m])) / sum by (route) (count_over_time(" + fx.selLogfmt + " | logfmt [5m])))" + loki := shape.run(t, fx, lokiURL, query) + proxy := shape.run(t, fx, proxyURL, query) + compareUnwrapSumSeries(t, query, loki, proxy) + assertInstantVector(t, query, fx) +} + +func assertInstantVector(t *testing.T, query string, fx *unwrapSumFixtures) { + t.Helper() + at := fx.s0.Add(unwrapSumShapes[len(unwrapSumShapes)-1].stop) + params := url.Values{"query": {query}, "time": {strconv.FormatInt(at.UnixNano(), 10)}} + status, body, resp := doJSONGET(t, proxyURL+"/loki/api/v1/query?"+params.Encode(), map[string]string{"X-Scope-OrgID": "0"}) + if status != http.StatusOK || extractMap(resp, "data")["resultType"] != "vector" { + t.Fatalf("%s: proxy answered %d %s, want a vector", query, status, strings.TrimSpace(string(body))) + } +} From 3f7371efcb88545fe917015a677e155c62a5ef19 Mon Sep 17 00:00:00 2001 From: szibis Date: Mon, 12 Oct 2026 13:11:49 +0200 Subject: [PATCH 12/18] fix: a top-level binary with a label_replace operand stays on the binary evaluator Only an aggregation routed from under a label_replace reads the expression inside it; a binary whose first operand is a label_replace is read whole again, as before. The differential also draws one-to-one ignoring() over a label one side lacks, and new tests pin a uniform offset on both sides. --- .../aggregation-over-binary-dropped.yaml | 2 +- internal/proxy/binary_aggregation.go | 5 ++ .../binary_aggregation_differential_test.go | 40 +++++++++++---- internal/proxy/binary_aggregation_test.go | 50 +++++++++++++++++++ internal/proxy/binary_logql.go | 17 ++++--- 5 files changed, 95 insertions(+), 19 deletions(-) diff --git a/conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml b/conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml index f624568f6..a12d8bec0 100644 --- a/conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml +++ b/conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml @@ -58,7 +58,7 @@ gap: area: 'internal/proxy binary_aggregation.go (proxyBinaryAggregation, foldBinaryAggregation); proxy.go query and query_range routing' loki: 'pkg/logql/evaluator.go:791 newBinOpStepEvaluator, :955 vectorBinop, :1096 resultMetric, :400-630 newVectorAggEvaluator / VectorAggEvaluator.Next; pkg/logql/syntax/ast.go:1944 MergeBinOp' proxy: 'the operands as native stats queries, the join and the fold over their aggregated series' - planned_test: 'TestCompat_AggregationOverBinary and TestCompat_AggregationOverBinaryMixedParsers (test/e2e-compat, group core); TestAggregationOverBinaryEdgeCases, TestAggregationOverBinaryRange, TestAggregationOverBinaryDifferential, TestAggregationOverBinaryKeepsEachOperandsParser and TestAggregationOverBinaryMixedOffset (internal/proxy)' + planned_test: 'TestCompat_AggregationOverBinary and TestCompat_AggregationOverBinaryMixedParsers (test/e2e-compat, group core); TestAggregationOverBinaryEdgeCases, TestAggregationOverBinaryRange, TestAggregationOverBinaryDifferential, TestAggregationOverBinaryKeepsEachOperandsParser, TestAggregationOverBinaryMixedOffset, TestAggregationOverBinaryUniformOffset and TestBinaryWithLabelReplaceOperandStaysBinary (internal/proxy)' note: 'The e2e is written and not yet run against the stack.' clusters: - f2066991 diff --git a/internal/proxy/binary_aggregation.go b/internal/proxy/binary_aggregation.go index 6c427a63c..e6ab3edb5 100644 --- a/internal/proxy/binary_aggregation.go +++ b/internal/proxy/binary_aggregation.go @@ -47,6 +47,11 @@ func (p *Proxy) proxyBinaryAggregation(w http.ResponseWriter, r *http.Request, r return false } chain, binary, ok := binaryAggregationChain(binaryRequestExpr(r)) + if !ok { + // Routed from under a label_replace, which the handler applies to the answer. + expr, _ := logqlpkg.Parse(stripOuterLabelReplace(binaryRequestQuery(r))) + chain, binary, ok = binaryAggregationChain(expr) + } if !ok { p.writeError(w, http.StatusBadRequest, "unsupported aggregation over a binary expression") return true diff --git a/internal/proxy/binary_aggregation_differential_test.go b/internal/proxy/binary_aggregation_differential_test.go index 8d23ffbcd..5cc284b5d 100644 --- a/internal/proxy/binary_aggregation_differential_test.go +++ b/internal/proxy/binary_aggregation_differential_test.go @@ -89,7 +89,7 @@ func refValue(op string, a, b float64) (float64, bool) { case "*": return a * b, true case "/": - return a / b, true // divisors are never zero here: Loki answers NaN, see the report + return a / b, true // the generator never divides by zero: Loki answers NaN there, the proxy ±Inf case ">": return a, a > b case "<": @@ -199,11 +199,20 @@ func (n *refNode) eval(rows []emuRow, t, window int64) ([]refSample, float64, bo continue } labels := s.labels - if !n.groupLeft && n.match == "on" { + if !n.groupLeft && n.match != "" { // one-to-one: resultMetric keeps on(), drops ignoring() labels = map[string]string{} - for _, name := range n.labels { - if v := s.labels[name]; v != "" { - labels[name] = v + if n.match == "on" { + for _, name := range n.labels { + if v := s.labels[name]; v != "" { + labels[name] = v + } + } + } else { + for k, v := range s.labels { + labels[k] = v + } + for _, name := range n.labels { + delete(labels, name) } } } @@ -289,7 +298,9 @@ func (a refAggregation) apply(in []refSample) []refSample { } // generateBinaryAggregation draws an expression: operands grouped by (job, pod) -// or by (job), joined on every label, on() or ignoring() with group_left, with +// or by (job), joined on every label, on() or ignoring() with group_left, +// ignoring() one-to-one (`sum by (job) (A) + ignoring(machine) sum by (job, +// machine) (B)`, one machine per job), with // a literal on either side, a comparison with or without bool, and binaries // nested in binaries; then one or two aggregations over it. A divisor is never // a value that can be zero. @@ -300,7 +311,7 @@ func generateBinaryAggregation(rnd *rand.Rand) (*refNode, []refAggregation) { literal := func(v float64) *refNode { return &refNode{scalar: &v} } full := []string{"job", "pod"} var expr *refNode - switch rnd.Intn(7) { + switch rnd.Intn(8) { case 0: // metric op metric on every label op := pick("+", "-", "*", "/") right := operand(full...) @@ -333,6 +344,13 @@ func generateBinaryAggregation(rnd *rand.Rand) (*refNode, []refAggregation) { match, labels = "ignoring", []string{"pod"} } expr = &refNode{op: pick("/", "*", "-"), match: match, labels: labels, groupLeft: true, l: operand(full...), r: divisor("job")} + case 7: // one-to-one ignoring() a label only one side has (one machine per job) + left := &refNode{fn: pick("count", "rate", "amount"), by: []string{"job"}} + if rnd.Intn(2) == 0 { + expr = &refNode{op: pick("+", "-", "*"), match: "ignoring", labels: []string{"machine"}, l: left, r: operand("job", "machine")} + } else { + expr = &refNode{op: "/", match: "ignoring", labels: []string{"machine"}, l: left, r: divisor("job", "machine")} + } case 5: // one-to-one on() the full label set: the answer keeps only those expr = &refNode{op: pick("+", "/"), match: "on", labels: full, l: operand(full...), r: divisor(full...)} default: // nested @@ -385,11 +403,11 @@ func TestAggregationOverBinaryDifferential(t *testing.T) { } for j := range s.rows { pod := s.rows[j].stream[len(`{app="uw",pod="`) : len(`{app="uw",pod="`)+1] - s.rows[j].stream = fmt.Sprintf(`{app="uw",job="%s",pod="%s"}`, jobs[pod], pod) - if version == "v1.40.0" && s.step > 0 && s.rows[j].ts%int64(s.step) == 0 { + s.rows[j].stream = fmt.Sprintf(`{app="uw",job="%s",machine="m-%[1]s",pod="%s"}`, jobs[pod], pod) + if version == "v1.40.0" && s.step > 0 && s.rows[j].ts%1e9 == 0 { // Before v1.45 VictoriaLogs has no stats_query_range offset: its - // epoch buckets [T, T+step) put a line exactly on T in the next - // tumbling window, not Loki's (T-step, T]. Registered apart. + // epoch buckets [T, T+b) put a line exactly on a bucket edge T in + // the next window, not Loki's (T-range, T], on any operand alone. s.rows[j].ts++ } } diff --git a/internal/proxy/binary_aggregation_test.go b/internal/proxy/binary_aggregation_test.go index dd8c01fd6..7b06cd05b 100644 --- a/internal/proxy/binary_aggregation_test.go +++ b/internal/proxy/binary_aggregation_test.go @@ -255,3 +255,53 @@ func TestAggregationOverBinaryMixedOffset(t *testing.T) { t.Fatalf("got %v, want %v", got, want) } } + +// A top-level binary whose first operand is a label_replace stays on the binary +// evaluator: only an aggregation routed from under a label_replace is read +// without it. +// +// conformance: semantics/aggregation-over-binary-dropped, loki_api_v1_query +func TestBinaryWithLabelReplaceOperandStaysBinary(t *testing.T) { + const count = `sum by (job, pod) (count_over_time({app="uw"}[1m]))` + for query, want := range map[string]map[string]float64{ + `label_replace(` + count + `, "dst", "$1", "job", "(.*)") / on (job, pod) ` + count: { + `{"job":"a","pod":"p1"}`: 1, `{"job":"a","pod":"p2"}`: 1, `{"job":"b","pod":"p3"}`: 1}, + `label_replace(` + count + `, "dst", "$1", "job", "(.*)") * on (job, pod) ` + count: { + `{"job":"a","pod":"p1"}`: 9, `{"job":"a","pod":"p2"}`: 1, `{"job":"b","pod":"p3"}`: 4}, + } { + emu := &vlEmulator{t: t, rows: binaryAggregationRows(), field: "amount", version: "v1.52.0"} + got, _ := binaryAggregationAnswer(t, emu, query, 0, binaryAggregationAt, 0) + flat := map[string]float64{} + for key, points := range got { + for _, v := range points { + flat[key] = v + } + } + if !reflect.DeepEqual(flat, want) { + t.Fatalf("%s:\n got %v\nwant %v", query, flat, want) + } + } +} + +// The same offset on both sides (the discovered shape's second form) shifts the +// windows and stamps the samples at the requested times, on both endpoints. +// +// conformance: semantics/aggregation-over-binary-dropped, loki_api_v1_query, loki_api_v1_query_range +func TestAggregationOverBinaryUniformOffset(t *testing.T) { + const ( + count = `sum by (job, pod) (count_over_time({app="uw"}[1m] offset 1h))` + amount = `sum by (job, pod) (sum_over_time({app="uw"} | unwrap amount [1m] offset 1h))` + rate = `sum by (job, pod) (rate({app="uw"} | unwrap amount [1m] offset 1h))` + ) + at := binaryAggregationAt + 3600e9 + emu := &vlEmulator{t: t, rows: binaryAggregationRows(), field: "amount", version: "v1.52.0"} + got, _ := binaryAggregationAnswer(t, emu, `sum by (job) (`+amount+` / `+count+`)`, 0, at, 0) + if want := map[string]map[int64]float64{`{"job":"a"}`: {at / 1e9: 6}, `{"job":"b"}`: {at / 1e9: 6}}; !reflect.DeepEqual(got, want) { + t.Fatalf("instant: got %v, want %v", got, want) + } + emu = &vlEmulator{t: t, rows: binaryAggregationRows(), field: "amount", version: "v1.52.0"} + got, _ = binaryAggregationAnswer(t, emu, `max(`+rate+` / `+count+`)`, at-60e9, at, time.Minute) + if want := map[string]map[int64]float64{`{}`: {at / 1e9: 6.0 / 60}}; !reflect.DeepEqual(got, want) { + t.Fatalf("range: got %v, want %v", got, want) + } +} diff --git a/internal/proxy/binary_logql.go b/internal/proxy/binary_logql.go index b8580d0ac..93c275497 100644 --- a/internal/proxy/binary_logql.go +++ b/internal/proxy/binary_logql.go @@ -63,20 +63,23 @@ func binaryExprForRequest(r *http.Request) *logqlpkg.BinOpExpr { } } -// binaryRequestExpr parses the request's query. A label_replace around it is -// applied to the answer by the handler, so the expression is the one inside. +// binaryRequestExpr parses the request's query. func binaryRequestExpr(r *http.Request) logqlpkg.Expr { + expr, err := logqlpkg.Parse(binaryRequestQuery(r)) + if err != nil { + return nil + } + return expr +} + +func binaryRequestQuery(r *http.Request) string { query := resolveGrafanaRangeTemplateTokens(r.FormValue("query"), r.FormValue("start"), r.FormValue("end"), r.FormValue("step")) // The outer handler has already shifted uniform offsets. Mixed offsets // remain on their individual operands and are applied by each child handler. if _, stripped, err := extractLogQLOffset(query); err == nil { query = stripped } - expr, err := logqlpkg.Parse(stripOuterLabelReplace(query)) - if err != nil { - return nil - } - return expr + return query } type binaryOperandResponse struct { From 9caaabb5b0ba4ab9a6a47d2f706738b8b3f21767 Mon Sep 17 00:00:00 2001 From: szibis Date: Mon, 12 Oct 2026 13:12:35 +0200 Subject: [PATCH 13/18] chore(conformance): visual fix proof page for aggregation-over-binary-dropped --- bench/visual/spec.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/bench/visual/spec.json b/bench/visual/spec.json index 13fe29413..982117e53 100644 --- a/bench/visual/spec.json +++ b/bench/visual/spec.json @@ -330,6 +330,19 @@ "semantics/unwrap-rate-ungrouped-sum-per-stream" ] }, + { + "id": "explore-sum-by-over-binary", + "kind": "explore", + "query": "sum by (service_name) (rate({service_name=\"api-gateway\"}[$__auto]) / 2)", + "covers": [ + "semantics/aggregation-over-binary-dropped", + "metric-series-identity", + "loki_api_v1_query_range" + ], + "fixes": [ + "semantics/aggregation-over-binary-dropped" + ] + }, { "id": "explore-label-browser", "kind": "label-browser", From cc7dcbc7ecec983e46679221b60fb24ec2d877ee Mon Sep 17 00:00:00 2001 From: szibis Date: Mon, 12 Oct 2026 13:12:57 +0200 Subject: [PATCH 14/18] chore(conformance): record #741 as the fix of aggregation-over-binary-dropped --- CHANGELOG.md | 2 +- .../cases/semantics/aggregation-over-binary-dropped.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e9eb13f8..4081d24aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,7 +89,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 the query names both `| json` and `| logfmt` (the shape the differential corpus found inside a `label_replace`), an offset on one side only is accepted as for a bare binary, and the series of an aggregation folded by the proxy come in - Loki's label order (#TBD). + Loki's label order (#741). ## [2.11.2] - 2026-10-12 diff --git a/conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml b/conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml index a12d8bec0..3e73cb2c7 100644 --- a/conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml +++ b/conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml @@ -52,7 +52,7 @@ compare: Loki's series and values: the binary per series, then the aggregation per step. gap: status: fixed - fixed_by: "#TBD" + fixed_by: "#741" impact: explore-visible effort: M area: 'internal/proxy binary_aggregation.go (proxyBinaryAggregation, foldBinaryAggregation); proxy.go query and query_range routing' From a009fa7c4c73a83e9642ba4564d908c5a0d8681e Mon Sep 17 00:00:00 2001 From: szibis Date: Mon, 12 Oct 2026 13:37:48 +0200 Subject: [PATCH 15/18] fix: a label transform around an aggregation over a binary applies to the folded answer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit label_replace and label_join around `sum by (job) (a / b)` (the discovered shape of cluster f2066991 on /query) route by the expression inside them on query_range as on query, and the transform applies to the folded answer; the transforms now apply to an instant vector as well as a matrix. A transform is stripped only when its call is the whole expression: `label_replace(a, ...) / b` is a binary whose operand applies its own. The series a fold makes keep every label set apart (escaped keys: a value holding `,` or `=` no longer folds with another set) and come in Loki's order, labels.Compare: name then value, label by label. The differential also starts half its range scenarios off the step grid. Registers semantics/binary-division-by-zero-nan (open): `x / 0` answers ±Inf where Loki answers NaN. --- CHANGELOG.md | 13 +- .../aggregation-over-binary-dropped.yaml | 30 ++- .../binary-division-by-zero-nan.yaml | 40 ++++ internal/proxy/binary_aggregation.go | 18 +- .../binary_aggregation_differential_test.go | 35 +++- internal/proxy/binary_aggregation_test.go | 188 +++++++++++++++--- internal/proxy/binary_logql.go | 6 +- internal/proxy/proxy.go | 8 + internal/proxy/vector_matching.go | 17 +- .../aggregation_over_binary_test.go | 66 +++++- 10 files changed, 354 insertions(+), 67 deletions(-) create mode 100644 conformance/registry/cases/semantics/binary-division-by-zero-nan.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 4081d24aa..ed3060d19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,10 +86,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 stays a native VictoriaLogs stats query through its own handler, the proxy joins the aggregated operand series and folds them per step, with no raw rows and work bounded by the operands' series. An operand keeps its own parser when - the query names both `| json` and `| logfmt` (the shape the differential - corpus found inside a `label_replace`), an offset on one side only is accepted as for a - bare binary, and the series of an aggregation folded by the proxy come in - Loki's label order (#741). + the query names both `| json` and `| logfmt`, and an offset on one side only + is accepted as for a bare binary. A `label_replace` or `label_join` around + such an aggregation (the shape the differential corpus found on /query) + routes by the expression inside it on query and query_range and applies to + the folded answer; the label transforms now apply to an instant vector too, + and a transform is stripped only when its call is the whole query + (`label_replace(a, ...) / b` is a binary whose operand applies its own). The series the proxy folds + (this, and every `without`) keep every label set apart, a value holding `,` + or `=` included, and come in Loki's order (`labels.Compare`) (#741). ## [2.11.2] - 2026-10-12 diff --git a/conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml b/conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml index 3e73cb2c7..69c5dd8a8 100644 --- a/conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml +++ b/conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml @@ -33,32 +33,42 @@ why: | with the aggregation's operator. The expression is read from the request, so an operand keeps its parser when the whole query names `| json` and `| logfmt` (the discovered cluster below). A mixed offset (one side only) is - accepted as for a bare binary, and the folded series are in Loki's label - order. + accepted as for a bare binary. + + A label_replace or label_join around the aggregation, the discovered shape + (cluster f2066991 on /query), routes by the expression inside it on both + endpoints and the transform applies to the folded answer; the transforms + now apply to an instant vector as well as a matrix. A transform is stripped + only when its call is the whole expression (`label_replace(a, ...) / b` is a + binary whose operand applies its own). The folded series keep every label set + apart (escaped keys: `a="b,c=d"` is not `a="b",c="d"`) and come in Loki's + order (labels.Compare: name then value, label by label). + + Owed after the PRs that own them land: the series limit counted on the folded + answer (semantics/non-sum-vector-aggregation-unreduced's fold) and NaN-skipping + min and max (semantics/binary-division-by-zero-nan). request: method: GET - path: /loki/api/v1/query_range + path: /loki/api/v1/query params: - query: 'sum by (job) (rate({app="api-gateway"}[1m]) / count_over_time({app="api-gateway"}[1m]))' - start: 'now-1h' - end: 'now' - step: '60' + query: 'label_replace( sum by (job) ( sum_over_time( {service_name="api-gateway"} |= "level=error" | json | avg=5 and bar<25ms | unwrap duration(latency) | __error__!~".*" [5m] ) / count_over_time({service_name="api-gateway"} | logfmt | label_format foo=bar[5m]) ), "foo", "$1", "service", "(.*):.*" )' headers: X-Scope-OrgID: "0" compare: mode: observed fields: [resultType, series, values] expect: | - Loki's series and values: the binary per series, then the aggregation per step. + Loki's vector: the binary per series, the aggregation per step, then the + label_replace; the same series and values as Loki. gap: status: fixed fixed_by: "#741" impact: explore-visible effort: M - area: 'internal/proxy binary_aggregation.go (proxyBinaryAggregation, foldBinaryAggregation); proxy.go query and query_range routing' + area: 'internal/proxy binary_aggregation.go (proxyBinaryAggregation, foldBinaryAggregation); proxy.go query and query_range routing; vector_matching.go (fold keys and order, label transforms on vectors); range_metric_compat.go stripOuterCalls' loki: 'pkg/logql/evaluator.go:791 newBinOpStepEvaluator, :955 vectorBinop, :1096 resultMetric, :400-630 newVectorAggEvaluator / VectorAggEvaluator.Next; pkg/logql/syntax/ast.go:1944 MergeBinOp' proxy: 'the operands as native stats queries, the join and the fold over their aggregated series' - planned_test: 'TestCompat_AggregationOverBinary and TestCompat_AggregationOverBinaryMixedParsers (test/e2e-compat, group core); TestAggregationOverBinaryEdgeCases, TestAggregationOverBinaryRange, TestAggregationOverBinaryDifferential, TestAggregationOverBinaryKeepsEachOperandsParser, TestAggregationOverBinaryMixedOffset, TestAggregationOverBinaryUniformOffset and TestBinaryWithLabelReplaceOperandStaysBinary (internal/proxy)' + planned_test: 'TestCompat_AggregationOverBinary, TestCompat_AggregationOverBinaryMixedParsers and TestCompat_AggregationOverBinaryUnderLabelTransforms (test/e2e-compat, group core); TestAggregationOverBinaryEdgeCases, TestAggregationOverBinaryRange, TestAggregationOverBinaryDifferential, TestAggregationOverBinaryKeepsEachOperandsParser, TestAggregationOverBinaryMixedOffset, TestAggregationOverBinaryUniformOffset, TestAggregationOverBinaryUnderLabelTransforms, TestBinaryWithLabelReplaceOperandStaysBinary, TestStripLabelTransformsWholeCallOnly and TestWithoutFoldLabelOrderAndKeys (internal/proxy)' note: 'The e2e is written and not yet run against the stack.' clusters: - f2066991 diff --git a/conformance/registry/cases/semantics/binary-division-by-zero-nan.yaml b/conformance/registry/cases/semantics/binary-division-by-zero-nan.yaml new file mode 100644 index 000000000..c0c64e840 --- /dev/null +++ b/conformance/registry/cases/semantics/binary-division-by-zero-nan.yaml @@ -0,0 +1,40 @@ +id: semantics/binary-division-by-zero-nan +title: A binary division or modulo by zero answers ±Inf where Loki answers NaN (open) +proves: + - loki_api_v1_query_range + - loki_api_v1_query +consumer: explore +why: | + Loki 3.7.7 guards both operators: `/` and `%` with a zero right-hand value + answer NaN (pkg/logql/syntax/ast.go:1978-2006 MergeBinOp, "guard against + divide by zero"), for a vector against a vector and against a literal + alike, and a NaN sample is encoded as "NaN". The proxy's binary evaluator + computes `a / b` and `math.Mod(a, b)` as Go floats (internal/proxy + binary_matching.go binarySampleValue), so `x / 0` answers "+Inf" or "-Inf" + (`0 / 0` and `x % 0` are NaN in both). A dashboard dividing by a count that + can be zero, or by a sum of values that can be zero, draws a different value. + Under an aggregation the difference carries on: Loki's min and max skip a NaN + after the first value of a group (pkg/logql/evaluator.go VectorAggEvaluator.Next, + `group.value < s.F || math.IsNaN(group.value)`), while the proxy's fold + propagates it (vector_matching.go withoutMerge.combine, math.Min/math.Max). +request: + method: GET + path: /loki/api/v1/query + params: + query: 'sum by (app) (count_over_time({app="api-gateway"}[1m])) / 0' + headers: + X-Scope-OrgID: "0" +compare: + mode: observed + fields: [values] + expect: | + "NaN" for every series, as Loki answers. +gap: + status: open + impact: explore-visible + effort: S + area: 'internal/proxy binary_matching.go binarySampleValue (/ and %); vector_matching.go withoutMerge.combine (min, max over NaN)' + loki: 'pkg/logql/syntax/ast.go:1978-2006 MergeBinOp; pkg/logql/evaluator.go VectorAggEvaluator.Next (min, max)' + proxy: '±Inf for x/0; min and max propagate NaN' + planned_test: 'TestCompat_BinaryDivisionByZero (test/e2e-compat, group core); a binarySampleValue edge table and the aggregation-over-binary differential with zero divisors (internal/proxy)' + clusters: [] diff --git a/internal/proxy/binary_aggregation.go b/internal/proxy/binary_aggregation.go index e6ab3edb5..9ba671232 100644 --- a/internal/proxy/binary_aggregation.go +++ b/internal/proxy/binary_aggregation.go @@ -36,6 +36,11 @@ func binaryAggregationChain(expr logqlpkg.Expr) ([]*logqlpkg.VectorAggregation, return chain, binary, !leftScalar || !rightScalar } +func isBinaryExpr(expr logqlpkg.Expr) bool { + _, ok := expr.(*logqlpkg.BinOpExpr) + return ok +} + // proxyBinaryAggregation answers a vector aggregation over a binary expression // when routed is one: each operand is its own VictoriaLogs stats query, joined // by the binary evaluator, and the aggregation folds the joined series per @@ -48,12 +53,19 @@ func (p *Proxy) proxyBinaryAggregation(w http.ResponseWriter, r *http.Request, r } chain, binary, ok := binaryAggregationChain(binaryRequestExpr(r)) if !ok { - // Routed from under a label_replace, which the handler applies to the answer. - expr, _ := logqlpkg.Parse(stripOuterLabelReplace(binaryRequestQuery(r))) + // Routed from under a label_replace or label_join, which the handler + // applies to the folded answer. + expr, _ := logqlpkg.Parse(stripLabelTransforms(binaryRequestQuery(r))) chain, binary, ok = binaryAggregationChain(expr) } if !ok { - p.writeError(w, http.StatusBadRequest, "unsupported aggregation over a binary expression") + // The request's own query does not parse as the routed one did: answer + // Loki's parse error for it. + msg := "unsupported aggregation over a binary expression" + if err := validateLogQLSyntax(binaryRequestQuery(r)); err != "" { + msg = truncateQueryError(err) + } + p.writeError(w, http.StatusBadRequest, msg) return true } buf := &bufferedResponseWriter{} diff --git a/internal/proxy/binary_aggregation_differential_test.go b/internal/proxy/binary_aggregation_differential_test.go index 5cc284b5d..317f9799d 100644 --- a/internal/proxy/binary_aggregation_differential_test.go +++ b/internal/proxy/binary_aggregation_differential_test.go @@ -72,6 +72,14 @@ func (n *refNode) query(window string) string { return q + " " + side(n.r) } +// joinsVectors reports whether the expression joins two vectors anywhere. +func (n *refNode) joinsVectors() bool { + if n.op == "" { + return false + } + return (n.l.scalar == nil && n.r.scalar == nil) || n.l.joinsVectors() || n.r.joinsVectors() +} + func refStreamLabels(stream string) map[string]string { out := map[string]string{} for _, m := range emuStreamLabelRE.FindAllStringSubmatch(stream, -1) { @@ -382,9 +390,9 @@ func generateBinaryAggregation(rnd *rand.Rand) (*refNode, []refAggregation) { // TestAggregationOverBinaryDifferential: over generated rows (three streams in // two jobs, values including zero, lines on window edges), generated windows -// (sliding, tumbling, longer and shorter than the step, instant) and generated -// expressions, the proxy's answer to an aggregation over a binary is Loki's: -// the same series, timestamps and values (to 1e-9 relative: sums of floats in +// (sliding, tumbling, longer and shorter than the step, starting on and off the +// step grid, instant) and generated expressions, the proxy's answer to an +// aggregation over a binary is Loki's: the same series, timestamps and values (to 1e-9 relative: sums of floats in // another order), on VictoriaLogs v1.52 and v1.40. // // conformance: semantics/aggregation-over-binary-dropped, loki_api_v1_query_range, loki_api_v1_query @@ -397,27 +405,40 @@ func TestAggregationOverBinaryDifferential(t *testing.T) { rnd := rand.New(rand.NewSource(int64(len(name))*6151 + 3)) for i := 0; i < 40; i++ { s := generateUnwrapScenario(rnd, "rate", "", instant) - if s.step > 0 { // the binary evaluates on the step-aligned axis + if s.step > 0 && rnd.Intn(2) == 0 { // the other half starts off the step grid s.start -= s.start % int64(s.step) s.end -= s.end % int64(s.step) } + expr, chain := generateBinaryAggregation(rnd) + times := s.times() + if s.step > 0 && expr.joinsVectors() { + // A join of two vectors evaluates on the step-aligned grid, as + // Loki's query frontend with align_queries_with_step (the + // compose stack's Loki); with a literal only, at + // start+k*step, as a Loki without it + // (TestBinaryMixedWindowsShareStepAlignedAxis). + aligned := s + aligned.start -= aligned.start % int64(s.step) + aligned.end -= aligned.end % int64(s.step) + times = aligned.times() + } for j := range s.rows { pod := s.rows[j].stream[len(`{app="uw",pod="`) : len(`{app="uw",pod="`)+1] s.rows[j].stream = fmt.Sprintf(`{app="uw",job="%s",machine="m-%[1]s",pod="%s"}`, jobs[pod], pod) if version == "v1.40.0" && s.step > 0 && s.rows[j].ts%1e9 == 0 { // Before v1.45 VictoriaLogs has no stats_query_range offset: its // epoch buckets [T, T+b) put a line exactly on a bucket edge T in - // the next window, not Loki's (T-range, T], on any operand alone. + // the next window, not Loki's (T-range, T], on any operand alone + // (versions/stats-bucket-label-v1.45). s.rows[j].ts++ } } - expr, chain := generateBinaryAggregation(rnd) query := expr.query(uwWindowText(s.window)) for _, agg := range chain { query = agg.wrap(query) } want := map[string]map[int64]float64{} - for _, ts := range s.times() { + for _, ts := range times { samples, _, _ := expr.eval(s.rows, ts, int64(s.window)) for _, agg := range chain { samples = agg.apply(samples) diff --git a/internal/proxy/binary_aggregation_test.go b/internal/proxy/binary_aggregation_test.go index 7b06cd05b..6c1f11a16 100644 --- a/internal/proxy/binary_aggregation_test.go +++ b/internal/proxy/binary_aggregation_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "net/url" "reflect" + "sort" "strconv" "strings" "testing" @@ -59,7 +60,7 @@ func binaryAggregationAnswer(t *testing.T, emu *vlEmulator, query string, start, } got := map[string]map[int64]float64{} for i, series := range resp.Data.Result { - if i > 0 && metricKeyStr(resp.Data.Result[i-1].Metric) >= metricKeyStr(series.Metric) { + if i > 0 && !testLokiLabelsLess(resp.Data.Result[i-1].Metric, series.Metric) { t.Fatalf("%s: series out of Loki's label order: %s", query, res.Body) } key := binaryLabelKey(series.Metric) @@ -209,7 +210,7 @@ func TestAggregationOverBinaryRange(t *testing.T) { } } -// The registry's discovered shape (cluster f2066991, inside its label_replace): +// The registry's discovered shape (cluster f2066991), with and without its label_replace: // one operand parses JSON, the other logfmt. Each operand reaches VictoriaLogs // with its own parser, though the whole query names both (the handler drops // one of two parsers it finds in a query), and the answer is a sample type on @@ -217,23 +218,25 @@ func TestAggregationOverBinaryRange(t *testing.T) { // // conformance: semantics/aggregation-over-binary-dropped, loki_api_v1_query, loki_api_v1_query_range func TestAggregationOverBinaryKeepsEachOperandsParser(t *testing.T) { - const query = `sum by (job) ( sum_over_time( {app="uw"} |= "level=error" | json | avg=5 and bar<25ms | unwrap duration(latency) | __error__!~".*" [5m] ) / count_over_time({app="uw"} | logfmt | label_format foo=bar[5m]) )` + const inner = `sum by (job) ( sum_over_time( {app="uw"} |= "level=error" | json | avg=5 and bar<25ms | unwrap duration(latency) | __error__!~".*" [5m] ) / count_over_time({app="uw"} | logfmt | label_format foo=bar[5m]) )` for _, step := range []time.Duration{0, time.Minute} { - emu := &vlEmulator{t: t, rows: binaryAggregationRows(), field: "amount", version: "v1.52.0"} - _, resultType := binaryAggregationAnswer(t, emu, query, binaryAggregationAt-300e9, binaryAggregationAt, step) - if want := map[bool]string{true: "vector", false: "matrix"}[step == 0]; resultType != want { - t.Fatalf("step=%v: resultType %q, want %q", step, resultType, want) - } - var jsonOperand, logfmtOperand bool - for _, call := range emu.calls { - if call.path == "/select/logsql/query" && strings.Contains(call.query, " / ") { - continue // the parser probe of the whole query string + for _, query := range []string{inner, `label_replace( ` + inner + `, "foo", "$1", "service", "(.*):.*" )`} { + emu := &vlEmulator{t: t, rows: binaryAggregationRows(), field: "amount", version: "v1.52.0"} + _, resultType := binaryAggregationAnswer(t, emu, query, binaryAggregationAt-300e9, binaryAggregationAt, step) + if want := map[bool]string{true: "vector", false: "matrix"}[step == 0]; resultType != want { + t.Fatalf("step=%v: resultType %q, want %q", step, resultType, want) + } + var jsonOperand, logfmtOperand bool + for _, call := range emu.calls { + if call.path == "/select/logsql/query" && strings.Contains(call.query, " / ") { + continue // the parser probe of the whole query string + } + jsonOperand = jsonOperand || (strings.Contains(call.query, "unpack_json") && strings.Contains(call.query, "level=error")) + logfmtOperand = logfmtOperand || (strings.Contains(call.query, "unpack_logfmt") && !strings.Contains(call.query, "level=error")) + } + if !jsonOperand || !logfmtOperand { + t.Fatalf("step=%v: an operand lost its parser (json %v, logfmt %v): %v", step, jsonOperand, logfmtOperand, emu.calls) } - jsonOperand = jsonOperand || (strings.Contains(call.query, "unpack_json") && strings.Contains(call.query, "level=error")) - logfmtOperand = logfmtOperand || (strings.Contains(call.query, "unpack_logfmt") && !strings.Contains(call.query, "level=error")) - } - if !jsonOperand || !logfmtOperand { - t.Fatalf("step=%v: an operand lost its parser (json %v, logfmt %v): %v", step, jsonOperand, logfmtOperand, emu.calls) } } } @@ -257,8 +260,8 @@ func TestAggregationOverBinaryMixedOffset(t *testing.T) { } // A top-level binary whose first operand is a label_replace stays on the binary -// evaluator: only an aggregation routed from under a label_replace is read -// without it. +// evaluator, and the operand applies its own transform: only a transform around +// the whole expression is applied to the binary's answer. // // conformance: semantics/aggregation-over-binary-dropped, loki_api_v1_query func TestBinaryWithLabelReplaceOperandStaysBinary(t *testing.T) { @@ -268,17 +271,24 @@ func TestBinaryWithLabelReplaceOperandStaysBinary(t *testing.T) { `{"job":"a","pod":"p1"}`: 1, `{"job":"a","pod":"p2"}`: 1, `{"job":"b","pod":"p3"}`: 1}, `label_replace(` + count + `, "dst", "$1", "job", "(.*)") * on (job, pod) ` + count: { `{"job":"a","pod":"p1"}`: 9, `{"job":"a","pod":"p2"}`: 1, `{"job":"b","pod":"p3"}`: 4}, + `label_replace(` + count + `, "dst", "$1", "job", "(.*)") * 2`: { + `{"dst":"a","job":"a","pod":"p1"}`: 6, `{"dst":"a","job":"a","pod":"p2"}`: 2, `{"dst":"b","job":"b","pod":"p3"}`: 4}, + `sum by (dst) (label_replace(` + count + `, "dst", "$1", "job", "(.*)") / ` + count + `)`: {}, + `sum by (dst) (label_replace(` + count + `, "dst", "$1", "job", "(.*)") / ignoring (dst) group_left ` + count + `)`: { + `{"dst":"a"}`: 2, `{"dst":"b"}`: 1}, } { - emu := &vlEmulator{t: t, rows: binaryAggregationRows(), field: "amount", version: "v1.52.0"} - got, _ := binaryAggregationAnswer(t, emu, query, 0, binaryAggregationAt, 0) - flat := map[string]float64{} - for key, points := range got { - for _, v := range points { - flat[key] = v + for _, step := range []time.Duration{0, time.Minute} { + emu := &vlEmulator{t: t, rows: binaryAggregationRows(), field: "amount", version: "v1.52.0"} + got, _ := binaryAggregationAnswer(t, emu, query, binaryAggregationAt, binaryAggregationAt, step) + flat := map[string]float64{} + for key, points := range got { + for _, v := range points { + flat[key] = v + } + } + if !reflect.DeepEqual(flat, want) { + t.Fatalf("step=%v %s:\n got %v\nwant %v", step, query, flat, want) } - } - if !reflect.DeepEqual(flat, want) { - t.Fatalf("%s:\n got %v\nwant %v", query, flat, want) } } } @@ -305,3 +315,125 @@ func TestAggregationOverBinaryUniformOffset(t *testing.T) { t.Fatalf("range: got %v, want %v", got, want) } } + +// A label_replace or label_join around the aggregation (the discovered shape, +// cluster f2066991) applies to the folded answer, on both endpoints: the +// aggregation is not handed to the translator, which kept its first operand. +// +// conformance: semantics/aggregation-over-binary-dropped, loki_api_v1_query, loki_api_v1_query_range +func TestAggregationOverBinaryUnderLabelTransforms(t *testing.T) { + const ( + count = `sum by (job, pod) (rate({app="uw"}[1m]))` + amount = `sum by (job, pod) (rate({app="uw"} | unwrap amount [1m]))` + ) + inner := `sum by (job) (` + amount + ` / ` + count + `)` + at := binaryAggregationAt / 1e9 + for _, tc := range []struct { + query string + want map[string]float64 + }{ + {`label_replace(` + inner + `, "dst", "$1", "job", "(.*)")`, map[string]float64{`{"dst":"a","job":"a"}`: 6, `{"dst":"b","job":"b"}`: 6}}, + {`label_join(` + inner + `, "dst", "-", "job", "job")`, map[string]float64{`{"dst":"a-a","job":"a"}`: 6, `{"dst":"b-b","job":"b"}`: 6}}, + {`label_replace(label_replace(` + inner + `, "dst", "$1", "job", "(.*)"), "job", "x", "", "")`, map[string]float64{`{"dst":"a","job":"x"}`: 6, `{"dst":"b","job":"x"}`: 6}}, + } { + for _, step := range []time.Duration{0, time.Minute} { + emu := &vlEmulator{t: t, rows: binaryAggregationRows(), field: "amount", version: "v1.52.0"} + got, _ := binaryAggregationAnswer(t, emu, tc.query, binaryAggregationAt, binaryAggregationAt, step) + want := map[string]map[int64]float64{} + for key, v := range tc.want { + want[key] = map[int64]float64{at: v} + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("step=%v %s:\n got %v\nwant %v", step, tc.query, got, want) + } + } + } +} + +// testLokiLabelsLess is labels.Compare(a, b) < 0 (Prometheus model/labels), +// written apart from the proxy's: the sorted (name, value) pairs compared in +// turn, a shorter list first when one is a prefix of the other. +func testLokiLabelsLess(a, b map[string]string) bool { + pairs := func(m map[string]string) [][2]string { + out := make([][2]string, 0, len(m)) + for k, v := range m { + out = append(out, [2]string{k, v}) + } + sort.Slice(out, func(i, j int) bool { return out[i][0] < out[j][0] }) + return out + } + pa, pb := pairs(a), pairs(b) + for i := range pa { + if i == len(pb) { + return false + } + if pa[i] != pb[i] { + if pa[i][0] != pb[i][0] { + return pa[i][0] < pb[i][0] + } + return pa[i][1] < pb[i][1] + } + } + return len(pa) < len(pb) +} + +// A fold keeps every label set apart, a label value with a comma and an equals +// sign included, and answers the series in Loki's label order, which is not the +// order of their `name=value,...` text. +// +// conformance: semantics/aggregation-over-binary-dropped +func TestWithoutFoldLabelOrderAndKeys(t *testing.T) { + for _, tc := range []struct { + name string + metric []map[string]string + want []map[string]string + }{ + {"a name that extends another", []map[string]string{{"a0": "y"}, {"a": "x"}}, []map[string]string{{"a": "x"}, {"a0": "y"}}}, + {"a value that extends another", []map[string]string{{"route": "/a b"}, {"route": "/a", "status": "200"}}, []map[string]string{{"route": "/a", "status": "200"}, {"route": "/a b"}}}, + {"a value with a comma and an equals sign", []map[string]string{{"a": "b", "c": "d"}, {"a": "b,c=d"}}, []map[string]string{{"a": "b", "c": "d"}, {"a": "b,c=d"}}}, + {"a prefix first", []map[string]string{{"a": "x", "b": "y"}, {"a": "x"}}, []map[string]string{{"a": "x"}, {"a": "x", "b": "y"}}}, + } { + for _, resultType := range []string{"vector", "matrix"} { + result := make([]map[string]any, 0, len(tc.metric)) + for _, metric := range tc.metric { + entry := map[string]any{"metric": metric, "value": []any{1.0, "1"}} + if resultType == "matrix" { + entry = map[string]any{"metric": metric, "values": [][]any{{1.0, "1"}}} + } + result = append(result, entry) + } + body, _ := json.Marshal(map[string]any{"status": "success", "data": map[string]any{"resultType": resultType, "result": result}}) + var resp struct { + Data struct { + Result []struct { + Metric map[string]string `json:"metric"` + Value []any `json:"value"` + Values [][]any `json:"values"` + } `json:"result"` + } `json:"data"` + } + if err := json.Unmarshal(applyWithoutGrouping(body, []string{"absent"}, "count"), &resp); err != nil { + t.Fatal(err) + } + var got []map[string]string + for _, series := range resp.Data.Result { + got = append(got, series.Metric) + point := series.Value + if resultType == "matrix" { + point = series.Values[0] + } + if point[1] != "1" { + t.Fatalf("%s %s: %v counts %v series, want 1 (folded with another)", tc.name, resultType, series.Metric, point[1]) + } + } + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("%s %s: got %v, want %v", tc.name, resultType, got, tc.want) + } + for i := 1; i < len(tc.want); i++ { + if !testLokiLabelsLess(tc.want[i-1], tc.want[i]) { + t.Fatalf("%s: the expectation is not in Loki's order", tc.name) + } + } + } + } +} diff --git a/internal/proxy/binary_logql.go b/internal/proxy/binary_logql.go index 93c275497..dfae140d0 100644 --- a/internal/proxy/binary_logql.go +++ b/internal/proxy/binary_logql.go @@ -72,8 +72,12 @@ func binaryRequestExpr(r *http.Request) logqlpkg.Expr { return expr } +// binaryRequestQuery is the request's query as the handler reads it before +// rewriting it: incomplete unwrap stubs dropped, Grafana range tokens resolved +// and a uniform offset stripped. func binaryRequestQuery(r *http.Request) string { - query := resolveGrafanaRangeTemplateTokens(r.FormValue("query"), r.FormValue("start"), r.FormValue("end"), r.FormValue("step")) + query := stripIncompleteUnwrapStubs(r.FormValue("query")) + query = resolveGrafanaRangeTemplateTokens(query, r.FormValue("start"), r.FormValue("end"), r.FormValue("step")) // The outer handler has already shifted uniform offsets. Mixed offsets // remain on their individual operands and are applied by each child handler. if _, stripped, err := extractLogQLOffset(query); err == nil { diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index c1ddaa04b..204414d79 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -2399,6 +2399,10 @@ func (p *Proxy) handleQueryRange(w http.ResponseWriter, r *http.Request) { r = cloneMetricQueryRequest(r, stripLabelTransforms(binaryRequestQuery(r))) } parsedForRouting, _ := logqlpkg.Parse(routedQuery) + if _, _, aggregated := binaryAggregationChain(parsedForRouting); routedQuery == logqlQuery && (aggregated || isBinaryExpr(parsedForRouting)) { + // The label transforms belong to an operand, which applies its own. + labelReplaceSpecs, labelJoinSpec = nil, nil + } if binOp, ok := parsedForRouting.(*logqlpkg.BinOpExpr); ok { // Binary metric expression: sum(rate(...)) / sum(rate(...)) // translateBinOpSide handles scalar literals (e.g. * 100) without translation. @@ -2682,6 +2686,10 @@ func (p *Proxy) handleQuery(w http.ResponseWriter, r *http.Request) { r = cloneMetricQueryRequest(r, stripLabelTransforms(binaryRequestQuery(r))) } parsedForRoutingQ, _ := logqlpkg.Parse(routedQuery) + if _, _, aggregated := binaryAggregationChain(parsedForRoutingQ); routedQuery == logqlQuery && (aggregated || isBinaryExpr(parsedForRoutingQ)) { + // The label transforms belong to an operand, which applies its own. + labelReplaceSpecs, labelJoinSpec = nil, nil + } if binOp, ok := parsedForRoutingQ.(*logqlpkg.BinOpExpr); ok { leftLogsql, leftErr := p.translateBinOpSide(r.Context(), binOp.Left) rightLogsql, rightErr := p.translateBinOpSide(r.Context(), binOp.Right) diff --git a/internal/proxy/vector_matching.go b/internal/proxy/vector_matching.go index 4d2c39e81..1899cb8e5 100644 --- a/internal/proxy/vector_matching.go +++ b/internal/proxy/vector_matching.go @@ -155,6 +155,7 @@ func applyWithoutVector(body []byte, exclude map[string]bool, merge withoutMerge ts interface{} } groups := make(map[string]*groupEntry) + labelSets := make(map[string]map[string]string) for _, series := range resp.Data.Result { // Strip excluded labels @@ -165,7 +166,8 @@ func applyWithoutVector(body []byte, exclude map[string]bool, merge withoutMerge } } - key := metricKeyStr(filtered) + key := binaryLabelKey(filtered) // escaped: `a="b,c=d"` is not `a="b",c="d"` + labelSets[key] = filtered val := 0.0 var ts interface{} if len(series.Value) >= 2 { @@ -190,7 +192,7 @@ func applyWithoutVector(body []byte, exclude map[string]bool, merge withoutMerge // Build result, in label order as Loki sorts the vector (pkg/logql/engine.go). result := []map[string]interface{}{} - for _, key := range slices.Sorted(maps.Keys(groups)) { + for _, key := range keysInLabelOrder(labelSets) { g := groups[key] result = append(result, map[string]interface{}{ "metric": g.metric, @@ -230,6 +232,7 @@ func applyWithoutMatrix(body []byte, exclude map[string]bool, merge withoutMerge order map[string]interface{} } groups := make(map[string]*groupedSeries) + labelSets := make(map[string]map[string]string) for _, series := range resp.Data.Result { filtered := make(map[string]string) for k, v := range series.Metric { @@ -237,7 +240,8 @@ func applyWithoutMatrix(body []byte, exclude map[string]bool, merge withoutMerge filtered[k] = v } } - key := metricKeyStr(filtered) + key := binaryLabelKey(filtered) // escaped: `a="b,c=d"` is not `a="b",c="d"` + labelSets[key] = filtered group := groups[key] if group == nil { metric := make(map[string]interface{}, len(filtered)) @@ -275,7 +279,7 @@ func applyWithoutMatrix(body []byte, exclude map[string]bool, merge withoutMerge } result := []map[string]interface{}{} - for _, key := range slices.Sorted(maps.Keys(groups)) { // label order, as Loki's sort.Sort(matrix) + for _, key := range keysInLabelOrder(labelSets) { // as Loki's sort.Sort(matrix) group := groups[key] type sampleTime struct { key string @@ -761,6 +765,11 @@ var sortCallRE = regexp.MustCompile(`(^|[^\w])sort(_desc)?\s*\(`) // Sortable: a sort or sort_desc anywhere in it). func queryIsSortable(logql string) bool { return sortCallRE.MatchString(logql) } +// keysInLabelOrder returns the keys of the label sets in Loki's series order. +func keysInLabelOrder(sets map[string]map[string]string) []string { + return slices.SortedFunc(maps.Keys(sets), func(a, b string) int { return compareLabelSets(sets[a], sets[b]) }) +} + // compareLabelSets is Prometheus' labels.Compare: label by label in name // order, the name first and then the value, a shorter prefix first. func compareLabelSets(a, b map[string]string) int { diff --git a/test/e2e-compat/aggregation_over_binary_test.go b/test/e2e-compat/aggregation_over_binary_test.go index d01794d1c..8900c3688 100644 --- a/test/e2e-compat/aggregation_over_binary_test.go +++ b/test/e2e-compat/aggregation_over_binary_test.go @@ -5,9 +5,11 @@ package e2e_compat import ( "net/http" "net/url" + "reflect" "strconv" "strings" "testing" + "time" ) // Loki evaluates a vector aggregation over a binary expression by joining the @@ -68,26 +70,70 @@ func TestCompat_AggregationOverBinary(t *testing.T) { } // TestCompat_AggregationOverBinaryMixedParsers: the registry's discovered shape -// (cluster f2066991, inside its label_replace): the operands parse with -// different parsers, JSON on one side and logfmt on the other, so the whole -// query names both; each operand keeps its own. An instant query only: a -// sum_over_time range query is registered apart +// (cluster f2066991): a label_replace around an aggregation over a binary whose +// operands parse with different parsers, JSON on one side and logfmt on the +// other, so the whole query names both; each operand keeps its own and the +// transform applies to the folded answer. The cluster's own /query is sent as +// recorded (over the setup data it may be empty on both, so it pins the answer's +// type and equality); the fixture's variant pins the values. An instant query +// only: a sum_over_time range query is registered apart // (semantics/unwrap-over-time-range-windows). // conformance: semantics/aggregation-over-binary-dropped, loki_api_v1_query func TestCompat_AggregationOverBinaryMixedParsers(t *testing.T) { fx := ensureUnwrapSumFixtures(t) fx.waitForLoki(t) + t.Run("cluster f2066991", func(t *testing.T) { + at := time.Now() + loki := unwrapInstant(t, lokiURL, aggregationOverBinaryCluster, at) + proxy := unwrapInstant(t, proxyURL, aggregationOverBinaryCluster, at) + if !reflect.DeepEqual(loki, proxy) { + t.Fatalf("Loki %v, proxy %v", loki, proxy) + } + assertInstantVectorAt(t, aggregationOverBinaryCluster, at) + }) shape := unwrapSumShape{name: "instant", window: "5m", stop: unwrapSumShapes[len(unwrapSumShapes)-1].stop} - query := "sum by (route) (sum by (route) (sum_over_time(" + fx.selJSON + " | json | unwrap amount [5m])) / sum by (route) (count_over_time(" + fx.selLogfmt + " | logfmt [5m])))" - loki := shape.run(t, fx, lokiURL, query) - proxy := shape.run(t, fx, proxyURL, query) - compareUnwrapSumSeries(t, query, loki, proxy) - assertInstantVector(t, query, fx) + inner := "sum by (route) (sum by (route) (sum_over_time(" + fx.selJSON + " | json | unwrap amount [5m])) / sum by (route) (count_over_time(" + fx.selLogfmt + " | logfmt [5m])))" + for _, query := range []string{inner, `label_replace(` + inner + `, "dst", "$1", "route", "/(.*)")`} { + loki := shape.run(t, fx, lokiURL, query) + proxy := shape.run(t, fx, proxyURL, query) + compareUnwrapSumSeries(t, query, loki, proxy) + assertInstantVector(t, query, fx) + } +} + +// aggregationOverBinaryCluster is the first query of cluster f2066991 +// (conformance/registry/generated/parity-discovery.json). +const aggregationOverBinaryCluster = `label_replace( sum by (job) ( sum_over_time( {service_name="api-gateway"} |= "level=error" | json | avg=5 and bar<25ms | unwrap duration(latency) | __error__!~".*" [5m] ) / count_over_time({service_name="api-gateway"} | logfmt | label_format foo=bar[5m]) ), "foo", "$1", "service", "(.*):.*" )` + +// TestCompat_AggregationOverBinaryUnderLabelTransforms: a label_replace or a +// label_join around an aggregation over a binary is applied to the folded +// answer, on every window shape and at an instant. +// conformance: semantics/aggregation-over-binary-dropped, loki_api_v1_query_range, loki_api_v1_query +func TestCompat_AggregationOverBinaryUnderLabelTransforms(t *testing.T) { + fx := ensureUnwrapSumFixtures(t) + fx.waitForLoki(t) + for _, shape := range unwrapSumShapes { + amount := "sum by (app, route) (rate(" + fx.selLogfmt + " | logfmt | unwrap amount [" + shape.window + "]))" + lines := "sum by (app, route) (rate(" + fx.selLogfmt + " | logfmt [" + shape.window + "]))" + inner := "sum by (route) (" + amount + " / " + lines + ")" + for _, query := range []string{ + `label_replace(` + inner + `, "dst", "$1", "route", "/(.*)")`, + `label_join(` + inner + `, "dst", "-", "route", "route")`, + } { + t.Run(shape.name+"/"+query, func(t *testing.T) { + compareUnwrapSumSeries(t, query, shape.run(t, fx, lokiURL, query), shape.run(t, fx, proxyURL, query)) + }) + } + } } func assertInstantVector(t *testing.T, query string, fx *unwrapSumFixtures) { t.Helper() - at := fx.s0.Add(unwrapSumShapes[len(unwrapSumShapes)-1].stop) + assertInstantVectorAt(t, query, fx.s0.Add(unwrapSumShapes[len(unwrapSumShapes)-1].stop)) +} + +func assertInstantVectorAt(t *testing.T, query string, at time.Time) { + t.Helper() params := url.Values{"query": {query}, "time": {strconv.FormatInt(at.UnixNano(), 10)}} status, body, resp := doJSONGET(t, proxyURL+"/loki/api/v1/query?"+params.Encode(), map[string]string{"X-Scope-OrgID": "0"}) if status != http.StatusOK || extractMap(resp, "data")["resultType"] != "vector" { From 7d0a132a7a5af1d54f82cd3264c0eb86319ec54b Mon Sep 17 00:00:00 2001 From: szibis Date: Mon, 12 Oct 2026 13:44:15 +0200 Subject: [PATCH 16/18] fix: a label transform clones the request from its own query, and the differential pins start+k*step Under a label_replace or label_join the request is cloned to the expression inside it from the request's own query, not from the handler's rewritten one, which may have lost the parser stage an operand needs (a query naming both `| json` and `| logfmt`). The aggregation differential's reference evaluates at start+k*step, as a default Loki does; a join of two vectors off the step grid answers on the aligned grid instead and is registered apart as semantics/binary-step-grid (open). A transform's warnings and statistics are pinned. --- CHANGELOG.md | 12 ++-- .../aggregation-over-binary-dropped.yaml | 16 +++-- .../cases/semantics/binary-step-grid.yaml | 49 +++++++++++++ conformance/registry/generated/wiring.json | 35 ++++++++-- conformance/reports/parity-gaps.md | 70 ++++++++++--------- internal/proxy/binary_aggregation.go | 8 +-- .../binary_aggregation_differential_test.go | 24 +++---- internal/proxy/binary_aggregation_test.go | 18 +++++ internal/proxy/vector_matching.go | 3 + .../aggregation_over_binary_test.go | 12 +++- 10 files changed, 177 insertions(+), 70 deletions(-) create mode 100644 conformance/registry/cases/semantics/binary-step-grid.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index ed3060d19..237cf27ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,12 +89,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 the query names both `| json` and `| logfmt`, and an offset on one side only is accepted as for a bare binary. A `label_replace` or `label_join` around such an aggregation (the shape the differential corpus found on /query) - routes by the expression inside it on query and query_range and applies to - the folded answer; the label transforms now apply to an instant vector too, - and a transform is stripped only when its call is the whole query - (`label_replace(a, ...) / b` is a binary whose operand applies its own). The series the proxy folds - (this, and every `without`) keep every label set apart, a value holding `,` - or `=` included, and come in Loki's order (`labels.Compare`) (#741). + is evaluated the same way and the transform applies to the folded answer, + with each operand still reading the request's own parsers; a transform inside + an operand (`sum(label_replace(a, ...) / b)`) is applied by that operand, not + again to the answer. The series the proxy folds (this, and every `without`) + keep every label set apart, a value holding `,` or `=` included, and come in + Loki's order (`labels.Compare`) (#741). ## [2.11.2] - 2026-10-12 diff --git a/conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml b/conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml index 69c5dd8a8..d96346481 100644 --- a/conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml +++ b/conformance/registry/cases/semantics/aggregation-over-binary-dropped.yaml @@ -37,13 +37,17 @@ why: | A label_replace or label_join around the aggregation, the discovered shape (cluster f2066991 on /query), routes by the expression inside it on both - endpoints and the transform applies to the folded answer; the transforms - now apply to an instant vector as well as a matrix. A transform is stripped - only when its call is the whole expression (`label_replace(a, ...) / b` is a - binary whose operand applies its own). The folded series keep every label set + endpoints and the transform applies to the folded answer. The request is + cloned to the expression inside the transform from the request's own query, + so an operand keeps its parser; a transform inside an operand is applied by + that operand, not again to the answer. The folded series keep every label set apart (escaped keys: `a="b,c=d"` is not `a="b",c="d"`) and come in Loki's order (labels.Compare: name then value, label by label). + A start off the step grid is pinned for a binary with a literal; a join of + two vectors answers on the aligned grid, registered apart + (semantics/binary-step-grid). + Owed after the PRs that own them land: the series limit counted on the folded answer (semantics/non-sum-vector-aggregation-unreduced's fold) and NaN-skipping min and max (semantics/binary-division-by-zero-nan). @@ -65,10 +69,10 @@ gap: fixed_by: "#741" impact: explore-visible effort: M - area: 'internal/proxy binary_aggregation.go (proxyBinaryAggregation, foldBinaryAggregation); proxy.go query and query_range routing; vector_matching.go (fold keys and order, label transforms on vectors); range_metric_compat.go stripOuterCalls' + area: 'internal/proxy binary_aggregation.go (proxyBinaryAggregation, foldBinaryAggregation); proxy.go query and query_range routing; vector_matching.go (fold keys and order)' loki: 'pkg/logql/evaluator.go:791 newBinOpStepEvaluator, :955 vectorBinop, :1096 resultMetric, :400-630 newVectorAggEvaluator / VectorAggEvaluator.Next; pkg/logql/syntax/ast.go:1944 MergeBinOp' proxy: 'the operands as native stats queries, the join and the fold over their aggregated series' - planned_test: 'TestCompat_AggregationOverBinary, TestCompat_AggregationOverBinaryMixedParsers and TestCompat_AggregationOverBinaryUnderLabelTransforms (test/e2e-compat, group core); TestAggregationOverBinaryEdgeCases, TestAggregationOverBinaryRange, TestAggregationOverBinaryDifferential, TestAggregationOverBinaryKeepsEachOperandsParser, TestAggregationOverBinaryMixedOffset, TestAggregationOverBinaryUniformOffset, TestAggregationOverBinaryUnderLabelTransforms, TestBinaryWithLabelReplaceOperandStaysBinary, TestStripLabelTransformsWholeCallOnly and TestWithoutFoldLabelOrderAndKeys (internal/proxy)' + planned_test: 'TestCompat_AggregationOverBinary, TestCompat_AggregationOverBinaryMixedParsers and TestCompat_AggregationOverBinaryUnderLabelTransforms (test/e2e-compat, group core); TestAggregationOverBinaryEdgeCases, TestAggregationOverBinaryRange, TestAggregationOverBinaryDifferential, TestAggregationOverBinaryKeepsEachOperandsParser, TestAggregationOverBinaryMixedOffset, TestAggregationOverBinaryUniformOffset, TestAggregationOverBinaryUnderLabelTransforms, TestBinaryWithLabelReplaceOperandStaysBinary, TestWithoutFoldLabelOrderAndKeys and TestLabelTransformsKeepVectorAndWarnings (internal/proxy)' note: 'The e2e is written and not yet run against the stack.' clusters: - f2066991 diff --git a/conformance/registry/cases/semantics/binary-step-grid.yaml b/conformance/registry/cases/semantics/binary-step-grid.yaml new file mode 100644 index 000000000..37905db7e --- /dev/null +++ b/conformance/registry/cases/semantics/binary-step-grid.yaml @@ -0,0 +1,49 @@ +id: semantics/binary-step-grid +title: A binary joining two vectors answers on the step-aligned grid, a binary with a literal at start+k*step (open) +proves: + - window-bounds-and-step-alignment + - loki_api_v1_query_range +consumer: explore +why: | + Loki 3.7.7 evaluates a range query at start+k*step (pkg/logql/engine.go, + the step evaluator from params.Start()); its query frontend moves start and + end down to step multiples first only with `align_queries_with_step: true` + (pkg/querier/queryrange/queryrangebase/step_align.go StepAlignMiddleware), + which the compose stack's Loki sets (test/e2e-compat/loki-local-config.yaml) + and a default Loki does not. + + The proxy evaluates a metric at start+k*step, as a default Loki does, except + where a binary joins two vectors: alignBinaryRangeRequest + (internal/proxy/binary_logql.go) moves that request to the aligned grid so a + sliding-window operand (start+k*step) and a tumbling one (VictoriaLogs' + epoch buckets) share an axis (TestBinaryMixedWindowsShareStepAlignedAxis). + A binary with a literal is not moved. So with a start off the step grid, + `sum(rate(a[1m]) / count_over_time(a[1m]))` answers at aligned times, which + is the stack Loki's answer but not a default Loki's, and `sum(rate(a[1m]) / + 2)` at start+k*step, which is a default Loki's but not the stack's. Grafana + sends step-aligned starts for most ranges, which hides it. +request: + method: GET + path: /loki/api/v1/query_range + params: + query: 'sum(rate({app="api-gateway"}[1m]) / count_over_time({app="api-gateway"}[1m]))' + start: '1800000017' + end: '1800000617' + step: '60' + headers: + X-Scope-OrgID: "0" +compare: + mode: observed + fields: [values] + expect: | + Samples at start+k*step on a default Loki; at step multiples with + align_queries_with_step. One rule for every binary, whichever is chosen. +gap: + status: open + impact: edge + effort: M + area: 'internal/proxy binary_logql.go alignBinaryRangeRequest (vector joins only)' + loki: 'pkg/logql/engine.go step evaluation at start+k*step; pkg/querier/queryrange/queryrangebase/step_align.go StepAlignMiddleware (align_queries_with_step)' + proxy: 'vector joins aligned, literal binaries and single metrics at start+k*step' + planned_test: 'TestAggregationOverBinaryDifferential with a start off the step grid for vector joins (internal/proxy; skipped there today)' + clusters: [] diff --git a/conformance/registry/generated/wiring.json b/conformance/registry/generated/wiring.json index 5483dc8a6..41c787153 100644 --- a/conformance/registry/generated/wiring.json +++ b/conformance/registry/generated/wiring.json @@ -181,6 +181,8 @@ "test/e2e-compat/metadata_scan_admission_test.go" ], "loki_api_v1_query": [ + "internal/proxy/binary_aggregation_differential_test.go", + "internal/proxy/binary_aggregation_test.go", "internal/proxy/instant_result_type_test.go", "internal/proxy/label_replace_semantics_test.go", "internal/proxy/parenthesized_range_test.go", @@ -188,6 +190,7 @@ "internal/proxy/tenant_query_limits_test.go", "internal/proxy/tumbling_axis_parity_test.go", "internal/proxy/unwrap_conversion_error_test.go", + "test/e2e-compat/aggregation_over_binary_test.go", "test/e2e-compat/extracted_suffix_collision_test.go", "test/e2e-compat/instant_result_type_test.go", "test/e2e-compat/label_replace_new_label_test.go", @@ -200,6 +203,8 @@ ], "loki_api_v1_query_range": [ "internal/proxy/backend_heavy_test.go", + "internal/proxy/binary_aggregation_differential_test.go", + "internal/proxy/binary_aggregation_test.go", "internal/proxy/drilldown_breakdown_exact_test.go", "internal/proxy/drilldown_breakdown_matrix_test.go", "internal/proxy/extracted_suffix_collision_test.go", @@ -215,6 +220,7 @@ "internal/proxy/unwrap_conversion_error_test.go", "internal/translator/extracted_suffix_test.go", "internal/translator/regexp_escaping_test.go", + "test/e2e-compat/aggregation_over_binary_test.go", "test/e2e-compat/drilldown_field_breakdown_test.go", "test/e2e-compat/extracted_suffix_collision_test.go", "test/e2e-compat/heavy_query_protection_test.go", @@ -360,6 +366,11 @@ "internal/proxy/labels_cache_warm_test.go", "test/e2e-compat/stack_profiles_test.go" ], + "semantics/aggregation-over-binary-dropped": [ + "internal/proxy/binary_aggregation_differential_test.go", + "internal/proxy/binary_aggregation_test.go", + "test/e2e-compat/aggregation_over_binary_test.go" + ], "semantics/backend-aborted-response": [ "internal/proxy/backend_response_guard_review_test.go", "internal/proxy/backend_response_guard_test.go" @@ -558,7 +569,7 @@ "test/e2e-compat/unwrap_ungrouped_sum_test.go" ] }, - "registry_items": 301, + "registry_items": 303, "routes_without_registry_item": [], "test_to_items": { "cmd/proxy/main_test.go": [ @@ -609,6 +620,16 @@ "operator-configurable-limits", "semantics/backend-aborted-response" ], + "internal/proxy/binary_aggregation_differential_test.go": [ + "loki_api_v1_query", + "loki_api_v1_query_range", + "semantics/aggregation-over-binary-dropped" + ], + "internal/proxy/binary_aggregation_test.go": [ + "loki_api_v1_query", + "loki_api_v1_query_range", + "semantics/aggregation-over-binary-dropped" + ], "internal/proxy/compat_cache_freshness_test.go": [ "semantics/metadata-answers-include-last-24h-like-loki" ], @@ -918,6 +939,11 @@ "metric-series-identity", "semantics/unwrap-rate-ungrouped-sum-per-stream" ], + "test/e2e-compat/aggregation_over_binary_test.go": [ + "loki_api_v1_query", + "loki_api_v1_query_range", + "semantics/aggregation-over-binary-dropped" + ], "test/e2e-compat/bad_request_status_parity_test.go": [ "loki-compatible-profile" ], @@ -1100,7 +1126,7 @@ "profiles/structured-metadata-keys-per-profile" ] }, - "tests_with_markers": 100, + "tests_with_markers": 103, "unknown_ids": {}, "unwired_items": [ "api_prom_label", @@ -1167,8 +1193,9 @@ "quality/labels-backfill-into-cached-bucket", "quality/stats-volume-freshness", "semantics/absent-over-time-semantics", - "semantics/aggregation-over-binary-dropped", + "semantics/binary-division-by-zero-nan", "semantics/binary-ignoring-matching", + "semantics/binary-step-grid", "semantics/count-values-unsupported", "semantics/detected-fields-extracted-suffix", "semantics/drop-keep-plain-name-after-parser", @@ -1207,5 +1234,5 @@ "versions/capabilities-floor-vs-matrix", "versions/rejected-stats-query-wording-vl-1.47" ], - "wired_items": 109 + "wired_items": 110 } diff --git a/conformance/reports/parity-gaps.md b/conformance/reports/parity-gaps.md index ea597c0a0..f48e9e3f1 100644 --- a/conformance/reports/parity-gaps.md +++ b/conformance/reports/parity-gaps.md @@ -35,13 +35,13 @@ Last run: 2026-10-05T20:16:15Z on `fcaccfcc`, window [1791220800, 1791224400]. | 14 | `semantics/absent-over-time-semantics` — absent_over_time answers when data exists and stays empty when it is absent, after a parser (open) | open | explore-visible | 8 | 6 | S | internal/translator absent_over_time (parser pipelines) | pkg/logql/evaluator.go:647-664, :1320 absentLabels | TestCompat_AbsentOverTimeWithParser (test/e2e-compat, group semantics) | | 15 | `semantics/stddev-stdvar-grouped-values` — stddev and stdvar by a label compute different values (open) | open | explore-visible | 7 | 5 | S | internal/translator stddev/stdvar outer aggregation (by-group fold) | pkg/logql/evaluator.go VectorAggEvaluator stddev/stdvar | TestCompat_StddevStdvarBy (test/e2e-compat, group semantics) | | 16 | `quality/label-values-chunk-window` — Loki lists label values of streams whose chunks overlap the window; the proxy only of streams with lines in it (open) | open | explore-visible | 2 | 2 | M | internal/proxy label values window (chunk-overlap semantics) | pkg/querier/querier.go Label (index lookups by chunk overlap) | TestCompat_LabelValuesWindowEdge (test/e2e-compat, group core) | -| 17 | `semantics/aggregation-over-binary-dropped` — An aggregation over a binary expression is translated to its first operand (open) | open | explore-visible | 2 | 1 | M | internal/translator extractOuterAggregation and internal/proxy handleQuery routing of an aggregation over a binary expression | pkg/logql/evaluator.go VectorAggEvaluator over a binary evaluator | TestCompat_AggregationOverBinary (test/e2e-compat, group semantics) | -| 18 | `semantics/pattern-line-filter` — An empty pattern line filter (`\|> ""`, `!> ""`) selects the opposite lines (open) | open | explore-visible | 2 | 2 | S | internal/translator pattern line filter | pkg/logql/syntax/lex.go:26,29 (NPA, PIPE_PATTERN); pkg/logql/log/filter.go pattern filter | TestCompat_PatternLineFilter (test/e2e-compat, group semantics) | -| 19 | `semantics/log-entries-differ-same-count` — A log query returns the same number of lines with different entries (open, cause not isolated) | open | explore-visible | 1 | 1 | S | to be isolated (log query paging / numeric label filters) | pkg/logql/log/label_filter.go numeric filters; pkg/iter entry ordering | TestCompat_NumericLabelFilterLines (test/e2e-compat, group semantics) | -| 20 | `profiles/label-filter-regexp-anchoring` — A label-filter regexp after a parser is not anchored the way Loki's is (open) | open | explore-visible | — | — | S | internal/translator label-filter regexp emission (anchor ^(?:re)$); external PR #572 open | pkg/logql/log/label_filter.go (regexp matcher on the whole value) | TestCompat_LabelFilterRegexpAnchoring (test/e2e-compat, group semantics) | -| 21 | `profiles/parsed-key-colliding-with-structured-metadata` — A parsed key named like structured metadata loses the metadata value in VictoriaLogs (open) | open | explore-visible | — | — | L | internal/translator json / logfmt unpack (original stored value) and internal/proxy row classification | pkg/logql/log/parser.go:152 HasInCategory(key, StructuredMetadataLabel) then duplicateSuffix | TestCompat_ParsedKeyAgainstStructuredMetadata (test/e2e-compat, group otel-edge) | -| 22 | `quality/label-cache-aged-labels-kept` — Labels that age out of the window are never dropped from the merged label cache (open) | open | explore-visible | — | — | S | internal/proxy/label_metadata.go:1259 mergeLabelsIntoCache | pkg/querier/querier.go Label (window-bounded) | TestE2E_LabelsAgeOutOfWindow (test/e2e-compat, group core) | -| 23 | `quality/labels-backfill-into-cached-bucket` — Rows backfilled into a past bucket the label inventory already cached with data appear in /labels only after its revalidation delay (open) | open | explore-visible | — | — | M | internal/proxy/metadata_inventory.go (bucket revalidation delay) | pkg/querier/querier.go Label (index read per request) | TestCompat_LabelsBackfillIntoCachedBucket (test/e2e-compat, group core) | +| 17 | `semantics/pattern-line-filter` — An empty pattern line filter (`\|> ""`, `!> ""`) selects the opposite lines (open) | open | explore-visible | 2 | 2 | S | internal/translator pattern line filter | pkg/logql/syntax/lex.go:26,29 (NPA, PIPE_PATTERN); pkg/logql/log/filter.go pattern filter | TestCompat_PatternLineFilter (test/e2e-compat, group semantics) | +| 18 | `semantics/log-entries-differ-same-count` — A log query returns the same number of lines with different entries (open, cause not isolated) | open | explore-visible | 1 | 1 | S | to be isolated (log query paging / numeric label filters) | pkg/logql/log/label_filter.go numeric filters; pkg/iter entry ordering | TestCompat_NumericLabelFilterLines (test/e2e-compat, group semantics) | +| 19 | `profiles/label-filter-regexp-anchoring` — A label-filter regexp after a parser is not anchored the way Loki's is (open) | open | explore-visible | — | — | S | internal/translator label-filter regexp emission (anchor ^(?:re)$); external PR #572 open | pkg/logql/log/label_filter.go (regexp matcher on the whole value) | TestCompat_LabelFilterRegexpAnchoring (test/e2e-compat, group semantics) | +| 20 | `profiles/parsed-key-colliding-with-structured-metadata` — A parsed key named like structured metadata loses the metadata value in VictoriaLogs (open) | open | explore-visible | — | — | L | internal/translator json / logfmt unpack (original stored value) and internal/proxy row classification | pkg/logql/log/parser.go:152 HasInCategory(key, StructuredMetadataLabel) then duplicateSuffix | TestCompat_ParsedKeyAgainstStructuredMetadata (test/e2e-compat, group otel-edge) | +| 21 | `quality/label-cache-aged-labels-kept` — Labels that age out of the window are never dropped from the merged label cache (open) | open | explore-visible | — | — | S | internal/proxy/label_metadata.go:1259 mergeLabelsIntoCache | pkg/querier/querier.go Label (window-bounded) | TestE2E_LabelsAgeOutOfWindow (test/e2e-compat, group core) | +| 22 | `quality/labels-backfill-into-cached-bucket` — Rows backfilled into a past bucket the label inventory already cached with data appear in /labels only after its revalidation delay (open) | open | explore-visible | — | — | M | internal/proxy/metadata_inventory.go (bucket revalidation delay) | pkg/querier/querier.go Label (index read per request) | TestCompat_LabelsBackfillIntoCachedBucket (test/e2e-compat, group core) | +| 23 | `semantics/binary-division-by-zero-nan` — A binary division or modulo by zero answers ±Inf where Loki answers NaN (open) | open | explore-visible | — | — | S | internal/proxy binary_matching.go binarySampleValue (/ and %); vector_matching.go withoutMerge.combine (min, max over NaN) | pkg/logql/syntax/ast.go:1978-2006 MergeBinOp; pkg/logql/evaluator.go VectorAggEvaluator.Next (min, max) | TestCompat_BinaryDivisionByZero (test/e2e-compat, group core); a binarySampleValue edge table and the aggregation-over-binary differentia... | | 24 | `semantics/drop-keep-plain-name-after-parser` — A log query's drop, keep or label_format source of a stream label's plain name after a parser acts on the stored field; Loki acts on the ... | open | explore-visible | — | — | S | internal/proxy stage-field exposure (drop and keep of a name a parser also reads) | pkg/logql/log/labels.go LabelsBuilder.Del / getWithCategory; pkg/logql/log/drop_labels.go, keep_labels.go | TestCompat_DropKeepPlainNameAfterParser (test/e2e-compat, group core) | | 25 | `semantics/extracted-ungrouped-metric-raw-rows` — An ungrouped range metric over a `\| json` pipeline filtered on name_extracted still reads raw rows and fails at the row limit (open) | open | explore-visible | — | — | L | internal/proxy bare-parser metric evaluator (fetchBareParserMetricSeries) and the ordered JSON pushdown (without grouping) | pkg/logql range aggregation keeps every label of the sample | TestCompat_ExtractedUngroupedMetricAtSixHours (test/e2e-compat, group core) | | 26 | `semantics/label-replace-duplicates-before-ranking` — topk, bottomk and stddev over label sets a label_replace made equal rank merged series on query_range (open) | open | explore-visible | — | — | S | internal/proxy metric_agg.go (handleRangeMetricPostAggregation) after transformSeriesLabels | pkg/logql/evaluator.go VectorAggEvaluator.Next; pkg/logql/engine.go:440-470 vectorsToSeries | TestCompat_LabelReplaceDuplicatesBeforeRanking (test/e2e-compat, group semantics) | @@ -70,33 +70,34 @@ Last run: 2026-10-05T20:16:15Z on `fcaccfcc`, window [1791220800, 1791224400]. | 49 | `parsed-identity/regexp-partial-match-series-count` — A partially-matching \| regexp returns about 3x Loki's series count | open | api-only | 2 | 2 | M | internal/proxy regexp capture series identity (non-matching lines) | pkg/logql/log/parser.go RegexpParser (no capture label for a non-matching line) | TestCompat_RegexpPartialMatchSeries (test/e2e-compat, group semantics) | | 50 | `profiles/json-extraction-list-missing-key` — A json extraction-list key absent from the line is dropped; Loki keeps it in the metric series (open) | open | api-only | 2 | 2 | S | internal/proxy stage-field exposure (extraction lists, metric path) | pkg/logql/log/parser.go:671-720 JSONExpressionParser.Process | TestCompat_JSONExtractionListMissingKey (test/e2e-compat, group semantics) | | 51 | `semantics/unpack-non-string-values` — `\| unpack` extracts non-string JSON values that Loki ignores (open) | open | api-only | 2 | 2 | S | internal/translator unpack (string values only) | pkg/logql/log/parser.go:789-820 UnpackParser.unpack | TestCompat_UnpackStringValuesOnly (test/e2e-compat, group semantics) | -| 52 | `semantics/extracted-derived-labels-raw-evaluator` — service_name_extracted and detected_level_extracted metrics keep the raw-row evaluator (open) | open | edge | — | — | S | internal/proxy orderedJSONPushdownLabelOK (service_name, detected_level) | pkg/logql/log/parser.go duplicateSuffix; service_name set by discover_service_name at ingest | TestCompat_ExtractedLabelMetricPushdown rows for service_name_extracted (test/e2e-compat, group core) | -| 53 | `semantics/extracted-metric-known-differences` — Two name_extracted differences against Loki remain in the metric pushdown and the raw evaluator (open) | open | edge | — | — | M | internal/proxy raw evaluator parsed-label naming (processWithStreamLabels) and translator.ExtractedLabelPipes | pkg/logql/log/parser.go duplicateSuffix and ParserHint.Extracted (a key an earlier extraction produced is skipped) | TestCompat_ExtractedBothKeysInLineOrder (test/e2e-compat, group core) | -| 54 | `semantics/unwrap-conversion-error-parser-divergence` — Where VictoriaLogs' logfmt parser reads another value than Loki's, an unwrap misses Loki's SampleExtractionErr or drops samples Loki keep... | open | edge | — | — | L | internal/translator logfmt unpacking (unpack_logfmt keeps the last value); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:379-438 LogfmtParser.Process; pkg/logql/log/logfmt/decode.go | TestUnwrapConversionErrorDifferential counts the first case; TestUnwrapLokiLabels pins the decoder; TestCompat_UnwrapConversionErrorParit... | -| 55 | `semantics/unwrap-conversion-error-postfilter-forms` — An unwrap whose post filter keeps only the error, compares a label, or combines filters with and/or answers 200 where Loki may fail with ... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorPostFilters; the translator's post-filter placement | pkg/logql/log/label_filter.go:90-100 BinaryLabelFilter.Process, 306-340 NumericLabelFilter.Process, 418-424 labelValue | TestUnwrapErrorProbePlans and TestUnwrapConversionErrorDifferential (internal/proxy) assert the detection stays off for these forms; flip... | -| 56 | `semantics/unwrap-conversion-error-stored-value-overwritten` — An unwrap over a label stored as structured metadata and also parsed from the line misses Loki's SampleExtractionErr (open) | open | edge | — | — | M | internal/translator parsed-key collision with stored fields (unpack keep_original_fields); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:151,809 duplicateSuffix | TestUnwrapConversionErrorDifferential counts the first case; TestCompat_UnwrapConversionErrorParity asserts the 200 side | -| 57 | `semantics/unwrap-conversion-error-undetected-shapes` — A few query shapes answer 200 where Loki fails with SampleExtractionErr, because no VictoriaLogs query of theirs carries the conversion c... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorProbes (opaque expressions), unwrapCountingQuery (pipes after stats), unwrapLokiLabe... | pkg/logql/evaluator.go:726-733 | TestCompat_UnwrapConversionErrorParity open/* asserts Loki's 400 and skips naming this case (absent_over_time, label_replace); TestUnwrap... | -| 58 | `semantics/unwrap-gate-parsefloat-divergences` — The unwrap sample gate drops a few values Loki's conversions accept or reject (open) | open | edge | — | — | S | internal/translator/unwrap_gate.go (UnwrapNumberPattern, UnwrapDurationPattern, UnwrapBytesPattern) | pkg/logql/log/metrics_extraction.go:316-338 convertFloat, convertDuration, convertBytes | TestUnwrapNumberPatternMatchesParseFloat (internal/translator) pins the known differences | -| 59 | `versions/stats-bucket-label-v1.45` — Stats-bucket metrics put a line on a bucket edge one bucket late on VictoriaLogs before v1.45 (open) | open | edge | — | — | M | stats-bucket relabel (internal/proxy range_metric_compat.go setSlidingStatsRangeParams / slidingStatsBucket; sliding_window_stats.go) | pkg/logql range vector (T-range, T] | TestRangeMetricCompatibilityJSONFilterPushdown on VictoriaLogs v1.40-v1.44 (version matrix); fix by relabelling edge lines or declining t... | -| 60 | `metric-identity/structured-metadata-splits-series` — Loki splits a stream's metric series per structured-metadata set; the proxy keeps the stream (owner-accepted deviation) | owner-kept | explore-visible | 173 | 37 | L | internal/translator series identity (stats by stream); VictoriaLogs cannot group by a row's remaining fields | pkg/logql/log/metrics_extraction.go:102-120 builder.Add(StructuredMetadataLabel) -> GroupedLabels | differential run cluster count (bench/parity) | -| 61 | `parsed-identity/dynamic-key-parsers` — `\| json` and `\| logfmt` metric series keep the stream identity by default (owner-accepted deviation, opt-in flag) | owner-kept | explore-visible | 95 | 29 | L | internal/translator parser series identity; -exact-parser-series-identity | pkg/logql/log/parser.go label extraction into the sample labels | differential run cluster count (bench/parity) | -| 62 | `profiles/label-filter-on-line-field` — A label filter on a line key no earlier stage exposes matches like Loki | owner-kept | explore-visible | — | — | M | internal/proxy label-filter refill (-label-filter-refill-max-pages); metric queries on keys no stage exposes | pkg/logql/log/label_filter.go (filter reads only labels the pipeline set) | TestCompat_LabelFilterOnLineField (exists, profile tests) | -| 63 | `profiles/detected-fields-no-extracted-suffix` — detected_fields lists service / service.name without Loki's _extracted suffix (owner decision) | owner-kept | drilldown-visible | — | — | S | internal/proxy/label_handlers.go handleDetectedFields | pkg/logql/log/parser.go:25 duplicateSuffix | bench/parity diff rule (documented facet, service fields only) | -| 64 | `profiles/json-line-stored-without-line` — A JSON line pushed without _msg is stored without its line (documented) | documented | explore-visible | — | — | L | client push configuration (disable_message_parsing=1); docs/KNOWN_ISSUES.md | pkg/distributor push keeps the line verbatim | cases/profiles/stage-field-exposure with disable_message_parsing pushes | -| 65 | `drilldown/detected-labels-sampled-cardinality` — detected_labels cardinality is sampled from the window's last minutes; Loki counts every stream its ingesters hold (documented) | documented | drilldown-visible | 20 | 2 | M | internal/proxy/patterns.go handleDetectedLabels | pkg/ingester/instance.go LabelsWithValues | bench/parity diff rule (documented facet) | -| 66 | `profiles/volume-detected-level-extension` — index/volume answers targetLabels=detected_level; Loki returns no volume for it (documented proxy extension) | documented | drilldown-visible | 17 | 2 | S | internal/proxy/volume.go | pkg/storage volume index (stream labels only) | differential run row volume by detected_level | -| 67 | `drilldown/patterns-mined-differently` — Patterns come from different miners on each side (documented) | documented | drilldown-visible | 15 | 3 | L | internal/proxy/patterns.go | pkg/pattern (ingester-side mining) | bench/visual history-dependent comparison (exists) | -| 68 | `profiles/volume-bytes-accounting` — Volume values count line bytes; Loki counts chunk bytes with structured metadata (documented) | documented | drilldown-visible | 4 | 3 | L | internal/proxy/volume.go; docs/KNOWN_ISSUES.md | pkg/storage volume index (chunk bytes incl. structured metadata) | bench/parity diff rule (documented facet) | -| 69 | `limits/drilldown-breakdown-keeps-busiest` — Over max_query_series a Drilldown breakdown keeps the busiest series; Loki keeps the first it meets (documented) | documented | drilldown-visible | 1 | 1 | S | internal/proxy series limit ranking | pkg/logql engine series limit | cases/limits/drilldown-partial-with-warning (exists) | -| 70 | `quality/index-stats-bytes-and-chunks` — index/stats bytes and chunks have no VictoriaLogs equivalent (documented) | documented | drilldown-visible | — | — | L | internal/proxy/label_handlers.go handleIndexStats; docs/KNOWN_ISSUES.md | pkg/storage index stats from chunk refs (bytes, chunks) | bench/parity diff rule (documented facet) | -| 71 | `profiles/grafana-stats-error-as-warning` — A failed stats query from Grafana answers 200 with warnings; Loki returns the error (documented) | documented | grafana-datasource | — | — | S | internal/proxy Grafana-sourced stats error handling | pkg/querier/queryrange error propagation (no warnings conversion) | TestCompat_GrafanaStatsErrorStatus (test/e2e-compat, fault injection) | -| 72 | `profiles/metadata-default-lookback` — Label endpoints without start/end default to a different lookback than Loki's 1h (documented) | documented | api-only | — | — | S | internal/config flag default -metadata-default-lookback | pkg/loghttp/params.go defaultSince = 1h | TestCompat_MetadataDefaultLookback (test/e2e-compat, group core) | -| 73 | `profiles/multi-tenant-volume-merge` — Multi-tenant volume returns per-tenant series with __tenant_id__; Loki sums the tenants' volumes (documented) | documented | api-only | — | — | S | internal/proxy/volume.go multi-tenant merge | pkg/querier/queryrange volume merge | TestCompat_MultiTenantVolume (test/e2e-compat, group tail-multitenancy) | -| 74 | `semantics/count-values-unsupported` — count_values() is rejected with 400 where Loki answers (documented) | documented | api-only | — | — | M | internal/translator + proxy-side post-aggregation of a stats matrix | pkg/logql/syntax OpTypeCountValues; vector evaluator | TestCompat_CountValues (test/e2e-compat, group semantics) | -| 75 | `semantics/eager-pipeline-validation` — Invalid pipeline stages fail with 400 regardless of data; Loki answers 200 empty over a range with no data (documented) | documented | edge | — | — | S | internal/translator validation | pkg/logql/log/fmt.go template build at pipeline construction | TestCompat_EagerValidationEmptyRange (test/e2e-compat, group semantics) | -| 76 | `semantics/ip-filter-precision` — ip() filters are approximated with regular expressions (documented) | documented | edge | — | — | M | internal/proxy/postprocess.go:84 ip filtering | pkg/logql/log/ip.go | TestCompat_IPFilterForms (test/e2e-compat, group semantics) | -| 77 | `drilldown/plugin-legend-sync-race` — Logs Drilldown crashes a breakdown page when a query answers before the panel module loads (upstream, not the proxy) | upstream | drilldown-visible | — | — | S | none (grafana-lokiexplore-app) | n/a (client race) | bench/visual capture waits for the panel module (already) | -| 78 | `quality/index-stats-loki-chunk-accounting` — Loki's index/stats counts chunks it holds twice; the proxy's counts equal Loki's own count_over_time (upstream) | upstream | drilldown-visible | — | — | S | none (Loki chunk index accounting) | pkg/storage stats from chunk refs (flushed and retained chunks both counted) | bench/parity cross-check (run.py loki_counts_itself) | +| 52 | `semantics/binary-step-grid` — A binary joining two vectors answers on the step-aligned grid, a binary with a literal at start+k*step (open) | open | edge | — | — | M | internal/proxy binary_logql.go alignBinaryRangeRequest (vector joins only) | pkg/logql/engine.go step evaluation at start+k*step; pkg/querier/queryrange/queryrangebase/step_align.go StepAlignMiddleware (align_queri... | TestAggregationOverBinaryDifferential with a start off the step grid for vector joins (internal/proxy; skipped there today) | +| 53 | `semantics/extracted-derived-labels-raw-evaluator` — service_name_extracted and detected_level_extracted metrics keep the raw-row evaluator (open) | open | edge | — | — | S | internal/proxy orderedJSONPushdownLabelOK (service_name, detected_level) | pkg/logql/log/parser.go duplicateSuffix; service_name set by discover_service_name at ingest | TestCompat_ExtractedLabelMetricPushdown rows for service_name_extracted (test/e2e-compat, group core) | +| 54 | `semantics/extracted-metric-known-differences` — Two name_extracted differences against Loki remain in the metric pushdown and the raw evaluator (open) | open | edge | — | — | M | internal/proxy raw evaluator parsed-label naming (processWithStreamLabels) and translator.ExtractedLabelPipes | pkg/logql/log/parser.go duplicateSuffix and ParserHint.Extracted (a key an earlier extraction produced is skipped) | TestCompat_ExtractedBothKeysInLineOrder (test/e2e-compat, group core) | +| 55 | `semantics/unwrap-conversion-error-parser-divergence` — Where VictoriaLogs' logfmt parser reads another value than Loki's, an unwrap misses Loki's SampleExtractionErr or drops samples Loki keep... | open | edge | — | — | L | internal/translator logfmt unpacking (unpack_logfmt keeps the last value); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:379-438 LogfmtParser.Process; pkg/logql/log/logfmt/decode.go | TestUnwrapConversionErrorDifferential counts the first case; TestUnwrapLokiLabels pins the decoder; TestCompat_UnwrapConversionErrorParit... | +| 56 | `semantics/unwrap-conversion-error-postfilter-forms` — An unwrap whose post filter keeps only the error, compares a label, or combines filters with and/or answers 200 where Loki may fail with ... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorPostFilters; the translator's post-filter placement | pkg/logql/log/label_filter.go:90-100 BinaryLabelFilter.Process, 306-340 NumericLabelFilter.Process, 418-424 labelValue | TestUnwrapErrorProbePlans and TestUnwrapConversionErrorDifferential (internal/proxy) assert the detection stays off for these forms; flip... | +| 57 | `semantics/unwrap-conversion-error-stored-value-overwritten` — An unwrap over a label stored as structured metadata and also parsed from the line misses Loki's SampleExtractionErr (open) | open | edge | — | — | M | internal/translator parsed-key collision with stored fields (unpack keep_original_fields); internal/proxy/unwrap_conversion_error.go | pkg/logql/log/parser.go:151,809 duplicateSuffix | TestUnwrapConversionErrorDifferential counts the first case; TestCompat_UnwrapConversionErrorParity asserts the 200 side | +| 58 | `semantics/unwrap-conversion-error-undetected-shapes` — A few query shapes answer 200 where Loki fails with SampleExtractionErr, because no VictoriaLogs query of theirs carries the conversion c... | open | edge | — | — | M | internal/proxy/unwrap_conversion_error.go unwrapErrorProbes (opaque expressions), unwrapCountingQuery (pipes after stats), unwrapLokiLabe... | pkg/logql/evaluator.go:726-733 | TestCompat_UnwrapConversionErrorParity open/* asserts Loki's 400 and skips naming this case (absent_over_time, label_replace); TestUnwrap... | +| 59 | `semantics/unwrap-gate-parsefloat-divergences` — The unwrap sample gate drops a few values Loki's conversions accept or reject (open) | open | edge | — | — | S | internal/translator/unwrap_gate.go (UnwrapNumberPattern, UnwrapDurationPattern, UnwrapBytesPattern) | pkg/logql/log/metrics_extraction.go:316-338 convertFloat, convertDuration, convertBytes | TestUnwrapNumberPatternMatchesParseFloat (internal/translator) pins the known differences | +| 60 | `versions/stats-bucket-label-v1.45` — Stats-bucket metrics put a line on a bucket edge one bucket late on VictoriaLogs before v1.45 (open) | open | edge | — | — | M | stats-bucket relabel (internal/proxy range_metric_compat.go setSlidingStatsRangeParams / slidingStatsBucket; sliding_window_stats.go) | pkg/logql range vector (T-range, T] | TestRangeMetricCompatibilityJSONFilterPushdown on VictoriaLogs v1.40-v1.44 (version matrix); fix by relabelling edge lines or declining t... | +| 61 | `metric-identity/structured-metadata-splits-series` — Loki splits a stream's metric series per structured-metadata set; the proxy keeps the stream (owner-accepted deviation) | owner-kept | explore-visible | 173 | 37 | L | internal/translator series identity (stats by stream); VictoriaLogs cannot group by a row's remaining fields | pkg/logql/log/metrics_extraction.go:102-120 builder.Add(StructuredMetadataLabel) -> GroupedLabels | differential run cluster count (bench/parity) | +| 62 | `parsed-identity/dynamic-key-parsers` — `\| json` and `\| logfmt` metric series keep the stream identity by default (owner-accepted deviation, opt-in flag) | owner-kept | explore-visible | 95 | 29 | L | internal/translator parser series identity; -exact-parser-series-identity | pkg/logql/log/parser.go label extraction into the sample labels | differential run cluster count (bench/parity) | +| 63 | `profiles/label-filter-on-line-field` — A label filter on a line key no earlier stage exposes matches like Loki | owner-kept | explore-visible | — | — | M | internal/proxy label-filter refill (-label-filter-refill-max-pages); metric queries on keys no stage exposes | pkg/logql/log/label_filter.go (filter reads only labels the pipeline set) | TestCompat_LabelFilterOnLineField (exists, profile tests) | +| 64 | `profiles/detected-fields-no-extracted-suffix` — detected_fields lists service / service.name without Loki's _extracted suffix (owner decision) | owner-kept | drilldown-visible | — | — | S | internal/proxy/label_handlers.go handleDetectedFields | pkg/logql/log/parser.go:25 duplicateSuffix | bench/parity diff rule (documented facet, service fields only) | +| 65 | `profiles/json-line-stored-without-line` — A JSON line pushed without _msg is stored without its line (documented) | documented | explore-visible | — | — | L | client push configuration (disable_message_parsing=1); docs/KNOWN_ISSUES.md | pkg/distributor push keeps the line verbatim | cases/profiles/stage-field-exposure with disable_message_parsing pushes | +| 66 | `drilldown/detected-labels-sampled-cardinality` — detected_labels cardinality is sampled from the window's last minutes; Loki counts every stream its ingesters hold (documented) | documented | drilldown-visible | 20 | 2 | M | internal/proxy/patterns.go handleDetectedLabels | pkg/ingester/instance.go LabelsWithValues | bench/parity diff rule (documented facet) | +| 67 | `profiles/volume-detected-level-extension` — index/volume answers targetLabels=detected_level; Loki returns no volume for it (documented proxy extension) | documented | drilldown-visible | 17 | 2 | S | internal/proxy/volume.go | pkg/storage volume index (stream labels only) | differential run row volume by detected_level | +| 68 | `drilldown/patterns-mined-differently` — Patterns come from different miners on each side (documented) | documented | drilldown-visible | 15 | 3 | L | internal/proxy/patterns.go | pkg/pattern (ingester-side mining) | bench/visual history-dependent comparison (exists) | +| 69 | `profiles/volume-bytes-accounting` — Volume values count line bytes; Loki counts chunk bytes with structured metadata (documented) | documented | drilldown-visible | 4 | 3 | L | internal/proxy/volume.go; docs/KNOWN_ISSUES.md | pkg/storage volume index (chunk bytes incl. structured metadata) | bench/parity diff rule (documented facet) | +| 70 | `limits/drilldown-breakdown-keeps-busiest` — Over max_query_series a Drilldown breakdown keeps the busiest series; Loki keeps the first it meets (documented) | documented | drilldown-visible | 1 | 1 | S | internal/proxy series limit ranking | pkg/logql engine series limit | cases/limits/drilldown-partial-with-warning (exists) | +| 71 | `quality/index-stats-bytes-and-chunks` — index/stats bytes and chunks have no VictoriaLogs equivalent (documented) | documented | drilldown-visible | — | — | L | internal/proxy/label_handlers.go handleIndexStats; docs/KNOWN_ISSUES.md | pkg/storage index stats from chunk refs (bytes, chunks) | bench/parity diff rule (documented facet) | +| 72 | `profiles/grafana-stats-error-as-warning` — A failed stats query from Grafana answers 200 with warnings; Loki returns the error (documented) | documented | grafana-datasource | — | — | S | internal/proxy Grafana-sourced stats error handling | pkg/querier/queryrange error propagation (no warnings conversion) | TestCompat_GrafanaStatsErrorStatus (test/e2e-compat, fault injection) | +| 73 | `profiles/metadata-default-lookback` — Label endpoints without start/end default to a different lookback than Loki's 1h (documented) | documented | api-only | — | — | S | internal/config flag default -metadata-default-lookback | pkg/loghttp/params.go defaultSince = 1h | TestCompat_MetadataDefaultLookback (test/e2e-compat, group core) | +| 74 | `profiles/multi-tenant-volume-merge` — Multi-tenant volume returns per-tenant series with __tenant_id__; Loki sums the tenants' volumes (documented) | documented | api-only | — | — | S | internal/proxy/volume.go multi-tenant merge | pkg/querier/queryrange volume merge | TestCompat_MultiTenantVolume (test/e2e-compat, group tail-multitenancy) | +| 75 | `semantics/count-values-unsupported` — count_values() is rejected with 400 where Loki answers (documented) | documented | api-only | — | — | M | internal/translator + proxy-side post-aggregation of a stats matrix | pkg/logql/syntax OpTypeCountValues; vector evaluator | TestCompat_CountValues (test/e2e-compat, group semantics) | +| 76 | `semantics/eager-pipeline-validation` — Invalid pipeline stages fail with 400 regardless of data; Loki answers 200 empty over a range with no data (documented) | documented | edge | — | — | S | internal/translator validation | pkg/logql/log/fmt.go template build at pipeline construction | TestCompat_EagerValidationEmptyRange (test/e2e-compat, group semantics) | +| 77 | `semantics/ip-filter-precision` — ip() filters are approximated with regular expressions (documented) | documented | edge | — | — | M | internal/proxy/postprocess.go:84 ip filtering | pkg/logql/log/ip.go | TestCompat_IPFilterForms (test/e2e-compat, group semantics) | +| 78 | `drilldown/plugin-legend-sync-race` — Logs Drilldown crashes a breakdown page when a query answers before the panel module loads (upstream, not the proxy) | upstream | drilldown-visible | — | — | S | none (grafana-lokiexplore-app) | n/a (client race) | bench/visual capture waits for the panel module (already) | +| 79 | `quality/index-stats-loki-chunk-accounting` — Loki's index/stats counts chunks it holds twice; the proxy's counts equal Loki's own count_over_time (upstream) | upstream | drilldown-visible | — | — | S | none (Loki chunk index accounting) | pkg/storage stats from chunk refs (flushed and retained chunks both counted) | bench/parity cross-check (run.py loki_counts_itself) | ## Fixed, awaiting the next run @@ -109,6 +110,7 @@ Last run: 2026-10-05T20:16:15Z on `fcaccfcc`, window [1791220800, 1791224400]. | `semantics/line-filter-or-alternatives` — Line filters with `or` alternatives match like Loki | #669 | 6 | 42 | | `semantics/plain-name-after-parser-reads-stream-label` — A filter or grouping on a name a parser also reads reads the parsed value; Loki reads the stream label (fixed) | #676 | 4 | 6 | | `semantics/label-replace-new-label` — label_replace adds its destination label to every series, instant and range (fixed) | #744 | 1 | 5 | +| `semantics/aggregation-over-binary-dropped` — A vector aggregation over a binary expression aggregates the whole binary, like Loki (fixed) | #741 | 1 | 2 | | `semantics/json-extracted-metric-pushdown` — A metric filtered or grouped on a name_extracted label after `\| json` is answered from VictoriaLogs stats (fixed) | #690 | 0 | 0 | | `semantics/labels-end-inclusive-like-loki` — /labels and /label/{name}/values include data at the request's end, so a zero-width request (Grafana "Show context") lists the row's labe... | #703 | 0 | 0 | | `semantics/unwrap-conversion-error` — An unwrap over a value its conversion rejects fails like Loki, with a 400 SampleExtractionErr (fixed) | #704 | 0 | 0 | diff --git a/internal/proxy/binary_aggregation.go b/internal/proxy/binary_aggregation.go index 9ba671232..69726c92e 100644 --- a/internal/proxy/binary_aggregation.go +++ b/internal/proxy/binary_aggregation.go @@ -51,13 +51,9 @@ func (p *Proxy) proxyBinaryAggregation(w http.ResponseWriter, r *http.Request, r if _, _, ok := binaryAggregationChain(routed); !ok { return false } + // Under a label_replace or label_join the handler has cloned the request + // to the expression inside it, and applies the transform to the answer. chain, binary, ok := binaryAggregationChain(binaryRequestExpr(r)) - if !ok { - // Routed from under a label_replace or label_join, which the handler - // applies to the folded answer. - expr, _ := logqlpkg.Parse(stripLabelTransforms(binaryRequestQuery(r))) - chain, binary, ok = binaryAggregationChain(expr) - } if !ok { // The request's own query does not parse as the routed one did: answer // Loki's parse error for it. diff --git a/internal/proxy/binary_aggregation_differential_test.go b/internal/proxy/binary_aggregation_differential_test.go index 317f9799d..b6607c701 100644 --- a/internal/proxy/binary_aggregation_differential_test.go +++ b/internal/proxy/binary_aggregation_differential_test.go @@ -403,6 +403,7 @@ func TestAggregationOverBinaryDifferential(t *testing.T) { name := fmt.Sprintf("version=%s/instant=%v", version, instant) t.Run(name, func(t *testing.T) { rnd := rand.New(rand.NewSource(int64(len(name))*6151 + 3)) + offGrid := 0 for i := 0; i < 40; i++ { s := generateUnwrapScenario(rnd, "rate", "", instant) if s.step > 0 && rnd.Intn(2) == 0 { // the other half starts off the step grid @@ -410,17 +411,10 @@ func TestAggregationOverBinaryDifferential(t *testing.T) { s.end -= s.end % int64(s.step) } expr, chain := generateBinaryAggregation(rnd) - times := s.times() - if s.step > 0 && expr.joinsVectors() { - // A join of two vectors evaluates on the step-aligned grid, as - // Loki's query frontend with align_queries_with_step (the - // compose stack's Loki); with a literal only, at - // start+k*step, as a Loki without it - // (TestBinaryMixedWindowsShareStepAlignedAxis). - aligned := s - aligned.start -= aligned.start % int64(s.step) - aligned.end -= aligned.end % int64(s.step) - times = aligned.times() + if s.step > 0 && s.start%int64(s.step) != 0 && expr.joinsVectors() { + // Loki evaluates at start+k*step; a join of two vectors evaluates + // on the step-aligned grid instead (semantics/binary-step-grid). + continue } for j := range s.rows { pod := s.rows[j].stream[len(`{app="uw",pod="`) : len(`{app="uw",pod="`)+1] @@ -438,7 +432,7 @@ func TestAggregationOverBinaryDifferential(t *testing.T) { query = agg.wrap(query) } want := map[string]map[int64]float64{} - for _, ts := range times { + for _, ts := range s.times() { samples, _, _ := expr.eval(s.rows, ts, int64(s.window)) for _, agg := range chain { samples = agg.apply(samples) @@ -455,6 +449,12 @@ func TestAggregationOverBinaryDifferential(t *testing.T) { got, _ := binaryAggregationAnswer(t, emu, query, s.start, s.end, s.step) label := fmt.Sprintf("%s #%d %s step=%v window=%v rows=%d", name, i, query, s.step, s.window, len(s.rows)) compareBinaryAggregation(t, label, got, want) + if s.step > 0 && s.start%int64(s.step) != 0 { + offGrid++ + } + } + if !instant && offGrid == 0 { + t.Fatal("no scenario started off the step grid") } }) } diff --git a/internal/proxy/binary_aggregation_test.go b/internal/proxy/binary_aggregation_test.go index 6c1f11a16..4643f1f0e 100644 --- a/internal/proxy/binary_aggregation_test.go +++ b/internal/proxy/binary_aggregation_test.go @@ -14,6 +14,7 @@ import ( "time" "github.com/ReliablyObserve/Loki-VL-proxy/internal/cache" + "github.com/ReliablyObserve/Loki-VL-proxy/internal/translator" ) // binaryAggregationAnswer sends query through the real handler against the @@ -393,6 +394,9 @@ func TestWithoutFoldLabelOrderAndKeys(t *testing.T) { {"a value with a comma and an equals sign", []map[string]string{{"a": "b", "c": "d"}, {"a": "b,c=d"}}, []map[string]string{{"a": "b", "c": "d"}, {"a": "b,c=d"}}}, {"a prefix first", []map[string]string{{"a": "x", "b": "y"}, {"a": "x"}}, []map[string]string{{"a": "x"}, {"a": "x", "b": "y"}}}, } { + if tc.name == "a value with a comma and an equals sign" && metricKeyStr(tc.metric[0]) != metricKeyStr(tc.metric[1]) { + t.Fatal("the two sets should share their name=value text, the key the fold no longer uses") + } for _, resultType := range []string{"vector", "matrix"} { result := make([]map[string]any, 0, len(tc.metric)) for _, metric := range tc.metric { @@ -437,3 +441,17 @@ func TestWithoutFoldLabelOrderAndKeys(t *testing.T) { } } } + +// A label transform rewrites the series of a vector or a matrix and keeps the +// answer's warnings and statistics. +// +// conformance: semantics/aggregation-over-binary-dropped +func TestLabelTransformsKeepVectorAndWarnings(t *testing.T) { + body := []byte(`{"status":"success","warnings":["partial"],"data":{"resultType":"vector","stats":{"summary":{"bytesProcessedPerSecond":1}},"result":[{"metric":{"job":"a"},"value":[1,"2"]}]}}`) + got := applyLabelJoin(applyLabelReplace(body, translator.LabelReplaceSpec{DstLabel: "dst", Replacement: "$1", SrcLabel: "job", Regex: "(.*)"}), translator.LabelJoinSpec{DstLabel: "both", Separator: "-", SrcLabels: []string{"job", "dst"}}) + want := `{"status":"success","warnings":["partial"],"data":{"resultType":"vector","stats":{"summary":{"bytesProcessedPerSecond":1}},"result":[{"metric":{"both":"a-a","dst":"a","job":"a"},"value":[1,"2"]}]}}` + var g, w any + if json.Unmarshal(got, &g) != nil || json.Unmarshal([]byte(want), &w) != nil || !reflect.DeepEqual(g, w) { + t.Fatalf("got %s\nwant %s", got, want) + } +} diff --git a/internal/proxy/vector_matching.go b/internal/proxy/vector_matching.go index 1899cb8e5..27cb514fc 100644 --- a/internal/proxy/vector_matching.go +++ b/internal/proxy/vector_matching.go @@ -570,6 +570,9 @@ func excludeKey(metric map[string]string, exclude map[string]bool) string { return strings.Join(parts, ",") } +// metricKeyStr is a label set as sorted `name=value` text joined by commas. It +// is not a unique key (a value may hold `,` or `=`): a fold keys by +// binaryLabelKey. func metricKeyStr(metric map[string]string) string { var parts []string for k, v := range metric { diff --git a/test/e2e-compat/aggregation_over_binary_test.go b/test/e2e-compat/aggregation_over_binary_test.go index 8900c3688..bde01f20b 100644 --- a/test/e2e-compat/aggregation_over_binary_test.go +++ b/test/e2e-compat/aggregation_over_binary_test.go @@ -3,9 +3,9 @@ package e2e_compat import ( + "math" "net/http" "net/url" - "reflect" "strconv" "strings" "testing" @@ -84,11 +84,19 @@ func TestCompat_AggregationOverBinaryMixedParsers(t *testing.T) { fx.waitForLoki(t) t.Run("cluster f2066991", func(t *testing.T) { at := time.Now() + // unwrapInstant asserts a success without warnings from each. loki := unwrapInstant(t, lokiURL, aggregationOverBinaryCluster, at) proxy := unwrapInstant(t, proxyURL, aggregationOverBinaryCluster, at) - if !reflect.DeepEqual(loki, proxy) { + if len(loki) != len(proxy) { t.Fatalf("Loki %v, proxy %v", loki, proxy) } + for key, wv := range loki { + w, werr := strconv.ParseFloat(wv, 64) + g, gerr := strconv.ParseFloat(proxy[key], 64) + if _, ok := proxy[key]; !ok || werr != nil || gerr != nil || math.Abs(g-w) > 1e-9*math.Max(1, math.Abs(w)) { + t.Fatalf("{%s}: Loki %s, proxy %q (Loki %v, proxy %v)", key, wv, proxy[key], loki, proxy) + } + } assertInstantVectorAt(t, aggregationOverBinaryCluster, at) }) shape := unwrapSumShape{name: "instant", window: "5m", stop: unwrapSumShapes[len(unwrapSumShapes)-1].stop} From 5158a0a643ce2f561c49eb8458e67b30903f35a4 Mon Sep 17 00:00:00 2001 From: szibis Date: Mon, 12 Oct 2026 13:59:51 +0200 Subject: [PATCH 17/18] refactor: use the binaryRequestQuery helper from the label_replace change --- internal/proxy/binary_logql.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/internal/proxy/binary_logql.go b/internal/proxy/binary_logql.go index dfae140d0..2e0764614 100644 --- a/internal/proxy/binary_logql.go +++ b/internal/proxy/binary_logql.go @@ -72,20 +72,6 @@ func binaryRequestExpr(r *http.Request) logqlpkg.Expr { return expr } -// binaryRequestQuery is the request's query as the handler reads it before -// rewriting it: incomplete unwrap stubs dropped, Grafana range tokens resolved -// and a uniform offset stripped. -func binaryRequestQuery(r *http.Request) string { - query := stripIncompleteUnwrapStubs(r.FormValue("query")) - query = resolveGrafanaRangeTemplateTokens(query, r.FormValue("start"), r.FormValue("end"), r.FormValue("step")) - // The outer handler has already shifted uniform offsets. Mixed offsets - // remain on their individual operands and are applied by each child handler. - if _, stripped, err := extractLogQLOffset(query); err == nil { - query = stripped - } - return query -} - type binaryOperandResponse struct { header http.Header body bytes.Buffer From 20b4391723aceead8c1fcadb91d2c25b9843a2c4 Mon Sep 17 00:00:00 2001 From: szibis Date: Mon, 12 Oct 2026 14:32:53 +0200 Subject: [PATCH 18/18] fix: an aggregation over a binary answers with a JSON content type --- internal/proxy/binary_aggregation.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/proxy/binary_aggregation.go b/internal/proxy/binary_aggregation.go index 69726c92e..9883bc1bf 100644 --- a/internal/proxy/binary_aggregation.go +++ b/internal/proxy/binary_aggregation.go @@ -73,10 +73,11 @@ func (p *Proxy) proxyBinaryAggregation(w http.ResponseWriter, r *http.Request, r } } copyHeaders(w.Header(), buf.Header()) + w.Header().Set("Content-Type", "application/json") if buf.code != 0 { w.WriteHeader(buf.code) } - _, _ = w.Write(body) // nosemgrep: go.lang.security.audit.xss.no-direct-write-to-responsewriter -- the binary evaluator's own JSON, Content-Type copied above + _, _ = w.Write(body) // nosemgrep: go.lang.security.audit.xss.no-direct-write-to-responsewriter -- the binary evaluator's own JSON return true }