From a329d32307b1f213566f16c6a5ba00e24a87c265 Mon Sep 17 00:00:00 2001 From: "talin.senner" Date: Sat, 30 May 2026 23:56:38 -0500 Subject: [PATCH 1/2] feat: inject a configurable pre-filter route on every HTTP server The upstream controller builds the Caddy config solely from Gateway/HTTPRoute resources and replaces the whole config on every admin-API /load. That discards the data plane's bootstrap pre-filter (geo-block via `blocker`, the Coraza WAF, and any `rate_limit`), so those edge policies never actually run once the controller reconciles. Add an optional pre-filter: when CADDY_PREFILTER_PATH names a file holding a JSON array of Caddy HTTP handler objects, prepend a single non-terminal, matcher-less Route carrying those handlers to every HTTP server. It runs before all route-derived handlers and falls through when it doesn't block, so geo/WAF/ rate_limit survive every reconcile. Handlers are emitted verbatim via a new RawHandler type (the plugins have no Go types here). Unset env = no-op (backwards compatible); a configured-but-unreadable/invalid file fails the reconcile loudly (data plane keeps its last-good config). Mount the policy as a ConfigMap and point CADDY_PREFILTER_PATH at it. --- internal/caddy/caddy.go | 55 +++++++++++++++ internal/caddy/prefilter_test.go | 90 ++++++++++++++++++++++++ internal/caddyv2/caddyhttp/rawhandler.go | 23 ++++++ 3 files changed, 168 insertions(+) create mode 100644 internal/caddy/prefilter_test.go create mode 100644 internal/caddyv2/caddyhttp/rawhandler.go diff --git a/internal/caddy/caddy.go b/internal/caddy/caddy.go index ad087ef..c1b8d43 100644 --- a/internal/caddy/caddy.go +++ b/internal/caddy/caddy.go @@ -5,7 +5,9 @@ package caddy import ( "encoding/json" + "fmt" "net/http" + "os" "strconv" "time" @@ -76,7 +78,24 @@ func (i *Input) Config() ([]byte, error) { } } if len(i.httpServers) > 0 { + // Load the optional pre-filter route (geo-block + WAF + rate_limit, + // etc.) once. It is prepended to every HTTP server so it runs before + // any route-derived handlers. The upstream controller replaces the + // whole Caddy config on every /load, which would otherwise discard the + // data plane's bootstrap pre-filter — injecting it here means the + // policy survives every reconcile. See preFilterRoute. + preRoute, err := preFilterRoute() + if err != nil { + return nil, err + } for _, s := range i.httpServers { + // Prepend the pre-filter so it runs first (before app routes and + // the catch-all). Its handlers are non-terminal pass-throughs that + // only write a response when they actively block a request. + if preRoute != nil { + s.Routes = append([]caddyhttp.Route{*preRoute}, s.Routes...) + } + // For all servers register a catch-all route that will match any // request that didn't already get handled. s.Routes = append(s.Routes, caddyhttp.Route{ @@ -118,6 +137,42 @@ func (i *Input) Config() ([]byte, error) { return json.Marshal(i.config) } +// PreFilterPathEnv is the environment variable naming a file that holds a JSON +// array of Caddy HTTP handler objects to run, in order, in front of every +// route on every HTTP server (e.g. blocker/coraza_waf/rate_limit). If unset or +// empty, no pre-filter is injected. +const PreFilterPathEnv = "CADDY_PREFILTER_PATH" + +// preFilterRoute reads the pre-filter handler list from the file named by +// PreFilterPathEnv and returns a single non-terminal, matcher-less Route +// carrying those handlers (so they run against every request and fall through +// when they don't block). Returns (nil, nil) when the env var is unset/empty. +// A configured-but-unreadable/invalid file returns an error so the controller +// fails the reconcile loudly rather than silently shipping an unprotected edge +// (the data plane keeps serving its last-good config in the meantime). +func preFilterRoute() (*caddyhttp.Route, error) { + path := os.Getenv(PreFilterPathEnv) + if path == "" { + return nil, nil + } + b, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading pre-filter config %q: %w", path, err) + } + var raw []json.RawMessage + if err := json.Unmarshal(b, &raw); err != nil { + return nil, fmt.Errorf("pre-filter config %q must be a JSON array of handler objects: %w", path, err) + } + if len(raw) == 0 { + return nil, nil + } + handlers := make([]caddyhttp.Handler, 0, len(raw)) + for _, h := range raw { + handlers = append(handlers, caddyhttp.RawHandler(h)) + } + return &caddyhttp.Route{Handlers: handlers}, nil +} + func (i *Input) handleListener(l gatewayv1.Listener) error { switch l.Protocol { case gatewayv1.HTTPProtocolType: diff --git a/internal/caddy/prefilter_test.go b/internal/caddy/prefilter_test.go new file mode 100644 index 0000000..d6807f9 --- /dev/null +++ b/internal/caddy/prefilter_test.go @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 Wildcard + +package caddy + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +func TestPreFilterRoute(t *testing.T) { + // Unset env -> no pre-filter, no error. + t.Setenv(PreFilterPathEnv, "") + if r, err := preFilterRoute(); err != nil || r != nil { + t.Fatalf("unset env: want (nil,nil), got (%v,%v)", r, err) + } + + // Empty array -> no route. + dir := t.TempDir() + empty := filepath.Join(dir, "empty.json") + if err := os.WriteFile(empty, []byte("[]"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv(PreFilterPathEnv, empty) + if r, err := preFilterRoute(); err != nil || r != nil { + t.Fatalf("empty array: want (nil,nil), got (%v,%v)", r, err) + } + + // Real handler list -> a matcher-less, non-terminal route that marshals + // the handlers verbatim under "handle". + p := filepath.Join(dir, "prefilter.json") + content := `[{"handler":"blocker","allow_countries":["US"]},{"handler":"rate_limit","rate_limits":{"per_ip":{"key":"{http.request.remote.host}","window":"1m","max_events":600}}}]` + if err := os.WriteFile(p, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv(PreFilterPathEnv, p) + r, err := preFilterRoute() + if err != nil { + t.Fatalf("valid file: unexpected error: %v", err) + } + if r == nil { + t.Fatal("valid file: expected a route, got nil") + } + if r.Terminal { + t.Error("pre-filter route must not be terminal") + } + if len(r.MatcherSets) != 0 { + t.Errorf("pre-filter route must have no matchers, got %d", len(r.MatcherSets)) + } + if len(r.Handlers) != 2 { + t.Fatalf("want 2 handlers, got %d", len(r.Handlers)) + } + b, err := json.Marshal(r) + if err != nil { + t.Fatalf("marshal route: %v", err) + } + got := string(b) + for _, want := range []string{`"handler":"blocker"`, `"handler":"rate_limit"`, `"max_events":600`} { + if !contains(got, want) { + t.Errorf("marshalled route missing %q; got %s", want, got) + } + } + + // Configured but unreadable -> error (fail loud). + t.Setenv(PreFilterPathEnv, filepath.Join(dir, "does-not-exist.json")) + if _, err := preFilterRoute(); err == nil { + t.Error("missing file: expected error, got nil") + } + + // Configured but invalid JSON -> error. + bad := filepath.Join(dir, "bad.json") + if err := os.WriteFile(bad, []byte("{not an array}"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv(PreFilterPathEnv, bad) + if _, err := preFilterRoute(); err == nil { + t.Error("invalid json: expected error, got nil") + } +} + +func contains(s, sub string) bool { + for i := 0; i+len(sub) <= len(s); i++ { + if s[i:i+len(sub)] == sub { + return true + } + } + return false +} diff --git a/internal/caddyv2/caddyhttp/rawhandler.go b/internal/caddyv2/caddyhttp/rawhandler.go new file mode 100644 index 0000000..c0a5312 --- /dev/null +++ b/internal/caddyv2/caddyhttp/rawhandler.go @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 Wildcard + +package caddyhttp + +import "encoding/json" + +// RawHandler is an HTTP handler expressed as raw JSON. It exists so the +// controller can inject handlers for Caddy modules it has no Go types for +// (e.g. the `blocker`, `coraza_waf`, and `rate_limit` plugins baked into the +// Caddy data-plane image) into a Route. The JSON is emitted verbatim, +// so the caller is responsible for ensuring it carries a valid `"handler"` +// field. +type RawHandler json.RawMessage + +func (RawHandler) IAmAHandler() {} + +func (h RawHandler) MarshalJSON() ([]byte, error) { + if len(h) == 0 { + return []byte("null"), nil + } + return h, nil +} From bdceabdc00ebccac17d48fab64e8c13bde56e39c Mon Sep 17 00:00:00 2001 From: "talin.senner" Date: Fri, 5 Jun 2026 10:13:17 -0500 Subject: [PATCH 2/2] fix(routechecks): scope route-kind check to the targeted listener CheckGatewayRouteKindAllowed iterated every listener on the Gateway and rejected a route as soon as ANY kind-restricted listener disallowed the route's kind -- without checking whether the route actually targets that listener. A single TCP listener with allowedRoutes.kinds=[TCPRoute] (e.g. an LDAPS L4 listener) therefore rejected EVERY HTTPRoute on a shared Gateway with reason NotAllowedByListeners ("route kind restrictions"), taking down all L7 routing. Skip listeners the route does not target via sectionName, mirroring the existing guard in CheckGatewayAllowedForNamespace. Routes without a sectionName keep their previous (evaluate-all) behavior. Adds a table test reproducing the incident. --- internal/routechecks/gateway.go | 11 ++ internal/routechecks/gateway_test.go | 159 +++++++++++++++++++++++++++ 2 files changed, 170 insertions(+) create mode 100644 internal/routechecks/gateway_test.go diff --git a/internal/routechecks/gateway.go b/internal/routechecks/gateway.go index ab09ab5..e559087 100644 --- a/internal/routechecks/gateway.go +++ b/internal/routechecks/gateway.go @@ -104,6 +104,17 @@ func CheckGatewayRouteKindAllowed(input Input, parentRef gatewayv1.ParentReferen continue } + // Only evaluate the listener(s) this route actually targets. A route + // pinned to a listener via sectionName must not be rejected because a + // *different* listener on the same Gateway restricts its allowed kinds. + // Without this guard, a single kind-restricted listener (e.g. a TCP + // listener with allowedRoutes.kinds=[TCPRoute]) rejects every HTTPRoute + // on the Gateway. Mirrors the sectionName guard in + // CheckGatewayAllowedForNamespace. + if parentRef.SectionName != nil && listener.Name != *parentRef.SectionName { + continue + } + allowed := false routeGVK := input.GetGVK() for _, kind := range listener.AllowedRoutes.Kinds { diff --git a/internal/routechecks/gateway_test.go b/internal/routechecks/gateway_test.go new file mode 100644 index 0000000..6fcf17c --- /dev/null +++ b/internal/routechecks/gateway_test.go @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2024 Matthew Penner + +package routechecks + +import ( + "context" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +// fakeInput is a minimal Input implementation for exercising the CheckGateway* +// functions without a live API server. Only the methods used by +// CheckGatewayRouteKindAllowed do anything meaningful. +type fakeInput struct { + gvk schema.GroupVersionKind + gw *gatewayv1.Gateway + conditions []metav1.Condition +} + +func (f *fakeInput) GetRules() []GenericRule { return nil } +func (f *fakeInput) GetNamespace() string { return f.gw.Namespace } +func (f *fakeInput) GetClient() client.Client { return nil } +func (f *fakeInput) GetContext() context.Context { return context.Background() } +func (f *fakeInput) GetGVK() schema.GroupVersionKind { return f.gvk } +func (f *fakeInput) GetGrants() []gatewayv1beta1.ReferenceGrant { return nil } +func (f *fakeInput) GetHostnames() []gatewayv1.Hostname { return nil } +func (f *fakeInput) GetGateway(_ gatewayv1.ParentReference) (*gatewayv1.Gateway, error) { + return f.gw, nil +} +func (f *fakeInput) SetParentCondition(_ gatewayv1.ParentReference, condition metav1.Condition) { + f.conditions = append(f.conditions, condition) +} +func (f *fakeInput) SetAllParentCondition(condition metav1.Condition) { + f.conditions = append(f.conditions, condition) +} + +func sectionPtr(s string) *gatewayv1.SectionName { + v := gatewayv1.SectionName(s) + return &v +} + +func groupPtr(s string) *gatewayv1.Group { + v := gatewayv1.Group(s) + return &v +} + +func kindRestrictedListener(name string, kind gatewayv1.Kind) gatewayv1.Listener { + return gatewayv1.Listener{ + Name: gatewayv1.SectionName(name), + AllowedRoutes: &gatewayv1.AllowedRoutes{ + Kinds: []gatewayv1.RouteGroupKind{ + {Group: groupPtr("gateway.networking.k8s.io"), Kind: kind}, + }, + }, + } +} + +// unrestrictedListener has no allowedRoutes.kinds, so it imposes no kind +// restriction (the kind check skips it). +func unrestrictedListener(name string) gatewayv1.Listener { + return gatewayv1.Listener{Name: gatewayv1.SectionName(name)} +} + +func gwWith(listeners ...gatewayv1.Listener) *gatewayv1.Gateway { + return &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "caddy-gateway"}, + Spec: gatewayv1.GatewaySpec{Listeners: listeners}, + } +} + +var ( + httpGVK = gatewayv1.SchemeGroupVersion.WithKind("HTTPRoute") + tcpGVK = gatewayv1alpha2.SchemeGroupVersion.WithKind("TCPRoute") +) + +func TestCheckGatewayRouteKindAllowed(t *testing.T) { + tests := []struct { + name string + gvk schema.GroupVersionKind + gw *gatewayv1.Gateway + sectionName *gatewayv1.SectionName + wantAllowed bool + }{ + { + // Regression guard: this is the exact 2026-06-05 incident. An HTTPRoute + // pinned to its own listener must NOT be rejected just because a + // *different* TCP listener restricts kinds to TCPRoute. + name: "httproute targeting own listener is allowed despite a kind-restricted TCP listener", + gvk: httpGVK, + gw: gwWith(kindRestrictedListener("ldaps", "TCPRoute"), unrestrictedListener("web")), + sectionName: sectionPtr("web"), + wantAllowed: true, + }, + { + // Did not over-correct: a route that genuinely targets a listener + // restricting to a different kind is still rejected. + name: "httproute targeting a TCPRoute-only listener is rejected", + gvk: httpGVK, + gw: gwWith(kindRestrictedListener("ldaps", "TCPRoute")), + sectionName: sectionPtr("ldaps"), + wantAllowed: false, + }, + { + // L4 path stays intact: the TCPRoute that targets the ldaps listener + // is still allowed after the fix. + name: "tcproute targeting the TCPRoute-only listener is allowed", + gvk: tcpGVK, + gw: gwWith(kindRestrictedListener("ldaps", "TCPRoute"), unrestrictedListener("web")), + sectionName: sectionPtr("ldaps"), + wantAllowed: true, + }, + { + // Backwards compatibility: a route with no sectionName is still + // evaluated against every kind-restricted listener (unchanged behavior). + name: "httproute without sectionName is still rejected by a kind-restricted listener", + gvk: httpGVK, + gw: gwWith(kindRestrictedListener("ldaps", "TCPRoute")), + sectionName: nil, + wantAllowed: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + in := &fakeInput{gvk: tt.gvk, gw: tt.gw} + parentRef := gatewayv1.ParentReference{ + Name: gatewayv1.ObjectName(tt.gw.Name), + SectionName: tt.sectionName, + } + + allowed, err := CheckGatewayRouteKindAllowed(in, parentRef) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if allowed != tt.wantAllowed { + t.Fatalf("allowed = %v, want %v", allowed, tt.wantAllowed) + } + // A rejection must set a condition; an allow must not. + if tt.wantAllowed && len(in.conditions) != 0 { + t.Fatalf("expected no parent condition on allow, got %d", len(in.conditions)) + } + if !tt.wantAllowed { + if len(in.conditions) == 0 { + t.Fatalf("expected a parent condition on reject, got none") + } + if got := in.conditions[0].Reason; got != string(gatewayv1.RouteReasonNotAllowedByListeners) { + t.Fatalf("reject reason = %q, want %q", got, gatewayv1.RouteReasonNotAllowedByListeners) + } + } + }) + } +}