Data files in bundles + adapter config/observability parity (ABI v4)

README.md

@@ -29,7 +29,7 @@ func Handle(req sdk.Request) sdk.Action {
 - **Registry**: S3-compatible object storage, local `file://`, or read-only `https://`
 - **Proxy**: Caddy handler module (`http.handlers.switchboard`), standalone `switchboard serve`, or embeddable Go middleware
 - **Runtime**: wazero with enforced timeouts, memory caps, and output validation
-- **Guests**: TinyGo WASI modules against a small host ABI (`switchboard/v3`)
+- **Guests**: TinyGo WASI modules against a small host ABI (`switchboard/v4`)
 
 ## Install
 
@@ -82,11 +82,11 @@ Highlights:
 
 ## Examples
 
-Real-world rules, each with a behavioral test suite, under [`examples/`](examples/): auth gating, legacy redirects, security headers, maintenance mode, and metadata-driven canary routing.
+Real-world rules, each with a behavioral test suite, under [`examples/`](examples/): auth gating, legacy redirects, security headers, maintenance mode, metadata-driven canary routing, a data-file-driven [IP allowlist](examples/ip-allowlist/), and a typed-config [feature-flags](examples/feature-flags/) rule.
 
 ## Limitations
 
-Rules cannot read request/response bodies, make network calls, or hold state. That's by design: it keeps decisions deterministic, replayable offline, and predictable when they fail. Switchboard is not a CDN, cache, or hosted control plane. Caddy is the reference adapter.
+Rules cannot read request/response bodies, make network calls, or hold state. They can read immutable [data files](docs/content/guides/data-files.mdx) baked into the bundle (allowlists, redirect maps, cohort tables), which keeps lookups deterministic and replayable without reintroducing state. Switchboard is not a CDN, cache, or hosted control plane. Caddy is the reference adapter.
 
 ## Prior art
 

abi/guest/guest.go

@@ -9,3 +9,24 @@ func CurrentRequest() sdk.Request {
 func Return(action sdk.Action) int32 {
 	return sdk.Return(action)
 }
+
+// Data accessors re-export the SDK's read-only bundled data files.
+type Set = sdk.Set
+
+func DataBytes(name string) []byte { return sdk.DataBytes(name) }
+
+func DataString(name string) string { return sdk.DataString(name) }
+
+func DataMap(name string) map[string]string { return sdk.DataMap(name) }
+
+func DataSet(name string) Set { return sdk.DataSet(name) }
+
+func DataLines(name string) []string { return sdk.DataLines(name) }
+
+func DataJSON(name string, v any) error { return sdk.DataJSON(name, v) }
+
+func DataJSONL(name string, out any) error { return sdk.DataJSONL(name, out) }
+
+func DataTOML(name string, v any) error { return sdk.DataTOML(name, v) }
+
+func DataCSV(name string) ([][]string, error) { return sdk.DataCSV(name) }

adapters/http/metrics.go

@@ -13,6 +13,8 @@ import (
 type Metrics struct {
 	invocations *prometheus.CounterVec
 	duration    prometheus.Histogram
+	reg         prometheus.Registerer
+	collectors  []prometheus.Collector
 }
 
 func NewMetrics(reg prometheus.Registerer, service *engine.Service) (*Metrics, error) {
@@ -60,12 +62,31 @@ func NewMetrics(reg prometheus.Registerer, service *engine.Service) (*Metrics, e
 	}
 	for _, collector := range collectors {
 		if err := reg.Register(collector); err != nil {
+			// Roll back partial registration so a retry can start clean.
+			for _, registered := range m.collectors {
+				reg.Unregister(registered)
+			}
 			return nil, err
 		}
+		m.collectors = append(m.collectors, collector)
 	}
+	m.reg = reg
 	return m, nil
 }
 
+// Close unregisters the collectors. Front-ends that register into a shared
+// registry (e.g. Caddy's default) call it on teardown so a reload can
+// re-register cleanly.
+func (m *Metrics) Close() {
+	if m == nil || m.reg == nil {
+		return
+	}
+	for _, collector := range m.collectors {
+		m.reg.Unregister(collector)
+	}
+	m.collectors = nil
+}
+
 func (m *Metrics) ObserveInvocation(decision switchboard.Decision, err error, elapsed time.Duration) {
 	if m == nil {
 		return

adapters/http/middleware.go

@@ -24,21 +24,17 @@ func Middleware(service *engine.Service, opts Options) func(http.Handler) http.H
 	if logger == nil {
 		logger = zap.NewNop()
 	}
+	instrumentation := Instrumentation{Metrics: opts.Metrics, OnDecision: opts.OnDecision}
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			start := time.Now()
 			result, err := service.InvokeWithFallback(r.Context(), RequestFromHTTP(r))
-			if opts.Metrics != nil {
-				opts.Metrics.ObserveInvocation(result.Action.Decision, err, time.Since(start))
-			}
+			instrumentation.Observe(r, result, err, time.Since(start))
 			if err != nil {
 				logger.Warn("switchboard rule invocation failed", zap.String("bundle_id", result.BundleID), zap.Error(err))
 				failUnavailable(w, r, next, opts.FailMode)
 				return
 			}
-			if opts.OnDecision != nil {
-				opts.OnDecision(r, result)
-			}
 			callNext, err := ApplyAction(w, r, result.Action)
 			if err != nil {
 				logger.Warn("unknown switchboard action", zap.String("decision", string(result.Action.Decision)), zap.Error(err))

adapters/http/observability.go

@@ -0,0 +1,42 @@
+package httpadapter
+
+import (
+	"encoding/json"
+	"net/http"
+	"time"
+
+	"github.com/ethndotsh/switchboard/engine"
+)
+
+// Instrumentation records metrics and surfaces decision metadata for a rule
+// invocation. Both the standalone Middleware and the Caddy handler route
+// through it so observability is identical across adapters.
+type Instrumentation struct {
+	Metrics    *Metrics
+	OnDecision func(r *http.Request, result engine.InvokeResult)
+}
+
+// Observe records the invocation's metrics and, on success, surfaces its
+// decision. It is safe to call on a zero Instrumentation.
+func (in Instrumentation) Observe(r *http.Request, result engine.InvokeResult, err error, elapsed time.Duration) {
+	if in.Metrics != nil {
+		in.Metrics.ObserveInvocation(result.Action.Decision, err, elapsed)
+	}
+	if err == nil && in.OnDecision != nil {
+		in.OnDecision(r, result)
+	}
+}
+
+// StatusHandler serves the service's status as JSON. Front-ends mount it at a
+// path of their choosing so the status contract is defined once.
+func StatusHandler(service *engine.Service) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		data, err := json.MarshalIndent(service.Status(), "", "  ")
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write(data)
+	})
+}

caddy/handler.go

@@ -4,13 +4,15 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"time"
 
 	caddyserver "github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 	"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	httpadapter "github.com/ethndotsh/switchboard/adapters/http"
 	"github.com/ethndotsh/switchboard/engine"
+	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/zap"
 )
 
@@ -38,16 +40,22 @@ type Switchboard struct {
 	MaxActionBytes     string `json:"max_action_bytes,omitempty"`
 	MaxHeaderOps       int    `json:"max_header_ops,omitempty"`
 	MaxResponseBody    string `json:"max_response_body,omitempty"`
+	MaxDataBytes       string `json:"max_data_bytes,omitempty"`
 	CacheDir           string `json:"cache_dir,omitempty"`
 	BootstrapFromCache string `json:"bootstrap_from_cache,omitempty"`
 	PoolAutoscale      string `json:"pool_autoscale,omitempty"`
 	PoolSize           int    `json:"pool_size,omitempty"`
 	MinPoolSize        int    `json:"min_pool_size,omitempty"`
 	MaxPoolSize        int    `json:"max_pool_size,omitempty"`
+	// StatusPath, when set, serves the switchboard status JSON at that request
+	// path instead of proxying the request downstream.
+	StatusPath string `json:"status_path,omitempty"`
 
-	service *engine.Service
-	manager *engine.RuntimeManager
-	logger  *zap.Logger
+	service       *engine.Service
+	manager       *engine.RuntimeManager
+	logger        *zap.Logger
+	metrics       *httpadapter.Metrics
+	statusHandler http.Handler
 }
 
 func (Switchboard) CaddyModule() caddyserver.ModuleInfo {
@@ -72,6 +80,7 @@ func (s *Switchboard) Provision(ctx caddyserver.Context) error {
 		MaxActionBytes:     s.MaxActionBytes,
 		MaxHeaderOps:       s.MaxHeaderOps,
 		MaxResponseBody:    s.MaxResponseBody,
+		MaxDataBytes:       s.MaxDataBytes,
 		CacheDir:           s.CacheDir,
 		BootstrapFromCache: s.BootstrapFromCache,
 		PoolAutoscale:      s.PoolAutoscale,
@@ -83,17 +92,33 @@ func (s *Switchboard) Provision(ctx caddyserver.Context) error {
 		return err
 	}
 	s.service = service
+	// Register switchboard_* metrics on Caddy's default registry (which backs
+	// the admin /metrics endpoint) so the Caddy path reports the same metrics
+	// as standalone serve.
+	metrics, err := httpadapter.NewMetrics(prometheus.DefaultRegisterer, service)
+	if err != nil {
+		s.logger.Warn("switchboard metrics registration failed", zap.Error(err))
+	} else {
+		s.metrics = metrics
+	}
+	s.statusHandler = httpadapter.StatusHandler(service)
 	return nil
 }
 
 func (s *Switchboard) Cleanup() error {
+	s.metrics.Close()
 	if s.service != nil {
 		return s.service.Close(context.Background())
 	}
 	return nil
 }
 
 func (s *Switchboard) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
+	if s.StatusPath != "" && r.URL.Path == s.StatusPath && s.statusHandler != nil {
+		s.statusHandler.ServeHTTP(w, r)
+		return nil
+	}
+
 	req := httpadapter.RequestFromHTTP(r)
 	// Prefer Caddy's client IP, which honors trusted_proxies.
 	if clientIP, ok := caddyhttp.GetVar(r.Context(), caddyhttp.ClientIPVarKey).(string); ok && clientIP != "" {
@@ -102,6 +127,7 @@ func (s *Switchboard) ServeHTTP(w http.ResponseWriter, r *http.Request, next cad
 
 	var result engine.InvokeResult
 	var err error
+	start := time.Now()
 	if s.service != nil {
 		result, err = s.service.InvokeWithFallback(r.Context(), req)
 	} else {
@@ -113,6 +139,7 @@ func (s *Switchboard) ServeHTTP(w http.ResponseWriter, r *http.Request, next cad
 			result.Action, err = current.Invoke(r.Context(), req)
 		}
 	}
+	s.metrics.ObserveInvocation(result.Action.Decision, err, time.Since(start))
 	if err != nil {
 		if s.logger != nil {
 			s.logger.Warn("switchboard rule invocation failed", zap.String("bundle_id", result.BundleID), zap.Error(err))
@@ -257,6 +284,10 @@ func (s *Switchboard) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 				err = positiveIntArg(&s.MaxHeaderOps, "max_header_ops")
 			case "max_response_body":
 				err = stringArg(&s.MaxResponseBody)
+			case "max_data_bytes":
+				err = stringArg(&s.MaxDataBytes)
+			case "status_path":
+				err = stringArg(&s.StatusPath)
 			case "cache_dir":
 				err = stringArg(&s.CacheDir)
 			case "bootstrap_from_cache":

caddy/metrics_integration_test.go

@@ -0,0 +1,90 @@
+package caddy
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	httpadapter "github.com/ethndotsh/switchboard/adapters/http"
+	"github.com/ethndotsh/switchboard/engine"
+	"github.com/ethndotsh/switchboard/internal/bundle"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// TestHandlerRecordsMetrics locks the Track 2 fix: the Caddy ServeHTTP path,
+// which previously bypassed the shared middleware, now records switchboard_*
+// invocation metrics.
+func TestHandlerRecordsMetrics(t *testing.T) {
+	dist := filepath.Join("..", "dist")
+	module, err := os.ReadFile(filepath.Join(dist, "module.wasm"))
+	if err != nil {
+		if os.IsNotExist(err) {
+			t.Skip("dist/module.wasm not present; run switchboard build for integration coverage")
+		}
+		t.Fatal(err)
+	}
+	manifestData, err := os.ReadFile(filepath.Join(dist, "manifest.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	manifest, err := bundle.ParseManifest(manifestData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	checksumData, err := os.ReadFile(filepath.Join(dist, "checksum.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	checksum := strings.TrimSpace(string(checksumData))
+
+	ctx := context.Background()
+	wasmRuntime, err := engine.NewWasmRuntime(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer wasmRuntime.Close(ctx)
+	runtime, err := engine.NewRuntime(ctx, wasmRuntime, bundle.Bundle{
+		ID:       manifest.Version,
+		Module:   module,
+		Manifest: manifest,
+		Checksum: checksum,
+	}, engine.InvokeLimits{Timeout: 500 * time.Millisecond}, 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reg := prometheus.NewRegistry()
+	metrics, err := httpadapter.NewMetrics(reg, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	handler := &Switchboard{FailMode: "open", manager: &engine.RuntimeManager{}, metrics: metrics}
+	handler.manager.Activate(runtime)
+
+	req := httptest.NewRequest(http.MethodGet, "/blocked", nil)
+	if err := handler.ServeHTTP(httptest.NewRecorder(), req, &nextHandler{}); err != nil {
+		t.Fatal(err)
+	}
+
+	families, err := reg.Gather()
+	if err != nil {
+		t.Fatal(err)
+	}
+	var total float64
+	for _, family := range families {
+		if family.GetName() != "switchboard_invocations_total" {
+			continue
+		}
+		for _, metric := range family.GetMetric() {
+			total += metric.GetCounter().GetValue()
+		}
+	}
+	if total == 0 {
+		t.Fatal("expected switchboard_invocations_total to be recorded via the Caddy handler")
+	}
+}