test(roles): exercise the disaggregated ingest and query pools

.gitignore

@@ -4,6 +4,9 @@
 *.test
 *.out
 
+# Ref-pinned sibling checkouts (E2E_SRC_CACHE)
+/.e2e-src/
+
 # Logs and scratch output
 *.log
 

Makefile

@@ -33,6 +33,18 @@ test-fast: ## Rebuild oteldb from local source and re-run against the EXISTING k
 test-no-build: ## Re-run against the existing cluster WITHOUT rebuilding/reloading images.
 	E2E_REUSE_CLUSTER=true E2E_SKIP_BUILD=true go test ./... $(GO_TEST_FLAGS)
 
+# Role pools need spec.ingest/spec.query (oteldb/operator#14, #15) and cmd/odbselect
+# (oteldb/oteldb#1266). Until those land, build the siblings from their branches; afterwards drop
+# the refs (or set them to main) and the target is just `E2E_ROLE_POOLS=1`.
+ROLE_OPERATOR_REF ?= claude/role-query-group
+ROLE_OTELDB_REF   ?= claude/1263-odbselect
+
+.PHONY: test-roles
+test-roles: ## Run the suite with the odbingest/odbselect role pools deployed as well.
+	E2E_ROLE_POOLS=1 \
+	OPERATOR_REF=$(ROLE_OPERATOR_REF) OTELDB_REF=$(ROLE_OTELDB_REF) \
+	go test ./... $(GO_TEST_FLAGS)
+
 .PHONY: cluster
 cluster: ## Create the kind cluster only.
 	kind create cluster --name $(KIND_CLUSTER) --config kind.yaml --wait 120s

README.md

@@ -100,6 +100,7 @@ shared-store cluster that isn't one.
 make test          # full run: build + fresh kind cluster + all specs + teardown
 make test-keep     # same, but leave the cluster up for inspection
 make test-fast     # rebuild oteldb from source, reuse an existing cluster (fast iteration)
+make test-roles    # full run plus the odbingest/odbselect role pools (see below)
 make test-no-build # reuse cluster AND already-loaded images (fastest; just re-run specs)
 make cluster       # create the kind cluster only
 make clean         # delete the kind cluster
@@ -118,8 +119,16 @@ make test-fast     # rebuild + reload oteldb, redeploy CR, re-run — repeat on
 |-----|---------|---------|
 | `OTELDB_SRC` | `/src/oteldb/oteldb` | oteldb source tree to build |
 | `OPERATOR_SRC` | `/src/oteldb/operator` | operator source tree to build/deploy |
+| `OTELDB_REF` | _unset_ | build oteldb from this git ref instead of `OTELDB_SRC` (branch, tag or SHA) |
+| `OPERATOR_REF` | _unset_ | build the operator from this git ref instead of `OPERATOR_SRC` |
+| `OTELDB_REPO` | `https://github.com/oteldb/oteldb.git` | repo cloned when `OTELDB_REF` is set |
+| `OPERATOR_REPO` | `https://github.com/oteldb/operator.git` | repo cloned when `OPERATOR_REF` is set |
+| `E2E_SRC_CACHE` | `.e2e-src` | where ref-pinned clones are cached between runs |
 | `OTELDB_IMAGE` | `oteldb/oteldb:e2e` | built oteldb image tag (must match `manifests/cluster.yaml` `spec.image`) |
 | `OPERATOR_IMAGE` | `oteldb/operator:e2e` | built operator image tag |
+| `INGEST_IMAGE` | `oteldb/odbingest:e2e` | built odbingest image tag (must match `manifests/role-pools.yaml`) |
+| `QUERY_IMAGE` | `oteldb/odbselect:e2e` | built odbselect image tag (must match `manifests/role-pools.yaml`) |
+| `E2E_ROLE_POOLS` | _unset_ | also deploy and exercise the `spec.ingest` / `spec.query` pools |
 | `KIND_CLUSTER` | `oteldb-e2e` | kind cluster name |
 | `KIND_CONFIG` | `kind.yaml` | kind cluster config |
 | `E2E_NAMESPACE` | `oteldb-e2e` | namespace for the cluster + etcd |
@@ -128,18 +137,50 @@ make test-fast     # rebuild + reload oteldb, redeploy CR, re-run — repeat on
 | `E2E_REUSE_CLUSTER` | _unset_ | assume the kind cluster already exists (no create/delete) |
 | `E2E_KEEP_CLUSTER` | _unset_ | leave the kind cluster running after the suite |
 
+## Role pools (disaggregated topology)
+
+By default the cluster is symmetric: every node ingests, stores and queries. Setting
+`E2E_ROLE_POOLS=1` merge-patches `manifests/role-pools.yaml` onto the same CR, adding a stateless
+`odbingest` pool (`spec.ingest`) and a stateless `odbselect` pool (`spec.query`), each a Deployment
+behind its own Service, each scaling independently of the storage nodes.
+
+`roles_test.go` then writes through the ingest pool's Service and reads back through the query
+pool's. That crossing is the whole point of the split and the one thing a symmetric suite cannot
+check: with one process doing both, a write-then-read proves nothing about the boundary. It also
+asserts the pools stay out of the storage ring — a pool pod that joined it would own shards whose
+data it does not have — and that losing a pod from each pool costs only capacity.
+
+The two binaries are built from the same oteldb source tree as `oteldb` itself
+(`./cmd/odbingest`, `./cmd/odbselect`), so the fact that neither is in a released image is not a
+problem here.
+
+This needs changes that are not on `main` yet — oteldb/operator#14 (`spec.ingest`),
+oteldb/operator#15 (`spec.query`, based on #14) and oteldb/oteldb#1266 (`cmd/odbselect`) — plus
+oteldb/operator#16 for `spec.cluster.privateBackend`. Point the suite at them with the ref knobs:
+
+```bash
+E2E_ROLE_POOLS=1 \
+OPERATOR_REF=claude/role-query-group \
+OTELDB_REF=claude/1263-odbselect \
+make test
+```
+
+Once they land, drop the two `*_REF` variables (or set them to `main`) and everything else is
+unchanged.
+
 ## Layout
 
 ```
 manifests/
   etcd-dev.yaml     dev etcd (single replica, emptyDir) — bring-your-own coordination
   cluster.yaml      the OtelDBCluster under test (file backend, RF=2, image pinned)
+  role-pools.yaml   merge patch adding the odbingest/odbselect pools (E2E_ROLE_POOLS)
 fast.Dockerfile     minimal image wrapping a host-built binary (fast rebuild path)
 kind.yaml           1 control-plane + 3 workers (room to spread replicas + survive a node loss)
 internal/
   cfg/              env-driven suite configuration
   shell/            command execution wrappers (docker/kind/kubectl/go/make)
-  build/            fast image path: host go build -> COPY image -> kind load
+  build/            fast image path: host go build -> COPY image -> kind load; ref-pinned checkouts
   kubeutil/         kubectl helpers: apply/wait/list/delete + Service port-forward
   otlp/             OTLP/gRPC ingest (traces/logs/metrics) + Tempo/Loki/Prometheus query clients
   nodeadmin/        per-node admin API client (/api/v1/storage): parts, shards, part-sync stats
@@ -148,4 +189,5 @@ helpers_test.go     shared port-forward + ingest/verify + flushed-part helpers
 normal_test.go      happy-path ingest/query specs
 durability_test.go  shared-nothing mode + flushed-part replication specs
 chaos_test.go       pod-kill and disk-loss recovery specs
+roles_test.go       disaggregated ingest/query pool specs (E2E_ROLE_POOLS)
 ```

fast.Dockerfile

@@ -4,17 +4,22 @@
 # local source edit is a `go build` + a single COPY layer — no in-container module download, no
 # full image rebuild.
 #
-# Used for both the oteldb server and the operator manager (BIN selects which). The binary is
-# installed at /<BIN>:
+# Used for the oteldb server, the operator manager and the odbingest/odbselect role binaries (BIN
+# selects which). The binary is installed at BOTH /<BIN> and /usr/local/bin/<BIN>, because the three
+# consumers disagree about where to look:
 #   - oteldb: the CR sets no command, so the container runs this image's ENTRYPOINT; the shell
 #     wrapper execs /oteldb and forwards the operator-appended args (e.g. --config=...).
 #   - operator: the manager Deployment overrides command with ["/manager"], bypassing the
 #     ENTRYPOINT, so /manager simply has to exist at the root.
+#   - odbingest / odbselect: their Deployments override command with the release image's absolute
+#     path, /usr/local/bin/<bin>, so the fast image has to mirror that layout or the container dies
+#     with "no such file or directory" before it logs anything.
 FROM alpine:3.20
 RUN apk add --no-cache ca-certificates
 
 ARG BIN=oteldb
-COPY ${BIN} /${BIN}
+COPY ${BIN} /usr/local/bin/${BIN}
+RUN ln -s /usr/local/bin/${BIN} /${BIN}
 
 # Create an /etc/passwd entry for the non-root UID. oteldb (go-faster/sdk resource detection)
 # calls user.Current(), which — in a CGO-disabled binary — needs a passwd entry to resolve the

internal/build/source.go

@@ -0,0 +1,53 @@
+package build
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo/v2" //nolint:revive,staticcheck // dot-import is the Ginkgo idiom.
+
+	"github.com/oteldb/e2e/internal/shell"
+)
+
+// Source resolves the source tree to build a sibling repo from.
+//
+// With ref empty it returns dir unchanged — the local working tree, which is the fast local loop
+// this suite is built around. With ref set it clones repoURL into a cache directory and checks that
+// ref out detached, so the suite can run against an unmerged branch (or a pinned commit) without
+// touching, or even needing, the developer's own checkout. The cache is reused across runs, so the
+// second run costs a fetch.
+//
+// The clone is blobless (--filter=blob:none): the whole history's trees are cheap and the build only
+// ever materializes one commit's blobs.
+func Source(name, dir, repoURL, ref, cacheDir string) (string, error) {
+	if ref == "" {
+		return dir, nil
+	}
+	if repoURL == "" {
+		return "", fmt.Errorf("%s: a ref (%q) was requested but no repository URL is configured", name, ref)
+	}
+
+	target := filepath.Join(cacheDir, name)
+	if _, err := os.Stat(filepath.Join(target, ".git")); err != nil {
+		if err := os.MkdirAll(cacheDir, 0o755); err != nil {
+			return "", fmt.Errorf("create source cache %s: %w", cacheDir, err)
+		}
+		By(fmt.Sprintf("cloning %s into %s", repoURL, target))
+		clone := shell.Command("git", "clone", "--quiet", "--filter=blob:none", repoURL, target)
+		if _, err := shell.Run(clone); err != nil {
+			return "", fmt.Errorf("clone %s: %w", repoURL, err)
+		}
+	}
+
+	By(fmt.Sprintf("checking out %s@%s", name, ref))
+	// Fetching the ref by name covers a branch, a tag and a full SHA alike, and FETCH_HEAD then
+	// resolves it without assuming a local branch exists or is current.
+	if _, err := shell.Run(shell.InDir(target, "git", "fetch", "--quiet", "origin", ref)); err != nil {
+		return "", fmt.Errorf("fetch %s %s: %w", repoURL, ref, err)
+	}
+	if _, err := shell.Run(shell.InDir(target, "git", "checkout", "--quiet", "--detach", "FETCH_HEAD")); err != nil {
+		return "", fmt.Errorf("checkout %s %s: %w", repoURL, ref, err)
+	}
+	return target, nil
+}

internal/cfg/cfg.go

@@ -12,10 +12,26 @@ type Config struct {
 	// OperatorSrc is the local operator source tree (Makefile-driven deploy).
 	OperatorSrc string
 
+	// OteldbRepo / OteldbRef and OperatorRepo / OperatorRef pin a sibling repo to a git ref
+	// instead of the local working tree. A non-empty ref makes the suite clone the repo into
+	// SrcCache and build that ref, so it can run against an unmerged branch before it lands and
+	// against "main" after, without editing anyone's checkout. Empty ref = use the local tree.
+	OteldbRepo   string
+	OteldbRef    string
+	OperatorRepo string
+	OperatorRef  string
+	// SrcCache is where ref-pinned clones live, reused across runs.
+	SrcCache string
+
 	// OteldbImage / OperatorImage are the locally built, kind-loaded image tags. They MUST match
 	// spec.image in manifests/cluster.yaml (oteldb) and the operator deploy IMG respectively.
 	OteldbImage   string
 	OperatorImage string
+	// IngestImage / QueryImage are the odbingest and odbselect images for the role pools. They
+	// MUST match spec.ingest.image / spec.query.image in manifests/cluster.yaml. They are separate
+	// binaries, so they cannot share the oteldb image the fast build path produces.
+	IngestImage string
+	QueryImage  string
 
 	// KindCluster is the kind cluster name.
 	KindCluster string
@@ -35,15 +51,26 @@ type Config struct {
 	KeepCluster bool
 	// ReuseCluster assumes the kind cluster already exists (no create, no delete).
 	ReuseCluster bool
+	// RolePools deploys and exercises the disaggregated topology: storage nodes plus an odbingest
+	// pool and an odbselect pool (spec.ingest / spec.query). Off by default until the operator and
+	// oteldb changes it needs are on main; see README.
+	RolePools bool
 }
 
 // Load resolves the suite config from the environment, applying defaults.
 func Load() Config {
 	return Config{
 		OteldbSrc:         shell.Getenv("OTELDB_SRC", "/src/oteldb/oteldb"),
 		OperatorSrc:       shell.Getenv("OPERATOR_SRC", "/src/oteldb/operator"),
+		OteldbRepo:        shell.Getenv("OTELDB_REPO", "https://github.com/oteldb/oteldb.git"),
+		OteldbRef:         shell.Getenv("OTELDB_REF", ""),
+		OperatorRepo:      shell.Getenv("OPERATOR_REPO", "https://github.com/oteldb/operator.git"),
+		OperatorRef:       shell.Getenv("OPERATOR_REF", ""),
+		SrcCache:          shell.Getenv("E2E_SRC_CACHE", ".e2e-src"),
 		OteldbImage:       shell.Getenv("OTELDB_IMAGE", "oteldb/oteldb:e2e"),
 		OperatorImage:     shell.Getenv("OPERATOR_IMAGE", "oteldb/operator:e2e"),
+		IngestImage:       shell.Getenv("INGEST_IMAGE", "oteldb/odbingest:e2e"),
+		QueryImage:        shell.Getenv("QUERY_IMAGE", "oteldb/odbselect:e2e"),
 		KindCluster:       shell.Getenv("KIND_CLUSTER", "oteldb-e2e"),
 		KindConfig:        shell.Getenv("KIND_CONFIG", "kind.yaml"),
 		Namespace:         shell.Getenv("E2E_NAMESPACE", "oteldb-e2e"),
@@ -52,5 +79,6 @@ func Load() Config {
 		SkipBuild:         shell.Truthy("E2E_SKIP_BUILD"),
 		KeepCluster:       shell.Truthy("E2E_KEEP_CLUSTER"),
 		ReuseCluster:      shell.Truthy("E2E_REUSE_CLUSTER"),
+		RolePools:         shell.Truthy("E2E_ROLE_POOLS"),
 	}
 }

internal/kubeutil/kubeutil.go

@@ -41,6 +41,13 @@ func Delete(namespace, path string) error {
 	return err
 }
 
+// PatchMerge applies a strategic-merge patch from a file to one resource.
+func PatchMerge(namespace, kind, name, path string) error {
+	_, err := Kubectl("patch", kind, name, "-n", namespace,
+		"--type=merge", "--patch-file", path)
+	return err
+}
+
 // EnsureNamespace creates the namespace if it does not already exist.
 func EnsureNamespace(ns string) error {
 	if _, err := Kubectl("get", "namespace", ns); err == nil {
@@ -51,12 +58,36 @@ func EnsureNamespace(ns string) error {
 }
 
 // WaitRollout blocks until a StatefulSet/Deployment rollout completes within timeout.
+//
+// It waits for the object to exist first. The workloads here are created by the operator in
+// response to the CR, so right after applying one there is a window where it does not exist yet —
+// and `kubectl rollout status` does not wait through that, it fails immediately with NotFound.
 func WaitRollout(namespace, kindName string, timeout time.Duration) error {
+	if err := WaitExists(namespace, kindName, timeout); err != nil {
+		return err
+	}
 	_, err := Kubectl("rollout", "status", kindName,
 		"-n", namespace, fmt.Sprintf("--timeout=%s", timeout))
 	return err
 }
 
+// WaitExists polls until a resource exists, or timeout elapses.
+func WaitExists(namespace, kindName string, timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	var last error
+	for {
+		if _, err := Kubectl("get", kindName, "-n", namespace); err == nil {
+			return nil
+		} else {
+			last = err
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("%s did not appear in %s: %w", kindName, timeout, last)
+		}
+		time.Sleep(2 * time.Second)
+	}
+}
+
 // WaitAvailable waits for a deployment to report Available (used for the operator manager).
 func WaitAvailable(namespace, deployment string, timeout time.Duration) error {
 	_, err := Kubectl("wait", "deployment/"+deployment,

manifests/role-pools.yaml

@@ -0,0 +1,32 @@
+# The disaggregated half of the topology, merge-patched onto manifests/cluster.yaml when
+# E2E_ROLE_POOLS is set: the same three storage nodes, plus a stateless odbingest pool and a
+# stateless odbselect pool that scale on their own. Neither pool is a ring member or holds data —
+# both watch etcd for membership and resolve shard owners locally.
+#
+# Images are the locally built, kind-loaded odbingest/odbselect binaries (internal/build). They must
+# be separate images: they are separate binaries, and odbselect ships in no released image yet. The
+# pools inherit spec.imagePullPolicy (IfNotPresent) from the cluster, so kind never reaches out.
+#
+# Two replicas each is the smallest count that still proves the pools are pools — a Service in front
+# of more than one pod, so a query can land on a different replica than the one that would have been
+# picked for the write.
+spec:
+  ingest:
+    replicas: 2
+    image: oteldb/odbingest:e2e
+    resources:
+      requests:
+        cpu: "100m"
+        memory: 128Mi
+      limits:
+        memory: 512Mi
+
+  query:
+    replicas: 2
+    image: oteldb/odbselect:e2e
+    resources:
+      requests:
+        cpu: "100m"
+        memory: 128Mi
+      limits:
+        memory: 512Mi