From 38fe3090e58f679b0ad641e2d961fee3339b4377 Mon Sep 17 00:00:00 2001 From: tdakkota Date: Tue, 18 Aug 2026 01:53:39 +0300 Subject: [PATCH] test(roles): exercise the disaggregated ingest and query pools Deploy an odbingest pool (spec.ingest) and an odbselect pool (spec.query) alongside the storage nodes, then write through the ingest pool's Service and read back through the query pool's for all three signals. That crossing is the point of the split and the one thing a symmetric suite cannot check: with one process doing both, a write-then-read proves nothing about the boundary. Also asserts the pools stay out of the storage ring and that losing a pod from each costs only capacity. Sibling repos can now be pinned to a git ref (OTELDB_REF / OPERATOR_REF), cloned into a cache instead of the local tree, so this runs against the unmerged branches now and main later. odbingest and odbselect go through the same fast build path as oteldb. Three fixes the pools exposed, all pre-existing: the fast image installed binaries only at /, but the pools' Deployments exec the release path /usr/local/bin/; WaitRollout raced the operator creating the workload, since `kubectl rollout status` fails NotFound rather than waiting; and `make deploy` leaves a reused cluster running the previous manager, because the image tag never changes. Gated behind E2E_ROLE_POOLS until oteldb/operator#14, #15 and oteldb/oteldb#1266 land. --- .gitignore | 3 + Makefile | 12 +++ README.md | 44 ++++++++- fast.Dockerfile | 11 ++- internal/build/source.go | 53 +++++++++++ internal/cfg/cfg.go | 28 ++++++ internal/kubeutil/kubeutil.go | 31 +++++++ manifests/role-pools.yaml | 32 +++++++ roles_test.go | 167 ++++++++++++++++++++++++++++++++++ suite_test.go | 56 ++++++++++++ 10 files changed, 433 insertions(+), 4 deletions(-) create mode 100644 internal/build/source.go create mode 100644 manifests/role-pools.yaml create mode 100644 roles_test.go diff --git a/.gitignore b/.gitignore index df32439..8b4f0b2 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ *.test *.out +# Ref-pinned sibling checkouts (E2E_SRC_CACHE) +/.e2e-src/ + # Logs and scratch output *.log diff --git a/Makefile b/Makefile index f73d42e..f6b2557 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,18 @@ test-fast: ## Rebuild oteldb from local source and re-run against the EXISTING k test-no-build: ## Re-run against the existing cluster WITHOUT rebuilding/reloading images. E2E_REUSE_CLUSTER=true E2E_SKIP_BUILD=true go test ./... $(GO_TEST_FLAGS) +# Role pools need spec.ingest/spec.query (oteldb/operator#14, #15) and cmd/odbselect +# (oteldb/oteldb#1266). Until those land, build the siblings from their branches; afterwards drop +# the refs (or set them to main) and the target is just `E2E_ROLE_POOLS=1`. +ROLE_OPERATOR_REF ?= claude/role-query-group +ROLE_OTELDB_REF ?= claude/1263-odbselect + +.PHONY: test-roles +test-roles: ## Run the suite with the odbingest/odbselect role pools deployed as well. + E2E_ROLE_POOLS=1 \ + OPERATOR_REF=$(ROLE_OPERATOR_REF) OTELDB_REF=$(ROLE_OTELDB_REF) \ + go test ./... $(GO_TEST_FLAGS) + .PHONY: cluster cluster: ## Create the kind cluster only. kind create cluster --name $(KIND_CLUSTER) --config kind.yaml --wait 120s diff --git a/README.md b/README.md index 7c60429..2734e78 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ shared-store cluster that isn't one. make test # full run: build + fresh kind cluster + all specs + teardown make test-keep # same, but leave the cluster up for inspection make test-fast # rebuild oteldb from source, reuse an existing cluster (fast iteration) +make test-roles # full run plus the odbingest/odbselect role pools (see below) make test-no-build # reuse cluster AND already-loaded images (fastest; just re-run specs) make cluster # create the kind cluster only make clean # delete the kind cluster @@ -118,8 +119,16 @@ make test-fast # rebuild + reload oteldb, redeploy CR, re-run — repeat on |-----|---------|---------| | `OTELDB_SRC` | `/src/oteldb/oteldb` | oteldb source tree to build | | `OPERATOR_SRC` | `/src/oteldb/operator` | operator source tree to build/deploy | +| `OTELDB_REF` | _unset_ | build oteldb from this git ref instead of `OTELDB_SRC` (branch, tag or SHA) | +| `OPERATOR_REF` | _unset_ | build the operator from this git ref instead of `OPERATOR_SRC` | +| `OTELDB_REPO` | `https://github.com/oteldb/oteldb.git` | repo cloned when `OTELDB_REF` is set | +| `OPERATOR_REPO` | `https://github.com/oteldb/operator.git` | repo cloned when `OPERATOR_REF` is set | +| `E2E_SRC_CACHE` | `.e2e-src` | where ref-pinned clones are cached between runs | | `OTELDB_IMAGE` | `oteldb/oteldb:e2e` | built oteldb image tag (must match `manifests/cluster.yaml` `spec.image`) | | `OPERATOR_IMAGE` | `oteldb/operator:e2e` | built operator image tag | +| `INGEST_IMAGE` | `oteldb/odbingest:e2e` | built odbingest image tag (must match `manifests/role-pools.yaml`) | +| `QUERY_IMAGE` | `oteldb/odbselect:e2e` | built odbselect image tag (must match `manifests/role-pools.yaml`) | +| `E2E_ROLE_POOLS` | _unset_ | also deploy and exercise the `spec.ingest` / `spec.query` pools | | `KIND_CLUSTER` | `oteldb-e2e` | kind cluster name | | `KIND_CONFIG` | `kind.yaml` | kind cluster config | | `E2E_NAMESPACE` | `oteldb-e2e` | namespace for the cluster + etcd | @@ -128,18 +137,50 @@ make test-fast # rebuild + reload oteldb, redeploy CR, re-run — repeat on | `E2E_REUSE_CLUSTER` | _unset_ | assume the kind cluster already exists (no create/delete) | | `E2E_KEEP_CLUSTER` | _unset_ | leave the kind cluster running after the suite | +## Role pools (disaggregated topology) + +By default the cluster is symmetric: every node ingests, stores and queries. Setting +`E2E_ROLE_POOLS=1` merge-patches `manifests/role-pools.yaml` onto the same CR, adding a stateless +`odbingest` pool (`spec.ingest`) and a stateless `odbselect` pool (`spec.query`), each a Deployment +behind its own Service, each scaling independently of the storage nodes. + +`roles_test.go` then writes through the ingest pool's Service and reads back through the query +pool's. That crossing is the whole point of the split and the one thing a symmetric suite cannot +check: with one process doing both, a write-then-read proves nothing about the boundary. It also +asserts the pools stay out of the storage ring — a pool pod that joined it would own shards whose +data it does not have — and that losing a pod from each pool costs only capacity. + +The two binaries are built from the same oteldb source tree as `oteldb` itself +(`./cmd/odbingest`, `./cmd/odbselect`), so the fact that neither is in a released image is not a +problem here. + +This needs changes that are not on `main` yet — oteldb/operator#14 (`spec.ingest`), +oteldb/operator#15 (`spec.query`, based on #14) and oteldb/oteldb#1266 (`cmd/odbselect`) — plus +oteldb/operator#16 for `spec.cluster.privateBackend`. Point the suite at them with the ref knobs: + +```bash +E2E_ROLE_POOLS=1 \ +OPERATOR_REF=claude/role-query-group \ +OTELDB_REF=claude/1263-odbselect \ +make test +``` + +Once they land, drop the two `*_REF` variables (or set them to `main`) and everything else is +unchanged. + ## Layout ``` manifests/ etcd-dev.yaml dev etcd (single replica, emptyDir) — bring-your-own coordination cluster.yaml the OtelDBCluster under test (file backend, RF=2, image pinned) + role-pools.yaml merge patch adding the odbingest/odbselect pools (E2E_ROLE_POOLS) fast.Dockerfile minimal image wrapping a host-built binary (fast rebuild path) kind.yaml 1 control-plane + 3 workers (room to spread replicas + survive a node loss) internal/ cfg/ env-driven suite configuration shell/ command execution wrappers (docker/kind/kubectl/go/make) - build/ fast image path: host go build -> COPY image -> kind load + build/ fast image path: host go build -> COPY image -> kind load; ref-pinned checkouts kubeutil/ kubectl helpers: apply/wait/list/delete + Service port-forward otlp/ OTLP/gRPC ingest (traces/logs/metrics) + Tempo/Loki/Prometheus query clients nodeadmin/ per-node admin API client (/api/v1/storage): parts, shards, part-sync stats @@ -148,4 +189,5 @@ helpers_test.go shared port-forward + ingest/verify + flushed-part helpers normal_test.go happy-path ingest/query specs durability_test.go shared-nothing mode + flushed-part replication specs chaos_test.go pod-kill and disk-loss recovery specs +roles_test.go disaggregated ingest/query pool specs (E2E_ROLE_POOLS) ``` diff --git a/fast.Dockerfile b/fast.Dockerfile index 92e7e49..ac7ccf4 100644 --- a/fast.Dockerfile +++ b/fast.Dockerfile @@ -4,17 +4,22 @@ # local source edit is a `go build` + a single COPY layer — no in-container module download, no # full image rebuild. # -# Used for both the oteldb server and the operator manager (BIN selects which). The binary is -# installed at /: +# Used for the oteldb server, the operator manager and the odbingest/odbselect role binaries (BIN +# selects which). The binary is installed at BOTH / and /usr/local/bin/, because the three +# consumers disagree about where to look: # - oteldb: the CR sets no command, so the container runs this image's ENTRYPOINT; the shell # wrapper execs /oteldb and forwards the operator-appended args (e.g. --config=...). # - operator: the manager Deployment overrides command with ["/manager"], bypassing the # ENTRYPOINT, so /manager simply has to exist at the root. +# - odbingest / odbselect: their Deployments override command with the release image's absolute +# path, /usr/local/bin/, so the fast image has to mirror that layout or the container dies +# with "no such file or directory" before it logs anything. FROM alpine:3.20 RUN apk add --no-cache ca-certificates ARG BIN=oteldb -COPY ${BIN} /${BIN} +COPY ${BIN} /usr/local/bin/${BIN} +RUN ln -s /usr/local/bin/${BIN} /${BIN} # Create an /etc/passwd entry for the non-root UID. oteldb (go-faster/sdk resource detection) # calls user.Current(), which — in a CGO-disabled binary — needs a passwd entry to resolve the diff --git a/internal/build/source.go b/internal/build/source.go new file mode 100644 index 0000000..a34e2a7 --- /dev/null +++ b/internal/build/source.go @@ -0,0 +1,53 @@ +package build + +import ( + "fmt" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" //nolint:revive,staticcheck // dot-import is the Ginkgo idiom. + + "github.com/oteldb/e2e/internal/shell" +) + +// Source resolves the source tree to build a sibling repo from. +// +// With ref empty it returns dir unchanged — the local working tree, which is the fast local loop +// this suite is built around. With ref set it clones repoURL into a cache directory and checks that +// ref out detached, so the suite can run against an unmerged branch (or a pinned commit) without +// touching, or even needing, the developer's own checkout. The cache is reused across runs, so the +// second run costs a fetch. +// +// The clone is blobless (--filter=blob:none): the whole history's trees are cheap and the build only +// ever materializes one commit's blobs. +func Source(name, dir, repoURL, ref, cacheDir string) (string, error) { + if ref == "" { + return dir, nil + } + if repoURL == "" { + return "", fmt.Errorf("%s: a ref (%q) was requested but no repository URL is configured", name, ref) + } + + target := filepath.Join(cacheDir, name) + if _, err := os.Stat(filepath.Join(target, ".git")); err != nil { + if err := os.MkdirAll(cacheDir, 0o755); err != nil { + return "", fmt.Errorf("create source cache %s: %w", cacheDir, err) + } + By(fmt.Sprintf("cloning %s into %s", repoURL, target)) + clone := shell.Command("git", "clone", "--quiet", "--filter=blob:none", repoURL, target) + if _, err := shell.Run(clone); err != nil { + return "", fmt.Errorf("clone %s: %w", repoURL, err) + } + } + + By(fmt.Sprintf("checking out %s@%s", name, ref)) + // Fetching the ref by name covers a branch, a tag and a full SHA alike, and FETCH_HEAD then + // resolves it without assuming a local branch exists or is current. + if _, err := shell.Run(shell.InDir(target, "git", "fetch", "--quiet", "origin", ref)); err != nil { + return "", fmt.Errorf("fetch %s %s: %w", repoURL, ref, err) + } + if _, err := shell.Run(shell.InDir(target, "git", "checkout", "--quiet", "--detach", "FETCH_HEAD")); err != nil { + return "", fmt.Errorf("checkout %s %s: %w", repoURL, ref, err) + } + return target, nil +} diff --git a/internal/cfg/cfg.go b/internal/cfg/cfg.go index aef6b58..9755013 100644 --- a/internal/cfg/cfg.go +++ b/internal/cfg/cfg.go @@ -12,10 +12,26 @@ type Config struct { // OperatorSrc is the local operator source tree (Makefile-driven deploy). OperatorSrc string + // OteldbRepo / OteldbRef and OperatorRepo / OperatorRef pin a sibling repo to a git ref + // instead of the local working tree. A non-empty ref makes the suite clone the repo into + // SrcCache and build that ref, so it can run against an unmerged branch before it lands and + // against "main" after, without editing anyone's checkout. Empty ref = use the local tree. + OteldbRepo string + OteldbRef string + OperatorRepo string + OperatorRef string + // SrcCache is where ref-pinned clones live, reused across runs. + SrcCache string + // OteldbImage / OperatorImage are the locally built, kind-loaded image tags. They MUST match // spec.image in manifests/cluster.yaml (oteldb) and the operator deploy IMG respectively. OteldbImage string OperatorImage string + // IngestImage / QueryImage are the odbingest and odbselect images for the role pools. They + // MUST match spec.ingest.image / spec.query.image in manifests/cluster.yaml. They are separate + // binaries, so they cannot share the oteldb image the fast build path produces. + IngestImage string + QueryImage string // KindCluster is the kind cluster name. KindCluster string @@ -35,6 +51,10 @@ type Config struct { KeepCluster bool // ReuseCluster assumes the kind cluster already exists (no create, no delete). ReuseCluster bool + // RolePools deploys and exercises the disaggregated topology: storage nodes plus an odbingest + // pool and an odbselect pool (spec.ingest / spec.query). Off by default until the operator and + // oteldb changes it needs are on main; see README. + RolePools bool } // Load resolves the suite config from the environment, applying defaults. @@ -42,8 +62,15 @@ func Load() Config { return Config{ OteldbSrc: shell.Getenv("OTELDB_SRC", "/src/oteldb/oteldb"), OperatorSrc: shell.Getenv("OPERATOR_SRC", "/src/oteldb/operator"), + OteldbRepo: shell.Getenv("OTELDB_REPO", "https://github.com/oteldb/oteldb.git"), + OteldbRef: shell.Getenv("OTELDB_REF", ""), + OperatorRepo: shell.Getenv("OPERATOR_REPO", "https://github.com/oteldb/operator.git"), + OperatorRef: shell.Getenv("OPERATOR_REF", ""), + SrcCache: shell.Getenv("E2E_SRC_CACHE", ".e2e-src"), OteldbImage: shell.Getenv("OTELDB_IMAGE", "oteldb/oteldb:e2e"), OperatorImage: shell.Getenv("OPERATOR_IMAGE", "oteldb/operator:e2e"), + IngestImage: shell.Getenv("INGEST_IMAGE", "oteldb/odbingest:e2e"), + QueryImage: shell.Getenv("QUERY_IMAGE", "oteldb/odbselect:e2e"), KindCluster: shell.Getenv("KIND_CLUSTER", "oteldb-e2e"), KindConfig: shell.Getenv("KIND_CONFIG", "kind.yaml"), Namespace: shell.Getenv("E2E_NAMESPACE", "oteldb-e2e"), @@ -52,5 +79,6 @@ func Load() Config { SkipBuild: shell.Truthy("E2E_SKIP_BUILD"), KeepCluster: shell.Truthy("E2E_KEEP_CLUSTER"), ReuseCluster: shell.Truthy("E2E_REUSE_CLUSTER"), + RolePools: shell.Truthy("E2E_ROLE_POOLS"), } } diff --git a/internal/kubeutil/kubeutil.go b/internal/kubeutil/kubeutil.go index f802f3c..aa97b5f 100644 --- a/internal/kubeutil/kubeutil.go +++ b/internal/kubeutil/kubeutil.go @@ -41,6 +41,13 @@ func Delete(namespace, path string) error { return err } +// PatchMerge applies a strategic-merge patch from a file to one resource. +func PatchMerge(namespace, kind, name, path string) error { + _, err := Kubectl("patch", kind, name, "-n", namespace, + "--type=merge", "--patch-file", path) + return err +} + // EnsureNamespace creates the namespace if it does not already exist. func EnsureNamespace(ns string) error { if _, err := Kubectl("get", "namespace", ns); err == nil { @@ -51,12 +58,36 @@ func EnsureNamespace(ns string) error { } // WaitRollout blocks until a StatefulSet/Deployment rollout completes within timeout. +// +// It waits for the object to exist first. The workloads here are created by the operator in +// response to the CR, so right after applying one there is a window where it does not exist yet — +// and `kubectl rollout status` does not wait through that, it fails immediately with NotFound. func WaitRollout(namespace, kindName string, timeout time.Duration) error { + if err := WaitExists(namespace, kindName, timeout); err != nil { + return err + } _, err := Kubectl("rollout", "status", kindName, "-n", namespace, fmt.Sprintf("--timeout=%s", timeout)) return err } +// WaitExists polls until a resource exists, or timeout elapses. +func WaitExists(namespace, kindName string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + var last error + for { + if _, err := Kubectl("get", kindName, "-n", namespace); err == nil { + return nil + } else { + last = err + } + if time.Now().After(deadline) { + return fmt.Errorf("%s did not appear in %s: %w", kindName, timeout, last) + } + time.Sleep(2 * time.Second) + } +} + // WaitAvailable waits for a deployment to report Available (used for the operator manager). func WaitAvailable(namespace, deployment string, timeout time.Duration) error { _, err := Kubectl("wait", "deployment/"+deployment, diff --git a/manifests/role-pools.yaml b/manifests/role-pools.yaml new file mode 100644 index 0000000..9b47cf7 --- /dev/null +++ b/manifests/role-pools.yaml @@ -0,0 +1,32 @@ +# The disaggregated half of the topology, merge-patched onto manifests/cluster.yaml when +# E2E_ROLE_POOLS is set: the same three storage nodes, plus a stateless odbingest pool and a +# stateless odbselect pool that scale on their own. Neither pool is a ring member or holds data — +# both watch etcd for membership and resolve shard owners locally. +# +# Images are the locally built, kind-loaded odbingest/odbselect binaries (internal/build). They must +# be separate images: they are separate binaries, and odbselect ships in no released image yet. The +# pools inherit spec.imagePullPolicy (IfNotPresent) from the cluster, so kind never reaches out. +# +# Two replicas each is the smallest count that still proves the pools are pools — a Service in front +# of more than one pod, so a query can land on a different replica than the one that would have been +# picked for the write. +spec: + ingest: + replicas: 2 + image: oteldb/odbingest:e2e + resources: + requests: + cpu: "100m" + memory: 128Mi + limits: + memory: 512Mi + + query: + replicas: 2 + image: oteldb/odbselect:e2e + resources: + requests: + cpu: "100m" + memory: 128Mi + limits: + memory: 512Mi diff --git a/roles_test.go b/roles_test.go new file mode 100644 index 0000000..dc4b7a4 --- /dev/null +++ b/roles_test.go @@ -0,0 +1,167 @@ +package e2e + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/oteldb/e2e/internal/kubeutil" + "github.com/oteldb/e2e/internal/otlp" +) + +// Role pool ports, as published by the operator's per-pool Services. +// +// The ingest pool folds OTLP/HTTP, Prometheus remote write and its health endpoints onto one +// listener inside the pod (:19291); the Service republishes it as 4318 as well, so an OTLP/HTTP +// client needs no special-casing. gRPC keeps its own listener. +// +// The query pool serves each read API on its own listener and publishes each enabled one. Health +// (13133) is a container port only, so it is not listed here. +const ( + portIngestOTLPGRPC = 4317 + portIngestOTLPHTTP = 4318 + portIngestPromRW = 19291 + + portQueryProm = 9090 + portQueryLoki = 3100 + portQueryTempo = 3200 + portQueryPyroscope = 4040 +) + +// ingestSelector / querySelector match each pool's pods. The operator selects on name+instance +// only (component is metadata), and each role carries its own binary's name. +func ingestSelector() string { + return "app.kubernetes.io/name=odbingest,app.kubernetes.io/instance=" + conf.ClusterName +} + +func querySelector() string { + return "app.kubernetes.io/name=odbselect,app.kubernetes.io/instance=" + conf.ClusterName +} + +// The disaggregated topology: storage nodes (spec.replicas) plus a stateless odbingest pool +// (spec.ingest) and a stateless odbselect pool (spec.query), each behind its own Service. +// +// The point of the split — and the thing a symmetric-only suite structurally cannot catch — is that +// the process accepting a write and the process answering the read are different processes on +// different pods, sharing nothing but etcd membership and the storage nodes behind them. So every +// spec here writes through one pool's Service and reads through the other's. +var _ = Describe("Role pools", Ordered, func() { + BeforeAll(func() { + if !conf.RolePools { + Skip("role pools disabled; set E2E_ROLE_POOLS=1 (see README for the refs it needs)") + } + }) + + It("brings up both pools", func() { + ingest, err := kubeutil.ReadyPodCount(conf.Namespace, ingestSelector()) + Expect(err).NotTo(HaveOccurred()) + Expect(ingest).To(Equal(2), "expected 2 Ready odbingest pods") + + query, err := kubeutil.ReadyPodCount(conf.Namespace, querySelector()) + Expect(err).NotTo(HaveOccurred()) + Expect(query).To(Equal(2), "expected 2 Ready odbselect pods") + }) + + It("keeps the pools off the storage ring", func(ctx SpecContext) { + // A pool pod that joined the ring would be a correctness bug, not just a topology one: it + // holds no data, so owning a shard would mean answering for data it does not have. The + // storage nodes' own view of membership is the authority. + pods, err := kubeutil.PodNames(conf.Namespace, selector()) + Expect(err).NotTo(HaveOccurred()) + Expect(pods).NotTo(BeEmpty()) + + st, err := nodeStorage(ctx, pods[0]) + Expect(err).NotTo(HaveOccurred()) + Expect(st.Engine.Cluster.Members).To(HaveLen(3), + "the ring has %d members; the ingest/query pools must not join it", len(st.Engine.Cluster.Members)) + }) + + It("ingests through the ingest pool and reads back through the query pool", func(ctx SpecContext) { + runID := "roles-" + randID() + + ingest, stopIngest := ingestEndpoint() + DeferCleanup(stopIngest) + clients, stopQuery := queryEndpoints() + DeferCleanup(stopQuery) + + By("emitting a trace, a log and a metric to the ingest pool's Service") + traceID, err := ingest.EmitTrace(ctx, "oteldb-e2e-traces", runID) + Expect(err).NotTo(HaveOccurred(), "Failed to emit trace through the ingest pool") + Expect(ingest.EmitLog(ctx, "oteldb-e2e-logs", runID)). + To(Succeed(), "Failed to emit log through the ingest pool") + Expect(ingest.EmitMetric(ctx, "oteldb-e2e-metrics", runID, 7)). + To(Succeed(), "Failed to emit metric through the ingest pool") + + By("reading the trace back through the query pool's Tempo API") + Eventually(func() (bool, error) { + return clients.TraceExists(ctx, traceID) + }).Should(BeTrue(), "trace %s written via odbingest is not queryable via odbselect", traceID) + + By("reading the log back through the query pool's Loki API") + Eventually(func() (bool, error) { + return clients.LogsContain(ctx, `{service_name="oteldb-e2e-logs"}`, runID) + }).Should(BeTrue(), "log for run=%s written via odbingest is not queryable via odbselect", runID) + + By("reading the metric back through the query pool's Prometheus API") + Eventually(func() (bool, error) { + return clients.SeriesExists(ctx, `{__name__=~"e2e.requests.*"}`) + }).Should(BeTrue(), "metric written via odbingest is not queryable via odbselect") + }, SpecTimeout(6*time.Minute)) + + It("survives losing an ingest pod and a query pod", func(ctx SpecContext) { + // The pools are stateless Deployments, so a lost pod costs nothing but capacity. Asserting + // it is what makes them worth splitting out: the failure domain is meant to be smaller than + // a storage node's, whose loss the chaos specs handle far more carefully. + runID := "roles-chaos-" + randID() + + for _, sel := range []string{ingestSelector(), querySelector()} { + pods, err := kubeutil.PodNames(conf.Namespace, sel) + Expect(err).NotTo(HaveOccurred()) + Expect(pods).NotTo(BeEmpty()) + Expect(kubeutil.DeletePodNow(conf.Namespace, pods[0])).To(Succeed()) + } + + By("waiting for both pools to become fully Ready again") + for _, p := range []struct { + deployment string + selector string + }{ + {conf.ClusterName + "-ingest", ingestSelector()}, + {conf.ClusterName + "-query", querySelector()}, + } { + Expect(kubeutil.WaitRollout(conf.Namespace, "deployment/"+p.deployment, 5*time.Minute)). + To(Succeed(), "%s did not roll out again", p.deployment) + Eventually(func() (int, error) { + return kubeutil.ReadyPodCount(conf.Namespace, p.selector) + }, 5*time.Minute, 5*time.Second).Should(Equal(2), "%s did not return to 2 Ready pods", p.deployment) + } + + By("writing through ingest and reading through query again after the disruption") + ingest, stopIngest := ingestEndpoint() + DeferCleanup(stopIngest) + clients, stopQuery := queryEndpoints() + DeferCleanup(stopQuery) + + Expect(ingest.EmitLog(ctx, "oteldb-e2e-logs", runID)).To(Succeed()) + Eventually(func() (bool, error) { + return clients.LogsContain(ctx, `{service_name="oteldb-e2e-logs"}`, runID) + }).Should(BeTrue(), "the pools did not recover the write-then-read path") + }, SpecTimeout(15*time.Minute)) +}) + +// ingestEndpoint port-forwards the ingest pool's Service and returns an OTLP/gRPC ingester. +func ingestEndpoint() (*otlp.Ingester, func()) { + svc := "service/" + conf.ClusterName + "-ingest" + ports, stop, err := kubeutil.PortForward(conf.Namespace, svc, portIngestOTLPGRPC) + Expect(err).NotTo(HaveOccurred(), "Failed to port-forward to the ingest pool") + return otlp.NewIngester(ports[portIngestOTLPGRPC]), stop +} + +// queryEndpoints port-forwards the query pool's Service and returns the read-side clients. +func queryEndpoints() (*otlp.Clients, func()) { + svc := "service/" + conf.ClusterName + "-query" + ports, stop, err := kubeutil.PortForward(conf.Namespace, svc, portQueryTempo, portQueryLoki, portQueryProm) + Expect(err).NotTo(HaveOccurred(), "Failed to port-forward to the query pool") + return otlp.NewClients(ports[portQueryTempo], ports[portQueryLoki], ports[portQueryProm]), stop +} diff --git a/suite_test.go b/suite_test.go index c4a7eb3..ce147bc 100644 --- a/suite_test.go +++ b/suite_test.go @@ -53,6 +53,7 @@ var _ = BeforeSuite(func() { SetDefaultEventuallyPollingInterval(2 * time.Second) ensureKindCluster() + resolveSources() if conf.SkipBuild { By("skipping image build (E2E_SKIP_BUILD set); reusing loaded images") @@ -64,6 +65,18 @@ var _ = BeforeSuite(func() { By("building and loading the operator image from local source (fast path)") Expect(build.BinaryImage(conf.OperatorSrc, "./cmd", "manager", conf.OperatorImage, conf.KindCluster)). To(Succeed(), "Failed to build/load operator image") + + if conf.RolePools { + // odbingest and odbselect are separate binaries, and odbselect ships in no released + // image yet — which is precisely why the suite builds from source. + By("building and loading the odbingest image from local source (fast path)") + Expect(build.BinaryImage(conf.OteldbSrc, "./cmd/odbingest", "odbingest", conf.IngestImage, conf.KindCluster)). + To(Succeed(), "Failed to build/load odbingest image") + + By("building and loading the odbselect image from local source (fast path)") + Expect(build.BinaryImage(conf.OteldbSrc, "./cmd/odbselect", "odbselect", conf.QueryImage, conf.KindCluster)). + To(Succeed(), "Failed to build/load odbselect image") + } } deployOperator() @@ -110,6 +123,22 @@ func ensureKindCluster() { Expect(err).NotTo(HaveOccurred(), "Failed to create kind cluster") } +// resolveSources swaps the configured local source trees for ref-pinned clones when OTELDB_REF / +// OPERATOR_REF are set, so the suite can run against branches that have not merged yet (and against +// main once they have) without touching a developer's checkout. +func resolveSources() { + oteldb, err := build.Source("oteldb", conf.OteldbSrc, conf.OteldbRepo, conf.OteldbRef, conf.SrcCache) + Expect(err).NotTo(HaveOccurred(), "Failed to resolve the oteldb source tree") + conf.OteldbSrc = oteldb + + operator, err := build.Source("operator", conf.OperatorSrc, conf.OperatorRepo, conf.OperatorRef, conf.SrcCache) + Expect(err).NotTo(HaveOccurred(), "Failed to resolve the operator source tree") + conf.OperatorSrc = operator + + _, _ = fmt.Fprintf(GinkgoWriter, "building oteldb from %s, operator from %s\n", + conf.OteldbSrc, conf.OperatorSrc) +} + // deployOperator installs the CRDs and controller-manager via the operator's Makefile. No webhooks // exist, so cert-manager is skipped. func deployOperator() { @@ -119,7 +148,18 @@ func deployOperator() { _, err := shell.Run(deploy) Expect(err).NotTo(HaveOccurred(), "Failed to deploy operator") + // The image tag is fixed, so a manager already running from a previous run keeps its old binary + // even though `kind load` replaced the image underneath it — `make deploy` sees an unchanged + // pod template and does nothing. On a fresh cluster that is moot; on a reused one (the fast + // loop, and any run that switches OPERATOR_REF) it silently tests the previous operator. + By("restarting the controller-manager onto the just-loaded image") + _, err = kubeutil.Kubectl("rollout", "restart", "deployment/operator-controller-manager", + "-n", conf.OperatorNamespace) + Expect(err).NotTo(HaveOccurred(), "Failed to restart the controller-manager") + By("waiting for the controller-manager to become Available") + Expect(kubeutil.WaitRollout(conf.OperatorNamespace, "deployment/operator-controller-manager", 3*time.Minute)). + To(Succeed(), "controller-manager did not roll out") Expect(kubeutil.WaitAvailable(conf.OperatorNamespace, "operator-controller-manager", 3*time.Minute)). To(Succeed(), "controller-manager did not become Available") } @@ -138,10 +178,26 @@ func deployEtcdAndCluster() { By("applying the OtelDBCluster") Expect(kubeutil.Apply(conf.Namespace, "manifests/cluster.yaml")).To(Succeed()) + if conf.RolePools { + // The pools are a merge patch rather than a second manifest so the storage half of the + // topology under test stays defined in exactly one place. + By("patching in the odbingest and odbselect role pools") + Expect(kubeutil.PatchMerge(conf.Namespace, "oteldbcluster", conf.ClusterName, "manifests/role-pools.yaml")). + To(Succeed(), "Failed to add the role pools to the OtelDBCluster") + } + By("waiting for the oteldb StatefulSet rollout") Expect(kubeutil.WaitRollout(conf.Namespace, "statefulset/"+conf.ClusterName, 5*time.Minute)). To(Succeed(), "oteldb StatefulSet did not roll out") + if conf.RolePools { + By("waiting for the ingest and query pool rollouts") + Expect(kubeutil.WaitRollout(conf.Namespace, "deployment/"+conf.ClusterName+"-ingest", 5*time.Minute)). + To(Succeed(), "odbingest pool did not roll out") + Expect(kubeutil.WaitRollout(conf.Namespace, "deployment/"+conf.ClusterName+"-query", 5*time.Minute)). + To(Succeed(), "odbselect pool did not roll out") + } + By("waiting for the OtelDBCluster to report Ready") Eventually(func() (string, error) { return kubeutil.ClusterPhase(conf.Namespace, conf.ClusterName)