diff --git a/.gitignore b/.gitignore index df32439..8b4f0b2 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ *.test *.out +# Ref-pinned sibling checkouts (E2E_SRC_CACHE) +/.e2e-src/ + # Logs and scratch output *.log diff --git a/Makefile b/Makefile index f73d42e..f6b2557 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,18 @@ test-fast: ## Rebuild oteldb from local source and re-run against the EXISTING k test-no-build: ## Re-run against the existing cluster WITHOUT rebuilding/reloading images. E2E_REUSE_CLUSTER=true E2E_SKIP_BUILD=true go test ./... $(GO_TEST_FLAGS) +# Role pools need spec.ingest/spec.query (oteldb/operator#14, #15) and cmd/odbselect +# (oteldb/oteldb#1266). Until those land, build the siblings from their branches; afterwards drop +# the refs (or set them to main) and the target is just `E2E_ROLE_POOLS=1`. +ROLE_OPERATOR_REF ?= claude/role-query-group +ROLE_OTELDB_REF ?= claude/1263-odbselect + +.PHONY: test-roles +test-roles: ## Run the suite with the odbingest/odbselect role pools deployed as well. + E2E_ROLE_POOLS=1 \ + OPERATOR_REF=$(ROLE_OPERATOR_REF) OTELDB_REF=$(ROLE_OTELDB_REF) \ + go test ./... $(GO_TEST_FLAGS) + .PHONY: cluster cluster: ## Create the kind cluster only. kind create cluster --name $(KIND_CLUSTER) --config kind.yaml --wait 120s diff --git a/README.md b/README.md index 7c60429..2734e78 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ shared-store cluster that isn't one. make test # full run: build + fresh kind cluster + all specs + teardown make test-keep # same, but leave the cluster up for inspection make test-fast # rebuild oteldb from source, reuse an existing cluster (fast iteration) +make test-roles # full run plus the odbingest/odbselect role pools (see below) make test-no-build # reuse cluster AND already-loaded images (fastest; just re-run specs) make cluster # create the kind cluster only make clean # delete the kind cluster @@ -118,8 +119,16 @@ make test-fast # rebuild + reload oteldb, redeploy CR, re-run — repeat on |-----|---------|---------| | `OTELDB_SRC` | `/src/oteldb/oteldb` | oteldb source tree to build | | `OPERATOR_SRC` | `/src/oteldb/operator` | operator source tree to build/deploy | +| `OTELDB_REF` | _unset_ | build oteldb from this git ref instead of `OTELDB_SRC` (branch, tag or SHA) | +| `OPERATOR_REF` | _unset_ | build the operator from this git ref instead of `OPERATOR_SRC` | +| `OTELDB_REPO` | `https://github.com/oteldb/oteldb.git` | repo cloned when `OTELDB_REF` is set | +| `OPERATOR_REPO` | `https://github.com/oteldb/operator.git` | repo cloned when `OPERATOR_REF` is set | +| `E2E_SRC_CACHE` | `.e2e-src` | where ref-pinned clones are cached between runs | | `OTELDB_IMAGE` | `oteldb/oteldb:e2e` | built oteldb image tag (must match `manifests/cluster.yaml` `spec.image`) | | `OPERATOR_IMAGE` | `oteldb/operator:e2e` | built operator image tag | +| `INGEST_IMAGE` | `oteldb/odbingest:e2e` | built odbingest image tag (must match `manifests/role-pools.yaml`) | +| `QUERY_IMAGE` | `oteldb/odbselect:e2e` | built odbselect image tag (must match `manifests/role-pools.yaml`) | +| `E2E_ROLE_POOLS` | _unset_ | also deploy and exercise the `spec.ingest` / `spec.query` pools | | `KIND_CLUSTER` | `oteldb-e2e` | kind cluster name | | `KIND_CONFIG` | `kind.yaml` | kind cluster config | | `E2E_NAMESPACE` | `oteldb-e2e` | namespace for the cluster + etcd | @@ -128,18 +137,50 @@ make test-fast # rebuild + reload oteldb, redeploy CR, re-run — repeat on | `E2E_REUSE_CLUSTER` | _unset_ | assume the kind cluster already exists (no create/delete) | | `E2E_KEEP_CLUSTER` | _unset_ | leave the kind cluster running after the suite | +## Role pools (disaggregated topology) + +By default the cluster is symmetric: every node ingests, stores and queries. Setting +`E2E_ROLE_POOLS=1` merge-patches `manifests/role-pools.yaml` onto the same CR, adding a stateless +`odbingest` pool (`spec.ingest`) and a stateless `odbselect` pool (`spec.query`), each a Deployment +behind its own Service, each scaling independently of the storage nodes. + +`roles_test.go` then writes through the ingest pool's Service and reads back through the query +pool's. That crossing is the whole point of the split and the one thing a symmetric suite cannot +check: with one process doing both, a write-then-read proves nothing about the boundary. It also +asserts the pools stay out of the storage ring — a pool pod that joined it would own shards whose +data it does not have — and that losing a pod from each pool costs only capacity. + +The two binaries are built from the same oteldb source tree as `oteldb` itself +(`./cmd/odbingest`, `./cmd/odbselect`), so the fact that neither is in a released image is not a +problem here. + +This needs changes that are not on `main` yet — oteldb/operator#14 (`spec.ingest`), +oteldb/operator#15 (`spec.query`, based on #14) and oteldb/oteldb#1266 (`cmd/odbselect`) — plus +oteldb/operator#16 for `spec.cluster.privateBackend`. Point the suite at them with the ref knobs: + +```bash +E2E_ROLE_POOLS=1 \ +OPERATOR_REF=claude/role-query-group \ +OTELDB_REF=claude/1263-odbselect \ +make test +``` + +Once they land, drop the two `*_REF` variables (or set them to `main`) and everything else is +unchanged. + ## Layout ``` manifests/ etcd-dev.yaml dev etcd (single replica, emptyDir) — bring-your-own coordination cluster.yaml the OtelDBCluster under test (file backend, RF=2, image pinned) + role-pools.yaml merge patch adding the odbingest/odbselect pools (E2E_ROLE_POOLS) fast.Dockerfile minimal image wrapping a host-built binary (fast rebuild path) kind.yaml 1 control-plane + 3 workers (room to spread replicas + survive a node loss) internal/ cfg/ env-driven suite configuration shell/ command execution wrappers (docker/kind/kubectl/go/make) - build/ fast image path: host go build -> COPY image -> kind load + build/ fast image path: host go build -> COPY image -> kind load; ref-pinned checkouts kubeutil/ kubectl helpers: apply/wait/list/delete + Service port-forward otlp/ OTLP/gRPC ingest (traces/logs/metrics) + Tempo/Loki/Prometheus query clients nodeadmin/ per-node admin API client (/api/v1/storage): parts, shards, part-sync stats @@ -148,4 +189,5 @@ helpers_test.go shared port-forward + ingest/verify + flushed-part helpers normal_test.go happy-path ingest/query specs durability_test.go shared-nothing mode + flushed-part replication specs chaos_test.go pod-kill and disk-loss recovery specs +roles_test.go disaggregated ingest/query pool specs (E2E_ROLE_POOLS) ``` diff --git a/fast.Dockerfile b/fast.Dockerfile index 92e7e49..ac7ccf4 100644 --- a/fast.Dockerfile +++ b/fast.Dockerfile @@ -4,17 +4,22 @@ # local source edit is a `go build` + a single COPY layer — no in-container module download, no # full image rebuild. # -# Used for both the oteldb server and the operator manager (BIN selects which). The binary is -# installed at /: +# Used for the oteldb server, the operator manager and the odbingest/odbselect role binaries (BIN +# selects which). The binary is installed at BOTH / and /usr/local/bin/, because the three +# consumers disagree about where to look: # - oteldb: the CR sets no command, so the container runs this image's ENTRYPOINT; the shell # wrapper execs /oteldb and forwards the operator-appended args (e.g. --config=...). # - operator: the manager Deployment overrides command with ["/manager"], bypassing the # ENTRYPOINT, so /manager simply has to exist at the root. +# - odbingest / odbselect: their Deployments override command with the release image's absolute +# path, /usr/local/bin/, so the fast image has to mirror that layout or the container dies +# with "no such file or directory" before it logs anything. FROM alpine:3.20 RUN apk add --no-cache ca-certificates ARG BIN=oteldb -COPY ${BIN} /${BIN} +COPY ${BIN} /usr/local/bin/${BIN} +RUN ln -s /usr/local/bin/${BIN} /${BIN} # Create an /etc/passwd entry for the non-root UID. oteldb (go-faster/sdk resource detection) # calls user.Current(), which — in a CGO-disabled binary — needs a passwd entry to resolve the diff --git a/internal/build/source.go b/internal/build/source.go new file mode 100644 index 0000000..a34e2a7 --- /dev/null +++ b/internal/build/source.go @@ -0,0 +1,53 @@ +package build + +import ( + "fmt" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" //nolint:revive,staticcheck // dot-import is the Ginkgo idiom. + + "github.com/oteldb/e2e/internal/shell" +) + +// Source resolves the source tree to build a sibling repo from. +// +// With ref empty it returns dir unchanged — the local working tree, which is the fast local loop +// this suite is built around. With ref set it clones repoURL into a cache directory and checks that +// ref out detached, so the suite can run against an unmerged branch (or a pinned commit) without +// touching, or even needing, the developer's own checkout. The cache is reused across runs, so the +// second run costs a fetch. +// +// The clone is blobless (--filter=blob:none): the whole history's trees are cheap and the build only +// ever materializes one commit's blobs. +func Source(name, dir, repoURL, ref, cacheDir string) (string, error) { + if ref == "" { + return dir, nil + } + if repoURL == "" { + return "", fmt.Errorf("%s: a ref (%q) was requested but no repository URL is configured", name, ref) + } + + target := filepath.Join(cacheDir, name) + if _, err := os.Stat(filepath.Join(target, ".git")); err != nil { + if err := os.MkdirAll(cacheDir, 0o755); err != nil { + return "", fmt.Errorf("create source cache %s: %w", cacheDir, err) + } + By(fmt.Sprintf("cloning %s into %s", repoURL, target)) + clone := shell.Command("git", "clone", "--quiet", "--filter=blob:none", repoURL, target) + if _, err := shell.Run(clone); err != nil { + return "", fmt.Errorf("clone %s: %w", repoURL, err) + } + } + + By(fmt.Sprintf("checking out %s@%s", name, ref)) + // Fetching the ref by name covers a branch, a tag and a full SHA alike, and FETCH_HEAD then + // resolves it without assuming a local branch exists or is current. + if _, err := shell.Run(shell.InDir(target, "git", "fetch", "--quiet", "origin", ref)); err != nil { + return "", fmt.Errorf("fetch %s %s: %w", repoURL, ref, err) + } + if _, err := shell.Run(shell.InDir(target, "git", "checkout", "--quiet", "--detach", "FETCH_HEAD")); err != nil { + return "", fmt.Errorf("checkout %s %s: %w", repoURL, ref, err) + } + return target, nil +} diff --git a/internal/cfg/cfg.go b/internal/cfg/cfg.go index aef6b58..9755013 100644 --- a/internal/cfg/cfg.go +++ b/internal/cfg/cfg.go @@ -12,10 +12,26 @@ type Config struct { // OperatorSrc is the local operator source tree (Makefile-driven deploy). OperatorSrc string + // OteldbRepo / OteldbRef and OperatorRepo / OperatorRef pin a sibling repo to a git ref + // instead of the local working tree. A non-empty ref makes the suite clone the repo into + // SrcCache and build that ref, so it can run against an unmerged branch before it lands and + // against "main" after, without editing anyone's checkout. Empty ref = use the local tree. + OteldbRepo string + OteldbRef string + OperatorRepo string + OperatorRef string + // SrcCache is where ref-pinned clones live, reused across runs. + SrcCache string + // OteldbImage / OperatorImage are the locally built, kind-loaded image tags. They MUST match // spec.image in manifests/cluster.yaml (oteldb) and the operator deploy IMG respectively. OteldbImage string OperatorImage string + // IngestImage / QueryImage are the odbingest and odbselect images for the role pools. They + // MUST match spec.ingest.image / spec.query.image in manifests/cluster.yaml. They are separate + // binaries, so they cannot share the oteldb image the fast build path produces. + IngestImage string + QueryImage string // KindCluster is the kind cluster name. KindCluster string @@ -35,6 +51,10 @@ type Config struct { KeepCluster bool // ReuseCluster assumes the kind cluster already exists (no create, no delete). ReuseCluster bool + // RolePools deploys and exercises the disaggregated topology: storage nodes plus an odbingest + // pool and an odbselect pool (spec.ingest / spec.query). Off by default until the operator and + // oteldb changes it needs are on main; see README. + RolePools bool } // Load resolves the suite config from the environment, applying defaults. @@ -42,8 +62,15 @@ func Load() Config { return Config{ OteldbSrc: shell.Getenv("OTELDB_SRC", "/src/oteldb/oteldb"), OperatorSrc: shell.Getenv("OPERATOR_SRC", "/src/oteldb/operator"), + OteldbRepo: shell.Getenv("OTELDB_REPO", "https://github.com/oteldb/oteldb.git"), + OteldbRef: shell.Getenv("OTELDB_REF", ""), + OperatorRepo: shell.Getenv("OPERATOR_REPO", "https://github.com/oteldb/operator.git"), + OperatorRef: shell.Getenv("OPERATOR_REF", ""), + SrcCache: shell.Getenv("E2E_SRC_CACHE", ".e2e-src"), OteldbImage: shell.Getenv("OTELDB_IMAGE", "oteldb/oteldb:e2e"), OperatorImage: shell.Getenv("OPERATOR_IMAGE", "oteldb/operator:e2e"), + IngestImage: shell.Getenv("INGEST_IMAGE", "oteldb/odbingest:e2e"), + QueryImage: shell.Getenv("QUERY_IMAGE", "oteldb/odbselect:e2e"), KindCluster: shell.Getenv("KIND_CLUSTER", "oteldb-e2e"), KindConfig: shell.Getenv("KIND_CONFIG", "kind.yaml"), Namespace: shell.Getenv("E2E_NAMESPACE", "oteldb-e2e"), @@ -52,5 +79,6 @@ func Load() Config { SkipBuild: shell.Truthy("E2E_SKIP_BUILD"), KeepCluster: shell.Truthy("E2E_KEEP_CLUSTER"), ReuseCluster: shell.Truthy("E2E_REUSE_CLUSTER"), + RolePools: shell.Truthy("E2E_ROLE_POOLS"), } } diff --git a/internal/kubeutil/kubeutil.go b/internal/kubeutil/kubeutil.go index f802f3c..aa97b5f 100644 --- a/internal/kubeutil/kubeutil.go +++ b/internal/kubeutil/kubeutil.go @@ -41,6 +41,13 @@ func Delete(namespace, path string) error { return err } +// PatchMerge applies a strategic-merge patch from a file to one resource. +func PatchMerge(namespace, kind, name, path string) error { + _, err := Kubectl("patch", kind, name, "-n", namespace, + "--type=merge", "--patch-file", path) + return err +} + // EnsureNamespace creates the namespace if it does not already exist. func EnsureNamespace(ns string) error { if _, err := Kubectl("get", "namespace", ns); err == nil { @@ -51,12 +58,36 @@ func EnsureNamespace(ns string) error { } // WaitRollout blocks until a StatefulSet/Deployment rollout completes within timeout. +// +// It waits for the object to exist first. The workloads here are created by the operator in +// response to the CR, so right after applying one there is a window where it does not exist yet — +// and `kubectl rollout status` does not wait through that, it fails immediately with NotFound. func WaitRollout(namespace, kindName string, timeout time.Duration) error { + if err := WaitExists(namespace, kindName, timeout); err != nil { + return err + } _, err := Kubectl("rollout", "status", kindName, "-n", namespace, fmt.Sprintf("--timeout=%s", timeout)) return err } +// WaitExists polls until a resource exists, or timeout elapses. +func WaitExists(namespace, kindName string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + var last error + for { + if _, err := Kubectl("get", kindName, "-n", namespace); err == nil { + return nil + } else { + last = err + } + if time.Now().After(deadline) { + return fmt.Errorf("%s did not appear in %s: %w", kindName, timeout, last) + } + time.Sleep(2 * time.Second) + } +} + // WaitAvailable waits for a deployment to report Available (used for the operator manager). func WaitAvailable(namespace, deployment string, timeout time.Duration) error { _, err := Kubectl("wait", "deployment/"+deployment, diff --git a/manifests/role-pools.yaml b/manifests/role-pools.yaml new file mode 100644 index 0000000..9b47cf7 --- /dev/null +++ b/manifests/role-pools.yaml @@ -0,0 +1,32 @@ +# The disaggregated half of the topology, merge-patched onto manifests/cluster.yaml when +# E2E_ROLE_POOLS is set: the same three storage nodes, plus a stateless odbingest pool and a +# stateless odbselect pool that scale on their own. Neither pool is a ring member or holds data — +# both watch etcd for membership and resolve shard owners locally. +# +# Images are the locally built, kind-loaded odbingest/odbselect binaries (internal/build). They must +# be separate images: they are separate binaries, and odbselect ships in no released image yet. The +# pools inherit spec.imagePullPolicy (IfNotPresent) from the cluster, so kind never reaches out. +# +# Two replicas each is the smallest count that still proves the pools are pools — a Service in front +# of more than one pod, so a query can land on a different replica than the one that would have been +# picked for the write. +spec: + ingest: + replicas: 2 + image: oteldb/odbingest:e2e + resources: + requests: + cpu: "100m" + memory: 128Mi + limits: + memory: 512Mi + + query: + replicas: 2 + image: oteldb/odbselect:e2e + resources: + requests: + cpu: "100m" + memory: 128Mi + limits: + memory: 512Mi diff --git a/roles_test.go b/roles_test.go new file mode 100644 index 0000000..dc4b7a4 --- /dev/null +++ b/roles_test.go @@ -0,0 +1,167 @@ +package e2e + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/oteldb/e2e/internal/kubeutil" + "github.com/oteldb/e2e/internal/otlp" +) + +// Role pool ports, as published by the operator's per-pool Services. +// +// The ingest pool folds OTLP/HTTP, Prometheus remote write and its health endpoints onto one +// listener inside the pod (:19291); the Service republishes it as 4318 as well, so an OTLP/HTTP +// client needs no special-casing. gRPC keeps its own listener. +// +// The query pool serves each read API on its own listener and publishes each enabled one. Health +// (13133) is a container port only, so it is not listed here. +const ( + portIngestOTLPGRPC = 4317 + portIngestOTLPHTTP = 4318 + portIngestPromRW = 19291 + + portQueryProm = 9090 + portQueryLoki = 3100 + portQueryTempo = 3200 + portQueryPyroscope = 4040 +) + +// ingestSelector / querySelector match each pool's pods. The operator selects on name+instance +// only (component is metadata), and each role carries its own binary's name. +func ingestSelector() string { + return "app.kubernetes.io/name=odbingest,app.kubernetes.io/instance=" + conf.ClusterName +} + +func querySelector() string { + return "app.kubernetes.io/name=odbselect,app.kubernetes.io/instance=" + conf.ClusterName +} + +// The disaggregated topology: storage nodes (spec.replicas) plus a stateless odbingest pool +// (spec.ingest) and a stateless odbselect pool (spec.query), each behind its own Service. +// +// The point of the split — and the thing a symmetric-only suite structurally cannot catch — is that +// the process accepting a write and the process answering the read are different processes on +// different pods, sharing nothing but etcd membership and the storage nodes behind them. So every +// spec here writes through one pool's Service and reads through the other's. +var _ = Describe("Role pools", Ordered, func() { + BeforeAll(func() { + if !conf.RolePools { + Skip("role pools disabled; set E2E_ROLE_POOLS=1 (see README for the refs it needs)") + } + }) + + It("brings up both pools", func() { + ingest, err := kubeutil.ReadyPodCount(conf.Namespace, ingestSelector()) + Expect(err).NotTo(HaveOccurred()) + Expect(ingest).To(Equal(2), "expected 2 Ready odbingest pods") + + query, err := kubeutil.ReadyPodCount(conf.Namespace, querySelector()) + Expect(err).NotTo(HaveOccurred()) + Expect(query).To(Equal(2), "expected 2 Ready odbselect pods") + }) + + It("keeps the pools off the storage ring", func(ctx SpecContext) { + // A pool pod that joined the ring would be a correctness bug, not just a topology one: it + // holds no data, so owning a shard would mean answering for data it does not have. The + // storage nodes' own view of membership is the authority. + pods, err := kubeutil.PodNames(conf.Namespace, selector()) + Expect(err).NotTo(HaveOccurred()) + Expect(pods).NotTo(BeEmpty()) + + st, err := nodeStorage(ctx, pods[0]) + Expect(err).NotTo(HaveOccurred()) + Expect(st.Engine.Cluster.Members).To(HaveLen(3), + "the ring has %d members; the ingest/query pools must not join it", len(st.Engine.Cluster.Members)) + }) + + It("ingests through the ingest pool and reads back through the query pool", func(ctx SpecContext) { + runID := "roles-" + randID() + + ingest, stopIngest := ingestEndpoint() + DeferCleanup(stopIngest) + clients, stopQuery := queryEndpoints() + DeferCleanup(stopQuery) + + By("emitting a trace, a log and a metric to the ingest pool's Service") + traceID, err := ingest.EmitTrace(ctx, "oteldb-e2e-traces", runID) + Expect(err).NotTo(HaveOccurred(), "Failed to emit trace through the ingest pool") + Expect(ingest.EmitLog(ctx, "oteldb-e2e-logs", runID)). + To(Succeed(), "Failed to emit log through the ingest pool") + Expect(ingest.EmitMetric(ctx, "oteldb-e2e-metrics", runID, 7)). + To(Succeed(), "Failed to emit metric through the ingest pool") + + By("reading the trace back through the query pool's Tempo API") + Eventually(func() (bool, error) { + return clients.TraceExists(ctx, traceID) + }).Should(BeTrue(), "trace %s written via odbingest is not queryable via odbselect", traceID) + + By("reading the log back through the query pool's Loki API") + Eventually(func() (bool, error) { + return clients.LogsContain(ctx, `{service_name="oteldb-e2e-logs"}`, runID) + }).Should(BeTrue(), "log for run=%s written via odbingest is not queryable via odbselect", runID) + + By("reading the metric back through the query pool's Prometheus API") + Eventually(func() (bool, error) { + return clients.SeriesExists(ctx, `{__name__=~"e2e.requests.*"}`) + }).Should(BeTrue(), "metric written via odbingest is not queryable via odbselect") + }, SpecTimeout(6*time.Minute)) + + It("survives losing an ingest pod and a query pod", func(ctx SpecContext) { + // The pools are stateless Deployments, so a lost pod costs nothing but capacity. Asserting + // it is what makes them worth splitting out: the failure domain is meant to be smaller than + // a storage node's, whose loss the chaos specs handle far more carefully. + runID := "roles-chaos-" + randID() + + for _, sel := range []string{ingestSelector(), querySelector()} { + pods, err := kubeutil.PodNames(conf.Namespace, sel) + Expect(err).NotTo(HaveOccurred()) + Expect(pods).NotTo(BeEmpty()) + Expect(kubeutil.DeletePodNow(conf.Namespace, pods[0])).To(Succeed()) + } + + By("waiting for both pools to become fully Ready again") + for _, p := range []struct { + deployment string + selector string + }{ + {conf.ClusterName + "-ingest", ingestSelector()}, + {conf.ClusterName + "-query", querySelector()}, + } { + Expect(kubeutil.WaitRollout(conf.Namespace, "deployment/"+p.deployment, 5*time.Minute)). + To(Succeed(), "%s did not roll out again", p.deployment) + Eventually(func() (int, error) { + return kubeutil.ReadyPodCount(conf.Namespace, p.selector) + }, 5*time.Minute, 5*time.Second).Should(Equal(2), "%s did not return to 2 Ready pods", p.deployment) + } + + By("writing through ingest and reading through query again after the disruption") + ingest, stopIngest := ingestEndpoint() + DeferCleanup(stopIngest) + clients, stopQuery := queryEndpoints() + DeferCleanup(stopQuery) + + Expect(ingest.EmitLog(ctx, "oteldb-e2e-logs", runID)).To(Succeed()) + Eventually(func() (bool, error) { + return clients.LogsContain(ctx, `{service_name="oteldb-e2e-logs"}`, runID) + }).Should(BeTrue(), "the pools did not recover the write-then-read path") + }, SpecTimeout(15*time.Minute)) +}) + +// ingestEndpoint port-forwards the ingest pool's Service and returns an OTLP/gRPC ingester. +func ingestEndpoint() (*otlp.Ingester, func()) { + svc := "service/" + conf.ClusterName + "-ingest" + ports, stop, err := kubeutil.PortForward(conf.Namespace, svc, portIngestOTLPGRPC) + Expect(err).NotTo(HaveOccurred(), "Failed to port-forward to the ingest pool") + return otlp.NewIngester(ports[portIngestOTLPGRPC]), stop +} + +// queryEndpoints port-forwards the query pool's Service and returns the read-side clients. +func queryEndpoints() (*otlp.Clients, func()) { + svc := "service/" + conf.ClusterName + "-query" + ports, stop, err := kubeutil.PortForward(conf.Namespace, svc, portQueryTempo, portQueryLoki, portQueryProm) + Expect(err).NotTo(HaveOccurred(), "Failed to port-forward to the query pool") + return otlp.NewClients(ports[portQueryTempo], ports[portQueryLoki], ports[portQueryProm]), stop +} diff --git a/suite_test.go b/suite_test.go index c4a7eb3..ce147bc 100644 --- a/suite_test.go +++ b/suite_test.go @@ -53,6 +53,7 @@ var _ = BeforeSuite(func() { SetDefaultEventuallyPollingInterval(2 * time.Second) ensureKindCluster() + resolveSources() if conf.SkipBuild { By("skipping image build (E2E_SKIP_BUILD set); reusing loaded images") @@ -64,6 +65,18 @@ var _ = BeforeSuite(func() { By("building and loading the operator image from local source (fast path)") Expect(build.BinaryImage(conf.OperatorSrc, "./cmd", "manager", conf.OperatorImage, conf.KindCluster)). To(Succeed(), "Failed to build/load operator image") + + if conf.RolePools { + // odbingest and odbselect are separate binaries, and odbselect ships in no released + // image yet — which is precisely why the suite builds from source. + By("building and loading the odbingest image from local source (fast path)") + Expect(build.BinaryImage(conf.OteldbSrc, "./cmd/odbingest", "odbingest", conf.IngestImage, conf.KindCluster)). + To(Succeed(), "Failed to build/load odbingest image") + + By("building and loading the odbselect image from local source (fast path)") + Expect(build.BinaryImage(conf.OteldbSrc, "./cmd/odbselect", "odbselect", conf.QueryImage, conf.KindCluster)). + To(Succeed(), "Failed to build/load odbselect image") + } } deployOperator() @@ -110,6 +123,22 @@ func ensureKindCluster() { Expect(err).NotTo(HaveOccurred(), "Failed to create kind cluster") } +// resolveSources swaps the configured local source trees for ref-pinned clones when OTELDB_REF / +// OPERATOR_REF are set, so the suite can run against branches that have not merged yet (and against +// main once they have) without touching a developer's checkout. +func resolveSources() { + oteldb, err := build.Source("oteldb", conf.OteldbSrc, conf.OteldbRepo, conf.OteldbRef, conf.SrcCache) + Expect(err).NotTo(HaveOccurred(), "Failed to resolve the oteldb source tree") + conf.OteldbSrc = oteldb + + operator, err := build.Source("operator", conf.OperatorSrc, conf.OperatorRepo, conf.OperatorRef, conf.SrcCache) + Expect(err).NotTo(HaveOccurred(), "Failed to resolve the operator source tree") + conf.OperatorSrc = operator + + _, _ = fmt.Fprintf(GinkgoWriter, "building oteldb from %s, operator from %s\n", + conf.OteldbSrc, conf.OperatorSrc) +} + // deployOperator installs the CRDs and controller-manager via the operator's Makefile. No webhooks // exist, so cert-manager is skipped. func deployOperator() { @@ -119,7 +148,18 @@ func deployOperator() { _, err := shell.Run(deploy) Expect(err).NotTo(HaveOccurred(), "Failed to deploy operator") + // The image tag is fixed, so a manager already running from a previous run keeps its old binary + // even though `kind load` replaced the image underneath it — `make deploy` sees an unchanged + // pod template and does nothing. On a fresh cluster that is moot; on a reused one (the fast + // loop, and any run that switches OPERATOR_REF) it silently tests the previous operator. + By("restarting the controller-manager onto the just-loaded image") + _, err = kubeutil.Kubectl("rollout", "restart", "deployment/operator-controller-manager", + "-n", conf.OperatorNamespace) + Expect(err).NotTo(HaveOccurred(), "Failed to restart the controller-manager") + By("waiting for the controller-manager to become Available") + Expect(kubeutil.WaitRollout(conf.OperatorNamespace, "deployment/operator-controller-manager", 3*time.Minute)). + To(Succeed(), "controller-manager did not roll out") Expect(kubeutil.WaitAvailable(conf.OperatorNamespace, "operator-controller-manager", 3*time.Minute)). To(Succeed(), "controller-manager did not become Available") } @@ -138,10 +178,26 @@ func deployEtcdAndCluster() { By("applying the OtelDBCluster") Expect(kubeutil.Apply(conf.Namespace, "manifests/cluster.yaml")).To(Succeed()) + if conf.RolePools { + // The pools are a merge patch rather than a second manifest so the storage half of the + // topology under test stays defined in exactly one place. + By("patching in the odbingest and odbselect role pools") + Expect(kubeutil.PatchMerge(conf.Namespace, "oteldbcluster", conf.ClusterName, "manifests/role-pools.yaml")). + To(Succeed(), "Failed to add the role pools to the OtelDBCluster") + } + By("waiting for the oteldb StatefulSet rollout") Expect(kubeutil.WaitRollout(conf.Namespace, "statefulset/"+conf.ClusterName, 5*time.Minute)). To(Succeed(), "oteldb StatefulSet did not roll out") + if conf.RolePools { + By("waiting for the ingest and query pool rollouts") + Expect(kubeutil.WaitRollout(conf.Namespace, "deployment/"+conf.ClusterName+"-ingest", 5*time.Minute)). + To(Succeed(), "odbingest pool did not roll out") + Expect(kubeutil.WaitRollout(conf.Namespace, "deployment/"+conf.ClusterName+"-query", 5*time.Minute)). + To(Succeed(), "odbselect pool did not roll out") + } + By("waiting for the OtelDBCluster to report Ready") Eventually(func() (string, error) { return kubeutil.ClusterPhase(conf.Namespace, conf.ClusterName)