diff --git a/README.md b/README.md index 696185e..99dbd02 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,50 @@ make test-fast # rebuild + reload oteldb, redeploy CR, re-run — repeat on | `E2E_SKIP_BUILD` | _unset_ | reuse already-loaded images (skip host compile + load) | | `E2E_REUSE_CLUSTER` | _unset_ | assume the kind cluster already exists (no create/delete) | | `E2E_KEEP_CLUSTER` | _unset_ | leave the kind cluster running after the suite | +| `E2E_REGISTRY` | _unset_ | `host:port` of a plain-HTTP registry to push built images to and mirror upstream pulls through (see below) | + +## Using a local registry (optional) + +By default the suite side-loads the images it builds with `kind load` and the nodes pull +everything else — the kind node image, the dev etcd, the operator's dependencies — straight from +upstream. Set `E2E_REGISTRY=host:port` and both change: built images are `docker push`ed there and +pulled back by the nodes, and the nodes' containerd mirrors `docker.io`, `ghcr.io`, `quay.io` and +`registry.k8s.io` through the same host. This is worth it on a machine that already runs a +pull-through cache, and useless otherwise — hence off by default. + +```bash +E2E_REGISTRY=10.0.0.1:5000 make test +``` + +The registry is assumed to speak plain HTTP, so the host must also be in the *host* docker daemon's +`insecure-registries` for the push (and for kind's own node-image pull) to work. + +`internal/registry` writes one `hosts.toml` per registry into `/etc/containerd/certs.d` on each +node — the directory `kind.yaml`'s `containerdConfigPatches` selects. That file is modelled on the +config running on a real cluster ([`tdakkota/nixfra`][nixfra], +`modules/registry-mirrors.nix`) rather than invented, because three of its properties are easy to +get wrong and expensive to debug: + +1. **`docker.io`'s upstream must be `https://registry-1.docker.io`, not `https://docker.io`.** + The latter is a website. containerd follows the fallback, gets HTML back, and caches it in the + content store as if it were a manifest; every later pull of that image fails with + `unexpected media type text/html for sha256:...`, and it stays broken across containerd restarts + until the poisoned blob is evicted by hand: + + ```bash + docker exec ctr -n k8s.io content rm sha256: + ``` + +2. **`override_path = true` is required.** The mirror is addressed as `/v2/`. + Without it containerd appends its own `/v2/` to that path and requests + `.../v2/docker.io/v2/library/alpine/manifests/3.21`, which 404s. + +3. **A pull-through mirror may re-digest what it caches.** Zot serves an `alpine` index digest that + differs from Docker Hub's, so a pull pinned to an upstream *digest* rather than a tag cannot be + satisfied from the mirror. Nothing to configure here — just do not expect digest pinning to + survive a mirror. + +[nixfra]: https://github.com/tdakkota/nixfra ## Layout @@ -85,6 +129,7 @@ fast.Dockerfile minimal image wrapping a host-built binary (fast rebuild pat kind.yaml 1 control-plane + 3 workers (room to spread replicas + survive a node loss) internal/ cfg/ env-driven suite configuration + registry/ optional local-registry mirror config for the kind nodes shell/ command execution wrappers (docker/kind/kubectl/go/make) build/ fast image path: host go build -> COPY image -> kind load kubeutil/ kubectl helpers: apply/wait/list/delete + Service port-forward diff --git a/internal/build/build.go b/internal/build/build.go index 4460a9d..ab4a57c 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -13,6 +13,7 @@ import ( . "github.com/onsi/ginkgo/v2" //nolint:revive,staticcheck // dot-import is the Ginkgo idiom. + "github.com/oteldb/e2e/internal/cfg" "github.com/oteldb/e2e/internal/shell" ) @@ -66,6 +67,17 @@ func BinaryImage(srcDir, cmdPkg, binName, image, kindCluster string) error { return fmt.Errorf("docker build %s: %w", image, err) } + // A tag naming a registry host (E2E_REGISTRY) is pushed there and pulled back by the nodes + // through the mirror internal/registry configured. A bare tag has nowhere to be pushed, so it + // is side-loaded into the nodes instead — the default. + if cfg.RegistryTagged(image) { + By(fmt.Sprintf("pushing %s", image)) + if _, err := shell.Run(shell.Command("docker", "push", image)); err != nil { + return fmt.Errorf("docker push %s: %w", image, err) + } + return nil + } + By(fmt.Sprintf("loading %s into kind cluster %s", image, kindCluster)) load := shell.Command("kind", "load", "docker-image", image, "--name", kindCluster) if _, err := shell.Run(load); err != nil { diff --git a/internal/cfg/cfg.go b/internal/cfg/cfg.go index aef6b58..998319e 100644 --- a/internal/cfg/cfg.go +++ b/internal/cfg/cfg.go @@ -3,7 +3,11 @@ // of a couple of env vars rather than code edits. package cfg -import "github.com/oteldb/e2e/internal/shell" +import ( + "strings" + + "github.com/oteldb/e2e/internal/shell" +) // Config holds the resolved suite configuration. type Config struct { @@ -12,11 +16,19 @@ type Config struct { // OperatorSrc is the local operator source tree (Makefile-driven deploy). OperatorSrc string - // OteldbImage / OperatorImage are the locally built, kind-loaded image tags. They MUST match - // spec.image in manifests/cluster.yaml (oteldb) and the operator deploy IMG respectively. + // OteldbImage / OperatorImage are the locally built image tags. They MUST match spec.image in + // manifests/cluster.yaml (oteldb) and the operator deploy IMG respectively — except under + // Registry, which prefixes both and overrides spec.image on the applied CR. OteldbImage string OperatorImage string + // Registry, when set (E2E_REGISTRY=host:port), is a plain-HTTP OCI registry that the suite + // pushes its built images to and points the kind nodes at, both for those images and as a + // pull-through mirror for the upstream registries. Empty — the default — keeps the + // `kind load` side-load path and leaves the nodes pulling from upstream directly, so a run + // without a registry behaves exactly as it always has. + Registry string + // KindCluster is the kind cluster name. KindCluster string // KindConfig is the path to the kind cluster config (multi-node). @@ -39,11 +51,13 @@ type Config struct { // Load resolves the suite config from the environment, applying defaults. func Load() Config { + registry := shell.Getenv("E2E_REGISTRY", "") return Config{ OteldbSrc: shell.Getenv("OTELDB_SRC", "/src/oteldb/oteldb"), OperatorSrc: shell.Getenv("OPERATOR_SRC", "/src/oteldb/operator"), - OteldbImage: shell.Getenv("OTELDB_IMAGE", "oteldb/oteldb:e2e"), - OperatorImage: shell.Getenv("OPERATOR_IMAGE", "oteldb/operator:e2e"), + OteldbImage: inRegistry(registry, shell.Getenv("OTELDB_IMAGE", "oteldb/oteldb:e2e")), + OperatorImage: inRegistry(registry, shell.Getenv("OPERATOR_IMAGE", "oteldb/operator:e2e")), + Registry: registry, KindCluster: shell.Getenv("KIND_CLUSTER", "oteldb-e2e"), KindConfig: shell.Getenv("KIND_CONFIG", "kind.yaml"), Namespace: shell.Getenv("E2E_NAMESPACE", "oteldb-e2e"), @@ -54,3 +68,21 @@ func Load() Config { ReuseCluster: shell.Truthy("E2E_REUSE_CLUSTER"), } } + +// inRegistry qualifies image with the configured registry host. It is a no-op when no registry is +// configured (the default) or when the caller already spelled a registry host into the tag, so an +// explicit OTELDB_IMAGE=some.host/oteldb:tag still wins. +func inRegistry(registry, image string) string { + if registry == "" || RegistryTagged(image) { + return image + } + return registry + "/" + image +} + +// RegistryTagged reports whether image names an explicit registry host, i.e. its first path +// component looks like a host (contains a dot or a port). "oteldb/oteldb:e2e" does not; +// "10.0.0.1:5000/oteldb/oteldb:e2e" does. +func RegistryTagged(image string) bool { + first, _, ok := strings.Cut(image, "/") + return ok && (strings.Contains(first, ".") || strings.Contains(first, ":")) +} diff --git a/internal/cfg/cfg_test.go b/internal/cfg/cfg_test.go new file mode 100644 index 0000000..c9514a2 --- /dev/null +++ b/internal/cfg/cfg_test.go @@ -0,0 +1,35 @@ +package cfg + +import "testing" + +func TestRegistryTagged(t *testing.T) { + for _, tt := range []struct { + image string + want bool + }{ + {"oteldb/oteldb:e2e", false}, + {"alpine:3.20", false}, + {"10.0.0.1:5000/oteldb/oteldb:e2e", true}, + {"ghcr.io/oteldb/oteldb:e2e", true}, + {"localhost:5000/x:y", true}, + } { + if got := RegistryTagged(tt.image); got != tt.want { + t.Errorf("RegistryTagged(%q) = %v, want %v", tt.image, got, tt.want) + } + } +} + +func TestInRegistry(t *testing.T) { + // No registry configured is the default and must not touch the tag: that is what keeps the + // suite on the `kind load` path. + if got := inRegistry("", "oteldb/oteldb:e2e"); got != "oteldb/oteldb:e2e" { + t.Errorf("unconfigured registry rewrote the tag: %q", got) + } + if got := inRegistry("10.0.0.1:5000", "oteldb/oteldb:e2e"); got != "10.0.0.1:5000/oteldb/oteldb:e2e" { + t.Errorf("inRegistry = %q", got) + } + // An explicitly host-qualified override wins over the configured registry. + if got := inRegistry("10.0.0.1:5000", "ghcr.io/oteldb/oteldb:e2e"); got != "ghcr.io/oteldb/oteldb:e2e" { + t.Errorf("inRegistry overrode an explicit host: %q", got) + } +} diff --git a/internal/registry/registry.go b/internal/registry/registry.go new file mode 100644 index 0000000..3db52d0 --- /dev/null +++ b/internal/registry/registry.go @@ -0,0 +1,117 @@ +// Package registry points a kind cluster's containerd at a local OCI registry: as a pull-through +// mirror for the upstream registries the suite's manifests reference, and as the home of the +// images the fast build path pushes. +// +// It is opt-in (E2E_REGISTRY). Without it the suite side-loads its images with `kind load` and the +// nodes pull everything else straight from upstream, exactly as they always have. +// +// The hosts.toml this writes is modelled on the containerd config running on the author's own +// cluster (tdakkota/nixfra, modules/registry-mirrors.nix) rather than invented here, because two +// of its details are load-bearing and easy to "simplify" back into a broken state: +// +// - docker.io's upstream MUST be https://registry-1.docker.io. "https://docker.io" is a website: +// containerd follows the fallback, gets HTML, and caches it in the content store as if it were +// a manifest. Every later pull of that image then fails with `unexpected media type text/html +// for sha256:...` — and it stays broken across containerd restarts until the poisoned blob is +// evicted by hand (`ctr -n k8s.io content rm sha256:...`). +// - override_path MUST be true. The mirror is addressed as /v2/, and without +// it containerd appends its own /v2/ to that path and requests +// .../v2/docker.io/v2/library/alpine/manifests/3.21, which 404s. +// +// A third property is worth knowing but cannot be configured around: a pull-through mirror may +// re-digest what it caches (Zot serves an alpine index digest that differs from Docker Hub's), so +// pulls pinned to an upstream digest rather than a tag are inherently fragile through one. +package registry + +import ( + "fmt" + "strings" + + . "github.com/onsi/ginkgo/v2" //nolint:revive,staticcheck // dot-import is the Ginkgo idiom. + + "github.com/oteldb/e2e/internal/shell" +) + +// mirrored are the upstream registries the suite's own manifests and images come from, mapped to +// the server containerd should fall back to when the mirror cannot serve a blob. The author's +// cluster mirrors a longer list; these are the ones an e2e run actually reaches for. +var mirrored = map[string]string{ + "docker.io": "https://registry-1.docker.io", + "ghcr.io": "https://ghcr.io", + "quay.io": "https://quay.io", + "registry.k8s.io": "https://registry.k8s.io", +} + +// certsDir is containerd's per-registry config directory, selected by the config_path set in +// kind.yaml's containerdConfigPatches. +const certsDir = "/etc/containerd/certs.d" + +// Configure writes a hosts.toml for every mirrored registry (and for the registry itself) onto +// each node of the kind cluster, then restarts containerd to pick them up. registry is a +// host:port addressed over plain HTTP. +func Configure(kindCluster, registry string) error { + nodes, err := shell.Output("kind", "get", "nodes", "--name", kindCluster) + if err != nil { + return fmt.Errorf("list kind nodes: %w", err) + } + + url := "http://" + registry + for _, node := range strings.Fields(nodes) { + By(fmt.Sprintf("pointing containerd on %s at %s", node, registry)) + for domain, upstream := range mirrored { + // The mirror lives under a per-domain path prefix, hence override_path. + if err := writeHosts(node, domain, upstream, url+"/v2/"+domain, false); err != nil { + return err + } + } + // The registry addressed as itself, so the images we push to it resolve by their own tag. + // No path prefix here, and plain HTTP means skip_verify. + if err := writeHosts(node, registry, url, url, true); err != nil { + return err + } + if _, err := shell.Run(shell.Command("docker", "exec", node, "systemctl", "restart", "containerd")); err != nil { + return fmt.Errorf("restart containerd on %s: %w", node, err) + } + } + return nil +} + +// writeHosts installs one registry's hosts.toml on a node. +func writeHosts(node, domain, upstream, host string, skipVerify bool) error { + dir := certsDir + "/" + domain + if _, err := shell.Run(shell.Command("docker", "exec", node, "mkdir", "-p", dir)); err != nil { + return fmt.Errorf("create %s on %s: %w", dir, node, err) + } + + cmd := shell.Command("docker", "exec", "-i", node, "sh", "-c", "cat > "+dir+"/hosts.toml") + cmd.Stdin = strings.NewReader(HostsTOML(upstream, host, skipVerify)) + if _, err := shell.Run(cmd); err != nil { + return fmt.Errorf("write %s/hosts.toml on %s: %w", dir, node, err) + } + return nil +} + +// HostsTOML renders one registry's containerd hosts.toml. upstream is the server containerd falls +// back to, host is the mirror endpoint, and skipVerify disables TLS verification for it. +func HostsTOML(upstream, host string, skipVerify bool) string { + return fmt.Sprintf(`server = %q + +[host.%q] + capabilities = ["pull", "resolve"] + override_path = true + skip_verify = %t +`, upstream, host, skipVerify) +} + +// Mirrors returns the mirror endpoint and hosts.toml for every registry Configure would write on a +// node, keyed by the containerd certs.d directory name. Exported so the rendering can be inspected +// and tested without a cluster. +func Mirrors(registry string) map[string]string { + url := "http://" + registry + out := make(map[string]string, len(mirrored)+1) + for domain, upstream := range mirrored { + out[domain] = HostsTOML(upstream, url+"/v2/"+domain, false) + } + out[registry] = HostsTOML(url, url, true) + return out +} diff --git a/internal/registry/registry_test.go b/internal/registry/registry_test.go new file mode 100644 index 0000000..a00983c --- /dev/null +++ b/internal/registry/registry_test.go @@ -0,0 +1,44 @@ +package registry + +import ( + "strings" + "testing" +) + +// The docker.io upstream and override_path are the two settings that silently break pulls when +// they regress, so they are asserted rather than left to review. See the package doc. +func TestMirrorsSpelling(t *testing.T) { + m := Mirrors("10.0.0.1:5000") + + docker, ok := m["docker.io"] + if !ok { + t.Fatal("docker.io is not mirrored") + } + if !strings.Contains(docker, `server = "https://registry-1.docker.io"`) { + t.Errorf("docker.io upstream must be registry-1.docker.io, got:\n%s", docker) + } + if strings.Contains(docker, `server = "https://docker.io"`) { + t.Error("docker.io upstream is a website; containerd caches its HTML as a manifest") + } + if !strings.Contains(docker, `[host."http://10.0.0.1:5000/v2/docker.io"]`) { + t.Errorf("docker.io mirror endpoint is wrong:\n%s", docker) + } + + for domain, toml := range m { + if !strings.Contains(toml, "override_path = true") { + t.Errorf("%s: override_path must be true or containerd doubles the /v2 prefix", domain) + } + if !strings.Contains(toml, `capabilities = ["pull", "resolve"]`) { + t.Errorf("%s: missing capabilities", domain) + } + } + + // The registry addressed as itself: no /v2/ prefix, and plain HTTP means skip_verify. + self, ok := m["10.0.0.1:5000"] + if !ok { + t.Fatal("the registry has no entry keyed on its own host") + } + if !strings.Contains(self, `[host."http://10.0.0.1:5000"]`) || !strings.Contains(self, "skip_verify = true") { + t.Errorf("registry self-entry is wrong:\n%s", self) + } +} diff --git a/kind.yaml b/kind.yaml index fe4e61f..8583544 100644 --- a/kind.yaml +++ b/kind.yaml @@ -2,8 +2,20 @@ # symmetric oteldb replicas (plus the dev etcd) can spread across nodes and pod anti-affinity / # topology-spread behaves like a real deployment. Chaos specs delete pods/PVCs and rely on the # StatefulSet rescheduling onto a surviving node. +# +# containerdConfigPatches only point containerd at its per-registry config directory. That +# directory is empty unless E2E_REGISTRY is set, in which case internal/registry drops a +# hosts.toml per registry into it after the cluster is created; with no registry configured this +# patch changes nothing and the nodes pull from upstream as before. kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 +containerdConfigPatches: + # Both spellings: containerd 2.x reads the CRI image service config, 1.x the grpc plugin. + - |- + [plugins."io.containerd.cri.v1.images".registry] + config_path = "/etc/containerd/certs.d" + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" nodes: - role: control-plane - role: worker diff --git a/suite_test.go b/suite_test.go index c4a7eb3..0d5c709 100644 --- a/suite_test.go +++ b/suite_test.go @@ -13,6 +13,7 @@ import ( "github.com/oteldb/e2e/internal/build" "github.com/oteldb/e2e/internal/cfg" "github.com/oteldb/e2e/internal/kubeutil" + "github.com/oteldb/e2e/internal/registry" "github.com/oteldb/e2e/internal/shell" ) @@ -53,6 +54,7 @@ var _ = BeforeSuite(func() { SetDefaultEventuallyPollingInterval(2 * time.Second) ensureKindCluster() + configureRegistry() if conf.SkipBuild { By("skipping image build (E2E_SKIP_BUILD set); reusing loaded images") @@ -110,6 +112,18 @@ func ensureKindCluster() { Expect(err).NotTo(HaveOccurred(), "Failed to create kind cluster") } +// configureRegistry points the kind nodes' containerd at the configured registry, both as a +// pull-through mirror and as the source of the images the build path pushes. No registry +// configured (the default) leaves containerd alone. +func configureRegistry() { + if conf.Registry == "" { + return + } + By(fmt.Sprintf("configuring the kind nodes to use registry %s", conf.Registry)) + Expect(registry.Configure(conf.KindCluster, conf.Registry)). + To(Succeed(), "Failed to point the kind nodes at %s", conf.Registry) +} + // deployOperator installs the CRDs and controller-manager via the operator's Makefile. No webhooks // exist, so cert-manager is skipped. func deployOperator() { @@ -138,6 +152,15 @@ func deployEtcdAndCluster() { By("applying the OtelDBCluster") Expect(kubeutil.Apply(conf.Namespace, "manifests/cluster.yaml")).To(Succeed()) + if conf.Registry != "" { + // cluster.yaml pins the bare image tag; under a registry the built image lives behind a + // host, so the CR has to name it too or the nodes would look for an image nobody pushed. + By("repointing the OtelDBCluster at the registry-hosted image") + _, err := kubeutil.Kubectl("patch", "oteldbcluster", conf.ClusterName, "-n", conf.Namespace, + "--type=merge", "-p", fmt.Sprintf(`{"spec":{"image":%q}}`, conf.OteldbImage)) + Expect(err).NotTo(HaveOccurred(), "Failed to repoint the OtelDBCluster at %s", conf.OteldbImage) + } + By("waiting for the oteldb StatefulSet rollout") Expect(kubeutil.WaitRollout(conf.Namespace, "statefulset/"+conf.ClusterName, 5*time.Minute)). To(Succeed(), "oteldb StatefulSet did not roll out")