feat(registry): optionally push images to and mirror pulls through a local registry

README.md

@@ -74,6 +74,50 @@ make test-fast     # rebuild + reload oteldb, redeploy CR, re-run — repeat on
 | `E2E_SKIP_BUILD` | _unset_ | reuse already-loaded images (skip host compile + load) |
 | `E2E_REUSE_CLUSTER` | _unset_ | assume the kind cluster already exists (no create/delete) |
 | `E2E_KEEP_CLUSTER` | _unset_ | leave the kind cluster running after the suite |
+| `E2E_REGISTRY` | _unset_ | `host:port` of a plain-HTTP registry to push built images to and mirror upstream pulls through (see below) |
+
+## Using a local registry (optional)
+
+By default the suite side-loads the images it builds with `kind load` and the nodes pull
+everything else — the kind node image, the dev etcd, the operator's dependencies — straight from
+upstream. Set `E2E_REGISTRY=host:port` and both change: built images are `docker push`ed there and
+pulled back by the nodes, and the nodes' containerd mirrors `docker.io`, `ghcr.io`, `quay.io` and
+`registry.k8s.io` through the same host. This is worth it on a machine that already runs a
+pull-through cache, and useless otherwise — hence off by default.
+
+```bash
+E2E_REGISTRY=10.0.0.1:5000 make test
+```
+
+The registry is assumed to speak plain HTTP, so the host must also be in the *host* docker daemon's
+`insecure-registries` for the push (and for kind's own node-image pull) to work.
+
+`internal/registry` writes one `hosts.toml` per registry into `/etc/containerd/certs.d` on each
+node — the directory `kind.yaml`'s `containerdConfigPatches` selects. That file is modelled on the
+config running on a real cluster ([`tdakkota/nixfra`][nixfra],
+`modules/registry-mirrors.nix`) rather than invented, because three of its properties are easy to
+get wrong and expensive to debug:
+
+1. **`docker.io`'s upstream must be `https://registry-1.docker.io`, not `https://docker.io`.**
+   The latter is a website. containerd follows the fallback, gets HTML back, and caches it in the
+   content store as if it were a manifest; every later pull of that image fails with
+   `unexpected media type text/html for sha256:...`, and it stays broken across containerd restarts
+   until the poisoned blob is evicted by hand:
+
+   ```bash
+   docker exec <node> ctr -n k8s.io content rm sha256:<digest>
+   ```
+
+2. **`override_path = true` is required.** The mirror is addressed as `<registry>/v2/<domain>`.
+   Without it containerd appends its own `/v2/<repo>` to that path and requests
+   `.../v2/docker.io/v2/library/alpine/manifests/3.21`, which 404s.
+
+3. **A pull-through mirror may re-digest what it caches.** Zot serves an `alpine` index digest that
+   differs from Docker Hub's, so a pull pinned to an upstream *digest* rather than a tag cannot be
+   satisfied from the mirror. Nothing to configure here — just do not expect digest pinning to
+   survive a mirror.
+
+[nixfra]: https://github.com/tdakkota/nixfra
 
 ## Layout
 
@@ -85,6 +129,7 @@ fast.Dockerfile     minimal image wrapping a host-built binary (fast rebuild pat
 kind.yaml           1 control-plane + 3 workers (room to spread replicas + survive a node loss)
 internal/
   cfg/              env-driven suite configuration
+  registry/         optional local-registry mirror config for the kind nodes
   shell/            command execution wrappers (docker/kind/kubectl/go/make)
   build/            fast image path: host go build -> COPY image -> kind load
   kubeutil/         kubectl helpers: apply/wait/list/delete + Service port-forward

internal/build/build.go

@@ -13,6 +13,7 @@ import (
 
 	. "github.com/onsi/ginkgo/v2" //nolint:revive,staticcheck // dot-import is the Ginkgo idiom.
 
+	"github.com/oteldb/e2e/internal/cfg"
 	"github.com/oteldb/e2e/internal/shell"
 )
 
@@ -66,6 +67,17 @@ func BinaryImage(srcDir, cmdPkg, binName, image, kindCluster string) error {
 		return fmt.Errorf("docker build %s: %w", image, err)
 	}
 
+	// A tag naming a registry host (E2E_REGISTRY) is pushed there and pulled back by the nodes
+	// through the mirror internal/registry configured. A bare tag has nowhere to be pushed, so it
+	// is side-loaded into the nodes instead — the default.
+	if cfg.RegistryTagged(image) {
+		By(fmt.Sprintf("pushing %s", image))
+		if _, err := shell.Run(shell.Command("docker", "push", image)); err != nil {
+			return fmt.Errorf("docker push %s: %w", image, err)
+		}
+		return nil
+	}
+
 	By(fmt.Sprintf("loading %s into kind cluster %s", image, kindCluster))
 	load := shell.Command("kind", "load", "docker-image", image, "--name", kindCluster)
 	if _, err := shell.Run(load); err != nil {

internal/cfg/cfg.go

@@ -3,7 +3,11 @@
 // of a couple of env vars rather than code edits.
 package cfg
 
-import "github.com/oteldb/e2e/internal/shell"
+import (
+	"strings"
+
+	"github.com/oteldb/e2e/internal/shell"
+)
 
 // Config holds the resolved suite configuration.
 type Config struct {
@@ -12,11 +16,19 @@ type Config struct {
 	// OperatorSrc is the local operator source tree (Makefile-driven deploy).
 	OperatorSrc string
 
-	// OteldbImage / OperatorImage are the locally built, kind-loaded image tags. They MUST match
-	// spec.image in manifests/cluster.yaml (oteldb) and the operator deploy IMG respectively.
+	// OteldbImage / OperatorImage are the locally built image tags. They MUST match spec.image in
+	// manifests/cluster.yaml (oteldb) and the operator deploy IMG respectively — except under
+	// Registry, which prefixes both and overrides spec.image on the applied CR.
 	OteldbImage   string
 	OperatorImage string
 
+	// Registry, when set (E2E_REGISTRY=host:port), is a plain-HTTP OCI registry that the suite
+	// pushes its built images to and points the kind nodes at, both for those images and as a
+	// pull-through mirror for the upstream registries. Empty — the default — keeps the
+	// `kind load` side-load path and leaves the nodes pulling from upstream directly, so a run
+	// without a registry behaves exactly as it always has.
+	Registry string
+
 	// KindCluster is the kind cluster name.
 	KindCluster string
 	// KindConfig is the path to the kind cluster config (multi-node).
@@ -39,11 +51,13 @@ type Config struct {
 
 // Load resolves the suite config from the environment, applying defaults.
 func Load() Config {
+	registry := shell.Getenv("E2E_REGISTRY", "")
 	return Config{
 		OteldbSrc:         shell.Getenv("OTELDB_SRC", "/src/oteldb/oteldb"),
 		OperatorSrc:       shell.Getenv("OPERATOR_SRC", "/src/oteldb/operator"),
-		OteldbImage:       shell.Getenv("OTELDB_IMAGE", "oteldb/oteldb:e2e"),
-		OperatorImage:     shell.Getenv("OPERATOR_IMAGE", "oteldb/operator:e2e"),
+		OteldbImage:       inRegistry(registry, shell.Getenv("OTELDB_IMAGE", "oteldb/oteldb:e2e")),
+		OperatorImage:     inRegistry(registry, shell.Getenv("OPERATOR_IMAGE", "oteldb/operator:e2e")),
+		Registry:          registry,
 		KindCluster:       shell.Getenv("KIND_CLUSTER", "oteldb-e2e"),
 		KindConfig:        shell.Getenv("KIND_CONFIG", "kind.yaml"),
 		Namespace:         shell.Getenv("E2E_NAMESPACE", "oteldb-e2e"),
@@ -54,3 +68,21 @@ func Load() Config {
 		ReuseCluster:      shell.Truthy("E2E_REUSE_CLUSTER"),
 	}
 }
+
+// inRegistry qualifies image with the configured registry host. It is a no-op when no registry is
+// configured (the default) or when the caller already spelled a registry host into the tag, so an
+// explicit OTELDB_IMAGE=some.host/oteldb:tag still wins.
+func inRegistry(registry, image string) string {
+	if registry == "" || RegistryTagged(image) {
+		return image
+	}
+	return registry + "/" + image
+}
+
+// RegistryTagged reports whether image names an explicit registry host, i.e. its first path
+// component looks like a host (contains a dot or a port). "oteldb/oteldb:e2e" does not;
+// "10.0.0.1:5000/oteldb/oteldb:e2e" does.
+func RegistryTagged(image string) bool {
+	first, _, ok := strings.Cut(image, "/")
+	return ok && (strings.Contains(first, ".") || strings.Contains(first, ":"))
+}

internal/cfg/cfg_test.go

@@ -0,0 +1,35 @@
+package cfg
+
+import "testing"
+
+func TestRegistryTagged(t *testing.T) {
+	for _, tt := range []struct {
+		image string
+		want  bool
+	}{
+		{"oteldb/oteldb:e2e", false},
+		{"alpine:3.20", false},
+		{"10.0.0.1:5000/oteldb/oteldb:e2e", true},
+		{"ghcr.io/oteldb/oteldb:e2e", true},
+		{"localhost:5000/x:y", true},
+	} {
+		if got := RegistryTagged(tt.image); got != tt.want {
+			t.Errorf("RegistryTagged(%q) = %v, want %v", tt.image, got, tt.want)
+		}
+	}
+}
+
+func TestInRegistry(t *testing.T) {
+	// No registry configured is the default and must not touch the tag: that is what keeps the
+	// suite on the `kind load` path.
+	if got := inRegistry("", "oteldb/oteldb:e2e"); got != "oteldb/oteldb:e2e" {
+		t.Errorf("unconfigured registry rewrote the tag: %q", got)
+	}
+	if got := inRegistry("10.0.0.1:5000", "oteldb/oteldb:e2e"); got != "10.0.0.1:5000/oteldb/oteldb:e2e" {
+		t.Errorf("inRegistry = %q", got)
+	}
+	// An explicitly host-qualified override wins over the configured registry.
+	if got := inRegistry("10.0.0.1:5000", "ghcr.io/oteldb/oteldb:e2e"); got != "ghcr.io/oteldb/oteldb:e2e" {
+		t.Errorf("inRegistry overrode an explicit host: %q", got)
+	}
+}

internal/registry/registry.go

@@ -0,0 +1,117 @@
+// Package registry points a kind cluster's containerd at a local OCI registry: as a pull-through
+// mirror for the upstream registries the suite's manifests reference, and as the home of the
+// images the fast build path pushes.
+//
+// It is opt-in (E2E_REGISTRY). Without it the suite side-loads its images with `kind load` and the
+// nodes pull everything else straight from upstream, exactly as they always have.
+//
+// The hosts.toml this writes is modelled on the containerd config running on the author's own
+// cluster (tdakkota/nixfra, modules/registry-mirrors.nix) rather than invented here, because two
+// of its details are load-bearing and easy to "simplify" back into a broken state:
+//
+//   - docker.io's upstream MUST be https://registry-1.docker.io. "https://docker.io" is a website:
+//     containerd follows the fallback, gets HTML, and caches it in the content store as if it were
+//     a manifest. Every later pull of that image then fails with `unexpected media type text/html
+//     for sha256:...` — and it stays broken across containerd restarts until the poisoned blob is
+//     evicted by hand (`ctr -n k8s.io content rm sha256:...`).
+//   - override_path MUST be true. The mirror is addressed as <registry>/v2/<domain>, and without
+//     it containerd appends its own /v2/<repo> to that path and requests
+//     .../v2/docker.io/v2/library/alpine/manifests/3.21, which 404s.
+//
+// A third property is worth knowing but cannot be configured around: a pull-through mirror may
+// re-digest what it caches (Zot serves an alpine index digest that differs from Docker Hub's), so
+// pulls pinned to an upstream digest rather than a tag are inherently fragile through one.
+package registry
+
+import (
+	"fmt"
+	"strings"
+
+	. "github.com/onsi/ginkgo/v2" //nolint:revive,staticcheck // dot-import is the Ginkgo idiom.
+
+	"github.com/oteldb/e2e/internal/shell"
+)
+
+// mirrored are the upstream registries the suite's own manifests and images come from, mapped to
+// the server containerd should fall back to when the mirror cannot serve a blob. The author's
+// cluster mirrors a longer list; these are the ones an e2e run actually reaches for.
+var mirrored = map[string]string{
+	"docker.io":       "https://registry-1.docker.io",
+	"ghcr.io":         "https://ghcr.io",
+	"quay.io":         "https://quay.io",
+	"registry.k8s.io": "https://registry.k8s.io",
+}
+
+// certsDir is containerd's per-registry config directory, selected by the config_path set in
+// kind.yaml's containerdConfigPatches.
+const certsDir = "/etc/containerd/certs.d"
+
+// Configure writes a hosts.toml for every mirrored registry (and for the registry itself) onto
+// each node of the kind cluster, then restarts containerd to pick them up. registry is a
+// host:port addressed over plain HTTP.
+func Configure(kindCluster, registry string) error {
+	nodes, err := shell.Output("kind", "get", "nodes", "--name", kindCluster)
+	if err != nil {
+		return fmt.Errorf("list kind nodes: %w", err)
+	}
+
+	url := "http://" + registry
+	for _, node := range strings.Fields(nodes) {
+		By(fmt.Sprintf("pointing containerd on %s at %s", node, registry))
+		for domain, upstream := range mirrored {
+			// The mirror lives under a per-domain path prefix, hence override_path.
+			if err := writeHosts(node, domain, upstream, url+"/v2/"+domain, false); err != nil {
+				return err
+			}
+		}
+		// The registry addressed as itself, so the images we push to it resolve by their own tag.
+		// No path prefix here, and plain HTTP means skip_verify.
+		if err := writeHosts(node, registry, url, url, true); err != nil {
+			return err
+		}
+		if _, err := shell.Run(shell.Command("docker", "exec", node, "systemctl", "restart", "containerd")); err != nil {
+			return fmt.Errorf("restart containerd on %s: %w", node, err)
+		}
+	}
+	return nil
+}
+
+// writeHosts installs one registry's hosts.toml on a node.
+func writeHosts(node, domain, upstream, host string, skipVerify bool) error {
+	dir := certsDir + "/" + domain
+	if _, err := shell.Run(shell.Command("docker", "exec", node, "mkdir", "-p", dir)); err != nil {
+		return fmt.Errorf("create %s on %s: %w", dir, node, err)
+	}
+
+	cmd := shell.Command("docker", "exec", "-i", node, "sh", "-c", "cat > "+dir+"/hosts.toml")
+	cmd.Stdin = strings.NewReader(HostsTOML(upstream, host, skipVerify))
+	if _, err := shell.Run(cmd); err != nil {
+		return fmt.Errorf("write %s/hosts.toml on %s: %w", dir, node, err)
+	}
+	return nil
+}
+
+// HostsTOML renders one registry's containerd hosts.toml. upstream is the server containerd falls
+// back to, host is the mirror endpoint, and skipVerify disables TLS verification for it.
+func HostsTOML(upstream, host string, skipVerify bool) string {
+	return fmt.Sprintf(`server = %q
+
+[host.%q]
+  capabilities = ["pull", "resolve"]
+  override_path = true
+  skip_verify = %t
+`, upstream, host, skipVerify)
+}
+
+// Mirrors returns the mirror endpoint and hosts.toml for every registry Configure would write on a
+// node, keyed by the containerd certs.d directory name. Exported so the rendering can be inspected
+// and tested without a cluster.
+func Mirrors(registry string) map[string]string {
+	url := "http://" + registry
+	out := make(map[string]string, len(mirrored)+1)
+	for domain, upstream := range mirrored {
+		out[domain] = HostsTOML(upstream, url+"/v2/"+domain, false)
+	}
+	out[registry] = HostsTOML(url, url, true)
+	return out
+}

internal/registry/registry_test.go

@@ -0,0 +1,44 @@
+package registry
+
+import (
+	"strings"
+	"testing"
+)
+
+// The docker.io upstream and override_path are the two settings that silently break pulls when
+// they regress, so they are asserted rather than left to review. See the package doc.
+func TestMirrorsSpelling(t *testing.T) {
+	m := Mirrors("10.0.0.1:5000")
+
+	docker, ok := m["docker.io"]
+	if !ok {
+		t.Fatal("docker.io is not mirrored")
+	}
+	if !strings.Contains(docker, `server = "https://registry-1.docker.io"`) {
+		t.Errorf("docker.io upstream must be registry-1.docker.io, got:\n%s", docker)
+	}
+	if strings.Contains(docker, `server = "https://docker.io"`) {
+		t.Error("docker.io upstream is a website; containerd caches its HTML as a manifest")
+	}
+	if !strings.Contains(docker, `[host."http://10.0.0.1:5000/v2/docker.io"]`) {
+		t.Errorf("docker.io mirror endpoint is wrong:\n%s", docker)
+	}
+
+	for domain, toml := range m {
+		if !strings.Contains(toml, "override_path = true") {
+			t.Errorf("%s: override_path must be true or containerd doubles the /v2 prefix", domain)
+		}
+		if !strings.Contains(toml, `capabilities = ["pull", "resolve"]`) {
+			t.Errorf("%s: missing capabilities", domain)
+		}
+	}
+
+	// The registry addressed as itself: no /v2/<domain> prefix, and plain HTTP means skip_verify.
+	self, ok := m["10.0.0.1:5000"]
+	if !ok {
+		t.Fatal("the registry has no entry keyed on its own host")
+	}
+	if !strings.Contains(self, `[host."http://10.0.0.1:5000"]`) || !strings.Contains(self, "skip_verify = true") {
+		t.Errorf("registry self-entry is wrong:\n%s", self)
+	}
+}

kind.yaml

@@ -2,8 +2,20 @@
 # symmetric oteldb replicas (plus the dev etcd) can spread across nodes and pod anti-affinity /
 # topology-spread behaves like a real deployment. Chaos specs delete pods/PVCs and rely on the
 # StatefulSet rescheduling onto a surviving node.
+#
+# containerdConfigPatches only point containerd at its per-registry config directory. That
+# directory is empty unless E2E_REGISTRY is set, in which case internal/registry drops a
+# hosts.toml per registry into it after the cluster is created; with no registry configured this
+# patch changes nothing and the nodes pull from upstream as before.
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+  # Both spellings: containerd 2.x reads the CRI image service config, 1.x the grpc plugin.
+  - |-
+    [plugins."io.containerd.cri.v1.images".registry]
+      config_path = "/etc/containerd/certs.d"
+    [plugins."io.containerd.grpc.v1.cri".registry]
+      config_path = "/etc/containerd/certs.d"
 nodes:
   - role: control-plane
   - role: worker