tests: pin k3d images for kubernetes worker

Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com>

Author: crazy-max

Dockerfile

@@ -15,6 +15,7 @@ ARG BUILDKIT_VERSION=v0.29.0
 ARG COMPOSE_VERSION=v5.1.0
 ARG UNDOCK_VERSION=0.9.0
 ARG K3D_VERSION=5.8.3
+ARG K3S_VERSION=v1.32.2-k3s1
 
 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
 FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS golatest
@@ -155,6 +156,9 @@ COPY --link --from=k3d /bin/k3d /usr/bin/
 COPY --link --from=binaries /buildx /usr/bin/
 RUN mkdir -p /usr/local/lib/docker/cli-plugins && ln -s /usr/bin/buildx /usr/local/lib/docker/cli-plugins/docker-buildx
 ENV TEST_DOCKER_EXTRA="docker@28.5=/opt/docker-alt-28,docker@27.5=/opt/docker-alt-27"
+ENV TEST_K3S_IMAGE="rancher/k3s:${K3S_VERSION}"
+ENV TEST_K3D_TOOLS_IMAGE="ghcr.io/k3d-io/k3d-tools:${K3D_VERSION}"
+ENV TEST_K3D_LOADBALANCER_IMAGE="ghcr.io/k3d-io/k3d-proxy:${K3D_VERSION}"
 
 FROM integration-test-base AS integration-test
 COPY . .

tests/helpers/k3d.go

@@ -36,24 +36,23 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd
 	createCtx, cancelCreate := context.WithTimeout(ctx, 90*time.Second)
 	defer cancelCreate()
 
-	cmd := exec.CommandContext(createCtx, k3dBin, "cluster", "create", clusterName,
+	args := []string{
+		"cluster", "create", clusterName,
 		"--wait",
-	)
-	cmd.Env = append(
-		os.Environ(),
-		"DOCKER_CONTEXT="+dockerAddress,
-	)
+	}
+	if image := KubernetesK3sImage(); image != "" {
+		args = append(args, "--image="+image)
+	}
+	cmd := exec.CommandContext(createCtx, k3dBin, args...)
+	cmd.Env = k3dEnv(dockerAddress)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		diag := KubernetesDiagnostics(clusterName, dockerAddress)
 		return "", "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s\n%s", clusterName, strings.TrimSpace(string(out)), diag)
 	}
 	deferF.Append(func() error {
 		cmd := exec.Command(k3dBin, "cluster", "delete", clusterName)
-		cmd.Env = append(
-			os.Environ(),
-			"DOCKER_CONTEXT="+dockerAddress,
-		)
+		cmd.Env = k3dEnv(dockerAddress)
 		out, err := cmd.CombinedOutput()
 		if err != nil {
 			return errors.Wrapf(err, "failed to delete k3d cluster %s: %s", clusterName, string(out))
@@ -65,10 +64,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd
 	defer cancelKubeconfig()
 
 	cmd = exec.CommandContext(kubeconfigCtx, k3dBin, "kubeconfig", "write", clusterName)
-	cmd.Env = append(
-		os.Environ(),
-		"DOCKER_CONTEXT="+dockerAddress,
-	)
+	cmd.Env = k3dEnv(dockerAddress)
 	out, err = cmd.CombinedOutput()
 	if err != nil {
 		diag := KubernetesDiagnostics(clusterName, dockerAddress)
@@ -78,3 +74,17 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd
 
 	return
 }
+
+func k3dEnv(dockerAddress string) []string {
+	env := append(
+		os.Environ(),
+		"DOCKER_CONTEXT="+dockerAddress,
+	)
+	if image := KubernetesK3DToolsImage(); image != "" {
+		env = append(env, "K3D_IMAGE_TOOLS="+image)
+	}
+	if image := KubernetesK3DLoadBalancerImage(); image != "" {
+		env = append(env, "K3D_IMAGE_LOADBALANCER="+image)
+	}
+	return env
+}

tests/helpers/kubernetes_diagnostics.go

@@ -23,6 +23,18 @@ func KubernetesBuildkitImage() string {
 	return "moby/buildkit:" + tag
 }
 
+func KubernetesK3sImage() string {
+	return os.Getenv("TEST_K3S_IMAGE")
+}
+
+func KubernetesK3DToolsImage() string {
+	return os.Getenv("TEST_K3D_TOOLS_IMAGE")
+}
+
+func KubernetesK3DLoadBalancerImage() string {
+	return os.Getenv("TEST_K3D_LOADBALANCER_IMAGE")
+}
+
 func KubernetesDiagnostics(clusterName, dockerContext string) string {
 	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
 	defer cancel()
@@ -56,16 +68,22 @@ func appendK3sServerDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterN
 	}
 
 	for _, nodeName := range nodeNames {
+		appendCommandOutput(ctx, buf, "docker inspect "+nodeName, "docker", []string{
+			"inspect",
+			"--format",
+			"Status={{.State.Status}} Health={{if .State.Health}}{{.State.Health.Status}}{{else}}<none>{{end}} Restarting={{.State.Restarting}} ExitCode={{.State.ExitCode}} Error={{.State.Error}}",
+			nodeName,
+		}, []string{"DOCKER_CONTEXT=" + dockerContext})
 		appendCommandOutput(ctx, buf, "docker logs "+nodeName, "docker", []string{"logs", "--tail", "80", nodeName}, []string{"DOCKER_CONTEXT=" + dockerContext})
 	}
 
 	for _, nodeName := range nodeNames {
 		if !strings.Contains(nodeName, "-server-") {
 			continue
 		}
-		appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s kubectl get pods", "docker", []string{"exec", nodeName, "k3s", "kubectl", "get", "pods", "-A", "-o", "wide"}, []string{"DOCKER_CONTEXT=" + dockerContext})
-		appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s kubectl get events", "docker", []string{"exec", nodeName, "k3s", "kubectl", "get", "events", "-A", "--sort-by=.lastTimestamp"}, []string{"DOCKER_CONTEXT=" + dockerContext})
-		appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s kubectl describe pods", "docker", []string{"exec", nodeName, "k3s", "kubectl", "describe", "pods", "-A"}, []string{"DOCKER_CONTEXT=" + dockerContext})
+		appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get pods", "docker", []string{"exec", nodeName, "kubectl", "get", "pods", "-A", "-o", "wide"}, []string{"DOCKER_CONTEXT=" + dockerContext})
+		appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get events", "docker", []string{"exec", nodeName, "kubectl", "get", "events", "-A", "--sort-by=.lastTimestamp"}, []string{"DOCKER_CONTEXT=" + dockerContext})
+		appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl describe pods", "docker", []string{"exec", nodeName, "kubectl", "describe", "pods", "-A"}, []string{"DOCKER_CONTEXT=" + dockerContext})
 		break
 	}
 }