Skip to content

Commit cedc3a2

Browse files
crazy-maxcrazy-max
committed
test kubernetes worker with k3d
Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com>
1 parent 581c232 commit cedc3a2

File tree

5 files changed

+210
-2
lines changed

5 files changed

+210
-2
lines changed

‎.github/workflows/build.yml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ jobs:
5454
worker:
5555
- docker-container
5656
- remote
57+
- kubernetes
5758
pkg:
5859
- ./tests
5960
mode:
@@ -105,14 +106,14 @@ jobs:
105106
fi
106107
testFlags="--run=//worker=$(echo "${{ matrix.worker }}" | sed 's/\+/\\+/g')$"
107108
case "${{ matrix.worker }}" in
108-
docker | docker+containerd | docker@* | docker+containerd@* | remote+multinode)
109+
docker | docker+containerd | docker@* | docker+containerd@* | remote+multinode | kubernetes)
109110
echo "TESTFLAGS=${{ env.TESTFLAGS_DOCKER }} $testFlags" >> $GITHUB_ENV
110111
;;
111112
*)
112113
echo "TESTFLAGS=${{ env.TESTFLAGS }} $testFlags" >> $GITHUB_ENV
113114
;;
114115
esac
115-
if [[ "${{ matrix.worker }}" == "docker"* || "${{ matrix.worker }}" == "remote+multinode" ]]; then
116+
if [[ "${{ matrix.worker }}" == "docker"* || "${{ matrix.worker }}" == "remote+multinode" || "${{ matrix.worker }}" == "kubernetes" ]]; then
116117
echo "TEST_DOCKERD=1" >> $GITHUB_ENV
117118
fi
118119
if [ "${{ matrix.mode }}" = "experimental" ]; then

‎Dockerfile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ ARG REGISTRY_VERSION=3.0.0
1414
ARG BUILDKIT_VERSION=v0.29.0
1515
ARG COMPOSE_VERSION=v5.1.0
1616
ARG UNDOCK_VERSION=0.9.0
17+
ARG K3D_VERSION=5.8.3
1718

1819
FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
1920
FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS golatest
@@ -27,6 +28,7 @@ FROM registry:$REGISTRY_VERSION AS registry
2728
FROM moby/buildkit:$BUILDKIT_VERSION AS buildkit
2829
FROM docker/compose-bin:$COMPOSE_VERSION AS compose
2930
FROM crazymax/undock:$UNDOCK_VERSION AS undock
31+
FROM ghcr.io/k3d-io/k3d:${K3D_VERSION} AS k3d
3032

3133
FROM golatest AS gobase
3234
COPY --from=xx / /
@@ -149,6 +151,7 @@ COPY --link --from=buildkit /usr/bin/buildkitd /usr/bin/
149151
COPY --link --from=buildkit /usr/bin/buildctl /usr/bin/
150152
COPY --link --from=compose /docker-compose /usr/bin/compose
151153
COPY --link --from=undock /usr/local/bin/undock /usr/bin/
154+
COPY --link --from=k3d /bin/k3d /usr/bin/
152155
COPY --link --from=binaries /buildx /usr/bin/
153156
RUN mkdir -p /usr/local/lib/docker/cli-plugins && ln -s /usr/bin/buildx /usr/local/lib/docker/cli-plugins/docker-buildx
154157
ENV TEST_DOCKER_EXTRA="docker@28.5=/opt/docker-alt-28,docker@27.5=/opt/docker-alt-27"

‎tests/helpers/k3d.go‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
package helpers
2+
3+
import (
4+
"context"
5+
"os"
6+
"os/exec"
7+
"strings"
8+
9+
"github.com/moby/buildkit/identity"
10+
"github.com/moby/buildkit/util/testutil/integration"
11+
"github.com/pkg/errors"
12+
)
13+
14+
const (
15+
k3dBin = "k3d"
16+
)
17+
18+
func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAddress string) (kubeConfig string, cl func() error, err error) {
19+
if _, err := exec.LookPath(k3dBin); err != nil {
20+
return "", nil, errors.Wrapf(err, "failed to lookup %s binary", k3dBin)
21+
}
22+
23+
deferF := &integration.MultiCloser{}
24+
cl = deferF.F()
25+
26+
defer func() {
27+
if err != nil {
28+
deferF.F()()
29+
cl = nil
30+
}
31+
}()
32+
33+
clusterName := "bk-" + identity.NewID()
34+
35+
cmd := exec.CommandContext(ctx, k3dBin, "cluster", "create", clusterName,
36+
"--wait",
37+
)
38+
cmd.Env = append(
39+
os.Environ(),
40+
"DOCKER_CONTEXT="+dockerAddress,
41+
)
42+
out, err := cmd.CombinedOutput()
43+
if err != nil {
44+
return "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s", clusterName, string(out))
45+
}
46+
deferF.Append(func() error {
47+
cmd := exec.Command(k3dBin, "cluster", "delete", clusterName)
48+
cmd.Env = append(
49+
os.Environ(),
50+
"DOCKER_CONTEXT="+dockerAddress,
51+
)
52+
out, err := cmd.CombinedOutput()
53+
if err != nil {
54+
return errors.Wrapf(err, "failed to delete k3d cluster %s: %s", clusterName, string(out))
55+
}
56+
return nil
57+
})
58+
59+
cmd = exec.CommandContext(ctx, k3dBin, "kubeconfig", "write", clusterName)
60+
cmd.Env = append(
61+
os.Environ(),
62+
"DOCKER_CONTEXT="+dockerAddress,
63+
)
64+
out, err = cmd.CombinedOutput()
65+
if err != nil {
66+
return "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s", clusterName, string(out))
67+
}
68+
kubeConfig = strings.TrimSpace(string(out))
69+
70+
return
71+
}

‎tests/integration_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ func init() {
1515
workers.InitDockerWorker()
1616
workers.InitDockerContainerWorker()
1717
workers.InitRemoteMultiNodeWorker()
18+
workers.InitKubernetesWorker()
1819
} else {
1920
workers.InitRemoteWorker()
2021
}

‎tests/workers/kubernetes.go‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
package workers
2+
3+
import (
4+
"context"
5+
"os"
6+
"os/exec"
7+
"path/filepath"
8+
"sync"
9+
10+
"github.com/docker/buildx/tests/helpers"
11+
"github.com/moby/buildkit/identity"
12+
"github.com/moby/buildkit/util/testutil/integration"
13+
"github.com/pkg/errors"
14+
)
15+
16+
func InitKubernetesWorker() {
17+
integration.Register(&kubernetesWorker{
18+
id: "kubernetes",
19+
})
20+
}
21+
22+
type kubernetesWorker struct {
23+
id string
24+
25+
unsupported []string
26+
27+
docker integration.Backend
28+
dockerClose func() error
29+
dockerErr error
30+
dockerOnce sync.Once
31+
32+
k3dConfig string
33+
k3dClose func() error
34+
k3dErr error
35+
k3dOnce sync.Once
36+
}
37+
38+
func (w *kubernetesWorker) Name() string {
39+
return w.id
40+
}
41+
42+
func (w *kubernetesWorker) Rootless() bool {
43+
return false
44+
}
45+
46+
func (w *kubernetesWorker) NetNSDetached() bool {
47+
return false
48+
}
49+
50+
func (w *kubernetesWorker) New(ctx context.Context, cfg *integration.BackendConfig) (integration.Backend, func() error, error) {
51+
w.dockerOnce.Do(func() {
52+
w.docker, w.dockerClose, w.dockerErr = dockerWorker{id: w.id}.New(ctx, cfg)
53+
})
54+
if w.dockerErr != nil {
55+
return w.docker, w.dockerClose, w.dockerErr
56+
}
57+
58+
w.k3dOnce.Do(func() {
59+
w.k3dConfig, w.k3dClose, w.k3dErr = helpers.NewK3dServer(ctx, cfg, w.docker.DockerAddress())
60+
})
61+
if w.k3dErr != nil {
62+
return nil, w.k3dClose, w.k3dErr
63+
}
64+
65+
cfgfile, release, err := integration.WriteConfig(cfg.DaemonConfig)
66+
if err != nil {
67+
return nil, nil, err
68+
}
69+
if release != nil {
70+
defer release()
71+
}
72+
defer os.RemoveAll(filepath.Dir(cfgfile))
73+
74+
name := "integration-kubernetes-" + identity.NewID()
75+
cmd := exec.CommandContext(ctx, "buildx", "create",
76+
"--bootstrap",
77+
"--name="+name,
78+
"--buildkitd-config="+cfgfile,
79+
"--driver=kubernetes",
80+
)
81+
cmd.Env = append(
82+
os.Environ(),
83+
"BUILDX_CONFIG=/tmp/buildx-"+name,
84+
"KUBECONFIG="+w.k3dConfig,
85+
)
86+
if err := cmd.Run(); err != nil {
87+
return nil, nil, errors.Wrapf(err, "failed to create buildx instance %s", name)
88+
}
89+
90+
cl := func() error {
91+
cmd := exec.CommandContext(context.Background(), "buildx", "rm", "-f", name)
92+
cmd.Env = append(
93+
os.Environ(),
94+
"BUILDX_CONFIG=/tmp/buildx-"+name,
95+
"KUBECONFIG="+w.k3dConfig,
96+
)
97+
return cmd.Run()
98+
}
99+
100+
return &backend{
101+
builder: name,
102+
unsupportedFeatures: w.unsupported,
103+
}, cl, nil
104+
}
105+
106+
func (w *kubernetesWorker) Close() error {
107+
setErr := func(dst *error, err error) {
108+
if err != nil && *dst == nil {
109+
*dst = err
110+
}
111+
}
112+
113+
var err error
114+
if c := w.k3dClose; c != nil {
115+
setErr(&err, c())
116+
}
117+
if c := w.dockerClose; c != nil {
118+
setErr(&err, c())
119+
}
120+
121+
// reset the worker to be ready to go again
122+
w.docker = nil
123+
w.dockerClose = nil
124+
w.dockerErr = nil
125+
w.dockerOnce = sync.Once{}
126+
w.k3dConfig = ""
127+
w.k3dClose = nil
128+
w.k3dErr = nil
129+
w.k3dOnce = sync.Once{}
130+
131+
return err
132+
}

0 commit comments

Comments
 (0)