Skip to content

Commit 3609204

Browse files
crazy-maxcrazy-max
committed
test kubernetes worker with k3d
Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com>
1 parent 1e50e8d commit 3609204

File tree

5 files changed

+173
-2
lines changed

5 files changed

+173
-2
lines changed

‎.github/workflows/build.yml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ jobs:
6060
worker:
6161
- docker-container
6262
- remote
63+
- kubernetes
6364
pkg:
6465
- ./tests
6566
mode:
@@ -106,14 +107,14 @@ jobs:
106107
fi
107108
testFlags="--run=//worker=$(echo "${{ matrix.worker }}" | sed 's/\+/\\+/g')$"
108109
case "${{ matrix.worker }}" in
109-
docker | docker+containerd | docker@* | docker+containerd@*)
110+
docker | docker+containerd | docker@* | docker+containerd@* | kubernetes)
110111
echo "TESTFLAGS=${{ env.TESTFLAGS_DOCKER }} $testFlags" >> $GITHUB_ENV
111112
;;
112113
*)
113114
echo "TESTFLAGS=${{ env.TESTFLAGS }} $testFlags" >> $GITHUB_ENV
114115
;;
115116
esac
116-
if [[ "${{ matrix.worker }}" == "docker"* ]]; then
117+
if [[ "${{ matrix.worker }}" == "docker"* ]] || [[ "${{ matrix.worker }}" == "kubernetes" ]]; then
117118
echo "TEST_DOCKERD=1" >> $GITHUB_ENV
118119
fi
119120
if [ "${{ matrix.mode }}" = "experimental" ]; then

‎Dockerfile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ ARG REGISTRY_VERSION=3.0.0
1414
ARG BUILDKIT_VERSION=v0.23.2
1515
ARG COMPOSE_VERSION=v2.39.0
1616
ARG UNDOCK_VERSION=0.9.0
17+
ARG K3D_VERSION=5.8.3
1718

1819
FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
1920
FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS golatest
@@ -27,6 +28,7 @@ FROM registry:$REGISTRY_VERSION AS registry
2728
FROM moby/buildkit:$BUILDKIT_VERSION AS buildkit
2829
FROM docker/compose-bin:$COMPOSE_VERSION AS compose
2930
FROM crazymax/undock:$UNDOCK_VERSION AS undock
31+
FROM ghcr.io/k3d-io/k3d:${K3D_VERSION} AS k3d
3032

3133
FROM golatest AS gobase
3234
COPY --from=xx / /
@@ -141,6 +143,7 @@ COPY --link --from=buildkit /usr/bin/buildkitd /usr/bin/
141143
COPY --link --from=buildkit /usr/bin/buildctl /usr/bin/
142144
COPY --link --from=compose /docker-compose /usr/bin/compose
143145
COPY --link --from=undock /usr/local/bin/undock /usr/bin/
146+
COPY --link --from=k3d /bin/k3d /usr/bin/
144147
COPY --link --from=binaries /buildx /usr/bin/
145148
RUN mkdir -p /usr/local/lib/docker/cli-plugins && ln -s /usr/bin/buildx /usr/local/lib/docker/cli-plugins/docker-buildx
146149
ENV TEST_DOCKER_EXTRA="docker@27.5=/opt/docker-alt-27,docker@26.1=/opt/docker-alt-26"

‎tests/helpers/k3d.go‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
package helpers
2+
3+
import (
4+
"os/exec"
5+
"strings"
6+
7+
"github.com/moby/buildkit/identity"
8+
"github.com/moby/buildkit/util/testutil/integration"
9+
"github.com/pkg/errors"
10+
)
11+
12+
const (
13+
k3dBin = "k3d"
14+
)
15+
16+
func NewK3dServer(cfg *integration.BackendConfig) (kubeConfig string, cl func() error, err error) {
17+
if _, err := exec.LookPath(k3dBin); err != nil {
18+
return "", nil, errors.Wrapf(err, "failed to lookup %s binary", k3dBin)
19+
}
20+
21+
deferF := &integration.MultiCloser{}
22+
cl = deferF.F()
23+
24+
defer func() {
25+
if err != nil {
26+
deferF.F()()
27+
cl = nil
28+
}
29+
}()
30+
31+
clusterName := "buildkit-" + identity.NewID()
32+
33+
stop, err := integration.StartCmd(exec.Command(k3dBin, "cluster", "create", clusterName,
34+
"--wait",
35+
), cfg.Logs)
36+
if err != nil {
37+
return "", nil, err
38+
}
39+
deferF.Append(stop)
40+
41+
cmd := exec.Command(k3dBin, "kubeconfig", "write", clusterName)
42+
out, err := cmd.CombinedOutput()
43+
if err != nil {
44+
return "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s", clusterName, string(out))
45+
}
46+
kubeConfig = strings.TrimSpace(string(out))
47+
48+
return
49+
}

‎tests/integration_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ func init() {
1414
if bkworkers.IsTestDockerd() {
1515
workers.InitDockerWorker()
1616
workers.InitDockerContainerWorker()
17+
workers.InitKubernetesWorker()
1718
} else {
1819
workers.InitRemoteWorker()
1920
}

‎tests/workers/kubernetes.go‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
package workers
2+
3+
import (
4+
"context"
5+
"os"
6+
"os/exec"
7+
"path/filepath"
8+
"sync"
9+
10+
"github.com/docker/buildx/tests/helpers"
11+
"github.com/moby/buildkit/identity"
12+
"github.com/moby/buildkit/util/testutil/integration"
13+
"github.com/pkg/errors"
14+
)
15+
16+
func InitKubernetesWorker() {
17+
integration.Register(&kubernetesWorker{
18+
id: "kubernetes",
19+
})
20+
}
21+
22+
type kubernetesWorker struct {
23+
id string
24+
25+
unsupported []string
26+
27+
docker integration.Backend
28+
dockerClose func() error
29+
dockerErr error
30+
dockerOnce sync.Once
31+
32+
k3dConfig string
33+
k3dClose func() error
34+
k3dErr error
35+
k3dOnce sync.Once
36+
}
37+
38+
func (w *kubernetesWorker) Name() string {
39+
return w.id
40+
}
41+
42+
func (w *kubernetesWorker) Rootless() bool {
43+
return false
44+
}
45+
46+
func (w *kubernetesWorker) NetNSDetached() bool {
47+
return false
48+
}
49+
50+
func (w *kubernetesWorker) New(ctx context.Context, cfg *integration.BackendConfig) (integration.Backend, func() error, error) {
51+
w.dockerOnce.Do(func() {
52+
w.docker, w.dockerClose, w.dockerErr = dockerWorker{id: w.id}.New(ctx, cfg)
53+
})
54+
if w.dockerErr != nil {
55+
return w.docker, w.dockerClose, w.dockerErr
56+
}
57+
58+
w.k3dOnce.Do(func() {
59+
w.k3dConfig, w.k3dClose, w.k3dErr = helpers.NewK3dServer(cfg)
60+
})
61+
if w.k3dErr != nil {
62+
return nil, w.k3dClose, w.k3dErr
63+
}
64+
65+
cfgfile, err := integration.WriteConfig(cfg.DaemonConfig)
66+
if err != nil {
67+
return nil, nil, err
68+
}
69+
defer os.RemoveAll(filepath.Dir(cfgfile))
70+
71+
name := "integration-kubernetes-" + identity.NewID()
72+
cmd := exec.Command("buildx", "create",
73+
"--bootstrap",
74+
"--name="+name,
75+
"--buildkitd-config="+cfgfile,
76+
"--driver=kubernetes",
77+
)
78+
cmd.Env = append(
79+
os.Environ(),
80+
"BUILDX_CONFIG=/tmp/buildx-"+name,
81+
"KUBECONFIG="+w.k3dConfig,
82+
)
83+
if err := cmd.Run(); err != nil {
84+
return nil, nil, errors.Wrapf(err, "failed to create buildx instance %s", name)
85+
}
86+
87+
cl := func() error {
88+
cmd := exec.Command("buildx", "rm", "-f", name)
89+
return cmd.Run()
90+
}
91+
92+
return &backend{
93+
builder: name,
94+
unsupportedFeatures: w.unsupported,
95+
}, cl, nil
96+
}
97+
98+
func (w *kubernetesWorker) Close() error {
99+
if c := w.k3dClose; c != nil {
100+
return c()
101+
}
102+
if c := w.dockerClose; c != nil {
103+
return c()
104+
}
105+
106+
// reset the worker to be ready to go again
107+
w.docker = nil
108+
w.dockerClose = nil
109+
w.dockerErr = nil
110+
w.dockerOnce = sync.Once{}
111+
w.k3dConfig = ""
112+
w.k3dClose = nil
113+
w.k3dErr = nil
114+
w.k3dOnce = sync.Once{}
115+
116+
return nil
117+
}

0 commit comments

Comments
 (0)