Skip to content

Commit 18c3dc2

Browse files
authored
Merge pull request #3724 from crazy-max/tests-multi-node
ci: run integration tests with the remote multi-node worker
2 parents 08152f5 + 176e497 commit 18c3dc2

7 files changed

Lines changed: 187 additions & 10 deletions

File tree

.github/workflows/build.yml

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -66,13 +66,18 @@ jobs:
6666
- ""
6767
- experimental
6868
include:
69-
- worker: docker
69+
- worker: remote+multinode
7070
pkg: ./tests
71-
- worker: docker+containerd # same as docker, but with containerd snapshotter
71+
- worker: remote+multinode
72+
pkg: ./tests
73+
mode: experimental
74+
- worker: docker
7275
pkg: ./tests
7376
- worker: docker
7477
pkg: ./tests
7578
mode: experimental
79+
- worker: docker+containerd # same as docker, but with containerd snapshotter
80+
pkg: ./tests
7681
- worker: docker+containerd # same as docker, but with containerd snapshotter
7782
pkg: ./tests
7883
mode: experimental
@@ -106,14 +111,14 @@ jobs:
106111
fi
107112
testFlags="--run=//worker=$(echo "${{ matrix.worker }}" | sed 's/\+/\\+/g')$"
108113
case "${{ matrix.worker }}" in
109-
docker | docker+containerd | docker@* | docker+containerd@*)
114+
docker | docker+containerd | docker@* | docker+containerd@* | remote+multinode)
110115
echo "TESTFLAGS=${{ env.TESTFLAGS_DOCKER }} $testFlags" >> $GITHUB_ENV
111116
;;
112117
*)
113118
echo "TESTFLAGS=${{ env.TESTFLAGS }} $testFlags" >> $GITHUB_ENV
114119
;;
115120
esac
116-
if [[ "${{ matrix.worker }}" == "docker"* ]]; then
121+
if [[ "${{ matrix.worker }}" == "docker"* || "${{ matrix.worker }}" == "remote+multinode" ]]; then
117122
echo "TEST_DOCKERD=1" >> $GITHUB_ENV
118123
fi
119124
if [ "${{ matrix.mode }}" = "experimental" ]; then

tests/history.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,11 @@ func testHistoryInspect(t *testing.T, sb integration.Sandbox) {
7777
}
7878

7979
func testHistoryLs(t *testing.T, sb integration.Sandbox) {
80+
if isRemoteMultiNodeWorker(sb) {
81+
// FIXME: "history ls" fails on multi nodes
82+
t.Skip("fails with multi nodes")
83+
}
84+
8085
ref := buildTestProject(t, sb)
8186
require.NotEmpty(t, ref.Ref)
8287

@@ -141,6 +146,11 @@ func testHistoryLsStoppedBuilder(t *testing.T, sb integration.Sandbox) {
141146
}
142147

143148
func testHistoryBuildName(t *testing.T, sb integration.Sandbox) {
149+
if isRemoteMultiNodeWorker(sb) {
150+
// FIXME: "history ls" fails on multi nodes
151+
t.Skip("fails with multi nodes")
152+
}
153+
144154
t.Run("override", func(t *testing.T) {
145155
dir := createTestProject(t)
146156
out, err := buildCmd(sb, withArgs("--build-arg=BUILDKIT_BUILD_NAME=foobar", "--metadata-file", filepath.Join(dir, "md.json"), dir))

tests/integration.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,15 @@ func isDockerContainerWorker(sb integration.Sandbox) bool {
141141
return name == "docker-container"
142142
}
143143

144+
func isRemoteWorker(sb integration.Sandbox) bool {
145+
name, _, _ := driverName(sb.Name())
146+
return name == "remote"
147+
}
148+
149+
func isRemoteMultiNodeWorker(sb integration.Sandbox) bool {
150+
return sb.Name() == "remote+multinode"
151+
}
152+
144153
func driverName(sbName string) (string, bool, bool) {
145154
name := sbName
146155
var hasVersion, hasFeature bool

tests/integration_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ func init() {
1414
if bkworkers.IsTestDockerd() {
1515
workers.InitDockerWorker()
1616
workers.InitDockerContainerWorker()
17+
workers.InitRemoteMultiNodeWorker()
1718
} else {
1819
workers.InitRemoteWorker()
1920
}

tests/policy_build.go

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -779,8 +779,7 @@ decision := {"allow": allow}
779779
for _, tc := range testCases {
780780
t.Run(tc.name, func(t *testing.T) {
781781
if tc.requiresHTTPChecksum {
782-
sbDriver, _, _ := driverName(sb.Name())
783-
if sbDriver != "remote" {
782+
if !isRemoteWorker(sb) {
784783
t.Skip("http checksum policy input requires remote driver")
785784
}
786785
skipNoCompatBuildKit(t, sb, ">= 0.26.3-0", "http checksum policy input")
@@ -1084,8 +1083,7 @@ decision := {"allow": allow}
10841083
for _, tc := range testCases {
10851084
t.Run(tc.name, func(t *testing.T) {
10861085
if tc.requiresGitResolve {
1087-
sbDriver, _, _ := driverName(sb.Name())
1088-
if sbDriver != "remote" {
1086+
if !isRemoteWorker(sb) {
10891087
t.Skip("git policy metadata requires remote driver")
10901088
}
10911089
}

tests/policy_eval.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -492,8 +492,7 @@ decision := {"allow": allow}
492492
for _, tc := range testCases {
493493
t.Run(tc.name, func(t *testing.T) {
494494
if tc.needsChecksum {
495-
sbDriver, _, _ := driverName(sb.Name())
496-
if sbDriver != "remote" {
495+
if !isRemoteWorker(sb) {
497496
t.Skip("http checksum policy eval requires remote driver")
498497
}
499498
skipNoCompatBuildKit(t, sb, ">= 0.26.3-0", "http checksum policy input")

tests/workers/remote-multinode.go

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
package workers
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"os"
7+
"os/exec"
8+
"path/filepath"
9+
"sync"
10+
11+
"github.com/docker/buildx/driver"
12+
"github.com/moby/buildkit/identity"
13+
"github.com/moby/buildkit/util/testutil/integration"
14+
"github.com/pkg/errors"
15+
)
16+
17+
func InitRemoteMultiNodeWorker() {
18+
integration.Register(&remoteMultiNodeWorker{
19+
id: "remote+multinode",
20+
})
21+
}
22+
23+
type remoteMultiNodeWorker struct {
24+
id string
25+
26+
unsupported []string
27+
28+
docker integration.Backend
29+
dockerClose func() error
30+
dockerErr error
31+
dockerOnce sync.Once
32+
}
33+
34+
func (w *remoteMultiNodeWorker) Name() string {
35+
return w.id
36+
}
37+
38+
func (w *remoteMultiNodeWorker) Rootless() bool {
39+
return false
40+
}
41+
42+
func (w *remoteMultiNodeWorker) NetNSDetached() bool {
43+
return false
44+
}
45+
46+
func (w *remoteMultiNodeWorker) New(ctx context.Context, cfg *integration.BackendConfig) (integration.Backend, func() error, error) {
47+
w.dockerOnce.Do(func() {
48+
w.docker, w.dockerClose, w.dockerErr = dockerWorker{id: w.id}.New(ctx, cfg)
49+
})
50+
if w.dockerErr != nil {
51+
return w.docker, w.dockerClose, w.dockerErr
52+
}
53+
54+
cfgfile, release, err := integration.WriteConfig(cfg.DaemonConfig)
55+
if err != nil {
56+
return nil, nil, err
57+
}
58+
if release != nil {
59+
defer release()
60+
}
61+
defer os.RemoveAll(filepath.Dir(cfgfile))
62+
63+
name := "integration-remote-multinode-" + identity.NewID()
64+
ctnBuilder0 := name + "-amd64"
65+
ctnBuilder1 := name + "-arm64"
66+
67+
run := func(ctx context.Context, args ...string) ([]byte, error) {
68+
cmd := exec.CommandContext(ctx, "buildx", args...)
69+
cmd.Env = append(
70+
os.Environ(),
71+
"BUILDX_CONFIG=/tmp/buildx-"+name,
72+
"DOCKER_CONTEXT="+w.docker.DockerAddress(),
73+
)
74+
return cmd.CombinedOutput()
75+
}
76+
77+
if out, err := run(ctx, "create",
78+
"--name="+ctnBuilder0,
79+
"--driver=docker-container",
80+
"--buildkitd-config="+cfgfile,
81+
"--driver-opt=network=host",
82+
"--platform=linux/amd64",
83+
); err != nil {
84+
return nil, nil, errors.Wrapf(err, "failed to create builder %s: %s", ctnBuilder0, string(out))
85+
}
86+
if out, err := run(ctx, "inspect", "--bootstrap", ctnBuilder0); err != nil {
87+
return nil, nil, errors.Wrapf(err, "failed to bootstrap builder %s: %s", ctnBuilder0, string(out))
88+
}
89+
90+
if out, err := run(ctx, "create",
91+
"--name="+ctnBuilder1,
92+
"--driver=docker-container",
93+
"--buildkitd-config="+cfgfile,
94+
"--driver-opt=network=host",
95+
"--platform=linux/arm64",
96+
); err != nil {
97+
return nil, nil, errors.Wrapf(err, "failed to create builder %s: %s", ctnBuilder1, string(out))
98+
}
99+
if out, err := run(ctx, "inspect", "--bootstrap", ctnBuilder1); err != nil {
100+
return nil, nil, errors.Wrapf(err, "failed to bootstrap builder %s: %s", ctnBuilder1, string(out))
101+
}
102+
103+
endpoint0 := fmt.Sprintf("docker-container://%s0", driver.BuilderName(ctnBuilder0))
104+
endpoint1 := fmt.Sprintf("docker-container://%s0", driver.BuilderName(ctnBuilder1))
105+
if out, err := run(ctx, "create", "--name="+name, "--driver=remote", endpoint0); err != nil {
106+
return nil, nil, errors.Wrapf(err, "failed to create builder %s: %s", name, string(out))
107+
}
108+
if out, err := run(ctx, "create", "--append", "--name="+name, endpoint1); err != nil {
109+
return nil, nil, errors.Wrapf(err, "failed to append builder %s: %s", name, string(out))
110+
}
111+
if out, err := run(ctx, "inspect", "--bootstrap", name); err != nil {
112+
return nil, nil, errors.Wrapf(err, "failed to bootstrap builder %s: %s", name, string(out))
113+
}
114+
115+
cl := func() error {
116+
runCleanup := func(args ...string) error {
117+
out, err := run(context.Background(), args...)
118+
if err != nil {
119+
return errors.Wrapf(err, "%s: %s", args[1], string(out))
120+
}
121+
return nil
122+
}
123+
124+
setErr := func(dst *error, err error) {
125+
if err != nil && *dst == nil {
126+
*dst = err
127+
}
128+
}
129+
130+
var err error
131+
setErr(&err, runCleanup("rm", "-f", name))
132+
setErr(&err, runCleanup("rm", "-f", ctnBuilder0))
133+
setErr(&err, runCleanup("rm", "-f", ctnBuilder1))
134+
return err
135+
}
136+
137+
return &backend{
138+
builder: name,
139+
context: w.docker.DockerAddress(),
140+
unsupportedFeatures: w.unsupported,
141+
}, cl, nil
142+
}
143+
144+
func (w *remoteMultiNodeWorker) Close() error {
145+
if c := w.dockerClose; c != nil {
146+
return c()
147+
}
148+
149+
w.docker = nil
150+
w.dockerClose = nil
151+
w.dockerErr = nil
152+
w.dockerOnce = sync.Once{}
153+
154+
return nil
155+
}

0 commit comments

Comments
 (0)