Skip to content

Commit c718b27

Browse files
crazy-maxcrazy-max
committed
diag
Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com>
1 parent b6e841c commit c718b27

3 files changed

Lines changed: 134 additions & 12 deletions

File tree

‎tests/helpers/k3d.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@ const (
1515
k3dBin = "k3d"
1616
)
1717

18-
func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAddress string) (kubeConfig string, cl func() error, err error) {
18+
func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAddress string) (clusterName, kubeConfig string, cl func() error, err error) {
1919
if _, err := exec.LookPath(k3dBin); err != nil {
20-
return "", nil, errors.Wrapf(err, "failed to lookup %s binary", k3dBin)
20+
return "", "", nil, errors.Wrapf(err, "failed to lookup %s binary", k3dBin)
2121
}
2222

2323
deferF := &integration.MultiCloser{}
@@ -30,7 +30,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd
3030
}
3131
}()
3232

33-
clusterName := "bk-" + identity.NewID()
33+
clusterName = "bk-" + identity.NewID()
3434

3535
cmd := exec.CommandContext(ctx, k3dBin, "cluster", "create", clusterName,
3636
"--wait",
@@ -41,7 +41,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd
4141
)
4242
out, err := cmd.CombinedOutput()
4343
if err != nil {
44-
return "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s", clusterName, string(out))
44+
return "", "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s", clusterName, string(out))
4545
}
4646
deferF.Append(func() error {
4747
cmd := exec.Command(k3dBin, "cluster", "delete", clusterName)
@@ -63,7 +63,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd
6363
)
6464
out, err = cmd.CombinedOutput()
6565
if err != nil {
66-
return "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s", clusterName, string(out))
66+
return "", "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s", clusterName, string(out))
6767
}
6868
kubeConfig = strings.TrimSpace(string(out))
6969

‎tests/helpers/kubernetes_diagnostics.go‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
package helpers
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"fmt"
7+
"os"
8+
"os/exec"
9+
"strings"
10+
"time"
11+
)
12+
13+
const defaultTestBuildkitTag = "buildx-stable-1"
14+
15+
func KubernetesBuildkitImage() string {
16+
if v := os.Getenv("TEST_BUILDKIT_IMAGE"); v != "" {
17+
return v
18+
}
19+
tag := os.Getenv("TEST_BUILDKIT_TAG")
20+
if tag == "" {
21+
tag = defaultTestBuildkitTag
22+
}
23+
return "moby/buildkit:" + tag
24+
}
25+
26+
func KubernetesDiagnostics(clusterName, dockerContext string) string {
27+
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
28+
defer cancel()
29+
30+
var buf bytes.Buffer
31+
appendK3dDiagnostics(ctx, &buf, clusterName, dockerContext)
32+
appendDockerDiagnostics(ctx, &buf, dockerContext)
33+
appendK3sServerDiagnostics(ctx, &buf, clusterName, dockerContext)
34+
return strings.TrimSpace(buf.String())
35+
}
36+
37+
func appendK3dDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterName, dockerContext string) {
38+
appendCommandOutput(ctx, buf, "k3d cluster list", "k3d", []string{"cluster", "list", clusterName}, []string{"DOCKER_CONTEXT=" + dockerContext})
39+
appendCommandOutput(ctx, buf, "k3d node list", "k3d", []string{"node", "list"}, []string{"DOCKER_CONTEXT=" + dockerContext})
40+
}
41+
42+
func appendDockerDiagnostics(ctx context.Context, buf *bytes.Buffer, dockerContext string) {
43+
args := []string{"ps", "-a", "--format", "{{.Names}}\t{{.Image}}\t{{.Status}}"}
44+
appendCommandOutput(ctx, buf, "docker ps", "docker", args, []string{"DOCKER_CONTEXT=" + dockerContext})
45+
}
46+
47+
func appendK3sServerDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterName, dockerContext string) {
48+
nodeNames, err := clusterNodeNames(ctx, clusterName, dockerContext)
49+
if err != nil {
50+
fmt.Fprintf(buf, "cluster node discovery error: %v\n", err)
51+
return
52+
}
53+
if len(nodeNames) == 0 {
54+
fmt.Fprintln(buf, "cluster node discovery: no matching k3d containers found")
55+
return
56+
}
57+
58+
for _, nodeName := range nodeNames {
59+
appendCommandOutput(ctx, buf, "docker logs "+nodeName, "docker", []string{"logs", "--tail", "80", nodeName}, []string{"DOCKER_CONTEXT=" + dockerContext})
60+
}
61+
62+
for _, nodeName := range nodeNames {
63+
if !strings.Contains(nodeName, "-server-") {
64+
continue
65+
}
66+
appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s kubectl get pods", "docker", []string{"exec", nodeName, "k3s", "kubectl", "get", "pods", "-A", "-o", "wide"}, []string{"DOCKER_CONTEXT=" + dockerContext})
67+
appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s kubectl get events", "docker", []string{"exec", nodeName, "k3s", "kubectl", "get", "events", "-A", "--sort-by=.lastTimestamp"}, []string{"DOCKER_CONTEXT=" + dockerContext})
68+
appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s kubectl describe pods", "docker", []string{"exec", nodeName, "k3s", "kubectl", "describe", "pods", "-A"}, []string{"DOCKER_CONTEXT=" + dockerContext})
69+
break
70+
}
71+
}
72+
73+
func clusterNodeNames(ctx context.Context, clusterName, dockerContext string) ([]string, error) {
74+
out, err := runCommand(ctx, "docker", []string{
75+
"ps", "-a",
76+
"--filter", "name=k3d-" + clusterName,
77+
"--format", "{{.Names}}",
78+
}, []string{"DOCKER_CONTEXT=" + dockerContext})
79+
if err != nil {
80+
return nil, err
81+
}
82+
var names []string
83+
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
84+
line = strings.TrimSpace(line)
85+
if line == "" {
86+
continue
87+
}
88+
names = append(names, line)
89+
}
90+
return names, nil
91+
}
92+
93+
func appendCommandOutput(ctx context.Context, buf *bytes.Buffer, title, name string, args []string, env []string) {
94+
out, err := runCommand(ctx, name, args, env)
95+
fmt.Fprintf(buf, "== %s ==\n", title)
96+
if err != nil {
97+
fmt.Fprintf(buf, "error: %v\n", err)
98+
}
99+
if strings.TrimSpace(out) == "" {
100+
fmt.Fprintln(buf, "<empty>")
101+
} else {
102+
fmt.Fprintf(buf, "%s\n", strings.TrimSpace(out))
103+
}
104+
fmt.Fprintln(buf)
105+
}
106+
107+
func runCommand(ctx context.Context, name string, args []string, env []string) (string, error) {
108+
cmd := exec.CommandContext(ctx, name, args...)
109+
if len(env) > 0 {
110+
cmd.Env = append(os.Environ(), env...)
111+
}
112+
out, err := cmd.CombinedOutput()
113+
return string(out), err
114+
}

‎tests/workers/kubernetes.go‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"os"
66
"os/exec"
7+
"strings"
78
"sync"
89

910
"github.com/docker/buildx/tests/helpers"
@@ -28,6 +29,7 @@ type kubernetesWorker struct {
2829
dockerErr error
2930
dockerOnce sync.Once
3031

32+
k3dName string
3133
k3dConfig string
3234
k3dClose func() error
3335
k3dErr error
@@ -55,25 +57,30 @@ func (w *kubernetesWorker) New(ctx context.Context, cfg *integration.BackendConf
5557
}
5658

5759
w.k3dOnce.Do(func() {
58-
w.k3dConfig, w.k3dClose, w.k3dErr = helpers.NewK3dServer(ctx, cfg, w.docker.DockerAddress())
60+
w.k3dName, w.k3dConfig, w.k3dClose, w.k3dErr = helpers.NewK3dServer(ctx, cfg, w.docker.DockerAddress())
5961
})
6062
if w.k3dErr != nil {
6163
return nil, w.k3dClose, w.k3dErr
6264
}
6365

6466
name := "integration-kubernetes-" + identity.NewID()
65-
// The generic integration harness injects a host-local registry mirror config.
66-
// That works for host-networked workers, but not for a BuildKit pod where
67-
// localhost points at the pod itself instead of the runner.
68-
cmd := exec.CommandContext(ctx, "buildx", "create", "--bootstrap", "--name="+name, "--driver=kubernetes")
67+
cmd := exec.CommandContext(ctx, "buildx", "create",
68+
"--bootstrap",
69+
"--name="+name,
70+
"--driver=kubernetes",
71+
"--driver-opt=image="+helpers.KubernetesBuildkitImage(),
72+
"--driver-opt=timeout=60s",
73+
)
6974
cmd.Env = append(
7075
os.Environ(),
7176
"BUILDX_CONFIG=/tmp/buildx-"+name,
7277
"DOCKER_CONTEXT="+w.docker.DockerAddress(),
7378
"KUBECONFIG="+w.k3dConfig,
7479
)
75-
if err := cmd.Run(); err != nil {
76-
return nil, nil, errors.Wrapf(err, "failed to create buildx instance %s", name)
80+
out, err := cmd.CombinedOutput()
81+
if err != nil {
82+
diag := helpers.KubernetesDiagnostics(w.k3dName, w.docker.DockerAddress())
83+
return nil, nil, errors.Wrapf(err, "failed to create buildx instance %s with image %s: %s\n%s", name, helpers.KubernetesBuildkitImage(), strings.TrimSpace(string(out)), diag)
7784
}
7885

7986
cl := func() error {
@@ -114,6 +121,7 @@ func (w *kubernetesWorker) Close() error {
114121
w.dockerClose = nil
115122
w.dockerErr = nil
116123
w.dockerOnce = sync.Once{}
124+
w.k3dName = ""
117125
w.k3dConfig = ""
118126
w.k3dClose = nil
119127
w.k3dErr = nil

0 commit comments

Comments
 (0)