From af1f060c11a4294748135e7ee182eb2e26721651 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Tue, 31 Mar 2026 23:25:04 +0200 Subject: [PATCH 01/14] test kubernetes worker with k3d Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .github/workflows/build.yml | 5 +- Dockerfile | 3 + tests/helpers/k3d.go | 71 +++++++++++++++++++++ tests/integration_test.go | 1 + tests/workers/kubernetes.go | 123 ++++++++++++++++++++++++++++++++++++ 5 files changed, 201 insertions(+), 2 deletions(-) create mode 100644 tests/helpers/k3d.go create mode 100644 tests/workers/kubernetes.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bd369d2b6ea0..db353a896aaf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,6 +53,7 @@ jobs: worker: - docker-container - remote + - kubernetes pkg: - ./tests mode: @@ -104,14 +105,14 @@ jobs: fi testFlags="--run=//worker=$(echo "${{ matrix.worker }}" | sed 's/\+/\\+/g')$" case "${{ matrix.worker }}" in - docker | docker+containerd | docker@* | docker+containerd@* | remote+multinode) + docker | docker+containerd | docker@* | docker+containerd@* | remote+multinode | kubernetes) echo "TESTFLAGS=${{ env.TESTFLAGS_DOCKER }} $testFlags" >> $GITHUB_ENV ;; *) echo "TESTFLAGS=${{ env.TESTFLAGS }} $testFlags" >> $GITHUB_ENV ;; esac - if [[ "${{ matrix.worker }}" == "docker"* || "${{ matrix.worker }}" == "remote+multinode" ]]; then + if [[ "${{ matrix.worker }}" == "docker"* || "${{ matrix.worker }}" == "remote+multinode" || "${{ matrix.worker }}" == "kubernetes" ]]; then echo "TEST_DOCKERD=1" >> $GITHUB_ENV fi if [ "${{ matrix.mode }}" = "experimental" ]; then diff --git a/Dockerfile b/Dockerfile index 00645b41efc0..4a80dd7c7037 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,7 @@ ARG REGISTRY_VERSION=3.0.0 ARG BUILDKIT_VERSION=v0.29.0 ARG COMPOSE_VERSION=v5.1.0 ARG UNDOCK_VERSION=0.9.0 +ARG K3D_VERSION=5.8.3 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS golatest @@ -27,6 +28,7 @@ FROM registry:$REGISTRY_VERSION AS registry FROM moby/buildkit:$BUILDKIT_VERSION AS buildkit FROM docker/compose-bin:$COMPOSE_VERSION AS compose FROM crazymax/undock:$UNDOCK_VERSION AS undock +FROM ghcr.io/k3d-io/k3d:${K3D_VERSION} AS k3d FROM golatest AS gobase COPY --from=xx / / @@ -149,6 +151,7 @@ COPY --link --from=buildkit /usr/bin/buildkitd /usr/bin/ COPY --link --from=buildkit /usr/bin/buildctl /usr/bin/ COPY --link --from=compose /docker-compose /usr/bin/compose COPY --link --from=undock /usr/local/bin/undock /usr/bin/ +COPY --link --from=k3d /bin/k3d /usr/bin/ COPY --link --from=binaries /buildx /usr/bin/ RUN mkdir -p /usr/local/lib/docker/cli-plugins && ln -s /usr/bin/buildx /usr/local/lib/docker/cli-plugins/docker-buildx ENV TEST_DOCKER_EXTRA="docker@28.5=/opt/docker-alt-28,docker@27.5=/opt/docker-alt-27" diff --git a/tests/helpers/k3d.go b/tests/helpers/k3d.go new file mode 100644 index 000000000000..ea67b896add0 --- /dev/null +++ b/tests/helpers/k3d.go @@ -0,0 +1,71 @@ +package helpers + +import ( + "context" + "os" + "os/exec" + "strings" + + "github.com/moby/buildkit/identity" + "github.com/moby/buildkit/util/testutil/integration" + "github.com/pkg/errors" +) + +const ( + k3dBin = "k3d" +) + +func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAddress string) (kubeConfig string, cl func() error, err error) { + if _, err := exec.LookPath(k3dBin); err != nil { + return "", nil, errors.Wrapf(err, "failed to lookup %s binary", k3dBin) + } + + deferF := &integration.MultiCloser{} + cl = deferF.F() + + defer func() { + if err != nil { + deferF.F()() + cl = nil + } + }() + + clusterName := "bk-" + identity.NewID() + + cmd := exec.CommandContext(ctx, k3dBin, "cluster", "create", clusterName, + "--wait", + ) + cmd.Env = append( + os.Environ(), + "DOCKER_CONTEXT="+dockerAddress, + ) + out, err := cmd.CombinedOutput() + if err != nil { + return "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s", clusterName, string(out)) + } + deferF.Append(func() error { + cmd := exec.Command(k3dBin, "cluster", "delete", clusterName) + cmd.Env = append( + os.Environ(), + "DOCKER_CONTEXT="+dockerAddress, + ) + out, err := cmd.CombinedOutput() + if err != nil { + return errors.Wrapf(err, "failed to delete k3d cluster %s: %s", clusterName, string(out)) + } + return nil + }) + + cmd = exec.CommandContext(ctx, k3dBin, "kubeconfig", "write", clusterName) + cmd.Env = append( + os.Environ(), + "DOCKER_CONTEXT="+dockerAddress, + ) + out, err = cmd.CombinedOutput() + if err != nil { + return "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s", clusterName, string(out)) + } + kubeConfig = strings.TrimSpace(string(out)) + + return +} diff --git a/tests/integration_test.go b/tests/integration_test.go index 574251dc5075..9ce1c8d9c128 100644 --- a/tests/integration_test.go +++ b/tests/integration_test.go @@ -15,6 +15,7 @@ func init() { workers.InitDockerWorker() workers.InitDockerContainerWorker() workers.InitRemoteMultiNodeWorker() + workers.InitKubernetesWorker() } else { workers.InitRemoteWorker() } diff --git a/tests/workers/kubernetes.go b/tests/workers/kubernetes.go new file mode 100644 index 000000000000..3a63cccb0ced --- /dev/null +++ b/tests/workers/kubernetes.go @@ -0,0 +1,123 @@ +package workers + +import ( + "context" + "os" + "os/exec" + "sync" + + "github.com/docker/buildx/tests/helpers" + "github.com/moby/buildkit/identity" + "github.com/moby/buildkit/util/testutil/integration" + "github.com/pkg/errors" +) + +func InitKubernetesWorker() { + integration.Register(&kubernetesWorker{ + id: "kubernetes", + }) +} + +type kubernetesWorker struct { + id string + + unsupported []string + + docker integration.Backend + dockerClose func() error + dockerErr error + dockerOnce sync.Once + + k3dConfig string + k3dClose func() error + k3dErr error + k3dOnce sync.Once +} + +func (w *kubernetesWorker) Name() string { + return w.id +} + +func (w *kubernetesWorker) Rootless() bool { + return false +} + +func (w *kubernetesWorker) NetNSDetached() bool { + return false +} + +func (w *kubernetesWorker) New(ctx context.Context, cfg *integration.BackendConfig) (integration.Backend, func() error, error) { + w.dockerOnce.Do(func() { + w.docker, w.dockerClose, w.dockerErr = dockerWorker{id: w.id}.New(ctx, cfg) + }) + if w.dockerErr != nil { + return w.docker, w.dockerClose, w.dockerErr + } + + w.k3dOnce.Do(func() { + w.k3dConfig, w.k3dClose, w.k3dErr = helpers.NewK3dServer(ctx, cfg, w.docker.DockerAddress()) + }) + if w.k3dErr != nil { + return nil, w.k3dClose, w.k3dErr + } + + name := "integration-kubernetes-" + identity.NewID() + // The generic integration harness injects a host-local registry mirror config. + // That works for host-networked workers, but not for a BuildKit pod where + // localhost points at the pod itself instead of the runner. + cmd := exec.CommandContext(ctx, "buildx", "create", "--bootstrap", "--name="+name, "--driver=kubernetes") + cmd.Env = append( + os.Environ(), + "BUILDX_CONFIG=/tmp/buildx-"+name, + "DOCKER_CONTEXT="+w.docker.DockerAddress(), + "KUBECONFIG="+w.k3dConfig, + ) + if err := cmd.Run(); err != nil { + return nil, nil, errors.Wrapf(err, "failed to create buildx instance %s", name) + } + + cl := func() error { + cmd := exec.CommandContext(context.Background(), "buildx", "rm", "-f", name) + cmd.Env = append( + os.Environ(), + "BUILDX_CONFIG=/tmp/buildx-"+name, + "DOCKER_CONTEXT="+w.docker.DockerAddress(), + "KUBECONFIG="+w.k3dConfig, + ) + return cmd.Run() + } + + return &backend{ + context: w.docker.DockerAddress(), + builder: name, + unsupportedFeatures: w.unsupported, + }, cl, nil +} + +func (w *kubernetesWorker) Close() error { + setErr := func(dst *error, err error) { + if err != nil && *dst == nil { + *dst = err + } + } + + var err error + if c := w.k3dClose; c != nil { + setErr(&err, c()) + } + if c := w.dockerClose; c != nil { + setErr(&err, c()) + } + + // reset the worker to be ready to go again + w.docker = nil + w.dockerClose = nil + w.dockerErr = nil + w.dockerOnce = sync.Once{} + w.k3dConfig = "" + w.k3dClose = nil + w.k3dErr = nil + w.k3dOnce = sync.Once{} + + return err +} From 5b3def3719afb2f4f1a8b12a445365d5bdcdcf76 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 1 Apr 2026 00:19:37 +0200 Subject: [PATCH 02/14] diag Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- tests/helpers/k3d.go | 10 +-- tests/helpers/kubernetes_diagnostics.go | 114 ++++++++++++++++++++++++ tests/workers/kubernetes.go | 22 +++-- 3 files changed, 134 insertions(+), 12 deletions(-) create mode 100644 tests/helpers/kubernetes_diagnostics.go diff --git a/tests/helpers/k3d.go b/tests/helpers/k3d.go index ea67b896add0..4672d820be25 100644 --- a/tests/helpers/k3d.go +++ b/tests/helpers/k3d.go @@ -15,9 +15,9 @@ const ( k3dBin = "k3d" ) -func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAddress string) (kubeConfig string, cl func() error, err error) { +func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAddress string) (clusterName, kubeConfig string, cl func() error, err error) { if _, err := exec.LookPath(k3dBin); err != nil { - return "", nil, errors.Wrapf(err, "failed to lookup %s binary", k3dBin) + return "", "", nil, errors.Wrapf(err, "failed to lookup %s binary", k3dBin) } deferF := &integration.MultiCloser{} @@ -30,7 +30,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd } }() - clusterName := "bk-" + identity.NewID() + clusterName = "bk-" + identity.NewID() cmd := exec.CommandContext(ctx, k3dBin, "cluster", "create", clusterName, "--wait", @@ -41,7 +41,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd ) out, err := cmd.CombinedOutput() if err != nil { - return "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s", clusterName, string(out)) + return "", "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s", clusterName, string(out)) } deferF.Append(func() error { cmd := exec.Command(k3dBin, "cluster", "delete", clusterName) @@ -63,7 +63,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd ) out, err = cmd.CombinedOutput() if err != nil { - return "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s", clusterName, string(out)) + return "", "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s", clusterName, string(out)) } kubeConfig = strings.TrimSpace(string(out)) diff --git a/tests/helpers/kubernetes_diagnostics.go b/tests/helpers/kubernetes_diagnostics.go new file mode 100644 index 000000000000..d296baf352c4 --- /dev/null +++ b/tests/helpers/kubernetes_diagnostics.go @@ -0,0 +1,114 @@ +package helpers + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "strings" + "time" +) + +const defaultTestBuildkitTag = "buildx-stable-1" + +func KubernetesBuildkitImage() string { + if v := os.Getenv("TEST_BUILDKIT_IMAGE"); v != "" { + return v + } + tag := os.Getenv("TEST_BUILDKIT_TAG") + if tag == "" { + tag = defaultTestBuildkitTag + } + return "moby/buildkit:" + tag +} + +func KubernetesDiagnostics(clusterName, dockerContext string) string { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + var buf bytes.Buffer + appendK3dDiagnostics(ctx, &buf, clusterName, dockerContext) + appendDockerDiagnostics(ctx, &buf, dockerContext) + appendK3sServerDiagnostics(ctx, &buf, clusterName, dockerContext) + return strings.TrimSpace(buf.String()) +} + +func appendK3dDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterName, dockerContext string) { + appendCommandOutput(ctx, buf, "k3d cluster list", "k3d", []string{"cluster", "list", clusterName}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "k3d node list", "k3d", []string{"node", "list"}, []string{"DOCKER_CONTEXT=" + dockerContext}) +} + +func appendDockerDiagnostics(ctx context.Context, buf *bytes.Buffer, dockerContext string) { + args := []string{"ps", "-a", "--format", "{{.Names}}\t{{.Image}}\t{{.Status}}"} + appendCommandOutput(ctx, buf, "docker ps", "docker", args, []string{"DOCKER_CONTEXT=" + dockerContext}) +} + +func appendK3sServerDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterName, dockerContext string) { + nodeNames, err := clusterNodeNames(ctx, clusterName, dockerContext) + if err != nil { + fmt.Fprintf(buf, "cluster node discovery error: %v\n", err) + return + } + if len(nodeNames) == 0 { + fmt.Fprintln(buf, "cluster node discovery: no matching k3d containers found") + return + } + + for _, nodeName := range nodeNames { + appendCommandOutput(ctx, buf, "docker logs "+nodeName, "docker", []string{"logs", "--tail", "80", nodeName}, []string{"DOCKER_CONTEXT=" + dockerContext}) + } + + for _, nodeName := range nodeNames { + if !strings.Contains(nodeName, "-server-") { + continue + } + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s kubectl get pods", "docker", []string{"exec", nodeName, "k3s", "kubectl", "get", "pods", "-A", "-o", "wide"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s kubectl get events", "docker", []string{"exec", nodeName, "k3s", "kubectl", "get", "events", "-A", "--sort-by=.lastTimestamp"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s kubectl describe pods", "docker", []string{"exec", nodeName, "k3s", "kubectl", "describe", "pods", "-A"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + break + } +} + +func clusterNodeNames(ctx context.Context, clusterName, dockerContext string) ([]string, error) { + out, err := runCommand(ctx, "docker", []string{ + "ps", "-a", + "--filter", "name=k3d-" + clusterName, + "--format", "{{.Names}}", + }, []string{"DOCKER_CONTEXT=" + dockerContext}) + if err != nil { + return nil, err + } + var names []string + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + names = append(names, line) + } + return names, nil +} + +func appendCommandOutput(ctx context.Context, buf *bytes.Buffer, title, name string, args []string, env []string) { + out, err := runCommand(ctx, name, args, env) + fmt.Fprintf(buf, "== %s ==\n", title) + if err != nil { + fmt.Fprintf(buf, "error: %v\n", err) + } + if strings.TrimSpace(out) == "" { + fmt.Fprintln(buf, "") + } else { + fmt.Fprintf(buf, "%s\n", strings.TrimSpace(out)) + } + fmt.Fprintln(buf) +} + +func runCommand(ctx context.Context, name string, args []string, env []string) (string, error) { + cmd := exec.CommandContext(ctx, name, args...) + if len(env) > 0 { + cmd.Env = append(os.Environ(), env...) + } + out, err := cmd.CombinedOutput() + return string(out), err +} diff --git a/tests/workers/kubernetes.go b/tests/workers/kubernetes.go index 3a63cccb0ced..ec9262418906 100644 --- a/tests/workers/kubernetes.go +++ b/tests/workers/kubernetes.go @@ -4,6 +4,7 @@ import ( "context" "os" "os/exec" + "strings" "sync" "github.com/docker/buildx/tests/helpers" @@ -28,6 +29,7 @@ type kubernetesWorker struct { dockerErr error dockerOnce sync.Once + k3dName string k3dConfig string k3dClose func() error k3dErr error @@ -55,25 +57,30 @@ func (w *kubernetesWorker) New(ctx context.Context, cfg *integration.BackendConf } w.k3dOnce.Do(func() { - w.k3dConfig, w.k3dClose, w.k3dErr = helpers.NewK3dServer(ctx, cfg, w.docker.DockerAddress()) + w.k3dName, w.k3dConfig, w.k3dClose, w.k3dErr = helpers.NewK3dServer(ctx, cfg, w.docker.DockerAddress()) }) if w.k3dErr != nil { return nil, w.k3dClose, w.k3dErr } name := "integration-kubernetes-" + identity.NewID() - // The generic integration harness injects a host-local registry mirror config. - // That works for host-networked workers, but not for a BuildKit pod where - // localhost points at the pod itself instead of the runner. - cmd := exec.CommandContext(ctx, "buildx", "create", "--bootstrap", "--name="+name, "--driver=kubernetes") + cmd := exec.CommandContext(ctx, "buildx", "create", + "--bootstrap", + "--name="+name, + "--driver=kubernetes", + "--driver-opt=image="+helpers.KubernetesBuildkitImage(), + "--driver-opt=timeout=60s", + ) cmd.Env = append( os.Environ(), "BUILDX_CONFIG=/tmp/buildx-"+name, "DOCKER_CONTEXT="+w.docker.DockerAddress(), "KUBECONFIG="+w.k3dConfig, ) - if err := cmd.Run(); err != nil { - return nil, nil, errors.Wrapf(err, "failed to create buildx instance %s", name) + out, err := cmd.CombinedOutput() + if err != nil { + diag := helpers.KubernetesDiagnostics(w.k3dName, w.docker.DockerAddress()) + return nil, nil, errors.Wrapf(err, "failed to create buildx instance %s with image %s: %s\n%s", name, helpers.KubernetesBuildkitImage(), strings.TrimSpace(string(out)), diag) } cl := func() error { @@ -114,6 +121,7 @@ func (w *kubernetesWorker) Close() error { w.dockerClose = nil w.dockerErr = nil w.dockerOnce = sync.Once{} + w.k3dName = "" w.k3dConfig = "" w.k3dClose = nil w.k3dErr = nil From cdb050b55f49f86d30bac37ea2116a8bf4e7af17 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 1 Apr 2026 08:29:28 +0200 Subject: [PATCH 03/14] k3d timeout Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- tests/helpers/k3d.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/helpers/k3d.go b/tests/helpers/k3d.go index 4672d820be25..44a51f25a391 100644 --- a/tests/helpers/k3d.go +++ b/tests/helpers/k3d.go @@ -5,6 +5,7 @@ import ( "os" "os/exec" "strings" + "time" "github.com/moby/buildkit/identity" "github.com/moby/buildkit/util/testutil/integration" @@ -32,7 +33,10 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd clusterName = "bk-" + identity.NewID() - cmd := exec.CommandContext(ctx, k3dBin, "cluster", "create", clusterName, + createCtx, cancelCreate := context.WithTimeout(ctx, 90*time.Second) + defer cancelCreate() + + cmd := exec.CommandContext(createCtx, k3dBin, "cluster", "create", clusterName, "--wait", ) cmd.Env = append( @@ -41,7 +45,8 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd ) out, err := cmd.CombinedOutput() if err != nil { - return "", "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s", clusterName, string(out)) + diag := KubernetesDiagnostics(clusterName, dockerAddress) + return "", "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s\n%s", clusterName, strings.TrimSpace(string(out)), diag) } deferF.Append(func() error { cmd := exec.Command(k3dBin, "cluster", "delete", clusterName) @@ -56,14 +61,18 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd return nil }) - cmd = exec.CommandContext(ctx, k3dBin, "kubeconfig", "write", clusterName) + kubeconfigCtx, cancelKubeconfig := context.WithTimeout(ctx, 30*time.Second) + defer cancelKubeconfig() + + cmd = exec.CommandContext(kubeconfigCtx, k3dBin, "kubeconfig", "write", clusterName) cmd.Env = append( os.Environ(), "DOCKER_CONTEXT="+dockerAddress, ) out, err = cmd.CombinedOutput() if err != nil { - return "", "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s", clusterName, string(out)) + diag := KubernetesDiagnostics(clusterName, dockerAddress) + return "", "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s\n%s", clusterName, strings.TrimSpace(string(out)), diag) } kubeConfig = strings.TrimSpace(string(out)) From 452ebbcfe5a8bf740d54e8554290d19e452e816e Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 1 Apr 2026 08:51:18 +0200 Subject: [PATCH 04/14] tests: pin k3d images for kubernetes worker Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- Dockerfile | 4 +++ tests/helpers/k3d.go | 38 ++++++++++++++++--------- tests/helpers/kubernetes_diagnostics.go | 24 ++++++++++++++-- 3 files changed, 49 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4a80dd7c7037..943a2cccbf3c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ ARG BUILDKIT_VERSION=v0.29.0 ARG COMPOSE_VERSION=v5.1.0 ARG UNDOCK_VERSION=0.9.0 ARG K3D_VERSION=5.8.3 +ARG K3S_VERSION=v1.32.2-k3s1 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS golatest @@ -155,6 +156,9 @@ COPY --link --from=k3d /bin/k3d /usr/bin/ COPY --link --from=binaries /buildx /usr/bin/ RUN mkdir -p /usr/local/lib/docker/cli-plugins && ln -s /usr/bin/buildx /usr/local/lib/docker/cli-plugins/docker-buildx ENV TEST_DOCKER_EXTRA="docker@28.5=/opt/docker-alt-28,docker@27.5=/opt/docker-alt-27" +ENV TEST_K3S_IMAGE="rancher/k3s:${K3S_VERSION}" +ENV TEST_K3D_TOOLS_IMAGE="ghcr.io/k3d-io/k3d-tools:${K3D_VERSION}" +ENV TEST_K3D_LOADBALANCER_IMAGE="ghcr.io/k3d-io/k3d-proxy:${K3D_VERSION}" FROM integration-test-base AS integration-test COPY . . diff --git a/tests/helpers/k3d.go b/tests/helpers/k3d.go index 44a51f25a391..92f3be37dba2 100644 --- a/tests/helpers/k3d.go +++ b/tests/helpers/k3d.go @@ -36,13 +36,15 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd createCtx, cancelCreate := context.WithTimeout(ctx, 90*time.Second) defer cancelCreate() - cmd := exec.CommandContext(createCtx, k3dBin, "cluster", "create", clusterName, + args := []string{ + "cluster", "create", clusterName, "--wait", - ) - cmd.Env = append( - os.Environ(), - "DOCKER_CONTEXT="+dockerAddress, - ) + } + if image := KubernetesK3sImage(); image != "" { + args = append(args, "--image="+image) + } + cmd := exec.CommandContext(createCtx, k3dBin, args...) + cmd.Env = k3dEnv(dockerAddress) out, err := cmd.CombinedOutput() if err != nil { diag := KubernetesDiagnostics(clusterName, dockerAddress) @@ -50,10 +52,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd } deferF.Append(func() error { cmd := exec.Command(k3dBin, "cluster", "delete", clusterName) - cmd.Env = append( - os.Environ(), - "DOCKER_CONTEXT="+dockerAddress, - ) + cmd.Env = k3dEnv(dockerAddress) out, err := cmd.CombinedOutput() if err != nil { return errors.Wrapf(err, "failed to delete k3d cluster %s: %s", clusterName, string(out)) @@ -65,10 +64,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd defer cancelKubeconfig() cmd = exec.CommandContext(kubeconfigCtx, k3dBin, "kubeconfig", "write", clusterName) - cmd.Env = append( - os.Environ(), - "DOCKER_CONTEXT="+dockerAddress, - ) + cmd.Env = k3dEnv(dockerAddress) out, err = cmd.CombinedOutput() if err != nil { diag := KubernetesDiagnostics(clusterName, dockerAddress) @@ -78,3 +74,17 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd return } + +func k3dEnv(dockerAddress string) []string { + env := append( + os.Environ(), + "DOCKER_CONTEXT="+dockerAddress, + ) + if image := KubernetesK3DToolsImage(); image != "" { + env = append(env, "K3D_IMAGE_TOOLS="+image) + } + if image := KubernetesK3DLoadBalancerImage(); image != "" { + env = append(env, "K3D_IMAGE_LOADBALANCER="+image) + } + return env +} diff --git a/tests/helpers/kubernetes_diagnostics.go b/tests/helpers/kubernetes_diagnostics.go index d296baf352c4..de124c01f637 100644 --- a/tests/helpers/kubernetes_diagnostics.go +++ b/tests/helpers/kubernetes_diagnostics.go @@ -23,6 +23,18 @@ func KubernetesBuildkitImage() string { return "moby/buildkit:" + tag } +func KubernetesK3sImage() string { + return os.Getenv("TEST_K3S_IMAGE") +} + +func KubernetesK3DToolsImage() string { + return os.Getenv("TEST_K3D_TOOLS_IMAGE") +} + +func KubernetesK3DLoadBalancerImage() string { + return os.Getenv("TEST_K3D_LOADBALANCER_IMAGE") +} + func KubernetesDiagnostics(clusterName, dockerContext string) string { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() @@ -56,6 +68,12 @@ func appendK3sServerDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterN } for _, nodeName := range nodeNames { + appendCommandOutput(ctx, buf, "docker inspect "+nodeName, "docker", []string{ + "inspect", + "--format", + "Status={{.State.Status}} Health={{if .State.Health}}{{.State.Health.Status}}{{else}}{{end}} Restarting={{.State.Restarting}} ExitCode={{.State.ExitCode}} Error={{.State.Error}}", + nodeName, + }, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker logs "+nodeName, "docker", []string{"logs", "--tail", "80", nodeName}, []string{"DOCKER_CONTEXT=" + dockerContext}) } @@ -63,9 +81,9 @@ func appendK3sServerDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterN if !strings.Contains(nodeName, "-server-") { continue } - appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s kubectl get pods", "docker", []string{"exec", nodeName, "k3s", "kubectl", "get", "pods", "-A", "-o", "wide"}, []string{"DOCKER_CONTEXT=" + dockerContext}) - appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s kubectl get events", "docker", []string{"exec", nodeName, "k3s", "kubectl", "get", "events", "-A", "--sort-by=.lastTimestamp"}, []string{"DOCKER_CONTEXT=" + dockerContext}) - appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s kubectl describe pods", "docker", []string{"exec", nodeName, "k3s", "kubectl", "describe", "pods", "-A"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get pods", "docker", []string{"exec", nodeName, "kubectl", "get", "pods", "-A", "-o", "wide"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get events", "docker", []string{"exec", nodeName, "kubectl", "get", "events", "-A", "--sort-by=.lastTimestamp"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl describe pods", "docker", []string{"exec", nodeName, "kubectl", "describe", "pods", "-A"}, []string{"DOCKER_CONTEXT=" + dockerContext}) break } } From 111b691ef4faaf1641664dc0ac9eaf96183205d6 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 1 Apr 2026 08:57:06 +0200 Subject: [PATCH 05/14] tests: redeclare k3d args in integration image Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 943a2cccbf3c..ff4eb460fd6f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -127,6 +127,8 @@ FROM binaries-$TARGETOS AS binaries ARG BUILDKIT_SBOM_SCAN_STAGE=true FROM gobase AS integration-test-base +ARG K3D_VERSION +ARG K3S_VERSION # https://github.com/docker/docker/blob/master/project/PACKAGERS.md#runtime-dependencies RUN apk add --no-cache \ bash \ From 87add3b80e1318f3c762dbb2ae571c7dd18d8f52 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 1 Apr 2026 09:07:09 +0200 Subject: [PATCH 06/14] tests: expand kubernetes worker diagnostics Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- tests/helpers/kubernetes_diagnostics.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/helpers/kubernetes_diagnostics.go b/tests/helpers/kubernetes_diagnostics.go index de124c01f637..3c5e2308f0da 100644 --- a/tests/helpers/kubernetes_diagnostics.go +++ b/tests/helpers/kubernetes_diagnostics.go @@ -71,7 +71,7 @@ func appendK3sServerDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterN appendCommandOutput(ctx, buf, "docker inspect "+nodeName, "docker", []string{ "inspect", "--format", - "Status={{.State.Status}} Health={{if .State.Health}}{{.State.Health.Status}}{{else}}{{end}} Restarting={{.State.Restarting}} ExitCode={{.State.ExitCode}} Error={{.State.Error}}", + "Status={{.State.Status}} Health={{if .State.Health}}{{.State.Health.Status}}{{else}}{{end}} Restarting={{.State.Restarting}} ExitCode={{.State.ExitCode}} Error={{.State.Error}} Privileged={{.HostConfig.Privileged}} Cgroupns={{.HostConfig.CgroupnsMode}}", nodeName, }, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker logs "+nodeName, "docker", []string{"logs", "--tail", "80", nodeName}, []string{"DOCKER_CONTEXT=" + dockerContext}) @@ -81,6 +81,11 @@ func appendK3sServerDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterN if !strings.Contains(nodeName, "-server-") { continue } + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" ps", "docker", []string{"exec", nodeName, "sh", "-c", "ps auxww"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" ss", "docker", []string{"exec", nodeName, "sh", "-c", "ss -lntp || netstat -lntp"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" cgroup", "docker", []string{"exec", nodeName, "sh", "-c", "cat /proc/1/cgroup && echo && mount | grep cgroup"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s files", "docker", []string{"exec", nodeName, "sh", "-c", "find /var/lib/rancher/k3s -maxdepth 3 -type f 2>/dev/null | sort"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s logs", "docker", []string{"exec", nodeName, "sh", "-c", "for f in /var/log/k3s.log /var/lib/rancher/k3s/agent/containerd/containerd.log /var/lib/rancher/k3s/server/logs/*; do if [ -f \"$f\" ]; then echo \"== $f ==\"; tail -n 200 \"$f\"; echo; fi; done"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get pods", "docker", []string{"exec", nodeName, "kubectl", "get", "pods", "-A", "-o", "wide"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get events", "docker", []string{"exec", nodeName, "kubectl", "get", "events", "-A", "--sort-by=.lastTimestamp"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl describe pods", "docker", []string{"exec", nodeName, "kubectl", "describe", "pods", "-A"}, []string{"DOCKER_CONTEXT=" + dockerContext}) From f4e46004f20e84f46453985d8f4c012d7d627b31 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 1 Apr 2026 09:17:18 +0200 Subject: [PATCH 07/14] tests: include nested dockerd logs in k3d failures Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- tests/helpers/k3d.go | 4 ++-- tests/helpers/kubernetes_diagnostics.go | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/helpers/k3d.go b/tests/helpers/k3d.go index 92f3be37dba2..47fcd13fb247 100644 --- a/tests/helpers/k3d.go +++ b/tests/helpers/k3d.go @@ -48,7 +48,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd out, err := cmd.CombinedOutput() if err != nil { diag := KubernetesDiagnostics(clusterName, dockerAddress) - return "", "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s\n%s", clusterName, strings.TrimSpace(string(out)), diag) + return "", "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s\n%s\nouter dockerd logs: %s", clusterName, strings.TrimSpace(string(out)), diag, integration.FormatLogs(cfg.Logs)) } deferF.Append(func() error { cmd := exec.Command(k3dBin, "cluster", "delete", clusterName) @@ -68,7 +68,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd out, err = cmd.CombinedOutput() if err != nil { diag := KubernetesDiagnostics(clusterName, dockerAddress) - return "", "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s\n%s", clusterName, strings.TrimSpace(string(out)), diag) + return "", "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s\n%s\nouter dockerd logs: %s", clusterName, strings.TrimSpace(string(out)), diag, integration.FormatLogs(cfg.Logs)) } kubeConfig = strings.TrimSpace(string(out)) diff --git a/tests/helpers/kubernetes_diagnostics.go b/tests/helpers/kubernetes_diagnostics.go index 3c5e2308f0da..76c1425acc7c 100644 --- a/tests/helpers/kubernetes_diagnostics.go +++ b/tests/helpers/kubernetes_diagnostics.go @@ -82,8 +82,10 @@ func appendK3sServerDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterN continue } appendCommandOutput(ctx, buf, "docker exec "+nodeName+" ps", "docker", []string{"exec", nodeName, "sh", "-c", "ps auxww"}, []string{"DOCKER_CONTEXT=" + dockerContext}) - appendCommandOutput(ctx, buf, "docker exec "+nodeName+" ss", "docker", []string{"exec", nodeName, "sh", "-c", "ss -lntp || netstat -lntp"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" sockets", "docker", []string{"exec", nodeName, "sh", "-c", "ss -lntp || netstat -lnt"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" cgroup", "docker", []string{"exec", nodeName, "sh", "-c", "cat /proc/1/cgroup && echo && mount | grep cgroup"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" env", "docker", []string{"exec", nodeName, "sh", "-c", "env | sort"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" entrypoint", "docker", []string{"exec", nodeName, "sh", "-c", "sed -n '1,200p' /bin/k3d-entrypoint.sh"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s files", "docker", []string{"exec", nodeName, "sh", "-c", "find /var/lib/rancher/k3s -maxdepth 3 -type f 2>/dev/null | sort"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s logs", "docker", []string{"exec", nodeName, "sh", "-c", "for f in /var/log/k3s.log /var/lib/rancher/k3s/agent/containerd/containerd.log /var/lib/rancher/k3s/server/logs/*; do if [ -f \"$f\" ]; then echo \"== $f ==\"; tail -n 200 \"$f\"; echo; fi; done"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get pods", "docker", []string{"exec", nodeName, "kubectl", "get", "pods", "-A", "-o", "wide"}, []string{"DOCKER_CONTEXT=" + dockerContext}) From 23b431f73737fba81c6d52a31d27d2ef35cbfc5e Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 1 Apr 2026 09:32:26 +0200 Subject: [PATCH 08/14] tests: capture k3s startup failures Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- tests/helpers/k3d.go | 1 + tests/helpers/kubernetes_diagnostics.go | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/helpers/k3d.go b/tests/helpers/k3d.go index 47fcd13fb247..fbb56748affe 100644 --- a/tests/helpers/k3d.go +++ b/tests/helpers/k3d.go @@ -39,6 +39,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd args := []string{ "cluster", "create", clusterName, "--wait", + "--k3s-arg=--debug@server:0", } if image := KubernetesK3sImage(); image != "" { args = append(args, "--image="+image) diff --git a/tests/helpers/kubernetes_diagnostics.go b/tests/helpers/kubernetes_diagnostics.go index 76c1425acc7c..5fa945e62cd7 100644 --- a/tests/helpers/kubernetes_diagnostics.go +++ b/tests/helpers/kubernetes_diagnostics.go @@ -86,6 +86,7 @@ func appendK3sServerDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterN appendCommandOutput(ctx, buf, "docker exec "+nodeName+" cgroup", "docker", []string{"exec", nodeName, "sh", "-c", "cat /proc/1/cgroup && echo && mount | grep cgroup"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" env", "docker", []string{"exec", nodeName, "sh", "-c", "env | sort"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" entrypoint", "docker", []string{"exec", nodeName, "sh", "-c", "sed -n '1,200p' /bin/k3d-entrypoint.sh"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" entrypoint logs", "docker", []string{"exec", nodeName, "sh", "-c", "for f in /var/log/k3d-entrypoints_*.log; do if [ -f \"$f\" ]; then echo \"== $f ==\"; tail -n 200 \"$f\"; echo; fi; done"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s files", "docker", []string{"exec", nodeName, "sh", "-c", "find /var/lib/rancher/k3s -maxdepth 3 -type f 2>/dev/null | sort"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s logs", "docker", []string{"exec", nodeName, "sh", "-c", "for f in /var/log/k3s.log /var/lib/rancher/k3s/agent/containerd/containerd.log /var/lib/rancher/k3s/server/logs/*; do if [ -f \"$f\" ]; then echo \"== $f ==\"; tail -n 200 \"$f\"; echo; fi; done"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get pods", "docker", []string{"exec", nodeName, "kubectl", "get", "pods", "-A", "-o", "wide"}, []string{"DOCKER_CONTEXT=" + dockerContext}) From 9d7a663913084fcd1f491193ecea5c16e416c363 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 1 Apr 2026 10:19:22 +0200 Subject: [PATCH 09/14] tests: refresh kubernetes k3s pin and timeouts Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- Dockerfile | 2 +- tests/helpers/k3d.go | 12 +++++++----- tests/helpers/kubernetes_diagnostics.go | 5 +++-- tests/workers/kubernetes.go | 2 +- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index ff4eb460fd6f..02b25c6ac6d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ ARG BUILDKIT_VERSION=v0.29.0 ARG COMPOSE_VERSION=v5.1.0 ARG UNDOCK_VERSION=0.9.0 ARG K3D_VERSION=5.8.3 -ARG K3S_VERSION=v1.32.2-k3s1 +ARG K3S_VERSION=v1.32.13-k3s1 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS golatest diff --git a/tests/helpers/k3d.go b/tests/helpers/k3d.go index fbb56748affe..6b204da214d0 100644 --- a/tests/helpers/k3d.go +++ b/tests/helpers/k3d.go @@ -33,7 +33,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd clusterName = "bk-" + identity.NewID() - createCtx, cancelCreate := context.WithTimeout(ctx, 90*time.Second) + createCtx, cancelCreate := context.WithTimeoutCause(ctx, 90*time.Second, errors.New("timed out creating k3d cluster")) defer cancelCreate() args := []string{ @@ -48,11 +48,13 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd cmd.Env = k3dEnv(dockerAddress) out, err := cmd.CombinedOutput() if err != nil { - diag := KubernetesDiagnostics(clusterName, dockerAddress) + diag := KubernetesDiagnostics(ctx, clusterName, dockerAddress) return "", "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s\n%s\nouter dockerd logs: %s", clusterName, strings.TrimSpace(string(out)), diag, integration.FormatLogs(cfg.Logs)) } deferF.Append(func() error { - cmd := exec.Command(k3dBin, "cluster", "delete", clusterName) + deleteCtx, cancelDelete := context.WithTimeoutCause(context.WithoutCancel(ctx), 30*time.Second, errors.New("timed out deleting k3d cluster")) + defer cancelDelete() + cmd := exec.CommandContext(deleteCtx, k3dBin, "cluster", "delete", clusterName) cmd.Env = k3dEnv(dockerAddress) out, err := cmd.CombinedOutput() if err != nil { @@ -61,14 +63,14 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd return nil }) - kubeconfigCtx, cancelKubeconfig := context.WithTimeout(ctx, 30*time.Second) + kubeconfigCtx, cancelKubeconfig := context.WithTimeoutCause(ctx, 30*time.Second, errors.New("timed out writing k3d kubeconfig")) defer cancelKubeconfig() cmd = exec.CommandContext(kubeconfigCtx, k3dBin, "kubeconfig", "write", clusterName) cmd.Env = k3dEnv(dockerAddress) out, err = cmd.CombinedOutput() if err != nil { - diag := KubernetesDiagnostics(clusterName, dockerAddress) + diag := KubernetesDiagnostics(ctx, clusterName, dockerAddress) return "", "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s\n%s\nouter dockerd logs: %s", clusterName, strings.TrimSpace(string(out)), diag, integration.FormatLogs(cfg.Logs)) } kubeConfig = strings.TrimSpace(string(out)) diff --git a/tests/helpers/kubernetes_diagnostics.go b/tests/helpers/kubernetes_diagnostics.go index 5fa945e62cd7..6c2224e45fcb 100644 --- a/tests/helpers/kubernetes_diagnostics.go +++ b/tests/helpers/kubernetes_diagnostics.go @@ -3,6 +3,7 @@ package helpers import ( "bytes" "context" + "errors" "fmt" "os" "os/exec" @@ -35,8 +36,8 @@ func KubernetesK3DLoadBalancerImage() string { return os.Getenv("TEST_K3D_LOADBALANCER_IMAGE") } -func KubernetesDiagnostics(clusterName, dockerContext string) string { - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) +func KubernetesDiagnostics(ctx context.Context, clusterName, dockerContext string) string { + ctx, cancel := context.WithTimeoutCause(context.WithoutCancel(ctx), 20*time.Second, errors.New("timed out collecting kubernetes diagnostics")) defer cancel() var buf bytes.Buffer diff --git a/tests/workers/kubernetes.go b/tests/workers/kubernetes.go index ec9262418906..ec5d725d3330 100644 --- a/tests/workers/kubernetes.go +++ b/tests/workers/kubernetes.go @@ -79,7 +79,7 @@ func (w *kubernetesWorker) New(ctx context.Context, cfg *integration.BackendConf ) out, err := cmd.CombinedOutput() if err != nil { - diag := helpers.KubernetesDiagnostics(w.k3dName, w.docker.DockerAddress()) + diag := helpers.KubernetesDiagnostics(ctx, w.k3dName, w.docker.DockerAddress()) return nil, nil, errors.Wrapf(err, "failed to create buildx instance %s with image %s: %s\n%s", name, helpers.KubernetesBuildkitImage(), strings.TrimSpace(string(out)), diag) } From b0285c815f3d6f111b24cdcafc7d9cbda67c05ee Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Fri, 3 Apr 2026 00:28:55 +0200 Subject: [PATCH 10/14] tests: extend k3d startup timeout Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- tests/helpers/k3d.go | 16 +++++++++++++--- tests/helpers/kubernetes_diagnostics.go | 9 +++++---- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/tests/helpers/k3d.go b/tests/helpers/k3d.go index 6b204da214d0..38f5050a4163 100644 --- a/tests/helpers/k3d.go +++ b/tests/helpers/k3d.go @@ -14,6 +14,10 @@ import ( const ( k3dBin = "k3d" + + k3dCreateTimeout = 3 * time.Minute + k3dKubeconfigTimeout = 30 * time.Second + k3dDeleteTimeout = 30 * time.Second ) func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAddress string) (clusterName, kubeConfig string, cl func() error, err error) { @@ -33,7 +37,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd clusterName = "bk-" + identity.NewID() - createCtx, cancelCreate := context.WithTimeoutCause(ctx, 90*time.Second, errors.New("timed out creating k3d cluster")) + createCtx, cancelCreate := context.WithTimeoutCause(ctx, k3dCreateTimeout, errors.New("timed out creating k3d cluster")) defer cancelCreate() args := []string{ @@ -48,11 +52,14 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd cmd.Env = k3dEnv(dockerAddress) out, err := cmd.CombinedOutput() if err != nil { + if cause := context.Cause(createCtx); cause != nil && cause != context.Canceled { + err = cause + } diag := KubernetesDiagnostics(ctx, clusterName, dockerAddress) return "", "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s\n%s\nouter dockerd logs: %s", clusterName, strings.TrimSpace(string(out)), diag, integration.FormatLogs(cfg.Logs)) } deferF.Append(func() error { - deleteCtx, cancelDelete := context.WithTimeoutCause(context.WithoutCancel(ctx), 30*time.Second, errors.New("timed out deleting k3d cluster")) + deleteCtx, cancelDelete := context.WithTimeoutCause(context.WithoutCancel(ctx), k3dDeleteTimeout, errors.New("timed out deleting k3d cluster")) defer cancelDelete() cmd := exec.CommandContext(deleteCtx, k3dBin, "cluster", "delete", clusterName) cmd.Env = k3dEnv(dockerAddress) @@ -63,13 +70,16 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd return nil }) - kubeconfigCtx, cancelKubeconfig := context.WithTimeoutCause(ctx, 30*time.Second, errors.New("timed out writing k3d kubeconfig")) + kubeconfigCtx, cancelKubeconfig := context.WithTimeoutCause(ctx, k3dKubeconfigTimeout, errors.New("timed out writing k3d kubeconfig")) defer cancelKubeconfig() cmd = exec.CommandContext(kubeconfigCtx, k3dBin, "kubeconfig", "write", clusterName) cmd.Env = k3dEnv(dockerAddress) out, err = cmd.CombinedOutput() if err != nil { + if cause := context.Cause(kubeconfigCtx); cause != nil && cause != context.Canceled { + err = cause + } diag := KubernetesDiagnostics(ctx, clusterName, dockerAddress) return "", "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s\n%s\nouter dockerd logs: %s", clusterName, strings.TrimSpace(string(out)), diag, integration.FormatLogs(cfg.Logs)) } diff --git a/tests/helpers/kubernetes_diagnostics.go b/tests/helpers/kubernetes_diagnostics.go index 6c2224e45fcb..21bd3aeb2e79 100644 --- a/tests/helpers/kubernetes_diagnostics.go +++ b/tests/helpers/kubernetes_diagnostics.go @@ -82,6 +82,7 @@ func appendK3sServerDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterN if !strings.Contains(nodeName, "-server-") { continue } + kubectl := "KUBECONFIG=/var/lib/rancher/k3s/server/cred/admin.kubeconfig kubectl" appendCommandOutput(ctx, buf, "docker exec "+nodeName+" ps", "docker", []string{"exec", nodeName, "sh", "-c", "ps auxww"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" sockets", "docker", []string{"exec", nodeName, "sh", "-c", "ss -lntp || netstat -lnt"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" cgroup", "docker", []string{"exec", nodeName, "sh", "-c", "cat /proc/1/cgroup && echo && mount | grep cgroup"}, []string{"DOCKER_CONTEXT=" + dockerContext}) @@ -90,9 +91,9 @@ func appendK3sServerDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterN appendCommandOutput(ctx, buf, "docker exec "+nodeName+" entrypoint logs", "docker", []string{"exec", nodeName, "sh", "-c", "for f in /var/log/k3d-entrypoints_*.log; do if [ -f \"$f\" ]; then echo \"== $f ==\"; tail -n 200 \"$f\"; echo; fi; done"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s files", "docker", []string{"exec", nodeName, "sh", "-c", "find /var/lib/rancher/k3s -maxdepth 3 -type f 2>/dev/null | sort"}, []string{"DOCKER_CONTEXT=" + dockerContext}) appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s logs", "docker", []string{"exec", nodeName, "sh", "-c", "for f in /var/log/k3s.log /var/lib/rancher/k3s/agent/containerd/containerd.log /var/lib/rancher/k3s/server/logs/*; do if [ -f \"$f\" ]; then echo \"== $f ==\"; tail -n 200 \"$f\"; echo; fi; done"}, []string{"DOCKER_CONTEXT=" + dockerContext}) - appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get pods", "docker", []string{"exec", nodeName, "kubectl", "get", "pods", "-A", "-o", "wide"}, []string{"DOCKER_CONTEXT=" + dockerContext}) - appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get events", "docker", []string{"exec", nodeName, "kubectl", "get", "events", "-A", "--sort-by=.lastTimestamp"}, []string{"DOCKER_CONTEXT=" + dockerContext}) - appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl describe pods", "docker", []string{"exec", nodeName, "kubectl", "describe", "pods", "-A"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get pods", "docker", []string{"exec", nodeName, "sh", "-c", kubectl + " get pods -A -o wide"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get events", "docker", []string{"exec", nodeName, "sh", "-c", kubectl + " get events -A --sort-by=.lastTimestamp"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl describe pods", "docker", []string{"exec", nodeName, "sh", "-c", kubectl + " describe pods -A"}, []string{"DOCKER_CONTEXT=" + dockerContext}) break } } @@ -107,7 +108,7 @@ func clusterNodeNames(ctx context.Context, clusterName, dockerContext string) ([ return nil, err } var names []string - for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + for line := range strings.SplitSeq(strings.TrimSpace(out), "\n") { line = strings.TrimSpace(line) if line == "" { continue From eebe66d1f055defd0c18632147985c1b2f54c330 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Fri, 3 Apr 2026 02:08:35 +0200 Subject: [PATCH 11/14] tests: fix k3s 1.32 startup in nested kubernetes CI Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- Dockerfile | 2 ++ hack/test-entrypoint.sh | 13 +++++++++++++ tests/helpers/k3d.go | 1 + 3 files changed, 16 insertions(+) create mode 100644 hack/test-entrypoint.sh diff --git a/Dockerfile b/Dockerfile index 02b25c6ac6d6..87aeb202a6b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -156,11 +156,13 @@ COPY --link --from=compose /docker-compose /usr/bin/compose COPY --link --from=undock /usr/local/bin/undock /usr/bin/ COPY --link --from=k3d /bin/k3d /usr/bin/ COPY --link --from=binaries /buildx /usr/bin/ +COPY --chmod=755 ./hack/test-entrypoint.sh /usr/bin/ RUN mkdir -p /usr/local/lib/docker/cli-plugins && ln -s /usr/bin/buildx /usr/local/lib/docker/cli-plugins/docker-buildx ENV TEST_DOCKER_EXTRA="docker@28.5=/opt/docker-alt-28,docker@27.5=/opt/docker-alt-27" ENV TEST_K3S_IMAGE="rancher/k3s:${K3S_VERSION}" ENV TEST_K3D_TOOLS_IMAGE="ghcr.io/k3d-io/k3d-tools:${K3D_VERSION}" ENV TEST_K3D_LOADBALANCER_IMAGE="ghcr.io/k3d-io/k3d-proxy:${K3D_VERSION}" +ENTRYPOINT ["/usr/bin/test-entrypoint.sh"] FROM integration-test-base AS integration-test COPY . . diff --git a/hack/test-entrypoint.sh b/hack/test-entrypoint.sh new file mode 100644 index 000000000000..cec0e7dbdd8a --- /dev/null +++ b/hack/test-entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +set -eu + +# Nested containers on cgroup v2 need controllers delegated from the root cgroup. +if [ -f /sys/fs/cgroup/cgroup.controllers ]; then + mkdir -p /sys/fs/cgroup/init + xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || : + sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \ + > /sys/fs/cgroup/cgroup.subtree_control +fi + +exec "$@" diff --git a/tests/helpers/k3d.go b/tests/helpers/k3d.go index 38f5050a4163..1ec97e661a81 100644 --- a/tests/helpers/k3d.go +++ b/tests/helpers/k3d.go @@ -44,6 +44,7 @@ func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAdd "cluster", "create", clusterName, "--wait", "--k3s-arg=--debug@server:0", + "--k3s-arg=--snapshotter=native@server:0", } if image := KubernetesK3sImage(); image != "" { args = append(args, "--image="+image) From cfc81386432ab68b411555b39943610fb0fae5ea Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Fri, 3 Apr 2026 02:40:46 +0200 Subject: [PATCH 12/14] tests: route kubernetes registries through the k3d gateway Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- tests/helpers/k3d.go | 22 +++++++++++++++++++ tests/workers/backend.go | 16 ++++++++++++++ tests/workers/kubernetes.go | 7 ++++++ .../util/testutil/integration/registry.go | 4 ++-- .../util/testutil/integration/sandbox.go | 3 +++ 5 files changed, 50 insertions(+), 2 deletions(-) diff --git a/tests/helpers/k3d.go b/tests/helpers/k3d.go index 1ec97e661a81..8c9a8ce018fa 100644 --- a/tests/helpers/k3d.go +++ b/tests/helpers/k3d.go @@ -14,10 +14,12 @@ import ( const ( k3dBin = "k3d" + dockerBin = "docker" k3dCreateTimeout = 3 * time.Minute k3dKubeconfigTimeout = 30 * time.Second k3dDeleteTimeout = 30 * time.Second + k3dInspectTimeout = 30 * time.Second ) func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAddress string) (clusterName, kubeConfig string, cl func() error, err error) { @@ -102,3 +104,23 @@ func k3dEnv(dockerAddress string) []string { } return env } + +func K3dNetworkGateway(ctx context.Context, clusterName, dockerAddress string) (string, error) { + inspectCtx, cancelInspect := context.WithTimeoutCause(ctx, k3dInspectTimeout, errors.New("timed out inspecting k3d network")) + defer cancelInspect() + + cmd := exec.CommandContext(inspectCtx, dockerBin, "network", "inspect", "k3d-"+clusterName, "--format", "{{(index .IPAM.Config 0).Gateway}}") + cmd.Env = append(os.Environ(), "DOCKER_CONTEXT="+dockerAddress) + out, err := cmd.CombinedOutput() + if err != nil { + if cause := context.Cause(inspectCtx); cause != nil && cause != context.Canceled { + err = cause + } + return "", errors.Wrapf(err, "failed to inspect k3d network %s: %s", clusterName, strings.TrimSpace(string(out))) + } + gateway := strings.TrimSpace(string(out)) + if gateway == "" { + return "", errors.Errorf("empty gateway for k3d network %s", clusterName) + } + return gateway, nil +} diff --git a/tests/workers/backend.go b/tests/workers/backend.go index 386bf7bd9f6b..4a4350f60a77 100644 --- a/tests/workers/backend.go +++ b/tests/workers/backend.go @@ -1,6 +1,7 @@ package workers import ( + "net" "os" "slices" "strings" @@ -11,6 +12,7 @@ import ( type backend struct { builder string context string + registryHost string unsupportedFeatures []string } @@ -48,6 +50,20 @@ func (s *backend) ExtraEnv() []string { return nil } +func (s *backend) RewriteRegistryAddress(in string) string { + if s.registryHost == "" { + return in + } + host, port, err := net.SplitHostPort(in) + if err != nil { + return in + } + if host != "localhost" && host != "127.0.0.1" { + return in + } + return net.JoinHostPort(s.registryHost, port) +} + func (s backend) Supports(feature string) bool { if enabledFeatures := os.Getenv("BUILDKIT_TEST_ENABLE_FEATURES"); enabledFeatures != "" { if slices.Contains(strings.Split(enabledFeatures, ","), feature) { diff --git a/tests/workers/kubernetes.go b/tests/workers/kubernetes.go index ec5d725d3330..8d49c1536573 100644 --- a/tests/workers/kubernetes.go +++ b/tests/workers/kubernetes.go @@ -31,6 +31,7 @@ type kubernetesWorker struct { k3dName string k3dConfig string + registry string k3dClose func() error k3dErr error k3dOnce sync.Once @@ -58,6 +59,10 @@ func (w *kubernetesWorker) New(ctx context.Context, cfg *integration.BackendConf w.k3dOnce.Do(func() { w.k3dName, w.k3dConfig, w.k3dClose, w.k3dErr = helpers.NewK3dServer(ctx, cfg, w.docker.DockerAddress()) + if w.k3dErr != nil { + return + } + w.registry, w.k3dErr = helpers.K3dNetworkGateway(ctx, w.k3dName, w.docker.DockerAddress()) }) if w.k3dErr != nil { return nil, w.k3dClose, w.k3dErr @@ -97,6 +102,7 @@ func (w *kubernetesWorker) New(ctx context.Context, cfg *integration.BackendConf return &backend{ context: w.docker.DockerAddress(), builder: name, + registryHost: w.registry, unsupportedFeatures: w.unsupported, }, cl, nil } @@ -123,6 +129,7 @@ func (w *kubernetesWorker) Close() error { w.dockerOnce = sync.Once{} w.k3dName = "" w.k3dConfig = "" + w.registry = "" w.k3dClose = nil w.k3dErr = nil w.k3dOnce = sync.Once{} diff --git a/vendor/github.com/moby/buildkit/util/testutil/integration/registry.go b/vendor/github.com/moby/buildkit/util/testutil/integration/registry.go index 0cbc7434266a..00fbb8926eb1 100644 --- a/vendor/github.com/moby/buildkit/util/testutil/integration/registry.go +++ b/vendor/github.com/moby/buildkit/util/testutil/integration/registry.go @@ -48,7 +48,7 @@ storage: filesystem: rootdirectory: %s http: - addr: 127.0.0.1:0 + addr: 0.0.0.0:0 `, filepath.Join(dir, "data")) if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(template), 0600); err != nil { @@ -79,7 +79,7 @@ http: } func detectPort(ctx context.Context, rc io.ReadCloser) (string, error) { - r := regexp.MustCompile(`listening on 127\.0\.0\.1:(\d+)`) + r := regexp.MustCompile(`listening on .*:(\d+)`) s := bufio.NewScanner(rc) found := make(chan struct{}) defer func() { diff --git a/vendor/github.com/moby/buildkit/util/testutil/integration/sandbox.go b/vendor/github.com/moby/buildkit/util/testutil/integration/sandbox.go index b3f770186b77..e7015d0a64fa 100644 --- a/vendor/github.com/moby/buildkit/util/testutil/integration/sandbox.go +++ b/vendor/github.com/moby/buildkit/util/testutil/integration/sandbox.go @@ -65,6 +65,9 @@ func (sb *sandbox) NewRegistry() (string, error) { return "", err } sb.cleanup.Append(cl) + if rewriter, ok := sb.Backend.(interface{ RewriteRegistryAddress(string) string }); ok { + url = rewriter.RewriteRegistryAddress(url) + } return url, nil } From d439f2352840fe5b652d51053bfe8826d1edd77f Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Fri, 3 Apr 2026 10:57:35 +0200 Subject: [PATCH 13/14] tests: mark k3d gateway registries as plain HTTP Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- tests/helpers/k3d.go | 16 ++++++++++++++++ tests/workers/kubernetes.go | 13 +++++++++++++ 2 files changed, 29 insertions(+) diff --git a/tests/helpers/k3d.go b/tests/helpers/k3d.go index 8c9a8ce018fa..dfc8e89a675b 100644 --- a/tests/helpers/k3d.go +++ b/tests/helpers/k3d.go @@ -2,6 +2,7 @@ package helpers import ( "context" + "fmt" "os" "os/exec" "strings" @@ -124,3 +125,18 @@ func K3dNetworkGateway(ctx context.Context, clusterName, dockerAddress string) ( } return gateway, nil } + +func K3dRegistryConfig(host string) integration.ConfigUpdater { + return k3dRegistryConfig(host) +} + +type k3dRegistryConfig string + +func (rc k3dRegistryConfig) UpdateConfigFile(in string) (string, func() error) { + return fmt.Sprintf(`%s + +[registry.%q] + http = true + insecure = true +`, in, string(rc)), nil +} diff --git a/tests/workers/kubernetes.go b/tests/workers/kubernetes.go index 8d49c1536573..de6f9ba3df1e 100644 --- a/tests/workers/kubernetes.go +++ b/tests/workers/kubernetes.go @@ -4,6 +4,7 @@ import ( "context" "os" "os/exec" + "path/filepath" "strings" "sync" @@ -68,10 +69,22 @@ func (w *kubernetesWorker) New(ctx context.Context, cfg *integration.BackendConf return nil, w.k3dClose, w.k3dErr } + daemonConfig := append([]integration.ConfigUpdater{}, cfg.DaemonConfig...) + daemonConfig = append(daemonConfig, helpers.K3dRegistryConfig(w.registry)) + cfgfile, release, err := integration.WriteConfig(daemonConfig) + if err != nil { + return nil, nil, err + } + if release != nil { + defer release() + } + defer os.RemoveAll(filepath.Dir(cfgfile)) + name := "integration-kubernetes-" + identity.NewID() cmd := exec.CommandContext(ctx, "buildx", "create", "--bootstrap", "--name="+name, + "--buildkitd-config="+cfgfile, "--driver=kubernetes", "--driver-opt=image="+helpers.KubernetesBuildkitImage(), "--driver-opt=timeout=60s", From be245b96c0b5d89023a77791b5ff3393faeae4b4 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Fri, 3 Apr 2026 11:43:03 +0200 Subject: [PATCH 14/14] tests: reserve k3d registry ports for kubernetes Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- tests/helpers/k3d.go | 55 +++++++++++++++++-- tests/helpers/k3d_test.go | 27 +++++++++ tests/workers/backend.go | 31 +++++++++++ tests/workers/kubernetes.go | 21 ++++--- .../util/testutil/integration/registry.go | 11 +++- .../util/testutil/integration/sandbox.go | 23 ++++++-- 6 files changed, 149 insertions(+), 19 deletions(-) create mode 100644 tests/helpers/k3d_test.go diff --git a/tests/helpers/k3d.go b/tests/helpers/k3d.go index dfc8e89a675b..29581735ba84 100644 --- a/tests/helpers/k3d.go +++ b/tests/helpers/k3d.go @@ -3,8 +3,10 @@ package helpers import ( "context" "fmt" + "net" "os" "os/exec" + "strconv" "strings" "time" @@ -14,13 +16,14 @@ import ( ) const ( - k3dBin = "k3d" + k3dBin = "k3d" dockerBin = "docker" k3dCreateTimeout = 3 * time.Minute k3dKubeconfigTimeout = 30 * time.Second k3dDeleteTimeout = 30 * time.Second k3dInspectTimeout = 30 * time.Second + K3dRegistryPortCount = 16 ) func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAddress string) (clusterName, kubeConfig string, cl func() error, err error) { @@ -126,17 +129,57 @@ func K3dNetworkGateway(ctx context.Context, clusterName, dockerAddress string) ( return gateway, nil } -func K3dRegistryConfig(host string) integration.ConfigUpdater { - return k3dRegistryConfig(host) +func ReserveK3dRegistryPorts(count int) ([]int, error) { + listeners := make([]net.Listener, 0, count) + ports := make([]int, 0, count) + defer func() { + for _, l := range listeners { + l.Close() + } + }() + + for i := 0; i < count; i++ { + l, err := net.Listen("tcp", "0.0.0.0:0") + if err != nil { + return nil, err + } + listeners = append(listeners, l) + addr, ok := l.Addr().(*net.TCPAddr) + if !ok { + return nil, errors.Errorf("unexpected registry listener address %T", l.Addr()) + } + ports = append(ports, addr.Port) + } + + return ports, nil +} + +func K3dRegistryConfig(host string, ports []int) integration.ConfigUpdater { + return k3dRegistryConfig{ + host: host, + ports: append([]int(nil), ports...), + } } -type k3dRegistryConfig string +type k3dRegistryConfig struct { + host string + ports []int +} func (rc k3dRegistryConfig) UpdateConfigFile(in string) (string, func() error) { - return fmt.Sprintf(`%s + if rc.host == "" || len(rc.ports) == 0 { + return in, nil + } + + var b strings.Builder + b.WriteString(in) + for _, port := range rc.ports { + fmt.Fprintf(&b, ` [registry.%q] http = true insecure = true -`, in, string(rc)), nil +`, net.JoinHostPort(rc.host, strconv.Itoa(port))) + } + return b.String(), nil } diff --git a/tests/helpers/k3d_test.go b/tests/helpers/k3d_test.go new file mode 100644 index 000000000000..6907bf9346b6 --- /dev/null +++ b/tests/helpers/k3d_test.go @@ -0,0 +1,27 @@ +package helpers + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestK3dRegistryConfigUsesHostPorts(t *testing.T) { + cfg, cleanup := K3dRegistryConfig("172.19.0.1", []int{40001, 40002}).UpdateConfigFile("debug = true") + if cleanup != nil { + t.Cleanup(func() { + require.NoError(t, cleanup()) + }) + } + + require.Contains(t, cfg, `debug = true`) + require.Contains(t, cfg, `[registry."172.19.0.1:40001"]`) + require.Contains(t, cfg, `[registry."172.19.0.1:40002"]`) + require.NotContains(t, cfg, `[registry."172.19.0.1"]`) +} + +func TestK3dRegistryConfigWithoutPortsIsNoop(t *testing.T) { + cfg, cleanup := K3dRegistryConfig("172.19.0.1", nil).UpdateConfigFile("debug = true") + require.Nil(t, cleanup) + require.Equal(t, "debug = true", cfg) +} diff --git a/tests/workers/backend.go b/tests/workers/backend.go index 4a4350f60a77..e3e8207b7104 100644 --- a/tests/workers/backend.go +++ b/tests/workers/backend.go @@ -1,10 +1,13 @@ package workers import ( + "fmt" "net" "os" "slices" + "strconv" "strings" + "sync" "github.com/moby/buildkit/util/testutil/integration" ) @@ -13,6 +16,9 @@ type backend struct { builder string context string registryHost string + registryPorts []int + registryPortIndex int + registryMu sync.Mutex unsupportedFeatures []string } @@ -50,6 +56,31 @@ func (s *backend) ExtraEnv() []string { return nil } +func (s *backend) NewRegistry() (string, func() error, error) { + if s.registryHost == "" || len(s.registryPorts) == 0 { + url, cl, err := integration.NewRegistry("") + if err != nil { + return "", nil, err + } + return s.RewriteRegistryAddress(url), cl, nil + } + + s.registryMu.Lock() + defer s.registryMu.Unlock() + + if s.registryPortIndex >= len(s.registryPorts) { + return "", nil, fmt.Errorf("exhausted kubernetes registry port pool") + } + port := s.registryPorts[s.registryPortIndex] + s.registryPortIndex++ + + _, cl, err := integration.NewRegistryAt("", net.JoinHostPort("0.0.0.0", strconv.Itoa(port))) + if err != nil { + return "", nil, err + } + return net.JoinHostPort(s.registryHost, strconv.Itoa(port)), cl, nil +} + func (s *backend) RewriteRegistryAddress(in string) string { if s.registryHost == "" { return in diff --git a/tests/workers/kubernetes.go b/tests/workers/kubernetes.go index de6f9ba3df1e..deeccc151e11 100644 --- a/tests/workers/kubernetes.go +++ b/tests/workers/kubernetes.go @@ -30,12 +30,13 @@ type kubernetesWorker struct { dockerErr error dockerOnce sync.Once - k3dName string - k3dConfig string - registry string - k3dClose func() error - k3dErr error - k3dOnce sync.Once + k3dName string + k3dConfig string + registry string + registryPorts []int + k3dClose func() error + k3dErr error + k3dOnce sync.Once } func (w *kubernetesWorker) Name() string { @@ -64,13 +65,17 @@ func (w *kubernetesWorker) New(ctx context.Context, cfg *integration.BackendConf return } w.registry, w.k3dErr = helpers.K3dNetworkGateway(ctx, w.k3dName, w.docker.DockerAddress()) + if w.k3dErr != nil { + return + } + w.registryPorts, w.k3dErr = helpers.ReserveK3dRegistryPorts(helpers.K3dRegistryPortCount) }) if w.k3dErr != nil { return nil, w.k3dClose, w.k3dErr } daemonConfig := append([]integration.ConfigUpdater{}, cfg.DaemonConfig...) - daemonConfig = append(daemonConfig, helpers.K3dRegistryConfig(w.registry)) + daemonConfig = append(daemonConfig, helpers.K3dRegistryConfig(w.registry, w.registryPorts)) cfgfile, release, err := integration.WriteConfig(daemonConfig) if err != nil { return nil, nil, err @@ -116,6 +121,7 @@ func (w *kubernetesWorker) New(ctx context.Context, cfg *integration.BackendConf context: w.docker.DockerAddress(), builder: name, registryHost: w.registry, + registryPorts: append([]int(nil), w.registryPorts...), unsupportedFeatures: w.unsupported, }, cl, nil } @@ -143,6 +149,7 @@ func (w *kubernetesWorker) Close() error { w.k3dName = "" w.k3dConfig = "" w.registry = "" + w.registryPorts = nil w.k3dClose = nil w.k3dErr = nil w.k3dOnce = sync.Once{} diff --git a/vendor/github.com/moby/buildkit/util/testutil/integration/registry.go b/vendor/github.com/moby/buildkit/util/testutil/integration/registry.go index 00fbb8926eb1..e5948d9a7d0e 100644 --- a/vendor/github.com/moby/buildkit/util/testutil/integration/registry.go +++ b/vendor/github.com/moby/buildkit/util/testutil/integration/registry.go @@ -15,6 +15,10 @@ import ( ) func NewRegistry(dir string) (url string, cl func() error, err error) { + return NewRegistryAt(dir, "0.0.0.0:0") +} + +func NewRegistryAt(dir, addr string) (url string, cl func() error, err error) { if err := LookupBinary("registry"); err != nil { return "", nil, err } @@ -42,14 +46,17 @@ func NewRegistry(dir string) (url string, cl func() error, err error) { if !errors.Is(err, os.ErrNotExist) { return "", nil, err } + if addr == "" { + addr = "0.0.0.0:0" + } template := fmt.Sprintf(`version: 0.1 loglevel: debug storage: filesystem: rootdirectory: %s http: - addr: 0.0.0.0:0 -`, filepath.Join(dir, "data")) + addr: %s +`, filepath.Join(dir, "data"), addr) if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(template), 0600); err != nil { return "", nil, err diff --git a/vendor/github.com/moby/buildkit/util/testutil/integration/sandbox.go b/vendor/github.com/moby/buildkit/util/testutil/integration/sandbox.go index e7015d0a64fa..40d7a6b0f795 100644 --- a/vendor/github.com/moby/buildkit/util/testutil/integration/sandbox.go +++ b/vendor/github.com/moby/buildkit/util/testutil/integration/sandbox.go @@ -60,14 +60,29 @@ func (sb *sandbox) ClearLogs() { } func (sb *sandbox) NewRegistry() (string, error) { - url, cl, err := NewRegistry("") + type registryProvider interface { + NewRegistry() (string, func() error, error) + } + + var ( + url string + cl func() error + err error + ) + if provider, ok := sb.Backend.(registryProvider); ok { + url, cl, err = provider.NewRegistry() + } else { + url, cl, err = NewRegistry("") + if err == nil { + if rewriter, ok := sb.Backend.(interface{ RewriteRegistryAddress(string) string }); ok { + url = rewriter.RewriteRegistryAddress(url) + } + } + } if err != nil { return "", err } sb.cleanup.Append(cl) - if rewriter, ok := sb.Backend.(interface{ RewriteRegistryAddress(string) string }); ok { - url = rewriter.RewriteRegistryAddress(url) - } return url, nil }