test kubernetes worker with k3d

.github/workflows/build.yml

@@ -53,6 +53,7 @@ jobs:
         worker:
           - docker-container
           - remote
+          - kubernetes
         pkg:
           - ./tests
         mode:
@@ -104,14 +105,14 @@ jobs:
           fi
           testFlags="--run=//worker=$(echo "${{ matrix.worker }}" | sed 's/\+/\\+/g')$"
           case "${{ matrix.worker }}" in
-            docker | docker+containerd | docker@* | docker+containerd@* | remote+multinode)
+            docker | docker+containerd | docker@* | docker+containerd@* | remote+multinode | kubernetes)
               echo "TESTFLAGS=${{ env.TESTFLAGS_DOCKER }} $testFlags" >> $GITHUB_ENV
               ;;
             *)
               echo "TESTFLAGS=${{ env.TESTFLAGS }} $testFlags" >> $GITHUB_ENV
               ;;
           esac
-          if [[ "${{ matrix.worker }}" == "docker"* || "${{ matrix.worker }}" == "remote+multinode" ]]; then
+          if [[ "${{ matrix.worker }}" == "docker"* || "${{ matrix.worker }}" == "remote+multinode" || "${{ matrix.worker }}" == "kubernetes" ]]; then
             echo "TEST_DOCKERD=1" >> $GITHUB_ENV
           fi
           if [ "${{ matrix.mode }}" = "experimental" ]; then

Dockerfile

@@ -14,6 +14,8 @@ ARG REGISTRY_VERSION=3.0.0
 ARG BUILDKIT_VERSION=v0.29.0
 ARG COMPOSE_VERSION=v5.1.0
 ARG UNDOCK_VERSION=0.9.0
+ARG K3D_VERSION=5.8.3
+ARG K3S_VERSION=v1.32.13-k3s1
 
 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
 FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS golatest
@@ -27,6 +29,7 @@ FROM registry:$REGISTRY_VERSION AS registry
 FROM moby/buildkit:$BUILDKIT_VERSION AS buildkit
 FROM docker/compose-bin:$COMPOSE_VERSION AS compose
 FROM crazymax/undock:$UNDOCK_VERSION AS undock
+FROM ghcr.io/k3d-io/k3d:${K3D_VERSION} AS k3d
 
 FROM golatest AS gobase
 COPY --from=xx / /
@@ -124,6 +127,8 @@ FROM binaries-$TARGETOS AS binaries
 ARG BUILDKIT_SBOM_SCAN_STAGE=true
 
 FROM gobase AS integration-test-base
+ARG K3D_VERSION
+ARG K3S_VERSION
 # https://github.com/docker/docker/blob/master/project/PACKAGERS.md#runtime-dependencies
 RUN apk add --no-cache \
       bash \
@@ -149,9 +154,15 @@ COPY --link --from=buildkit /usr/bin/buildkitd /usr/bin/
 COPY --link --from=buildkit /usr/bin/buildctl /usr/bin/
 COPY --link --from=compose /docker-compose /usr/bin/compose
 COPY --link --from=undock /usr/local/bin/undock /usr/bin/
+COPY --link --from=k3d /bin/k3d /usr/bin/
 COPY --link --from=binaries /buildx /usr/bin/
+COPY --chmod=755 ./hack/test-entrypoint.sh /usr/bin/
 RUN mkdir -p /usr/local/lib/docker/cli-plugins && ln -s /usr/bin/buildx /usr/local/lib/docker/cli-plugins/docker-buildx
 ENV TEST_DOCKER_EXTRA="docker@28.5=/opt/docker-alt-28,docker@27.5=/opt/docker-alt-27"
+ENV TEST_K3S_IMAGE="rancher/k3s:${K3S_VERSION}"
+ENV TEST_K3D_TOOLS_IMAGE="ghcr.io/k3d-io/k3d-tools:${K3D_VERSION}"
+ENV TEST_K3D_LOADBALANCER_IMAGE="ghcr.io/k3d-io/k3d-proxy:${K3D_VERSION}"
+ENTRYPOINT ["/usr/bin/test-entrypoint.sh"]
 
 FROM integration-test-base AS integration-test
 COPY . .

hack/test-entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+set -eu
+
+# Nested containers on cgroup v2 need controllers delegated from the root cgroup.
+if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
+	mkdir -p /sys/fs/cgroup/init
+	xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || :
+	sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \
+		> /sys/fs/cgroup/cgroup.subtree_control
+fi
+
+exec "$@"

tests/helpers/k3d.go

@@ -0,0 +1,185 @@
+package helpers
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/moby/buildkit/identity"
+	"github.com/moby/buildkit/util/testutil/integration"
+	"github.com/pkg/errors"
+)
+
+const (
+	k3dBin    = "k3d"
+	dockerBin = "docker"
+
+	k3dCreateTimeout     = 3 * time.Minute
+	k3dKubeconfigTimeout = 30 * time.Second
+	k3dDeleteTimeout     = 30 * time.Second
+	k3dInspectTimeout    = 30 * time.Second
+	K3dRegistryPortCount = 16
+)
+
+func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAddress string) (clusterName, kubeConfig string, cl func() error, err error) {
+	if _, err := exec.LookPath(k3dBin); err != nil {
+		return "", "", nil, errors.Wrapf(err, "failed to lookup %s binary", k3dBin)
+	}
+
+	deferF := &integration.MultiCloser{}
+	cl = deferF.F()
+
+	defer func() {
+		if err != nil {
+			deferF.F()()
+			cl = nil
+		}
+	}()
+
+	clusterName = "bk-" + identity.NewID()
+
+	createCtx, cancelCreate := context.WithTimeoutCause(ctx, k3dCreateTimeout, errors.New("timed out creating k3d cluster"))
+	defer cancelCreate()
+
+	args := []string{
+		"cluster", "create", clusterName,
+		"--wait",
+		"--k3s-arg=--debug@server:0",
+		"--k3s-arg=--snapshotter=native@server:0",
+	}
+	if image := KubernetesK3sImage(); image != "" {
+		args = append(args, "--image="+image)
+	}
+	cmd := exec.CommandContext(createCtx, k3dBin, args...)
+	cmd.Env = k3dEnv(dockerAddress)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		if cause := context.Cause(createCtx); cause != nil && cause != context.Canceled {
+			err = cause
+		}
+		diag := KubernetesDiagnostics(ctx, clusterName, dockerAddress)
+		return "", "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s\n%s\nouter dockerd logs: %s", clusterName, strings.TrimSpace(string(out)), diag, integration.FormatLogs(cfg.Logs))
+	}
+	deferF.Append(func() error {
+		deleteCtx, cancelDelete := context.WithTimeoutCause(context.WithoutCancel(ctx), k3dDeleteTimeout, errors.New("timed out deleting k3d cluster"))
+		defer cancelDelete()
+		cmd := exec.CommandContext(deleteCtx, k3dBin, "cluster", "delete", clusterName)
+		cmd.Env = k3dEnv(dockerAddress)
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			return errors.Wrapf(err, "failed to delete k3d cluster %s: %s", clusterName, string(out))
+		}
+		return nil
+	})
+
+	kubeconfigCtx, cancelKubeconfig := context.WithTimeoutCause(ctx, k3dKubeconfigTimeout, errors.New("timed out writing k3d kubeconfig"))
+	defer cancelKubeconfig()
+
+	cmd = exec.CommandContext(kubeconfigCtx, k3dBin, "kubeconfig", "write", clusterName)
+	cmd.Env = k3dEnv(dockerAddress)
+	out, err = cmd.CombinedOutput()
+	if err != nil {
+		if cause := context.Cause(kubeconfigCtx); cause != nil && cause != context.Canceled {
+			err = cause
+		}
+		diag := KubernetesDiagnostics(ctx, clusterName, dockerAddress)
+		return "", "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s\n%s\nouter dockerd logs: %s", clusterName, strings.TrimSpace(string(out)), diag, integration.FormatLogs(cfg.Logs))
+	}
+	kubeConfig = strings.TrimSpace(string(out))
+
+	return
+}
+
+func k3dEnv(dockerAddress string) []string {
+	env := append(
+		os.Environ(),
+		"DOCKER_CONTEXT="+dockerAddress,
+	)
+	if image := KubernetesK3DToolsImage(); image != "" {
+		env = append(env, "K3D_IMAGE_TOOLS="+image)
+	}
+	if image := KubernetesK3DLoadBalancerImage(); image != "" {
+		env = append(env, "K3D_IMAGE_LOADBALANCER="+image)
+	}
+	return env
+}
+
+func K3dNetworkGateway(ctx context.Context, clusterName, dockerAddress string) (string, error) {
+	inspectCtx, cancelInspect := context.WithTimeoutCause(ctx, k3dInspectTimeout, errors.New("timed out inspecting k3d network"))
+	defer cancelInspect()
+
+	cmd := exec.CommandContext(inspectCtx, dockerBin, "network", "inspect", "k3d-"+clusterName, "--format", "{{(index .IPAM.Config 0).Gateway}}")
+	cmd.Env = append(os.Environ(), "DOCKER_CONTEXT="+dockerAddress)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		if cause := context.Cause(inspectCtx); cause != nil && cause != context.Canceled {
+			err = cause
+		}
+		return "", errors.Wrapf(err, "failed to inspect k3d network %s: %s", clusterName, strings.TrimSpace(string(out)))
+	}
+	gateway := strings.TrimSpace(string(out))
+	if gateway == "" {
+		return "", errors.Errorf("empty gateway for k3d network %s", clusterName)
+	}
+	return gateway, nil
+}
+
+func ReserveK3dRegistryPorts(count int) ([]int, error) {
+	listeners := make([]net.Listener, 0, count)
+	ports := make([]int, 0, count)
+	defer func() {
+		for _, l := range listeners {
+			l.Close()
+		}
+	}()
+
+	for i := 0; i < count; i++ {
+		l, err := net.Listen("tcp", "0.0.0.0:0")
+		if err != nil {
+			return nil, err
+		}
+		listeners = append(listeners, l)
+		addr, ok := l.Addr().(*net.TCPAddr)
+		if !ok {
+			return nil, errors.Errorf("unexpected registry listener address %T", l.Addr())
+		}
+		ports = append(ports, addr.Port)
+	}
+
+	return ports, nil
+}
+
+func K3dRegistryConfig(host string, ports []int) integration.ConfigUpdater {
+	return k3dRegistryConfig{
+		host:  host,
+		ports: append([]int(nil), ports...),
+	}
+}
+
+type k3dRegistryConfig struct {
+	host  string
+	ports []int
+}
+
+func (rc k3dRegistryConfig) UpdateConfigFile(in string) (string, func() error) {
+	if rc.host == "" || len(rc.ports) == 0 {
+		return in, nil
+	}
+
+	var b strings.Builder
+	b.WriteString(in)
+	for _, port := range rc.ports {
+		fmt.Fprintf(&b, `
+
+[registry.%q]
+  http = true
+  insecure = true
+`, net.JoinHostPort(rc.host, strconv.Itoa(port)))
+	}
+	return b.String(), nil
+}

tests/helpers/k3d_test.go

@@ -0,0 +1,27 @@
+package helpers
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestK3dRegistryConfigUsesHostPorts(t *testing.T) {
+	cfg, cleanup := K3dRegistryConfig("172.19.0.1", []int{40001, 40002}).UpdateConfigFile("debug = true")
+	if cleanup != nil {
+		t.Cleanup(func() {
+			require.NoError(t, cleanup())
+		})
+	}
+
+	require.Contains(t, cfg, `debug = true`)
+	require.Contains(t, cfg, `[registry."172.19.0.1:40001"]`)
+	require.Contains(t, cfg, `[registry."172.19.0.1:40002"]`)
+	require.NotContains(t, cfg, `[registry."172.19.0.1"]`)
+}
+
+func TestK3dRegistryConfigWithoutPortsIsNoop(t *testing.T) {
+	cfg, cleanup := K3dRegistryConfig("172.19.0.1", nil).UpdateConfigFile("debug = true")
+	require.Nil(t, cleanup)
+	require.Equal(t, "debug = true", cfg)
+}