diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bd369d2b6ea0..db353a896aaf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,6 +53,7 @@ jobs: worker: - docker-container - remote + - kubernetes pkg: - ./tests mode: @@ -104,14 +105,14 @@ jobs: fi testFlags="--run=//worker=$(echo "${{ matrix.worker }}" | sed 's/\+/\\+/g')$" case "${{ matrix.worker }}" in - docker | docker+containerd | docker@* | docker+containerd@* | remote+multinode) + docker | docker+containerd | docker@* | docker+containerd@* | remote+multinode | kubernetes) echo "TESTFLAGS=${{ env.TESTFLAGS_DOCKER }} $testFlags" >> $GITHUB_ENV ;; *) echo "TESTFLAGS=${{ env.TESTFLAGS }} $testFlags" >> $GITHUB_ENV ;; esac - if [[ "${{ matrix.worker }}" == "docker"* || "${{ matrix.worker }}" == "remote+multinode" ]]; then + if [[ "${{ matrix.worker }}" == "docker"* || "${{ matrix.worker }}" == "remote+multinode" || "${{ matrix.worker }}" == "kubernetes" ]]; then echo "TEST_DOCKERD=1" >> $GITHUB_ENV fi if [ "${{ matrix.mode }}" = "experimental" ]; then diff --git a/Dockerfile b/Dockerfile index 00645b41efc0..87aeb202a6b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,8 @@ ARG REGISTRY_VERSION=3.0.0 ARG BUILDKIT_VERSION=v0.29.0 ARG COMPOSE_VERSION=v5.1.0 ARG UNDOCK_VERSION=0.9.0 +ARG K3D_VERSION=5.8.3 +ARG K3S_VERSION=v1.32.13-k3s1 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS golatest @@ -27,6 +29,7 @@ FROM registry:$REGISTRY_VERSION AS registry FROM moby/buildkit:$BUILDKIT_VERSION AS buildkit FROM docker/compose-bin:$COMPOSE_VERSION AS compose FROM crazymax/undock:$UNDOCK_VERSION AS undock +FROM ghcr.io/k3d-io/k3d:${K3D_VERSION} AS k3d FROM golatest AS gobase COPY --from=xx / / @@ -124,6 +127,8 @@ FROM binaries-$TARGETOS AS binaries ARG BUILDKIT_SBOM_SCAN_STAGE=true FROM gobase AS integration-test-base +ARG K3D_VERSION +ARG K3S_VERSION # https://github.com/docker/docker/blob/master/project/PACKAGERS.md#runtime-dependencies RUN apk add --no-cache \ bash \ @@ -149,9 +154,15 @@ COPY --link --from=buildkit /usr/bin/buildkitd /usr/bin/ COPY --link --from=buildkit /usr/bin/buildctl /usr/bin/ COPY --link --from=compose /docker-compose /usr/bin/compose COPY --link --from=undock /usr/local/bin/undock /usr/bin/ +COPY --link --from=k3d /bin/k3d /usr/bin/ COPY --link --from=binaries /buildx /usr/bin/ +COPY --chmod=755 ./hack/test-entrypoint.sh /usr/bin/ RUN mkdir -p /usr/local/lib/docker/cli-plugins && ln -s /usr/bin/buildx /usr/local/lib/docker/cli-plugins/docker-buildx ENV TEST_DOCKER_EXTRA="docker@28.5=/opt/docker-alt-28,docker@27.5=/opt/docker-alt-27" +ENV TEST_K3S_IMAGE="rancher/k3s:${K3S_VERSION}" +ENV TEST_K3D_TOOLS_IMAGE="ghcr.io/k3d-io/k3d-tools:${K3D_VERSION}" +ENV TEST_K3D_LOADBALANCER_IMAGE="ghcr.io/k3d-io/k3d-proxy:${K3D_VERSION}" +ENTRYPOINT ["/usr/bin/test-entrypoint.sh"] FROM integration-test-base AS integration-test COPY . . diff --git a/hack/test-entrypoint.sh b/hack/test-entrypoint.sh new file mode 100644 index 000000000000..cec0e7dbdd8a --- /dev/null +++ b/hack/test-entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +set -eu + +# Nested containers on cgroup v2 need controllers delegated from the root cgroup. +if [ -f /sys/fs/cgroup/cgroup.controllers ]; then + mkdir -p /sys/fs/cgroup/init + xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || : + sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \ + > /sys/fs/cgroup/cgroup.subtree_control +fi + +exec "$@" diff --git a/tests/helpers/k3d.go b/tests/helpers/k3d.go new file mode 100644 index 000000000000..29581735ba84 --- /dev/null +++ b/tests/helpers/k3d.go @@ -0,0 +1,185 @@ +package helpers + +import ( + "context" + "fmt" + "net" + "os" + "os/exec" + "strconv" + "strings" + "time" + + "github.com/moby/buildkit/identity" + "github.com/moby/buildkit/util/testutil/integration" + "github.com/pkg/errors" +) + +const ( + k3dBin = "k3d" + dockerBin = "docker" + + k3dCreateTimeout = 3 * time.Minute + k3dKubeconfigTimeout = 30 * time.Second + k3dDeleteTimeout = 30 * time.Second + k3dInspectTimeout = 30 * time.Second + K3dRegistryPortCount = 16 +) + +func NewK3dServer(ctx context.Context, cfg *integration.BackendConfig, dockerAddress string) (clusterName, kubeConfig string, cl func() error, err error) { + if _, err := exec.LookPath(k3dBin); err != nil { + return "", "", nil, errors.Wrapf(err, "failed to lookup %s binary", k3dBin) + } + + deferF := &integration.MultiCloser{} + cl = deferF.F() + + defer func() { + if err != nil { + deferF.F()() + cl = nil + } + }() + + clusterName = "bk-" + identity.NewID() + + createCtx, cancelCreate := context.WithTimeoutCause(ctx, k3dCreateTimeout, errors.New("timed out creating k3d cluster")) + defer cancelCreate() + + args := []string{ + "cluster", "create", clusterName, + "--wait", + "--k3s-arg=--debug@server:0", + "--k3s-arg=--snapshotter=native@server:0", + } + if image := KubernetesK3sImage(); image != "" { + args = append(args, "--image="+image) + } + cmd := exec.CommandContext(createCtx, k3dBin, args...) + cmd.Env = k3dEnv(dockerAddress) + out, err := cmd.CombinedOutput() + if err != nil { + if cause := context.Cause(createCtx); cause != nil && cause != context.Canceled { + err = cause + } + diag := KubernetesDiagnostics(ctx, clusterName, dockerAddress) + return "", "", nil, errors.Wrapf(err, "failed to create k3d cluster %s: %s\n%s\nouter dockerd logs: %s", clusterName, strings.TrimSpace(string(out)), diag, integration.FormatLogs(cfg.Logs)) + } + deferF.Append(func() error { + deleteCtx, cancelDelete := context.WithTimeoutCause(context.WithoutCancel(ctx), k3dDeleteTimeout, errors.New("timed out deleting k3d cluster")) + defer cancelDelete() + cmd := exec.CommandContext(deleteCtx, k3dBin, "cluster", "delete", clusterName) + cmd.Env = k3dEnv(dockerAddress) + out, err := cmd.CombinedOutput() + if err != nil { + return errors.Wrapf(err, "failed to delete k3d cluster %s: %s", clusterName, string(out)) + } + return nil + }) + + kubeconfigCtx, cancelKubeconfig := context.WithTimeoutCause(ctx, k3dKubeconfigTimeout, errors.New("timed out writing k3d kubeconfig")) + defer cancelKubeconfig() + + cmd = exec.CommandContext(kubeconfigCtx, k3dBin, "kubeconfig", "write", clusterName) + cmd.Env = k3dEnv(dockerAddress) + out, err = cmd.CombinedOutput() + if err != nil { + if cause := context.Cause(kubeconfigCtx); cause != nil && cause != context.Canceled { + err = cause + } + diag := KubernetesDiagnostics(ctx, clusterName, dockerAddress) + return "", "", nil, errors.Wrapf(err, "failed to write kubeconfig for cluster %s: %s\n%s\nouter dockerd logs: %s", clusterName, strings.TrimSpace(string(out)), diag, integration.FormatLogs(cfg.Logs)) + } + kubeConfig = strings.TrimSpace(string(out)) + + return +} + +func k3dEnv(dockerAddress string) []string { + env := append( + os.Environ(), + "DOCKER_CONTEXT="+dockerAddress, + ) + if image := KubernetesK3DToolsImage(); image != "" { + env = append(env, "K3D_IMAGE_TOOLS="+image) + } + if image := KubernetesK3DLoadBalancerImage(); image != "" { + env = append(env, "K3D_IMAGE_LOADBALANCER="+image) + } + return env +} + +func K3dNetworkGateway(ctx context.Context, clusterName, dockerAddress string) (string, error) { + inspectCtx, cancelInspect := context.WithTimeoutCause(ctx, k3dInspectTimeout, errors.New("timed out inspecting k3d network")) + defer cancelInspect() + + cmd := exec.CommandContext(inspectCtx, dockerBin, "network", "inspect", "k3d-"+clusterName, "--format", "{{(index .IPAM.Config 0).Gateway}}") + cmd.Env = append(os.Environ(), "DOCKER_CONTEXT="+dockerAddress) + out, err := cmd.CombinedOutput() + if err != nil { + if cause := context.Cause(inspectCtx); cause != nil && cause != context.Canceled { + err = cause + } + return "", errors.Wrapf(err, "failed to inspect k3d network %s: %s", clusterName, strings.TrimSpace(string(out))) + } + gateway := strings.TrimSpace(string(out)) + if gateway == "" { + return "", errors.Errorf("empty gateway for k3d network %s", clusterName) + } + return gateway, nil +} + +func ReserveK3dRegistryPorts(count int) ([]int, error) { + listeners := make([]net.Listener, 0, count) + ports := make([]int, 0, count) + defer func() { + for _, l := range listeners { + l.Close() + } + }() + + for i := 0; i < count; i++ { + l, err := net.Listen("tcp", "0.0.0.0:0") + if err != nil { + return nil, err + } + listeners = append(listeners, l) + addr, ok := l.Addr().(*net.TCPAddr) + if !ok { + return nil, errors.Errorf("unexpected registry listener address %T", l.Addr()) + } + ports = append(ports, addr.Port) + } + + return ports, nil +} + +func K3dRegistryConfig(host string, ports []int) integration.ConfigUpdater { + return k3dRegistryConfig{ + host: host, + ports: append([]int(nil), ports...), + } +} + +type k3dRegistryConfig struct { + host string + ports []int +} + +func (rc k3dRegistryConfig) UpdateConfigFile(in string) (string, func() error) { + if rc.host == "" || len(rc.ports) == 0 { + return in, nil + } + + var b strings.Builder + b.WriteString(in) + for _, port := range rc.ports { + fmt.Fprintf(&b, ` + +[registry.%q] + http = true + insecure = true +`, net.JoinHostPort(rc.host, strconv.Itoa(port))) + } + return b.String(), nil +} diff --git a/tests/helpers/k3d_test.go b/tests/helpers/k3d_test.go new file mode 100644 index 000000000000..6907bf9346b6 --- /dev/null +++ b/tests/helpers/k3d_test.go @@ -0,0 +1,27 @@ +package helpers + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestK3dRegistryConfigUsesHostPorts(t *testing.T) { + cfg, cleanup := K3dRegistryConfig("172.19.0.1", []int{40001, 40002}).UpdateConfigFile("debug = true") + if cleanup != nil { + t.Cleanup(func() { + require.NoError(t, cleanup()) + }) + } + + require.Contains(t, cfg, `debug = true`) + require.Contains(t, cfg, `[registry."172.19.0.1:40001"]`) + require.Contains(t, cfg, `[registry."172.19.0.1:40002"]`) + require.NotContains(t, cfg, `[registry."172.19.0.1"]`) +} + +func TestK3dRegistryConfigWithoutPortsIsNoop(t *testing.T) { + cfg, cleanup := K3dRegistryConfig("172.19.0.1", nil).UpdateConfigFile("debug = true") + require.Nil(t, cleanup) + require.Equal(t, "debug = true", cfg) +} diff --git a/tests/helpers/kubernetes_diagnostics.go b/tests/helpers/kubernetes_diagnostics.go new file mode 100644 index 000000000000..21bd3aeb2e79 --- /dev/null +++ b/tests/helpers/kubernetes_diagnostics.go @@ -0,0 +1,142 @@ +package helpers + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "os/exec" + "strings" + "time" +) + +const defaultTestBuildkitTag = "buildx-stable-1" + +func KubernetesBuildkitImage() string { + if v := os.Getenv("TEST_BUILDKIT_IMAGE"); v != "" { + return v + } + tag := os.Getenv("TEST_BUILDKIT_TAG") + if tag == "" { + tag = defaultTestBuildkitTag + } + return "moby/buildkit:" + tag +} + +func KubernetesK3sImage() string { + return os.Getenv("TEST_K3S_IMAGE") +} + +func KubernetesK3DToolsImage() string { + return os.Getenv("TEST_K3D_TOOLS_IMAGE") +} + +func KubernetesK3DLoadBalancerImage() string { + return os.Getenv("TEST_K3D_LOADBALANCER_IMAGE") +} + +func KubernetesDiagnostics(ctx context.Context, clusterName, dockerContext string) string { + ctx, cancel := context.WithTimeoutCause(context.WithoutCancel(ctx), 20*time.Second, errors.New("timed out collecting kubernetes diagnostics")) + defer cancel() + + var buf bytes.Buffer + appendK3dDiagnostics(ctx, &buf, clusterName, dockerContext) + appendDockerDiagnostics(ctx, &buf, dockerContext) + appendK3sServerDiagnostics(ctx, &buf, clusterName, dockerContext) + return strings.TrimSpace(buf.String()) +} + +func appendK3dDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterName, dockerContext string) { + appendCommandOutput(ctx, buf, "k3d cluster list", "k3d", []string{"cluster", "list", clusterName}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "k3d node list", "k3d", []string{"node", "list"}, []string{"DOCKER_CONTEXT=" + dockerContext}) +} + +func appendDockerDiagnostics(ctx context.Context, buf *bytes.Buffer, dockerContext string) { + args := []string{"ps", "-a", "--format", "{{.Names}}\t{{.Image}}\t{{.Status}}"} + appendCommandOutput(ctx, buf, "docker ps", "docker", args, []string{"DOCKER_CONTEXT=" + dockerContext}) +} + +func appendK3sServerDiagnostics(ctx context.Context, buf *bytes.Buffer, clusterName, dockerContext string) { + nodeNames, err := clusterNodeNames(ctx, clusterName, dockerContext) + if err != nil { + fmt.Fprintf(buf, "cluster node discovery error: %v\n", err) + return + } + if len(nodeNames) == 0 { + fmt.Fprintln(buf, "cluster node discovery: no matching k3d containers found") + return + } + + for _, nodeName := range nodeNames { + appendCommandOutput(ctx, buf, "docker inspect "+nodeName, "docker", []string{ + "inspect", + "--format", + "Status={{.State.Status}} Health={{if .State.Health}}{{.State.Health.Status}}{{else}}{{end}} Restarting={{.State.Restarting}} ExitCode={{.State.ExitCode}} Error={{.State.Error}} Privileged={{.HostConfig.Privileged}} Cgroupns={{.HostConfig.CgroupnsMode}}", + nodeName, + }, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker logs "+nodeName, "docker", []string{"logs", "--tail", "80", nodeName}, []string{"DOCKER_CONTEXT=" + dockerContext}) + } + + for _, nodeName := range nodeNames { + if !strings.Contains(nodeName, "-server-") { + continue + } + kubectl := "KUBECONFIG=/var/lib/rancher/k3s/server/cred/admin.kubeconfig kubectl" + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" ps", "docker", []string{"exec", nodeName, "sh", "-c", "ps auxww"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" sockets", "docker", []string{"exec", nodeName, "sh", "-c", "ss -lntp || netstat -lnt"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" cgroup", "docker", []string{"exec", nodeName, "sh", "-c", "cat /proc/1/cgroup && echo && mount | grep cgroup"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" env", "docker", []string{"exec", nodeName, "sh", "-c", "env | sort"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" entrypoint", "docker", []string{"exec", nodeName, "sh", "-c", "sed -n '1,200p' /bin/k3d-entrypoint.sh"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" entrypoint logs", "docker", []string{"exec", nodeName, "sh", "-c", "for f in /var/log/k3d-entrypoints_*.log; do if [ -f \"$f\" ]; then echo \"== $f ==\"; tail -n 200 \"$f\"; echo; fi; done"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s files", "docker", []string{"exec", nodeName, "sh", "-c", "find /var/lib/rancher/k3s -maxdepth 3 -type f 2>/dev/null | sort"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" k3s logs", "docker", []string{"exec", nodeName, "sh", "-c", "for f in /var/log/k3s.log /var/lib/rancher/k3s/agent/containerd/containerd.log /var/lib/rancher/k3s/server/logs/*; do if [ -f \"$f\" ]; then echo \"== $f ==\"; tail -n 200 \"$f\"; echo; fi; done"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get pods", "docker", []string{"exec", nodeName, "sh", "-c", kubectl + " get pods -A -o wide"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl get events", "docker", []string{"exec", nodeName, "sh", "-c", kubectl + " get events -A --sort-by=.lastTimestamp"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + appendCommandOutput(ctx, buf, "docker exec "+nodeName+" kubectl describe pods", "docker", []string{"exec", nodeName, "sh", "-c", kubectl + " describe pods -A"}, []string{"DOCKER_CONTEXT=" + dockerContext}) + break + } +} + +func clusterNodeNames(ctx context.Context, clusterName, dockerContext string) ([]string, error) { + out, err := runCommand(ctx, "docker", []string{ + "ps", "-a", + "--filter", "name=k3d-" + clusterName, + "--format", "{{.Names}}", + }, []string{"DOCKER_CONTEXT=" + dockerContext}) + if err != nil { + return nil, err + } + var names []string + for line := range strings.SplitSeq(strings.TrimSpace(out), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + names = append(names, line) + } + return names, nil +} + +func appendCommandOutput(ctx context.Context, buf *bytes.Buffer, title, name string, args []string, env []string) { + out, err := runCommand(ctx, name, args, env) + fmt.Fprintf(buf, "== %s ==\n", title) + if err != nil { + fmt.Fprintf(buf, "error: %v\n", err) + } + if strings.TrimSpace(out) == "" { + fmt.Fprintln(buf, "") + } else { + fmt.Fprintf(buf, "%s\n", strings.TrimSpace(out)) + } + fmt.Fprintln(buf) +} + +func runCommand(ctx context.Context, name string, args []string, env []string) (string, error) { + cmd := exec.CommandContext(ctx, name, args...) + if len(env) > 0 { + cmd.Env = append(os.Environ(), env...) + } + out, err := cmd.CombinedOutput() + return string(out), err +} diff --git a/tests/integration_test.go b/tests/integration_test.go index 574251dc5075..9ce1c8d9c128 100644 --- a/tests/integration_test.go +++ b/tests/integration_test.go @@ -15,6 +15,7 @@ func init() { workers.InitDockerWorker() workers.InitDockerContainerWorker() workers.InitRemoteMultiNodeWorker() + workers.InitKubernetesWorker() } else { workers.InitRemoteWorker() } diff --git a/tests/workers/backend.go b/tests/workers/backend.go index 386bf7bd9f6b..e3e8207b7104 100644 --- a/tests/workers/backend.go +++ b/tests/workers/backend.go @@ -1,9 +1,13 @@ package workers import ( + "fmt" + "net" "os" "slices" + "strconv" "strings" + "sync" "github.com/moby/buildkit/util/testutil/integration" ) @@ -11,6 +15,10 @@ import ( type backend struct { builder string context string + registryHost string + registryPorts []int + registryPortIndex int + registryMu sync.Mutex unsupportedFeatures []string } @@ -48,6 +56,45 @@ func (s *backend) ExtraEnv() []string { return nil } +func (s *backend) NewRegistry() (string, func() error, error) { + if s.registryHost == "" || len(s.registryPorts) == 0 { + url, cl, err := integration.NewRegistry("") + if err != nil { + return "", nil, err + } + return s.RewriteRegistryAddress(url), cl, nil + } + + s.registryMu.Lock() + defer s.registryMu.Unlock() + + if s.registryPortIndex >= len(s.registryPorts) { + return "", nil, fmt.Errorf("exhausted kubernetes registry port pool") + } + port := s.registryPorts[s.registryPortIndex] + s.registryPortIndex++ + + _, cl, err := integration.NewRegistryAt("", net.JoinHostPort("0.0.0.0", strconv.Itoa(port))) + if err != nil { + return "", nil, err + } + return net.JoinHostPort(s.registryHost, strconv.Itoa(port)), cl, nil +} + +func (s *backend) RewriteRegistryAddress(in string) string { + if s.registryHost == "" { + return in + } + host, port, err := net.SplitHostPort(in) + if err != nil { + return in + } + if host != "localhost" && host != "127.0.0.1" { + return in + } + return net.JoinHostPort(s.registryHost, port) +} + func (s backend) Supports(feature string) bool { if enabledFeatures := os.Getenv("BUILDKIT_TEST_ENABLE_FEATURES"); enabledFeatures != "" { if slices.Contains(strings.Split(enabledFeatures, ","), feature) { diff --git a/tests/workers/kubernetes.go b/tests/workers/kubernetes.go new file mode 100644 index 000000000000..deeccc151e11 --- /dev/null +++ b/tests/workers/kubernetes.go @@ -0,0 +1,158 @@ +package workers + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + + "github.com/docker/buildx/tests/helpers" + "github.com/moby/buildkit/identity" + "github.com/moby/buildkit/util/testutil/integration" + "github.com/pkg/errors" +) + +func InitKubernetesWorker() { + integration.Register(&kubernetesWorker{ + id: "kubernetes", + }) +} + +type kubernetesWorker struct { + id string + + unsupported []string + + docker integration.Backend + dockerClose func() error + dockerErr error + dockerOnce sync.Once + + k3dName string + k3dConfig string + registry string + registryPorts []int + k3dClose func() error + k3dErr error + k3dOnce sync.Once +} + +func (w *kubernetesWorker) Name() string { + return w.id +} + +func (w *kubernetesWorker) Rootless() bool { + return false +} + +func (w *kubernetesWorker) NetNSDetached() bool { + return false +} + +func (w *kubernetesWorker) New(ctx context.Context, cfg *integration.BackendConfig) (integration.Backend, func() error, error) { + w.dockerOnce.Do(func() { + w.docker, w.dockerClose, w.dockerErr = dockerWorker{id: w.id}.New(ctx, cfg) + }) + if w.dockerErr != nil { + return w.docker, w.dockerClose, w.dockerErr + } + + w.k3dOnce.Do(func() { + w.k3dName, w.k3dConfig, w.k3dClose, w.k3dErr = helpers.NewK3dServer(ctx, cfg, w.docker.DockerAddress()) + if w.k3dErr != nil { + return + } + w.registry, w.k3dErr = helpers.K3dNetworkGateway(ctx, w.k3dName, w.docker.DockerAddress()) + if w.k3dErr != nil { + return + } + w.registryPorts, w.k3dErr = helpers.ReserveK3dRegistryPorts(helpers.K3dRegistryPortCount) + }) + if w.k3dErr != nil { + return nil, w.k3dClose, w.k3dErr + } + + daemonConfig := append([]integration.ConfigUpdater{}, cfg.DaemonConfig...) + daemonConfig = append(daemonConfig, helpers.K3dRegistryConfig(w.registry, w.registryPorts)) + cfgfile, release, err := integration.WriteConfig(daemonConfig) + if err != nil { + return nil, nil, err + } + if release != nil { + defer release() + } + defer os.RemoveAll(filepath.Dir(cfgfile)) + + name := "integration-kubernetes-" + identity.NewID() + cmd := exec.CommandContext(ctx, "buildx", "create", + "--bootstrap", + "--name="+name, + "--buildkitd-config="+cfgfile, + "--driver=kubernetes", + "--driver-opt=image="+helpers.KubernetesBuildkitImage(), + "--driver-opt=timeout=60s", + ) + cmd.Env = append( + os.Environ(), + "BUILDX_CONFIG=/tmp/buildx-"+name, + "DOCKER_CONTEXT="+w.docker.DockerAddress(), + "KUBECONFIG="+w.k3dConfig, + ) + out, err := cmd.CombinedOutput() + if err != nil { + diag := helpers.KubernetesDiagnostics(ctx, w.k3dName, w.docker.DockerAddress()) + return nil, nil, errors.Wrapf(err, "failed to create buildx instance %s with image %s: %s\n%s", name, helpers.KubernetesBuildkitImage(), strings.TrimSpace(string(out)), diag) + } + + cl := func() error { + cmd := exec.CommandContext(context.Background(), "buildx", "rm", "-f", name) + cmd.Env = append( + os.Environ(), + "BUILDX_CONFIG=/tmp/buildx-"+name, + "DOCKER_CONTEXT="+w.docker.DockerAddress(), + "KUBECONFIG="+w.k3dConfig, + ) + return cmd.Run() + } + + return &backend{ + context: w.docker.DockerAddress(), + builder: name, + registryHost: w.registry, + registryPorts: append([]int(nil), w.registryPorts...), + unsupportedFeatures: w.unsupported, + }, cl, nil +} + +func (w *kubernetesWorker) Close() error { + setErr := func(dst *error, err error) { + if err != nil && *dst == nil { + *dst = err + } + } + + var err error + if c := w.k3dClose; c != nil { + setErr(&err, c()) + } + if c := w.dockerClose; c != nil { + setErr(&err, c()) + } + + // reset the worker to be ready to go again + w.docker = nil + w.dockerClose = nil + w.dockerErr = nil + w.dockerOnce = sync.Once{} + w.k3dName = "" + w.k3dConfig = "" + w.registry = "" + w.registryPorts = nil + w.k3dClose = nil + w.k3dErr = nil + w.k3dOnce = sync.Once{} + + return err +} diff --git a/vendor/github.com/moby/buildkit/util/testutil/integration/registry.go b/vendor/github.com/moby/buildkit/util/testutil/integration/registry.go index 0cbc7434266a..e5948d9a7d0e 100644 --- a/vendor/github.com/moby/buildkit/util/testutil/integration/registry.go +++ b/vendor/github.com/moby/buildkit/util/testutil/integration/registry.go @@ -15,6 +15,10 @@ import ( ) func NewRegistry(dir string) (url string, cl func() error, err error) { + return NewRegistryAt(dir, "0.0.0.0:0") +} + +func NewRegistryAt(dir, addr string) (url string, cl func() error, err error) { if err := LookupBinary("registry"); err != nil { return "", nil, err } @@ -42,14 +46,17 @@ func NewRegistry(dir string) (url string, cl func() error, err error) { if !errors.Is(err, os.ErrNotExist) { return "", nil, err } + if addr == "" { + addr = "0.0.0.0:0" + } template := fmt.Sprintf(`version: 0.1 loglevel: debug storage: filesystem: rootdirectory: %s http: - addr: 127.0.0.1:0 -`, filepath.Join(dir, "data")) + addr: %s +`, filepath.Join(dir, "data"), addr) if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(template), 0600); err != nil { return "", nil, err @@ -79,7 +86,7 @@ http: } func detectPort(ctx context.Context, rc io.ReadCloser) (string, error) { - r := regexp.MustCompile(`listening on 127\.0\.0\.1:(\d+)`) + r := regexp.MustCompile(`listening on .*:(\d+)`) s := bufio.NewScanner(rc) found := make(chan struct{}) defer func() { diff --git a/vendor/github.com/moby/buildkit/util/testutil/integration/sandbox.go b/vendor/github.com/moby/buildkit/util/testutil/integration/sandbox.go index b3f770186b77..40d7a6b0f795 100644 --- a/vendor/github.com/moby/buildkit/util/testutil/integration/sandbox.go +++ b/vendor/github.com/moby/buildkit/util/testutil/integration/sandbox.go @@ -60,7 +60,25 @@ func (sb *sandbox) ClearLogs() { } func (sb *sandbox) NewRegistry() (string, error) { - url, cl, err := NewRegistry("") + type registryProvider interface { + NewRegistry() (string, func() error, error) + } + + var ( + url string + cl func() error + err error + ) + if provider, ok := sb.Backend.(registryProvider); ok { + url, cl, err = provider.NewRegistry() + } else { + url, cl, err = NewRegistry("") + if err == nil { + if rewriter, ok := sb.Backend.(interface{ RewriteRegistryAddress(string) string }); ok { + url = rewriter.RewriteRegistryAddress(url) + } + } + } if err != nil { return "", err }