From 132aa9e6da5c40a202ec5cc8d7fdd10dfda7c078 Mon Sep 17 00:00:00 2001 From: Samuel Gaist Date: Tue, 10 Mar 2026 18:42:08 +0100 Subject: [PATCH 1/4] feat: implement wrapper to retry pack build in tests Miniconda tests sometimes failed during the pack build phase. It's a transitive error that is related to Docker that usually does not happen on a second run. The wrapper will retry the build immediately. --- integration/helpers.go | 34 +++++++++++++++++++ .../installers/miniconda_default_test.go | 20 +++++++---- .../installers/miniconda_offline_test.go | 17 +++++++--- .../installers/miniconda_reuse_layer_test.go | 34 ++++++++++++------- .../installers/miniconda_versions_test.go | 18 ++++++---- 5 files changed, 94 insertions(+), 29 deletions(-) diff --git a/integration/helpers.go b/integration/helpers.go index a49bd28..3acb0db 100644 --- a/integration/helpers.go +++ b/integration/helpers.go @@ -5,6 +5,13 @@ package integration_helpers +import ( + "fmt" + "testing" + + "github.com/paketo-buildpacks/occam" +) + type Buildpack struct { ID string Name string @@ -58,3 +65,30 @@ func DependenciesForId(dependencies []Dependency, id string) []Dependency { return output } + +func NewRetryBuild(t *testing.T, retry int) RetryBuild { + return RetryBuild{t, retry} +} + +type RetryBuild struct { + t *testing.T + retry int +} + +func (r *RetryBuild) Build(packBuild occam.PackBuild, name string, source string) (occam.Image, fmt.Stringer, error) { + var image occam.Image + var logs fmt.Stringer + var err error + + for i := range r.retry { + image, logs, err = packBuild.Execute(name, source) + if err == nil { + return image, logs, err + } else { + r.t.Logf("Build failed: %v\n", err) + r.t.Logf("Retry %v\n", i) + } + } + + return image, logs, err +} diff --git a/integration/installers/miniconda_default_test.go b/integration/installers/miniconda_default_test.go index 842fbd0..ce8b051 100644 --- a/integration/installers/miniconda_default_test.go +++ b/integration/installers/miniconda_default_test.go @@ -16,6 +16,8 @@ import ( . "github.com/onsi/gomega" . "github.com/paketo-buildpacks/occam/matchers" + + integration_helpers "github.com/paketo-buildpacks/python-installers/integration" ) func minicondaTestDefault(t *testing.T, context spec.G, it spec.S) { @@ -23,6 +25,8 @@ func minicondaTestDefault(t *testing.T, context spec.G, it spec.S) { Expect = NewWithT(t).Expect Eventually = NewWithT(t).Eventually + retryBuild = integration_helpers.NewRetryBuild(t, 3) + pack occam.Pack docker occam.Docker ) @@ -64,13 +68,15 @@ func minicondaTestDefault(t *testing.T, context spec.G, it spec.S) { err error ) - image, logs, err = pack.WithNoColor().Build. + image, logs, err = retryBuild.Build(pack.WithNoColor().Build. WithPullPolicy("never"). WithBuildpacks( settings.Buildpacks.PythonInstallers.Online, settings.Buildpacks.BuildPlan.Online, - ). - Execute(name, source) + ), + name, + source, + ) Expect(err).ToNot(HaveOccurred(), logs.String) Expect(logs).To(ContainLines( @@ -134,7 +140,7 @@ func minicondaTestDefault(t *testing.T, context spec.G, it spec.S) { source, err = occam.Source(filepath.Join("testdata", "conda", "miniconda_app")) Expect(err).NotTo(HaveOccurred()) - image, logs, err = pack.WithNoColor().Build. + image, logs, err = retryBuild.Build(pack.WithNoColor().Build. WithPullPolicy("never"). WithBuildpacks( settings.Buildpacks.PythonInstallers.Online, @@ -143,8 +149,10 @@ func minicondaTestDefault(t *testing.T, context spec.G, it spec.S) { WithEnv(map[string]string{ "BP_LOG_LEVEL": "DEBUG", }). - WithSBOMOutputDir(sbomDir). - Execute(name, source) + WithSBOMOutputDir(sbomDir), + name, + source, + ) Expect(err).ToNot(HaveOccurred(), logs.String) container, err = docker.Container.Run. diff --git a/integration/installers/miniconda_offline_test.go b/integration/installers/miniconda_offline_test.go index ecf7076..0930e2a 100644 --- a/integration/installers/miniconda_offline_test.go +++ b/integration/installers/miniconda_offline_test.go @@ -14,14 +14,19 @@ import ( "github.com/sclevine/spec" . "github.com/onsi/gomega" + + integration_helpers "github.com/paketo-buildpacks/python-installers/integration" ) func minicondaTestOffline(t *testing.T, context spec.G, it spec.S) { var ( Expect = NewWithT(t).Expect Eventually = NewWithT(t).Eventually - pack occam.Pack - docker occam.Docker + + retryBuild = integration_helpers.NewRetryBuild(t, 3) + + pack occam.Pack + docker occam.Docker ) it.Before(func() { @@ -58,14 +63,16 @@ func minicondaTestOffline(t *testing.T, context spec.G, it spec.S) { var err error var logs fmt.Stringer - image, logs, err = pack.WithNoColor().Build. + image, logs, err = retryBuild.Build(pack.WithNoColor().Build. WithPullPolicy("never"). WithBuildpacks( settings.Buildpacks.PythonInstallers.Offline, settings.Buildpacks.BuildPlan.Online, ). - WithNetwork("none"). - Execute(name, source) + WithNetwork("none"), + name, + source, + ) Expect(err).NotTo(HaveOccurred(), logs.String()) diff --git a/integration/installers/miniconda_reuse_layer_test.go b/integration/installers/miniconda_reuse_layer_test.go index d39d55c..9e52d60 100644 --- a/integration/installers/miniconda_reuse_layer_test.go +++ b/integration/installers/miniconda_reuse_layer_test.go @@ -25,6 +25,8 @@ func minicondaTestLayerReuse(t *testing.T, context spec.G, it spec.S) { Expect = NewWithT(t).Expect Eventually = NewWithT(t).Eventually + retryBuild = integration_helpers.NewRetryBuild(t, 3) + pack occam.Pack docker occam.Docker @@ -77,13 +79,15 @@ func minicondaTestLayerReuse(t *testing.T, context spec.G, it spec.S) { secondContainer occam.Container ) - firstImage, logs, err = pack.WithNoColor().Build. + firstImage, logs, err = retryBuild.Build(pack.WithNoColor().Build. WithPullPolicy("never"). WithBuildpacks( settings.Buildpacks.PythonInstallers.Online, settings.Buildpacks.BuildPlan.Online, - ). - Execute(name, source) + ), + name, + source, + ) Expect(err).ToNot(HaveOccurred(), logs.String) imageIDs[firstImage.ID] = struct{}{} @@ -94,13 +98,15 @@ func minicondaTestLayerReuse(t *testing.T, context spec.G, it spec.S) { containerIDs[firstContainer.ID] = struct{}{} - secondImage, logs, err = pack.WithNoColor().Build. + secondImage, logs, err = retryBuild.Build(pack.WithNoColor().Build. WithPullPolicy("never"). WithBuildpacks( settings.Buildpacks.PythonInstallers.Online, settings.Buildpacks.BuildPlan.Online, - ). - Execute(name, source) + ), + name, + source, + ) Expect(err).ToNot(HaveOccurred(), logs.String) imageIDs[secondImage.ID] = struct{}{} @@ -151,14 +157,16 @@ func minicondaTestLayerReuse(t *testing.T, context spec.G, it spec.S) { dependencies := integration_helpers.DependenciesForId(buildpackInfo.Metadata.Dependencies, "miniconda3") - firstImage, logs, err = pack.WithNoColor().Build. + firstImage, logs, err = retryBuild.Build(pack.WithNoColor().Build. WithPullPolicy("never"). WithBuildpacks( settings.Buildpacks.PythonInstallers.Online, settings.Buildpacks.BuildPlan.Online, ). - WithEnv(map[string]string{"BP_MINICONDA_VERSION": dependencies[0].Version}). - Execute(name, source) + WithEnv(map[string]string{"BP_MINICONDA_VERSION": dependencies[0].Version}), + name, + source, + ) Expect(err).ToNot(HaveOccurred(), logs.String) imageIDs[firstImage.ID] = struct{}{} @@ -169,14 +177,16 @@ func minicondaTestLayerReuse(t *testing.T, context spec.G, it spec.S) { containerIDs[firstContainer.ID] = struct{}{} - secondImage, logs, err = pack.WithNoColor().Build. + secondImage, logs, err = retryBuild.Build(pack.WithNoColor().Build. WithPullPolicy("never"). WithBuildpacks( settings.Buildpacks.PythonInstallers.Online, settings.Buildpacks.BuildPlan.Online, ). - WithEnv(map[string]string{"BP_MINICONDA_VERSION": dependencies[2].Version}). - Execute(name, source) + WithEnv(map[string]string{"BP_MINICONDA_VERSION": dependencies[2].Version}), + name, + source, + ) Expect(err).ToNot(HaveOccurred(), logs.String) imageIDs[secondImage.ID] = struct{}{} diff --git a/integration/installers/miniconda_versions_test.go b/integration/installers/miniconda_versions_test.go index 2a1f6d8..f719843 100644 --- a/integration/installers/miniconda_versions_test.go +++ b/integration/installers/miniconda_versions_test.go @@ -25,6 +25,8 @@ func minicondaTestVersions(t *testing.T, context spec.G, it spec.S) { Expect = NewWithT(t).Expect Eventually = NewWithT(t).Eventually + retryBuild = integration_helpers.NewRetryBuild(t, 3) + pack occam.Pack docker occam.Docker ) @@ -76,14 +78,16 @@ func minicondaTestVersions(t *testing.T, context spec.G, it spec.S) { Expect(firstMinicondaVersion).NotTo(Equal(secondMinicondaVersion)) - firstImage, firstLogs, err := pack.WithNoColor().Build. + firstImage, firstLogs, err := retryBuild.Build(pack.WithNoColor().Build. WithPullPolicy("never"). WithBuildpacks( settings.Buildpacks.PythonInstallers.Online, settings.Buildpacks.BuildPlan.Online, ). - WithEnv(map[string]string{miniconda.EnvVersion: firstMinicondaVersion}). - Execute(name, source) + WithEnv(map[string]string{miniconda.EnvVersion: firstMinicondaVersion}), + name, + source, + ) Expect(err).ToNot(HaveOccurred(), firstLogs.String) imagesMap[firstImage.ID] = nil @@ -105,14 +109,16 @@ func minicondaTestVersions(t *testing.T, context spec.G, it spec.S) { return cLogs.String() }).Should(ContainSubstring(fmt.Sprintf(`conda %s`, firstMinicondaVersion))) - secondImage, secondLogs, err := pack.WithNoColor().Build. + secondImage, secondLogs, err := retryBuild.Build(pack.WithNoColor().Build. WithPullPolicy("never"). WithBuildpacks( settings.Buildpacks.PythonInstallers.Online, settings.Buildpacks.BuildPlan.Online, ). - WithEnv(map[string]string{miniconda.EnvVersion: secondMinicondaVersion}). - Execute(name, source) + WithEnv(map[string]string{miniconda.EnvVersion: secondMinicondaVersion}), + name, + source, + ) Expect(err).ToNot(HaveOccurred(), secondLogs.String) imagesMap[secondImage.ID] = nil From c3e52173b4c974d3a918286a2d448e25654ee468 Mon Sep 17 00:00:00 2001 From: Samuel Gaist Date: Tue, 10 Mar 2026 21:28:15 +0100 Subject: [PATCH 2/4] feat(BuildRetry): accumulate errors when a retry occurs That way, if all the retry fails, the developer will have the full picture of failures, not just the last one. --- integration/helpers.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/integration/helpers.go b/integration/helpers.go index 3acb0db..409c1a6 100644 --- a/integration/helpers.go +++ b/integration/helpers.go @@ -6,6 +6,7 @@ package integration_helpers import ( + "errors" "fmt" "testing" @@ -78,17 +79,19 @@ type RetryBuild struct { func (r *RetryBuild) Build(packBuild occam.PackBuild, name string, source string) (occam.Image, fmt.Stringer, error) { var image occam.Image var logs fmt.Stringer - var err error + var errs error for i := range r.retry { + var err error image, logs, err = packBuild.Execute(name, source) if err == nil { return image, logs, err } else { + errs = errors.Join(errs, err) r.t.Logf("Build failed: %v\n", err) r.t.Logf("Retry %v\n", i) } } - return image, logs, err + return image, logs, errs } From 1400f3b6d6ca2485216522d10845eeee084f94de Mon Sep 17 00:00:00 2001 From: Samuel Gaist Date: Tue, 10 Mar 2026 22:21:07 +0100 Subject: [PATCH 3/4] refactor: improve retry handling - loop runs retry + 1 time (0 is initial attempt, rest is retry) - Show retry message at the start of the loop starting the first retry round --- integration/helpers.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/integration/helpers.go b/integration/helpers.go index 409c1a6..f83d346 100644 --- a/integration/helpers.go +++ b/integration/helpers.go @@ -81,7 +81,10 @@ func (r *RetryBuild) Build(packBuild occam.PackBuild, name string, source string var logs fmt.Stringer var errs error - for i := range r.retry { + for i := range r.retry + 1 { + if i > 0 { + r.t.Logf("Retry %v\n", i) + } var err error image, logs, err = packBuild.Execute(name, source) if err == nil { @@ -89,7 +92,6 @@ func (r *RetryBuild) Build(packBuild occam.PackBuild, name string, source string } else { errs = errors.Join(errs, err) r.t.Logf("Build failed: %v\n", err) - r.t.Logf("Retry %v\n", i) } } From 1a78b3d3a0de9559209507918e137f75ceac8d05 Mon Sep 17 00:00:00 2001 From: Samuel Gaist Date: Tue, 10 Mar 2026 22:28:54 +0100 Subject: [PATCH 4/4] chore: rename Build to Execute This matches more precisely with the logic of PackBuild. --- integration/helpers.go | 2 +- integration/installers/miniconda_default_test.go | 4 ++-- integration/installers/miniconda_offline_test.go | 2 +- integration/installers/miniconda_reuse_layer_test.go | 8 ++++---- integration/installers/miniconda_versions_test.go | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/integration/helpers.go b/integration/helpers.go index f83d346..ef013df 100644 --- a/integration/helpers.go +++ b/integration/helpers.go @@ -76,7 +76,7 @@ type RetryBuild struct { retry int } -func (r *RetryBuild) Build(packBuild occam.PackBuild, name string, source string) (occam.Image, fmt.Stringer, error) { +func (r *RetryBuild) Execute(packBuild occam.PackBuild, name string, source string) (occam.Image, fmt.Stringer, error) { var image occam.Image var logs fmt.Stringer var errs error diff --git a/integration/installers/miniconda_default_test.go b/integration/installers/miniconda_default_test.go index ce8b051..8504675 100644 --- a/integration/installers/miniconda_default_test.go +++ b/integration/installers/miniconda_default_test.go @@ -68,7 +68,7 @@ func minicondaTestDefault(t *testing.T, context spec.G, it spec.S) { err error ) - image, logs, err = retryBuild.Build(pack.WithNoColor().Build. + image, logs, err = retryBuild.Execute(pack.WithNoColor().Build. WithPullPolicy("never"). WithBuildpacks( settings.Buildpacks.PythonInstallers.Online, @@ -140,7 +140,7 @@ func minicondaTestDefault(t *testing.T, context spec.G, it spec.S) { source, err = occam.Source(filepath.Join("testdata", "conda", "miniconda_app")) Expect(err).NotTo(HaveOccurred()) - image, logs, err = retryBuild.Build(pack.WithNoColor().Build. + image, logs, err = retryBuild.Execute(pack.WithNoColor().Build. WithPullPolicy("never"). WithBuildpacks( settings.Buildpacks.PythonInstallers.Online, diff --git a/integration/installers/miniconda_offline_test.go b/integration/installers/miniconda_offline_test.go index 0930e2a..cafe6cf 100644 --- a/integration/installers/miniconda_offline_test.go +++ b/integration/installers/miniconda_offline_test.go @@ -63,7 +63,7 @@ func minicondaTestOffline(t *testing.T, context spec.G, it spec.S) { var err error var logs fmt.Stringer - image, logs, err = retryBuild.Build(pack.WithNoColor().Build. + image, logs, err = retryBuild.Execute(pack.WithNoColor().Build. WithPullPolicy("never"). WithBuildpacks( settings.Buildpacks.PythonInstallers.Offline, diff --git a/integration/installers/miniconda_reuse_layer_test.go b/integration/installers/miniconda_reuse_layer_test.go index 9e52d60..62c65a4 100644 --- a/integration/installers/miniconda_reuse_layer_test.go +++ b/integration/installers/miniconda_reuse_layer_test.go @@ -79,7 +79,7 @@ func minicondaTestLayerReuse(t *testing.T, context spec.G, it spec.S) { secondContainer occam.Container ) - firstImage, logs, err = retryBuild.Build(pack.WithNoColor().Build. + firstImage, logs, err = retryBuild.Execute(pack.WithNoColor().Build. WithPullPolicy("never"). WithBuildpacks( settings.Buildpacks.PythonInstallers.Online, @@ -98,7 +98,7 @@ func minicondaTestLayerReuse(t *testing.T, context spec.G, it spec.S) { containerIDs[firstContainer.ID] = struct{}{} - secondImage, logs, err = retryBuild.Build(pack.WithNoColor().Build. + secondImage, logs, err = retryBuild.Execute(pack.WithNoColor().Build. WithPullPolicy("never"). WithBuildpacks( settings.Buildpacks.PythonInstallers.Online, @@ -157,7 +157,7 @@ func minicondaTestLayerReuse(t *testing.T, context spec.G, it spec.S) { dependencies := integration_helpers.DependenciesForId(buildpackInfo.Metadata.Dependencies, "miniconda3") - firstImage, logs, err = retryBuild.Build(pack.WithNoColor().Build. + firstImage, logs, err = retryBuild.Execute(pack.WithNoColor().Build. WithPullPolicy("never"). WithBuildpacks( settings.Buildpacks.PythonInstallers.Online, @@ -177,7 +177,7 @@ func minicondaTestLayerReuse(t *testing.T, context spec.G, it spec.S) { containerIDs[firstContainer.ID] = struct{}{} - secondImage, logs, err = retryBuild.Build(pack.WithNoColor().Build. + secondImage, logs, err = retryBuild.Execute(pack.WithNoColor().Build. WithPullPolicy("never"). WithBuildpacks( settings.Buildpacks.PythonInstallers.Online, diff --git a/integration/installers/miniconda_versions_test.go b/integration/installers/miniconda_versions_test.go index f719843..f4b0ed8 100644 --- a/integration/installers/miniconda_versions_test.go +++ b/integration/installers/miniconda_versions_test.go @@ -78,7 +78,7 @@ func minicondaTestVersions(t *testing.T, context spec.G, it spec.S) { Expect(firstMinicondaVersion).NotTo(Equal(secondMinicondaVersion)) - firstImage, firstLogs, err := retryBuild.Build(pack.WithNoColor().Build. + firstImage, firstLogs, err := retryBuild.Execute(pack.WithNoColor().Build. WithPullPolicy("never"). WithBuildpacks( settings.Buildpacks.PythonInstallers.Online, @@ -109,7 +109,7 @@ func minicondaTestVersions(t *testing.T, context spec.G, it spec.S) { return cLogs.String() }).Should(ContainSubstring(fmt.Sprintf(`conda %s`, firstMinicondaVersion))) - secondImage, secondLogs, err := retryBuild.Build(pack.WithNoColor().Build. + secondImage, secondLogs, err := retryBuild.Execute(pack.WithNoColor().Build. WithPullPolicy("never"). WithBuildpacks( settings.Buildpacks.PythonInstallers.Online,