test(e2e): skip the global addon uninstall on disposable clusters (#6780)

Signed-off-by: Alexander Chernov <alexander@chernov.it>

.github/workflows/e2e-reusable.yml

@@ -231,6 +231,10 @@ jobs:
       # both into the e2e pod; entrypoint.sh runs ginkgo with them.
       TEST_SUITES: ${{ matrix.suite }}
       GINKGO_LABELS: ${{ matrix.labels }}
+      # The kind cluster goes away with the runner, so uninstalling the global
+      # addons costs about a minute and buys nothing. Safe because TEST_SUITES
+      # above is a single suite; the helper refuses this for several.
+      E2E_SKIP_GLOBAL_TEARDOWN: "true"
     steps:
       - uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
         with:

docs/contributing/process.md

@@ -91,6 +91,10 @@ run managed tests.
 make test.e2e GINKGO_LABELS='gcp&&!managed'
 ```
 
+`E2E_SKIP_GLOBAL_TEARDOWN=true` leaves the ESO release installed when the suite
+finishes, saving about a minute. Only for a throwaway cluster: it applies to
+whatever your kube context points at. `make test.managed` clears it.
+
 #### Managed Kubernetes e2e tests
 
 There's another suite of e2e tests that integrate with managed Kubernetes offerings.

e2e/Makefile

@@ -8,6 +8,9 @@ DOCKER_BUILD_ARGS     ?=
 export E2E_IMAGE_NAME ?= ghcr.io/external-secrets/external-secrets-e2e
 export GINKGO_LABELS ?= !managed
 export TEST_SUITES ?= provider generator flux argocd
+# Leave the global addons installed on the way out. Only for a cluster you are
+# about to delete. Off by default; CI sets it per leg.
+export E2E_SKIP_GLOBAL_TEARDOWN ?=
 
 export OCI_IMAGE_NAME = ghcr.io/external-secrets/external-secrets
 
@@ -46,6 +49,9 @@ test.run: ## Load prebuilt image tarballs into kind and run the e2e suite
 	kind load image-archive --name="external-secrets" $(E2E_ARTIFACT_DIR)/e2e.tar
 	./run.sh
 
+# Runs against the current kube context, which is not ours to leave dirty.
+# override beats both an inherited and a command-line value.
+test.managed: override E2E_SKIP_GLOBAL_TEARDOWN =
 test.managed: e2e-image ## Run e2e tests against current kube context
 	$(MAKE) -C ../ docker.build \
 		VERSION=$(VERSION) \

e2e/README.md

@@ -21,7 +21,7 @@ scoped per provider, and how to add or enable a provider.
 | `e2e/suites/provider/cases/import.go` | Blank-imports every provider case into the single `provider.test` binary. Providers are told apart at run time by Ginkgo label. |
 | `e2e/matrix.yaml` | Source of truth for the fan-out: one `area` (leg) per provider, with its suite, label filter, secret groups, and trigger paths. |
 | `e2e/matrix.py` | Validates the matrix (`check`), emits the CI matrix JSON (`json`), and prints the per-leg credential plan (`plan`). |
-| `e2e/run.sh` | Host-side launcher. Runs `kubectl run` to start the e2e pod, forwarding `TEST_SUITES`, `GINKGO_LABELS`, and the (scoped) credentials as pod env. |
+| `e2e/run.sh` | Host-side launcher. Runs `kubectl run` to start the e2e pod, forwarding `TEST_SUITES`, `GINKGO_LABELS`, `E2E_SKIP_GLOBAL_TEARDOWN`, and the (scoped) credentials as pod env. |
 | `e2e/entrypoint.sh` | In-pod entry (image `CMD`). Loops over `TEST_SUITES` and runs `ginkgo -label-filter="$GINKGO_LABELS"` against each `<suite>.test`. |
 | `.github/workflows/e2e.yml` | Non-managed e2e. Fans out into per-provider legs. Owns the `e2e-required` gate. |
 | `.github/workflows/e2e-reusable.yml` | The reusable build + matrix-test pipeline that `e2e.yml` calls. |
@@ -157,6 +157,13 @@ make -C e2e matrix.plan
 
 # run a single provider locally (overrides the Makefile defaults)
 make -C e2e test.run TEST_SUITES=provider GINKGO_LABELS="vault && !managed"
+
+# leave the global addons installed, for a cluster you are about to delete.
+# Saves about a minute; the kind legs set it, e2e-managed.yml does not.
+# Refused (stderr) when TEST_SUITES names several suites, and that guard sees
+# only its own process, so two single-suite runs on one cluster still collide.
+make -C e2e test.run TEST_SUITES=provider GINKGO_LABELS="vault && !managed" \
+  E2E_SKIP_GLOBAL_TEARDOWN=true
 ```
 
 ## Adding or enabling a provider

e2e/framework/addon/addon.go

@@ -17,8 +17,11 @@ limitations under the License.
 package addon
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
+	"strconv"
+	"strings"
 
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
@@ -78,6 +81,46 @@ func UninstallGlobalAddons() {
 	}
 }
 
+const skipGlobalTeardownVar = "E2E_SKIP_GLOBAL_TEARDOWN"
+
+// SkipGlobalTeardown reports whether to leave the global addons installed, for a
+// cluster that is about to be discarded. Off unless asked for, and refused when
+// several suites share the cluster, since two of them install the same release.
+func SkipGlobalTeardown() bool {
+	raw, ok := os.LookupEnv(skipGlobalTeardownVar)
+	if !ok || raw == "" {
+		return false
+	}
+	skip, err := strconv.ParseBool(raw)
+	if err != nil {
+		// Failing here would unwind the whole AfterSuite, losing the teardown
+		// and the logs, so fall back to tearing down and say so.
+		teardownLogf("%s is not a boolean (%q), so the teardown will run: %v",
+			skipGlobalTeardownVar, raw, err)
+		return false
+	}
+	if !skip {
+		return false
+	}
+	// Only sees this process. Two separate single-suite runs against one cluster
+	// would still collide.
+	if suites := strings.Fields(os.Getenv("TEST_SUITES")); len(suites) > 1 {
+		teardownLogf("%s ignored: suites %q share one cluster, so the global "+
+			"addons have to come out between them", skipGlobalTeardownVar,
+			strings.Join(suites, " "))
+		return false
+	}
+	teardownLogf("%s set: leaving the global addons installed for the cluster to "+
+		"be discarded with", skipGlobalTeardownVar)
+	return true
+}
+
+// teardownLogf logs to stderr, not log.Logf: ginkgo drops GinkgoWriter output
+// for a passing node without -v, and these lines must survive a green run.
+func teardownLogf(format string, args ...any) {
+	fmt.Fprintf(os.Stderr, format+"\n", args...)
+}
+
 // AssetDir returns the path to the k8s asset directory
 // which holds the helm charts, vault and conjur configuration.
 // It starts at the cwd, and walks its way up to the root.

e2e/run.sh

@@ -85,5 +85,6 @@ kubectl run --rm \
   --env="GRAFANA_TOKEN=${GRAFANA_TOKEN:-}" \
   --env="VERSION=${VERSION}" \
   --env="TEST_SUITES=${TEST_SUITES}" \
+  --env="E2E_SKIP_GLOBAL_TEARDOWN=${E2E_SKIP_GLOBAL_TEARDOWN:-}" \
   --overrides='{ "apiVersion": "v1", "spec":{"serviceAccountName": "external-secrets-e2e"}}' \
   e2e --image=${E2E_IMAGE_NAME}:${VERSION}

e2e/suites/argocd/suite_test.go

@@ -41,18 +41,21 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 var _ = SynchronizedAfterSuite(func() {
 	// noop
 }, func() {
-	_, _, cl := util.NewConfig()
-	By("Deleting any pending generator states")
-	generatorStates := &genv1alpha1.GeneratorStateList{}
-	err := cl.List(GinkgoT().Context(), generatorStates)
-	Expect(err).ToNot(HaveOccurred())
-	for _, generatorState := range generatorStates.Items {
-		err = cl.Delete(GinkgoT().Context(), &generatorState)
+	// The pre-deletion serves only the uninstall, so it is skipped with it.
+	if !addon.SkipGlobalTeardown() {
+		_, _, cl := util.NewConfig()
+		By("Deleting any pending generator states")
+		generatorStates := &genv1alpha1.GeneratorStateList{}
+		err := cl.List(GinkgoT().Context(), generatorStates)
 		Expect(err).ToNot(HaveOccurred())
-	}
+		for _, generatorState := range generatorStates.Items {
+			err = cl.Delete(GinkgoT().Context(), &generatorState)
+			Expect(err).ToNot(HaveOccurred())
+		}
 
-	By("Cleaning up global addons")
-	addon.UninstallGlobalAddons()
+		By("Cleaning up global addons")
+		addon.UninstallGlobalAddons()
+	}
 	if CurrentSpecReport().Failed() {
 		addon.PrintLogs()
 	}

e2e/suites/flux/suite_test.go

@@ -41,23 +41,28 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 var _ = SynchronizedAfterSuite(func() {
 	// noop
 }, func() {
-	cfg := &addon.Config{}
-	cfg.KubeConfig, cfg.KubeClientSet, cfg.CRClient = util.NewConfig()
-	By("Deleting any pending generator states")
-	generatorStates := &genv1alpha1.GeneratorStateList{}
-	err := cfg.CRClient.List(GinkgoT().Context(), generatorStates)
-	Expect(err).ToNot(HaveOccurred())
-	for _, generatorState := range generatorStates.Items {
-		err = cfg.CRClient.Delete(GinkgoT().Context(), &generatorState)
+	// uninstallFlux is gated too: on its own it deletes the flux-system namespace
+	// while helm-controller is still needed to clear the HelmRelease finalizer
+	// that UninstallGlobalAddons clears first, and kubectl waits 168h on that.
+	if !addon.SkipGlobalTeardown() {
+		cfg := &addon.Config{}
+		cfg.KubeConfig, cfg.KubeClientSet, cfg.CRClient = util.NewConfig()
+		By("Deleting any pending generator states")
+		generatorStates := &genv1alpha1.GeneratorStateList{}
+		err := cfg.CRClient.List(GinkgoT().Context(), generatorStates)
 		Expect(err).ToNot(HaveOccurred())
-	}
+		for _, generatorState := range generatorStates.Items {
+			err = cfg.CRClient.Delete(GinkgoT().Context(), &generatorState)
+			Expect(err).ToNot(HaveOccurred())
+		}
 
-	By("Cleaning up global addons")
-	addon.UninstallGlobalAddons()
+		By("Cleaning up global addons")
+		addon.UninstallGlobalAddons()
+		uninstallFlux()
+	}
 	if CurrentSpecReport().Failed() {
 		addon.PrintLogs()
 	}
-	uninstallFlux()
 })
 
 func TestE2E(t *testing.T) {

e2e/suites/generator/suite_test.go

@@ -44,18 +44,21 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 var _ = SynchronizedAfterSuite(func() {
 	// noop
 }, func() {
-	cfg := &addon.Config{}
-	cfg.KubeConfig, cfg.KubeClientSet, cfg.CRClient = util.NewConfig()
-	By("Deleting any pending generator states")
-	generatorStates := &genv1alpha1.GeneratorStateList{}
-	err := cfg.CRClient.List(GinkgoT().Context(), generatorStates)
-	Expect(err).ToNot(HaveOccurred())
-	for _, generatorState := range generatorStates.Items {
-		err = cfg.CRClient.Delete(GinkgoT().Context(), &generatorState)
+	// The pre-deletion serves only the uninstall, so it is skipped with it.
+	if !addon.SkipGlobalTeardown() {
+		cfg := &addon.Config{}
+		cfg.KubeConfig, cfg.KubeClientSet, cfg.CRClient = util.NewConfig()
+		By("Deleting any pending generator states")
+		generatorStates := &genv1alpha1.GeneratorStateList{}
+		err := cfg.CRClient.List(GinkgoT().Context(), generatorStates)
 		Expect(err).ToNot(HaveOccurred())
+		for _, generatorState := range generatorStates.Items {
+			err = cfg.CRClient.Delete(GinkgoT().Context(), &generatorState)
+			Expect(err).ToNot(HaveOccurred())
+		}
+		By("Cleaning up global addons")
+		addon.UninstallGlobalAddons()
 	}
-	By("Cleaning up global addons")
-	addon.UninstallGlobalAddons()
 	if CurrentSpecReport().Failed() {
 		addon.PrintLogs()
 	}

e2e/suites/provider/suite_test.go

@@ -44,29 +44,32 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 var _ = SynchronizedAfterSuite(func() {
 	// noop
 }, func() {
-	cfg := &addon.Config{}
-	cfg.KubeConfig, cfg.KubeClientSet, cfg.CRClient = util.NewConfig()
+	// The pre-deletions serve only the uninstall, so they are skipped with it.
+	if !addon.SkipGlobalTeardown() {
+		cfg := &addon.Config{}
+		cfg.KubeConfig, cfg.KubeClientSet, cfg.CRClient = util.NewConfig()
 
-	By("Deleting any pending generator states")
-	generatorStates := &genv1alpha1.GeneratorStateList{}
-	err := cfg.CRClient.List(GinkgoT().Context(), generatorStates)
-	Expect(err).ToNot(HaveOccurred())
-	for _, generatorState := range generatorStates.Items {
-		err = cfg.CRClient.Delete(GinkgoT().Context(), &generatorState)
+		By("Deleting any pending generator states")
+		generatorStates := &genv1alpha1.GeneratorStateList{}
+		err := cfg.CRClient.List(GinkgoT().Context(), generatorStates)
 		Expect(err).ToNot(HaveOccurred())
-	}
+		for _, generatorState := range generatorStates.Items {
+			err = cfg.CRClient.Delete(GinkgoT().Context(), &generatorState)
+			Expect(err).ToNot(HaveOccurred())
+		}
 
-	By("Deleting all ClusterExternalSecrets")
-	externalSecretsList := &v1.ClusterExternalSecretList{}
-	err = cfg.CRClient.List(GinkgoT().Context(), externalSecretsList)
-	Expect(err).ToNot(HaveOccurred())
-	for _, externalSecret := range externalSecretsList.Items {
-		err = cfg.CRClient.Delete(GinkgoT().Context(), &externalSecret)
+		By("Deleting all ClusterExternalSecrets")
+		externalSecretsList := &v1.ClusterExternalSecretList{}
+		err = cfg.CRClient.List(GinkgoT().Context(), externalSecretsList)
 		Expect(err).ToNot(HaveOccurred())
-	}
+		for _, externalSecret := range externalSecretsList.Items {
+			err = cfg.CRClient.Delete(GinkgoT().Context(), &externalSecret)
+			Expect(err).ToNot(HaveOccurred())
+		}
 
-	By("Cleaning up global addons")
-	addon.UninstallGlobalAddons()
+		By("Cleaning up global addons")
+		addon.UninstallGlobalAddons()
+	}
 	if CurrentSpecReport().Failed() {
 		addon.PrintLogs()
 	}