From 226ef3d8ec8d9f97867ce6bd8cb4085b7a20efa8 Mon Sep 17 00:00:00 2001 From: Moritz Clasmeier Date: Tue, 27 Oct 2026 14:23:50 +0100 Subject: [PATCH 01/16] Discover namespaces from cluster for teardown instead of relying on manifest Teardown no longer loads the on-cluster roxie manifest or assembles a deploy config. Instead, it discovers Central and SecuredCluster CRs directly from the cluster using `kubectl get -A`, and add-on teardown discovers Helm releases by prefix. Parallelize waiting for PVCs and other resources on teardown --- cmd/teardown.go | 13 +- internal/deployer/addons.go | 31 ++--- internal/deployer/deployer.go | 217 +++++++++++++++++++++------------- internal/k8s/resource.go | 41 +++++++ 4 files changed, 197 insertions(+), 105 deletions(-) diff --git a/cmd/teardown.go b/cmd/teardown.go index 37f01b94..b5f69cf0 100644 --- a/cmd/teardown.go +++ b/cmd/teardown.go @@ -2,7 +2,6 @@ package main import ( "context" - "errors" "fmt" "time" @@ -10,7 +9,6 @@ import ( "github.com/stackrox/roxie/internal/component" "github.com/stackrox/roxie/internal/deployer" "github.com/stackrox/roxie/internal/env" - "github.com/stackrox/roxie/internal/k8s" log "github.com/stackrox/roxie/internal/logger" "github.com/stackrox/roxie/internal/manifest" ) @@ -60,16 +58,7 @@ func runTeardown(cmd *cobra.Command, args []string) error { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() - var clusterConfig *deployer.Config - clusterManifest, err := manifest.LoadManifestSecret(ctx) - if errors.Is(err, k8s.ErrResourceNotFound) { - log.Infof("No roxie manifest found in cluster, proceeding without it") - } else if err != nil { - log.Warningf("Failed to load manifest from cluster: %v", err) - } else { - clusterConfig = &clusterManifest.Config - } - + clusterConfig := new(deployer.DefaultConfig()) deploySettings, err := assembleConfigForCommand(clusterConfig, deploySettingsFromArgs, skipUserConfig) if err != nil { return err diff --git a/internal/deployer/addons.go b/internal/deployer/addons.go index 8123d8f0..5f2adce1 100644 --- a/internal/deployer/addons.go +++ b/internal/deployer/addons.go @@ -8,6 +8,7 @@ import ( "regexp" "slices" + "github.com/stackrox/roxie/internal/helm" log "github.com/stackrox/roxie/internal/logger" ) @@ -52,24 +53,26 @@ func (d *Deployer) deployAddOns(ctx context.Context, addOns []AddOn) error { return nil } -func (d *Deployer) teardownAddOns(ctx context.Context, addOns []AddOn) { - if len(addOns) == 0 { - return +// teardownDiscoveredAddOns uninstalls add-on Helm releases in the given +// namespace. Add-ons are lifecycle-aligned with Central, so the caller +// passes the Central namespace. +func (d *Deployer) teardownDiscoveredAddOns(ctx context.Context, namespace string) error { + releases, err := helm.ListByPrefix(ctx, addOnReleasePrefix, namespace) + if err != nil { + return fmt.Errorf("listing add-on Helm releases in namespace %s: %w", namespace, err) + } + if len(releases) == 0 { + return nil } - log.Infof("Tearing down %d add-on(s)...", len(addOns)) - - for _, addon := range addOns { - if err := addon.Teardown(ctx); err != nil { - if !addon.IsOptional() { - log.Errorf("Failed to tear down non-optional add-on %q: %v", addon.Name(), err) - } else { - log.Warningf("Failed to tear down optional add-on %q: %v", addon.Name(), err) - } - } else { - log.Successf("Add-on %q torn down", addon.Name()) + log.Infof("Tearing down %d add-on(s)...", len(releases)) + for _, release := range releases { + if err := helm.Uninstall(ctx, release, namespace); err != nil { + return fmt.Errorf("tearing down add-on release %q: %w", release, err) } + log.Successf("Add-on release %q torn down", release) } + return nil } // ResolveEnabledAddOns returns the enabled add-ons sorted by descending priority (name for ties at zero). diff --git a/internal/deployer/deployer.go b/internal/deployer/deployer.go index e5b75310..98fc21d4 100644 --- a/internal/deployer/deployer.go +++ b/internal/deployer/deployer.go @@ -3,10 +3,10 @@ package deployer import ( "context" "crypto/rand" + "errors" "fmt" "os" "os/exec" - "slices" "strconv" "strings" "sync" @@ -14,6 +14,7 @@ import ( "time" "github.com/fatih/color" + "golang.org/x/sync/errgroup" "github.com/stackrox/roxie/internal/component" "github.com/stackrox/roxie/internal/containerrt" @@ -89,11 +90,11 @@ func (d *Deployer) deleteResources(ctx context.Context, namespace string, resour } // Expects that reconciliation for the RHACS operator is paused. -func (d *Deployer) deleteCentralResources(ctx context.Context) error { +func (d *Deployer) deleteCentralResources(ctx context.Context, namespace string) error { log.Info("Deleting Central resources") crExists := true - if _, err := k8s.RetrieveResourceFromCluster(ctx, d.config.Central.Namespace, "central", "stackrox-central-services"); err != nil { + if _, err := k8s.RetrieveResourceFromCluster(ctx, namespace, "central", "stackrox-central-services"); err != nil { if !k8s.IsResourceNotFound(err) { return fmt.Errorf("retrieving Central CR: %w", err) } @@ -102,12 +103,12 @@ func (d *Deployer) deleteCentralResources(ctx context.Context) error { if crExists { log.Info("Removing any pause-reconcile annotation from Central") - if err := d.removePauseReconcileAnnotation(ctx, "central", "stackrox-central-services", d.config.Central.Namespace); err != nil { + if err := d.removePauseReconcileAnnotation(ctx, "central", "stackrox-central-services", namespace); err != nil { return err } log.Debug("Removed any pause-reconcile annotation from Central") - err := d.deleteResource(ctx, d.config.Central.Namespace, "central", "stackrox-central-services", "--wait") + err := d.deleteResource(ctx, namespace, "central", "stackrox-central-services", "--wait") if err != nil { return err } @@ -116,6 +117,7 @@ func (d *Deployer) deleteCentralResources(ctx context.Context) error { log.Info("Deletion of Central resources requested, but Central CR is not present anymore") } + var group errgroup.Group for _, resource := range []ResourceToDelete{ {Name: "central-db", Kind: "pvc"}, {Name: "central-db-backup", Kind: "pvc"}, @@ -127,36 +129,42 @@ func (d *Deployer) deleteCentralResources(ctx context.Context) error { {Name: "scanner-db-password", Kind: "secret", OwnerName: centralCrName}, {Name: "stackrox-central-helm", Kind: "configmap"}, } { - log.Dimf("Attempting to delete %s/%s", resource.Kind, resource.Name) - if resource.OwnerName != "" { - // Avoid deletion if the resource does not have the expected owner. - // (e.g. in case central and secured cluster are deployed into the same namespace). - obj, err := k8s.RetrieveResourceFromCluster(ctx, d.config.Central.Namespace, resource.Kind, resource.Name) - if err != nil { - if !k8s.IsResourceNotFound(err) { - log.Warningf("Failed to retrieve %s/%s for owner checking: %v. Skipping deletion. Deployment might be affected.", resource.Kind, resource.Name, err) + group.Go(func() error { + log.Dimf("%s: Attempting to delete %s/%s", namespace, resource.Kind, resource.Name) + if resource.OwnerName != "" { + // Avoid deletion if the resource does not have the expected owner. + // (e.g. in case central and secured cluster are deployed into the same namespace). + obj, err := k8s.RetrieveResourceFromCluster(ctx, namespace, resource.Kind, resource.Name) + if err != nil { + if !k8s.IsResourceNotFound(err) { + log.Warningf("%s: Failed to retrieve %s/%s for owner checking: %v. Skipping deletion. Deployment might be affected.", namespace, resource.Kind, resource.Name, err) + return fmt.Errorf("retrieving resource %s/%s from namespace %s: %w", resource.Kind, resource.Name, namespace, err) + } + return nil // resource not found + } + if k8s.ResourceNotOwnedByName(obj, resource.OwnerName) { + log.Dimf("%s: Skipping deletion of %s/%s: not owned by %s", namespace, resource.Kind, resource.Name, resource.OwnerName) + return nil } - continue } - if k8s.ResourceNotOwnedByName(obj, resource.OwnerName) { - log.Dimf("Skipping deletion of %s/%s: not owned by %s", resource.Kind, resource.Name, resource.OwnerName) - continue + + if err := d.deleteResource(ctx, namespace, resource.Kind, resource.Name); err != nil { + return fmt.Errorf("failed to delete %s/%s in namepace %s: %w", resource.Kind, resource.Name, namespace, err) } - } - if err := d.deleteResource(ctx, d.config.Central.Namespace, resource.Kind, resource.Name); err != nil { - return fmt.Errorf("failed to delete %s/%s: %w", resource.Kind, resource.Name, err) - } + log.Dimf("%s: Deleted %s/%s", namespace, resource.Kind, resource.Name) + return nil + }) } - return nil + return group.Wait() } -func (d *Deployer) deleteSecuredClusterResources(ctx context.Context) error { +func (d *Deployer) deleteSecuredClusterResources(ctx context.Context, namespace string) error { log.Info("Deleting SecuredCluster resources") crExists := true - if _, err := k8s.RetrieveResourceFromCluster(ctx, d.config.SecuredCluster.Namespace, "securedcluster", "stackrox-secured-cluster-services"); err != nil { + if _, err := k8s.RetrieveResourceFromCluster(ctx, namespace, "securedcluster", securedClusterCrName); err != nil { if !k8s.IsResourceNotFound(err) { return fmt.Errorf("retrieving SecuredCluster CR: %w", err) } @@ -165,12 +173,12 @@ func (d *Deployer) deleteSecuredClusterResources(ctx context.Context) error { if crExists { log.Info("Removing any pause-reconcile annotation from SecuredCluster") - if err := d.removePauseReconcileAnnotation(ctx, "securedcluster", "stackrox-secured-cluster-services", d.config.SecuredCluster.Namespace); err != nil { + if err := d.removePauseReconcileAnnotation(ctx, "securedcluster", securedClusterCrName, namespace); err != nil { return err } log.Debug("Removed any pause-reconcile annotation from SecuredCluster") - err := d.deleteResource(ctx, d.config.SecuredCluster.Namespace, "securedcluster", "stackrox-secured-cluster-services", "--wait") + err := d.deleteResource(ctx, namespace, "securedcluster", securedClusterCrName, "--wait") if err != nil { return err } @@ -180,6 +188,7 @@ func (d *Deployer) deleteSecuredClusterResources(ctx context.Context) error { } // Delete resources, which are treated special. + group, ctx := errgroup.WithContext(ctx) for _, resource := range []ResourceToDelete{ {Name: "cluster-registration-secret", Kind: "secret"}, // scanner-v4-db is not managed by the operator's ReconcilePVCExtension, so its ownerReference @@ -190,28 +199,35 @@ func (d *Deployer) deleteSecuredClusterResources(ctx context.Context) error { // when both are deployed into the same namespace. {Name: "scanner-db-password", Kind: "secret", OwnerName: securedClusterCrName}, } { - log.Dimf("Attempting to delete %s/%s", resource.Kind, resource.Name) - if resource.OwnerName != "" { - // Avoid deletion if the resource does not have the expected owner. - // (e.g. in case central and secured cluster are deployed into the same namespace). - obj, err := k8s.RetrieveResourceFromCluster(ctx, d.config.SecuredCluster.Namespace, resource.Kind, resource.Name) - if err != nil { - if !k8s.IsResourceNotFound(err) { - log.Warningf("Failed to retrieve %s/%s for owner checking: %v. Skipping deletion. Deployment might be affected.", resource.Kind, resource.Name, err) + group.Go(func() error { + log.Dimf("%s: Attempting to delete %s/%s", namespace, resource.Kind, resource.Name) + if resource.OwnerName != "" { + // Avoid deletion if the resource does not have the expected owner. + // (e.g. in case central and secured cluster are deployed into the same namespace). + obj, err := k8s.RetrieveResourceFromCluster(ctx, namespace, resource.Kind, resource.Name) + if err != nil { + if !k8s.IsResourceNotFound(err) { + log.Warningf("%s: Failed to retrieve %s/%s for owner checking: %v. Skipping deletion. Deployment might be affected.", namespace, resource.Kind, resource.Name, err) + return fmt.Errorf("retrieving resource %s/%s from namespace %s: %w", resource.Kind, resource.Name, namespace, err) + } + return nil // resource not found + } + if k8s.ResourceNotOwnedByName(obj, resource.OwnerName) { + log.Dimf("%s: Skipping deletion of %s/%s: not owned by %s", namespace, resource.Kind, resource.Name, resource.OwnerName) + return nil } - continue } - if k8s.ResourceNotOwnedByName(obj, resource.OwnerName) { - log.Dimf("Skipping deletion of %s/%s: not owned by %s", resource.Kind, resource.Name, resource.OwnerName) - continue + + if err := d.deleteResource(ctx, namespace, resource.Kind, resource.Name); err != nil { + return fmt.Errorf("failed to delete %s/%s in namespace %s: %w", resource.Kind, resource.Name, namespace, err) } - } - if err := d.deleteResource(ctx, d.config.SecuredCluster.Namespace, resource.Kind, resource.Name); err != nil { - return fmt.Errorf("failed to delete %s/%s: %w", resource.Kind, resource.Name, err) - } + + log.Dimf("%s: Deleted %s/%s", namespace, resource.Kind, resource.Name) + return nil + }) } - return nil + return group.Wait() } func (d *Deployer) SetConfig(config Config) { @@ -420,7 +436,7 @@ func (d *Deployer) deployCentral(ctx context.Context) error { log.Infof("Central resource profile: %s", d.config.Central.ResourceProfile) if d.namespaceExists(d.config.Central.Namespace) { log.Info("Existing Central deployment found, tearing down...") - if err := d.teardownCentral(ctx); err != nil { + if err := d.teardownCentralWithAddOns(ctx); err != nil { log.Warningf("Error during teardown: %v", err) } } @@ -455,27 +471,17 @@ func (d *Deployer) deploySecuredCluster(ctx context.Context) error { func (d *Deployer) Teardown(ctx context.Context, components component.Component) error { log.Infof("Starting teardown of %s", components) - if components.IncludesAddOns() { - enabledAddOns, err := d.ResolveEnabledAddOns() - if err != nil { - return fmt.Errorf("resolving enabled add-ons: %w", err) - } - slices.Reverse(enabledAddOns) // Tearing down in opposite order as deploying. - d.teardownAddOns(ctx, enabledAddOns) - } - switch components { case component.Central: - return d.teardownCentral(ctx) + return d.teardownCentralWithAddOns(ctx) + case component.AddOns: + return d.teardownAddOns(ctx) case component.SecuredCluster: return d.teardownSecuredCluster(ctx) case component.Operator: return d.teardownOperator(ctx) case component.Both, component.All: - // Tear down components in parallel for better performance var wg sync.WaitGroup - - // Always tear down central and sensor wg.Add(2) go func() { @@ -487,7 +493,7 @@ func (d *Deployer) Teardown(ctx context.Context, components component.Component) go func() { defer wg.Done() - if err := d.teardownCentral(ctx); err != nil { + if err := d.teardownCentralWithAddOns(ctx); err != nil { log.Warningf("Error tearing down central: %v", err) } }() @@ -502,62 +508,115 @@ func (d *Deployer) Teardown(ctx context.Context, components component.Component) } } return nil - case component.AddOns: - return nil default: return fmt.Errorf("unknown component: %s", components) } } -func (d *Deployer) teardownCentral(ctx context.Context) error { - log.Infof("🗑️ Tearing down central in namespace %s", d.config.Central.Namespace) +// teardownCentralWithAddOns discovers the Central CR namespace from the +// cluster and tears down add-ons and Central resources in that namespace. +func (d *Deployer) teardownCentralWithAddOns(ctx context.Context) error { + namespace, err := k8s.DiscoverCRNamespace(ctx, "central", centralCrName) + if err != nil { + return fmt.Errorf("discovering Central namespace: %w", err) + } + if namespace == "" { + log.Infof("No Central CR found on cluster, skipping Central teardown") + return d.teardownAddOns(ctx) + } + + addOnsErr := d.teardownDiscoveredAddOns(ctx, namespace) + if addOnsErr != nil { + addOnsErr = fmt.Errorf("tearing down add-ons: %w", addOnsErr) + } + + centralErr := d.teardownCentral(ctx, namespace) + + return errors.Join(addOnsErr, centralErr) +} - if !d.namespaceExists(d.config.Central.Namespace) { - log.Infof("Namespace %s doesn't exist, skipping", d.config.Central.Namespace) +func (d *Deployer) teardownAddOns(ctx context.Context) error { + namespace, err := k8s.DiscoverCRNamespace(ctx, "central", centralCrName) + if err != nil { + return fmt.Errorf("discovering Central namespace: %w", err) + } + // Add-ons are Helm releases owned by roxie, not by the operator. Fall back to + // the configured namespace so they can get cleaned up even if the Central CR is gone. + if namespace == "" && d.config.Central.Namespace != "" { + namespace = d.config.Central.Namespace + if !d.namespaceExists(namespace) { + log.Infof("No Central CR found and namespace %s does not exist, skipping add-on teardown", namespace) + return nil + } + log.Dimf("No Central CR found, falling back to configured namespace %s for add-on teardown", namespace) + } + if namespace == "" { + log.Infof("No Central CR found and no configured namespace, skipping add-on teardown") return nil } + return d.teardownDiscoveredAddOns(ctx, namespace) +} + +func (d *Deployer) teardownCentral(ctx context.Context, namespace string) error { + log.Infof("🗑️ Tearing down central in namespace %s", namespace) d.portForward.Stop() d.stopDetachedPortForward() // Add pause-reconcile annotation to not have the operator interfere during resource deletion. - if d.doesResourceExist(ctx, "central", "stackrox-central-services", d.config.Central.Namespace) { - if err := d.addPauseReconcileAnnotation(ctx, "central", "stackrox-central-services", d.config.Central.Namespace); err != nil { + if d.doesResourceExist(ctx, "central", centralCrName, namespace) { + if err := d.addPauseReconcileAnnotation(ctx, "central", centralCrName, namespace); err != nil { log.Warningf("Error adding pause-reconcile annotation: %v", err) } } log.Info("⏳ Waiting for Central resources to be fully deleted...") - if err := d.deleteCentralResources(ctx); err != nil { + if err := d.deleteCentralResources(ctx, namespace); err != nil { return fmt.Errorf("failed to delete Central resources: %w", err) } - log.Successf("✓ Central resources in namespace %s have been deleted", d.config.Central.Namespace) + log.Successf("✓ Central resources in namespace %s have been deleted", namespace) return nil } func (d *Deployer) teardownSecuredCluster(ctx context.Context) error { - log.Infof("🗑️ Tearing down secured cluster in namespace %s", d.config.SecuredCluster.Namespace) - - if !d.namespaceExists(d.config.SecuredCluster.Namespace) { - log.Infof("Namespace %s doesn't exist, skipping", d.config.SecuredCluster.Namespace) + namespaces, err := k8s.DiscoverCRNamespaces(ctx, "securedcluster", securedClusterCrName) + if err != nil { + return fmt.Errorf("discovering SecuredCluster namespaces: %w", err) + } + if len(namespaces) == 0 { + log.Infof("No SecuredCluster CRs found on cluster, skipping SecuredCluster teardown") return nil } - if d.doesResourceExist(ctx, "securedcluster", "stackrox-secured-cluster-services", d.config.SecuredCluster.Namespace) { - // Add pause-reconcile annotation to not have the operator interfere during resource deletion. - if err := d.addPauseReconcileAnnotation(ctx, "securedcluster", "stackrox-secured-cluster-services", d.config.SecuredCluster.Namespace); err != nil { + var group errgroup.Group + for _, ns := range namespaces { + group.Go(func() error { + if err := d.teardownSecuredClusterInNamespace(ctx, ns); err != nil { + log.Warningf("Error tearing down SecuredCluster in namespace %s: %v", ns, err) + return fmt.Errorf("tearing down SecuredCluster in namespace %s: %w", ns, err) + } + return nil + }) + } + return group.Wait() +} + +func (d *Deployer) teardownSecuredClusterInNamespace(ctx context.Context, namespace string) error { + log.Infof("🗑️ Tearing down secured cluster in namespace %s", namespace) + + if d.doesResourceExist(ctx, "securedcluster", securedClusterCrName, namespace) { + if err := d.addPauseReconcileAnnotation(ctx, "securedcluster", securedClusterCrName, namespace); err != nil { log.Warningf("Error adding pause-reconcile annotation: %v", err) } } log.Info("⏳ Waiting for SecuredCluster resources to be fully deleted...") - err := d.deleteSecuredClusterResources(ctx) - if err != nil { - return fmt.Errorf("failed to delete SecuredCluster resources: %w", err) + if err := d.deleteSecuredClusterResources(ctx, namespace); err != nil { + return fmt.Errorf("failed to delete SecuredCluster resources in namespace %s: %w", namespace, err) } - log.Successf("✓ SecuredCluster resources in namespace %s have been deleted", d.config.SecuredCluster.Namespace) + log.Successf("✓ SecuredCluster resources in namespace %s have been deleted", namespace) return nil } diff --git a/internal/k8s/resource.go b/internal/k8s/resource.go index 67a04ae5..90a72867 100644 --- a/internal/k8s/resource.go +++ b/internal/k8s/resource.go @@ -62,6 +62,47 @@ func IsResourceNotFound(err error) bool { return errors.Is(err, ErrResourceNotFound) } +// DiscoverCRNamespaces returns the namespaces that contain a custom resource of +// the given type and name. It returns (nil, nil) when the CR type is not +// registered on the cluster or no matching instances are found. +func DiscoverCRNamespaces(ctx context.Context, crType, crName string) ([]string, error) { + jsonpathExpr := fmt.Sprintf(`{.items[?(@.metadata.name=="%s")].metadata.namespace}`, crName) + result, err := RunKubectl(ctx, KubectlOptions{ + Args: []string{"get", crType, "-A", "-o", "jsonpath=" + jsonpathExpr}, + }) + if err != nil { + stderr := strings.ToLower(result.Stderr) + if strings.Contains(stderr, "the server doesn't have a resource type") { + return nil, nil + } + return nil, fmt.Errorf("discovering %s/%s: %w", crType, crName, err) + } + output := strings.TrimSpace(result.Stdout) + if output == "" { + return nil, nil + } + namespaces := strings.Fields(output) + log.Infof("Discovered %s/%s in namespace(s): %s", crType, crName, strings.Join(namespaces, ", ")) + return namespaces, nil +} + +// DiscoverCRNamespace discovers the single namespace containing the named custom +// resource. It returns ("", nil) when no instance is found, and an error when +// the CR exists in more than one namespace. +func DiscoverCRNamespace(ctx context.Context, crType, crName string) (string, error) { + namespaces, err := DiscoverCRNamespaces(ctx, crType, crName) + if err != nil { + return "", err + } + if len(namespaces) > 1 { + return "", fmt.Errorf("found %s/%s in multiple namespaces: %s", crType, crName, strings.Join(namespaces, ", ")) + } + if len(namespaces) == 0 { + return "", nil + } + return namespaces[0], nil +} + // ResourceNotOwnedByName checks if the given unstructured object does not have an owner reference with the specified owner name. // The negative version of this function is better suited for our needs, because there can be multiple owner references and // we are only interested in checking if a given resource is NOT owned by a specific owner. From 62dd09f1a92838e64e0534fcc2418dd22b69e596 Mon Sep 17 00:00:00 2001 From: Moritz Clasmeier Date: Fri, 30 Oct 2026 07:36:34 +0100 Subject: [PATCH 02/16] go mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 1a0133a7..f4071d50 100644 --- a/go.mod +++ b/go.mod @@ -144,7 +144,7 @@ require ( go.opentelemetry.io/otel/trace v1.45.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect golang.org/x/net v0.60.0 // indirect - golang.org/x/sync v0.23.0 // indirect + golang.org/x/sync v0.23.0 golang.org/x/sys v0.49.0 // indirect golang.org/x/text v0.42.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect From b99ee3c743ed812bbf1224f9da3c31628272c09c Mon Sep 17 00:00:00 2001 From: Moritz Clasmeier Date: Fri, 30 Oct 2026 16:17:15 +0100 Subject: [PATCH 03/16] Teardown addons independently, even if some fail --- internal/deployer/addons.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/deployer/addons.go b/internal/deployer/addons.go index 5f2adce1..975a04b2 100644 --- a/internal/deployer/addons.go +++ b/internal/deployer/addons.go @@ -66,13 +66,19 @@ func (d *Deployer) teardownDiscoveredAddOns(ctx context.Context, namespace strin } log.Infof("Tearing down %d add-on(s)...", len(releases)) + var errs []error for _, release := range releases { + if err := ctx.Err(); err != nil { + errs = append(errs, err) + break + } if err := helm.Uninstall(ctx, release, namespace); err != nil { - return fmt.Errorf("tearing down add-on release %q: %w", release, err) + errs = append(errs, fmt.Errorf("tearing down add-on release %q: %w", release, err)) + continue } log.Successf("Add-on release %q torn down", release) } - return nil + return errors.Join(errs...) } // ResolveEnabledAddOns returns the enabled add-ons sorted by descending priority (name for ties at zero). From 58410e42cbda926834c994d2471e2fe6689c7c51 Mon Sep 17 00:00:00 2001 From: Moritz Clasmeier <111092021+mclasmeier@users.noreply.github.com> Date: Fri, 30 Oct 2026 16:11:40 +0100 Subject: [PATCH 04/16] Update internal/deployer/deployer.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- internal/deployer/deployer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/deployer/deployer.go b/internal/deployer/deployer.go index 98fc21d4..f8f6e5ab 100644 --- a/internal/deployer/deployer.go +++ b/internal/deployer/deployer.go @@ -149,7 +149,7 @@ func (d *Deployer) deleteCentralResources(ctx context.Context, namespace string) } if err := d.deleteResource(ctx, namespace, resource.Kind, resource.Name); err != nil { - return fmt.Errorf("failed to delete %s/%s in namepace %s: %w", resource.Kind, resource.Name, namespace, err) + return fmt.Errorf("failed to delete %s/%s in namespace %s: %w", resource.Kind, resource.Name, namespace, err) } log.Dimf("%s: Deleted %s/%s", namespace, resource.Kind, resource.Name) From 5f552c09c3570521001ac963585dc453d889a189 Mon Sep 17 00:00:00 2001 From: Moritz Clasmeier Date: Tue, 3 Nov 2026 16:11:57 +0100 Subject: [PATCH 05/16] Fall back to configured namespace for teardown retry When a previous teardown deleted the CR but failed during resource cleanup (PVCs, secrets), a retry could not discover the namespace from the now-deleted CR and would skip cleanup entirely. Use the configured namespace as a fallback when the CR is gone but the namespace still exists, matching the pattern already used by teardownAddOns. Co-Authored-By: Claude Opus 4.6 --- internal/deployer/deployer.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/internal/deployer/deployer.go b/internal/deployer/deployer.go index f8f6e5ab..0ca41c2c 100644 --- a/internal/deployer/deployer.go +++ b/internal/deployer/deployer.go @@ -521,8 +521,13 @@ func (d *Deployer) teardownCentralWithAddOns(ctx context.Context) error { return fmt.Errorf("discovering Central namespace: %w", err) } if namespace == "" { - log.Infof("No Central CR found on cluster, skipping Central teardown") - return d.teardownAddOns(ctx) + if d.config.Central.Namespace != "" && d.namespaceExists(d.config.Central.Namespace) { + namespace = d.config.Central.Namespace + log.Dimf("No Central CR found, falling back to configured namespace %s for resource cleanup", namespace) + } else { + log.Infof("No Central CR found on cluster, skipping Central teardown") + return d.teardownAddOns(ctx) + } } addOnsErr := d.teardownDiscoveredAddOns(ctx, namespace) @@ -585,8 +590,13 @@ func (d *Deployer) teardownSecuredCluster(ctx context.Context) error { return fmt.Errorf("discovering SecuredCluster namespaces: %w", err) } if len(namespaces) == 0 { - log.Infof("No SecuredCluster CRs found on cluster, skipping SecuredCluster teardown") - return nil + if d.config.SecuredCluster.Namespace != "" && d.namespaceExists(d.config.SecuredCluster.Namespace) { + namespaces = []string{d.config.SecuredCluster.Namespace} + log.Dimf("No SecuredCluster CRs found, falling back to configured namespace %s for resource cleanup", d.config.SecuredCluster.Namespace) + } else { + log.Infof("No SecuredCluster CRs found on cluster, skipping SecuredCluster teardown") + return nil + } } var group errgroup.Group From 5c043956b9ef491664cdd45209d8871f5bf033f2 Mon Sep 17 00:00:00 2001 From: Moritz Clasmeier Date: Wed, 11 Nov 2026 15:45:37 +0100 Subject: [PATCH 06/16] Include transitional Helm release states in add-on discovery ListByPrefix defaulted to ListDeployed, so releases stuck in pending or uninstalling states after an interrupted operation were invisible to subsequent teardown runs. Co-Authored-By: Claude Opus 4.6 --- internal/helm/helm.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/helm/helm.go b/internal/helm/helm.go index b9b0359d..e3876036 100644 --- a/internal/helm/helm.go +++ b/internal/helm/helm.go @@ -201,6 +201,9 @@ func doListByPrefix(ctx context.Context, prefix, namespace string) ([]string, er list := action.NewList(cfg) list.Filter = "^" + regexp.QuoteMeta(prefix) + list.StateMask = action.ListDeployed | action.ListFailed | + action.ListPendingInstall | action.ListPendingUpgrade | action.ListPendingRollback | + action.ListUninstalling results, err := list.Run() if err != nil { From 947c8485740ad039ac5fc42eb5868f997c8ec871 Mon Sep 17 00:00:00 2001 From: Moritz Clasmeier Date: Wed, 18 Nov 2026 13:54:28 +0100 Subject: [PATCH 07/16] Preserve reverse-priority add-on teardown via Helm release labels Discovered add-on releases carry no install-order information, so teardown fell back to Helm's alphabetical order, contradicting the documented priority semantics. Store each add-on's priority as a Helm release label and uninstall in reverse installation order. Releases without a valid label (installed by older roxie versions) are treated as lowest priority. Co-Authored-By: Claude Opus 5.5 --- internal/deployer/addons.go | 31 +++++++++++++++++++--- internal/deployer/addons_helm_chart.go | 7 +++++ internal/deployer/addons_test.go | 36 ++++++++++++++++++++++++++ internal/helm/helm.go | 22 +++++++++++----- 4 files changed, 87 insertions(+), 9 deletions(-) diff --git a/internal/deployer/addons.go b/internal/deployer/addons.go index 975a04b2..b61ba074 100644 --- a/internal/deployer/addons.go +++ b/internal/deployer/addons.go @@ -7,6 +7,7 @@ import ( "fmt" "regexp" "slices" + "strconv" "github.com/stackrox/roxie/internal/helm" log "github.com/stackrox/roxie/internal/logger" @@ -65,6 +66,8 @@ func (d *Deployer) teardownDiscoveredAddOns(ctx context.Context, namespace strin return nil } + sortReleasesForTeardown(releases) + log.Infof("Tearing down %d add-on(s)...", len(releases)) var errs []error for _, release := range releases { @@ -72,15 +75,37 @@ func (d *Deployer) teardownDiscoveredAddOns(ctx context.Context, namespace strin errs = append(errs, err) break } - if err := helm.Uninstall(ctx, release, namespace); err != nil { - errs = append(errs, fmt.Errorf("tearing down add-on release %q: %w", release, err)) + if err := helm.Uninstall(ctx, release.Name, namespace); err != nil { + errs = append(errs, fmt.Errorf("tearing down add-on release %q: %w", release.Name, err)) continue } - log.Successf("Add-on release %q torn down", release) + log.Successf("Add-on release %q torn down", release.Name) } return errors.Join(errs...) } +// sortReleasesForTeardown orders add-on releases in reverse installation order +// (see sortAddOns): ascending priority, ties broken by descending name. +func sortReleasesForTeardown(releases []helm.Release) { + slices.SortFunc(releases, func(a, b helm.Release) int { + return cmp.Or( + cmp.Compare(releasePriority(a), releasePriority(b)), + cmp.Compare(b.Name, a.Name), // ties: descending name + ) + }) +} + +// releasePriority returns the priority stored on the release, falling back to the +// lowest priority if the label is missing or invalid (e.g. releases installed by +// older roxie versions). +func releasePriority(release helm.Release) uint64 { + prio, err := strconv.ParseUint(release.Labels[addOnPriorityLabel], 10, 0) + if err != nil { + return 0 + } + return prio +} + // ResolveEnabledAddOns returns the enabled add-ons sorted by descending priority (name for ties at zero). func (d *Deployer) ResolveEnabledAddOns() ([]AddOn, error) { return resolveEnabledAddOns(d.config.Central) diff --git a/internal/deployer/addons_helm_chart.go b/internal/deployer/addons_helm_chart.go index 3943fca3..b9a647f4 100644 --- a/internal/deployer/addons_helm_chart.go +++ b/internal/deployer/addons_helm_chart.go @@ -3,6 +3,7 @@ package deployer import ( "context" "fmt" + "strconv" "github.com/stackrox/roxie/internal/helm" log "github.com/stackrox/roxie/internal/logger" @@ -11,6 +12,9 @@ import ( const ( addOnReleasePrefix = "roxie-addon-" maxHelmReleaseNameLen = 53 + // addOnPriorityLabel is stored on the add-on's Helm release, so that teardown can + // restore the reverse installation order without consulting local config. + addOnPriorityLabel = "roxie.stackrox.io/addon-priority" ) type helmAddOn struct { @@ -54,6 +58,9 @@ func newHelmAddOn( } opts.ReleaseName = releaseName opts.Namespace = namespace + opts.Labels = map[string]string{ + addOnPriorityLabel: strconv.FormatUint(uint64(commonProperties.Priority), 10), + } return &helmAddOn{ name: name, releaseName: releaseName, diff --git a/internal/deployer/addons_test.go b/internal/deployer/addons_test.go index d749efcc..53b307cc 100644 --- a/internal/deployer/addons_test.go +++ b/internal/deployer/addons_test.go @@ -4,6 +4,7 @@ import ( "strings" "testing" + "github.com/stackrox/roxie/internal/helm" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -86,6 +87,7 @@ func TestResolveAddOns(t *testing.T) { assert.Equal(t, "example", ha.installOpts.ChartName) assert.Equal(t, "1.0.0", ha.installOpts.ChartVersion) assert.Equal(t, "acs-central", ha.installOpts.Namespace) + assert.Equal(t, map[string]string{addOnPriorityLabel: "0"}, ha.installOpts.Labels) }, }, { @@ -244,3 +246,37 @@ func TestResolveAddOns(t *testing.T) { }) } } + +func TestSortReleasesForTeardown(t *testing.T) { + withPrio := func(name, prio string) helm.Release { + return helm.Release{Name: name, Labels: map[string]string{"owner": "helm", addOnPriorityLabel: prio}} + } + releases := []helm.Release{ + {Name: "roxie-addon-legacy-b"}, + withPrio("roxie-addon-hi-prio", "10"), + withPrio("roxie-addon-alpha", "0"), + withPrio("roxie-addon-garbage", "not-a-number"), + withPrio("roxie-addon-mid-prio", "5"), + withPrio("roxie-addon-hi-prio-2", "10"), + {Name: "roxie-addon-legacy-a"}, + withPrio("roxie-addon-zebra", "0"), + } + + sortReleasesForTeardown(releases) + + names := make([]string, len(releases)) + for i, r := range releases { + names[i] = r.Name + } + // Exact reverse of the installation order; unlabeled/invalid releases count as lowest priority. + assert.Equal(t, []string{ + "roxie-addon-zebra", + "roxie-addon-legacy-b", + "roxie-addon-legacy-a", + "roxie-addon-garbage", + "roxie-addon-alpha", + "roxie-addon-mid-prio", + "roxie-addon-hi-prio-2", + "roxie-addon-hi-prio", + }, names) +} diff --git a/internal/helm/helm.go b/internal/helm/helm.go index e3876036..2165493b 100644 --- a/internal/helm/helm.go +++ b/internal/helm/helm.go @@ -26,6 +26,14 @@ type InstallOptions struct { ChartVersion string Namespace string Values map[string]any + // Labels are attached to the Helm release (stored on the release's storage Secret). + Labels map[string]string +} + +// Release is a minimal view of an installed Helm release. +type Release struct { + Name string + Labels map[string]string } const ( @@ -72,9 +80,9 @@ func Uninstall(ctx context.Context, releaseName, namespace string) error { }) } -// ListByPrefix returns the names of Helm releases whose name starts with the given prefix. -func ListByPrefix(ctx context.Context, prefix, namespace string) ([]string, error) { - var releases []string +// ListByPrefix returns the Helm releases whose name starts with the given prefix. +func ListByPrefix(ctx context.Context, prefix, namespace string) ([]Release, error) { + var releases []Release err := executeHelmActionWithRetries(ctx, "list", func(ctx context.Context) error { @@ -154,6 +162,7 @@ func doInstall(ctx context.Context, opts InstallOptions) error { install.ReleaseName = opts.ReleaseName install.Namespace = opts.Namespace install.Wait = false + install.Labels = opts.Labels chart, err := loader.Load(chartPath) if err != nil { @@ -170,6 +179,7 @@ func doUpgrade(ctx context.Context, cfg *action.Configuration, opts InstallOptio upgrade := action.NewUpgrade(cfg) upgrade.Namespace = opts.Namespace upgrade.Wait = false + upgrade.Labels = opts.Labels chart, err := loader.Load(chartPath) if err != nil { @@ -193,7 +203,7 @@ func doUninstall(ctx context.Context, releaseName, namespace string) error { return err } -func doListByPrefix(ctx context.Context, prefix, namespace string) ([]string, error) { +func doListByPrefix(ctx context.Context, prefix, namespace string) ([]Release, error) { cfg, err := newActionConfig(ctx, namespace) if err != nil { return nil, err @@ -210,9 +220,9 @@ func doListByPrefix(ctx context.Context, prefix, namespace string) ([]string, er return nil, err } - var releases []string + var releases []Release for _, r := range results { - releases = append(releases, r.Name) + releases = append(releases, Release{Name: r.Name, Labels: r.Labels}) } return releases, nil } From cecbd0458f0aa30d727febba851789b33f86e5e9 Mon Sep 17 00:00:00 2001 From: Moritz Clasmeier Date: Wed, 18 Nov 2026 14:27:38 +0100 Subject: [PATCH 08/16] Scope redeploy cleanup to the configured namespace deployCentral and deploySecuredCluster reused the discovery-based teardown, which searches the whole cluster. Redeploying Central into namespace B could delete a Central (incl. PVCs and add-ons) living in namespace A, and redeploying a SecuredCluster tore down every SecuredCluster on the cluster. Restrict redeploy cleanup to the target namespace; cluster-wide discovery remains limited to `roxie teardown`. Co-Authored-By: Claude Opus 5.5 --- internal/deployer/deployer.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/internal/deployer/deployer.go b/internal/deployer/deployer.go index 0ca41c2c..e7b77502 100644 --- a/internal/deployer/deployer.go +++ b/internal/deployer/deployer.go @@ -434,9 +434,15 @@ func (d *Deployer) prepareCredentials(ctx context.Context) error { func (d *Deployer) deployCentral(ctx context.Context) error { log.Infof("Deploying Central to namespace %s", d.config.Central.Namespace) log.Infof("Central resource profile: %s", d.config.Central.ResourceProfile) - if d.namespaceExists(d.config.Central.Namespace) { + if ns := d.config.Central.Namespace; d.namespaceExists(ns) { log.Info("Existing Central deployment found, tearing down...") - if err := d.teardownCentralWithAddOns(ctx); err != nil { + // Only clean up the namespace we are about to deploy into. Unlike explicit + // `roxie teardown`, a redeployment must not use cluster-wide discovery, which + // could find (and delete) a Central, including its PVCs and add-ons, living in + // a different namespace. + addOnsErr := d.teardownDiscoveredAddOns(ctx, ns) + centralErr := d.teardownCentral(ctx, ns) + if err := errors.Join(addOnsErr, centralErr); err != nil { log.Warningf("Error during teardown: %v", err) } } @@ -458,9 +464,13 @@ func (d *Deployer) deployCentral(ctx context.Context) error { func (d *Deployer) deploySecuredCluster(ctx context.Context) error { log.Infof("Deploying SecuredCluster to namespace %s", d.config.SecuredCluster.Namespace) log.Infof("SecuredCluster resource profile: %s", d.config.SecuredCluster.ResourceProfile) - if d.namespaceExists(d.config.SecuredCluster.Namespace) { + if ns := d.config.SecuredCluster.Namespace; d.namespaceExists(ns) { log.Info("Existing SecuredCluster deployment found, tearing down...") - if err := d.teardownSecuredCluster(ctx); err != nil { + // Only clean up the namespace we are about to deploy into. Unlike explicit + // `roxie teardown`, a redeployment must not use cluster-wide discovery, which + // would tear down every SecuredCluster on the cluster, not just the one being + // replaced. + if err := d.teardownSecuredClusterInNamespace(ctx, ns); err != nil { log.Warningf("Error during teardown: %v", err) } } From 4d1b7aaaba364739ba899a334cf2f4bcc5c5cd99 Mon Sep 17 00:00:00 2001 From: Moritz Clasmeier Date: Wed, 18 Nov 2026 14:35:07 +0100 Subject: [PATCH 09/16] Propagate errors from combined teardown For `both`/`all`, component teardown errors were only logged and Teardown returned nil, so the CLI deleted the manifest and reported "Teardown complete!" despite failures. Collect the per-component errors and return them joined, keeping the existing ordering (Central and SecuredCluster in parallel, operator afterwards). Co-Authored-By: Claude Opus 5.5 --- internal/deployer/deployer.go | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/internal/deployer/deployer.go b/internal/deployer/deployer.go index e7b77502..1c3a7676 100644 --- a/internal/deployer/deployer.go +++ b/internal/deployer/deployer.go @@ -491,33 +491,32 @@ func (d *Deployer) Teardown(ctx context.Context, components component.Component) case component.Operator: return d.teardownOperator(ctx) case component.Both, component.All: + // Each component is torn down independently; errors are collected rather than + // aborting early, so that one failing component does not prevent cleanup of the + // others. They are returned to the caller so that a partial failure is not + // reported as a successful teardown. + var securedClusterErr, centralErr, operatorErr error var wg sync.WaitGroup - wg.Add(2) - - go func() { - defer wg.Done() + wg.Go(func() { if err := d.teardownSecuredCluster(ctx); err != nil { - log.Warningf("Error tearing down secured cluster: %v", err) + securedClusterErr = fmt.Errorf("tearing down secured cluster: %w", err) } - }() - - go func() { - defer wg.Done() + }) + wg.Go(func() { if err := d.teardownCentralWithAddOns(ctx); err != nil { - log.Warningf("Error tearing down central: %v", err) + centralErr = fmt.Errorf("tearing down central: %w", err) } - }() - + }) wg.Wait() // Tear down the operator strictly after Central/SecuredCluster are gone, // because the operator manages finalizers on their custom resources. if components == component.All { if err := d.teardownOperator(ctx); err != nil { - log.Warningf("Error tearing down operator: %v", err) + operatorErr = fmt.Errorf("tearing down operator: %w", err) } } - return nil + return errors.Join(securedClusterErr, centralErr, operatorErr) default: return fmt.Errorf("unknown component: %s", components) } From 2675bff2e6adbdc364047938e7d6069f0aba7505 Mon Sep 17 00:00:00 2001 From: Moritz Clasmeier Date: Wed, 18 Nov 2026 14:45:31 +0100 Subject: [PATCH 10/16] Only treat missing CR type as absent when API discovery is complete kubectl reports "the server doesn't have a resource type" also when API discovery is only partially successful. DiscoverCRNamespaces took that as proof that no CRs exist, letting teardown silently skip cleanup. Confirm with a full `kubectl api-resources` before returning an empty result; otherwise propagate the error. Co-Authored-By: Claude Opus 5.5 --- internal/k8s/resource.go | 8 ++++++++ internal/k8s/resource_integration_test.go | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/internal/k8s/resource.go b/internal/k8s/resource.go index 90a72867..5974ebd2 100644 --- a/internal/k8s/resource.go +++ b/internal/k8s/resource.go @@ -73,6 +73,14 @@ func DiscoverCRNamespaces(ctx context.Context, crType, crName string) ([]string, if err != nil { stderr := strings.ToLower(result.Stderr) if strings.Contains(stderr, "the server doesn't have a resource type") { + // kubectl also reports a missing resource type when API discovery is only + // partially successful (e.g. an unavailable aggregated API server). Only + // trust the message if full API discovery succeeds; otherwise teardown + // would silently skip cleanup of CRs that do exist. + if _, discoveryErr := RunKubectl(ctx, KubectlOptions{Args: []string{"api-resources", "-o", "name"}}); discoveryErr != nil { + return nil, fmt.Errorf("discovering %s/%s: resource type reported as missing, but API discovery is incomplete: %w", + crType, crName, errors.Join(err, discoveryErr)) + } return nil, nil } return nil, fmt.Errorf("discovering %s/%s: %w", crType, crName, err) diff --git a/internal/k8s/resource_integration_test.go b/internal/k8s/resource_integration_test.go index 0f335aec..eb4fa848 100644 --- a/internal/k8s/resource_integration_test.go +++ b/internal/k8s/resource_integration_test.go @@ -28,3 +28,9 @@ func TestRetrieveResourceFromCluster_Found(t *testing.T) { assert.Nil(t, err, "Expected no error to be returned when resource is found") assert.Equal(t, "default", obj.GetName(), "Expected resource name to match") } + +func TestDiscoverCRNamespaces_UnknownResourceType(t *testing.T) { + namespaces, err := DiscoverCRNamespaces(context.Background(), "roxienonexistentkind", "whatever") + assert.NoError(t, err, "Expected an unregistered resource type to be reported as absent, not as an error") + assert.Empty(t, namespaces) +} From e961f8bf5f5868fec299ced605bd9d73330a7f71 Mon Sep 17 00:00:00 2001 From: Moritz Clasmeier Date: Wed, 18 Nov 2026 15:01:53 +0100 Subject: [PATCH 11/16] Adapt e2e Helm helpers to ListByPrefix returning releases Co-Authored-By: Claude Opus 5.5 --- tests/e2e/helpers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/helpers.go b/tests/e2e/helpers.go index 193943ef..4a296745 100644 --- a/tests/e2e/helpers.go +++ b/tests/e2e/helpers.go @@ -407,7 +407,7 @@ func verifyHelmReleaseExists(t *testing.T, ctx context.Context, releaseName, nam releases, err := helm.ListByPrefix(ctx, releaseName, namespace) require.NoError(t, err, "listing Helm releases in namespace %s", namespace) - if !slices.Contains(releases, releaseName) { + if !slices.ContainsFunc(releases, func(r helm.Release) bool { return r.Name == releaseName }) { t.Fatalf("Helm release %s does not exist in namespace %s", releaseName, namespace) } } @@ -417,7 +417,7 @@ func verifyHelmReleaseNotExists(t *testing.T, ctx context.Context, releaseName, releases, err := helm.ListByPrefix(ctx, releaseName, namespace) require.NoError(t, err, "listing Helm releases in namespace %s", namespace) - if slices.Contains(releases, releaseName) { + if slices.ContainsFunc(releases, func(r helm.Release) bool { return r.Name == releaseName }) { t.Fatalf("Helm release %s already exists in namespace %s", releaseName, namespace) } } From 747604710f558f720a697196f848caede4de86ac Mon Sep 17 00:00:00 2001 From: Moritz Clasmeier Date: Thu, 19 Nov 2026 10:04:33 +0100 Subject: [PATCH 12/16] Restrict teardown to explicitly configured namespaces --- cmd/teardown.go | 5 +++ internal/deployer/deployer.go | 70 ++++++++++++++++++++--------------- 2 files changed, 45 insertions(+), 30 deletions(-) diff --git a/cmd/teardown.go b/cmd/teardown.go index b5f69cf0..c4424bf0 100644 --- a/cmd/teardown.go +++ b/cmd/teardown.go @@ -59,6 +59,11 @@ func runTeardown(cmd *cobra.Command, args []string) error { defer cancel() clusterConfig := new(deployer.DefaultConfig()) + // A configured namespace restricts teardown to that namespace. Drop the built-in + // default namespaces, so that only explicit user configuration (user config, --set, ...) + // restricts teardown and roxie deployments are otherwise discovered cluster-wide. + clusterConfig.Central.Namespace = "" + clusterConfig.SecuredCluster.Namespace = "" deploySettings, err := assembleConfigForCommand(clusterConfig, deploySettingsFromArgs, skipUserConfig) if err != nil { return err diff --git a/internal/deployer/deployer.go b/internal/deployer/deployer.go index 1c3a7676..b628b510 100644 --- a/internal/deployer/deployer.go +++ b/internal/deployer/deployer.go @@ -440,9 +440,7 @@ func (d *Deployer) deployCentral(ctx context.Context) error { // `roxie teardown`, a redeployment must not use cluster-wide discovery, which // could find (and delete) a Central, including its PVCs and add-ons, living in // a different namespace. - addOnsErr := d.teardownDiscoveredAddOns(ctx, ns) - centralErr := d.teardownCentral(ctx, ns) - if err := errors.Join(addOnsErr, centralErr); err != nil { + if err := d.teardownCentralWithAddOnsInNamespace(ctx, ns); err != nil { log.Warningf("Error during teardown: %v", err) } } @@ -522,23 +520,30 @@ func (d *Deployer) Teardown(ctx context.Context, components component.Component) } } -// teardownCentralWithAddOns discovers the Central CR namespace from the -// cluster and tears down add-ons and Central resources in that namespace. +// teardownCentralWithAddOns tears down add-ons and Central resources. If a Central +// namespace is configured (only explicit user configuration, see runTeardown), only that +// namespace is cleaned up; otherwise the Central CR namespace is discovered from the cluster. func (d *Deployer) teardownCentralWithAddOns(ctx context.Context) error { + if ns := d.config.Central.Namespace; ns != "" { + if !d.namespaceExists(ns) { + log.Infof("Namespace %s doesn't exist, skipping Central teardown", ns) + return nil + } + return d.teardownCentralWithAddOnsInNamespace(ctx, ns) + } + namespace, err := k8s.DiscoverCRNamespace(ctx, "central", centralCrName) if err != nil { return fmt.Errorf("discovering Central namespace: %w", err) } if namespace == "" { - if d.config.Central.Namespace != "" && d.namespaceExists(d.config.Central.Namespace) { - namespace = d.config.Central.Namespace - log.Dimf("No Central CR found, falling back to configured namespace %s for resource cleanup", namespace) - } else { - log.Infof("No Central CR found on cluster, skipping Central teardown") - return d.teardownAddOns(ctx) - } + log.Infof("No Central CR found on cluster, skipping Central teardown") + return nil } + return d.teardownCentralWithAddOnsInNamespace(ctx, namespace) +} +func (d *Deployer) teardownCentralWithAddOnsInNamespace(ctx context.Context, namespace string) error { addOnsErr := d.teardownDiscoveredAddOns(ctx, namespace) if addOnsErr != nil { addOnsErr = fmt.Errorf("tearing down add-ons: %w", addOnsErr) @@ -550,22 +555,20 @@ func (d *Deployer) teardownCentralWithAddOns(ctx context.Context) error { } func (d *Deployer) teardownAddOns(ctx context.Context) error { + if ns := d.config.Central.Namespace; ns != "" { + if !d.namespaceExists(ns) { + log.Infof("Namespace %s doesn't exist, skipping add-on teardown", ns) + return nil + } + return d.teardownDiscoveredAddOns(ctx, ns) + } + namespace, err := k8s.DiscoverCRNamespace(ctx, "central", centralCrName) if err != nil { return fmt.Errorf("discovering Central namespace: %w", err) } - // Add-ons are Helm releases owned by roxie, not by the operator. Fall back to - // the configured namespace so they can get cleaned up even if the Central CR is gone. - if namespace == "" && d.config.Central.Namespace != "" { - namespace = d.config.Central.Namespace - if !d.namespaceExists(namespace) { - log.Infof("No Central CR found and namespace %s does not exist, skipping add-on teardown", namespace) - return nil - } - log.Dimf("No Central CR found, falling back to configured namespace %s for add-on teardown", namespace) - } if namespace == "" { - log.Infof("No Central CR found and no configured namespace, skipping add-on teardown") + log.Infof("No Central CR found and no namespace configured, skipping add-on teardown") return nil } return d.teardownDiscoveredAddOns(ctx, namespace) @@ -593,19 +596,26 @@ func (d *Deployer) teardownCentral(ctx context.Context, namespace string) error return nil } +// teardownSecuredCluster tears down SecuredCluster resources. If a SecuredCluster +// namespace is configured (only explicit user configuration, see runTeardown), only that +// namespace is cleaned up; otherwise all SecuredCluster CRs on the cluster are discovered +// and torn down. func (d *Deployer) teardownSecuredCluster(ctx context.Context) error { + if ns := d.config.SecuredCluster.Namespace; ns != "" { + if !d.namespaceExists(ns) { + log.Infof("Namespace %s doesn't exist, skipping SecuredCluster teardown", ns) + return nil + } + return d.teardownSecuredClusterInNamespace(ctx, ns) + } + namespaces, err := k8s.DiscoverCRNamespaces(ctx, "securedcluster", securedClusterCrName) if err != nil { return fmt.Errorf("discovering SecuredCluster namespaces: %w", err) } if len(namespaces) == 0 { - if d.config.SecuredCluster.Namespace != "" && d.namespaceExists(d.config.SecuredCluster.Namespace) { - namespaces = []string{d.config.SecuredCluster.Namespace} - log.Dimf("No SecuredCluster CRs found, falling back to configured namespace %s for resource cleanup", d.config.SecuredCluster.Namespace) - } else { - log.Infof("No SecuredCluster CRs found on cluster, skipping SecuredCluster teardown") - return nil - } + log.Infof("No SecuredCluster CRs found on cluster, skipping SecuredCluster teardown") + return nil } var group errgroup.Group From 47c13eea3cba870f530c77ac41d1314375dbf846 Mon Sep 17 00:00:00 2001 From: Moritz Clasmeier Date: Wed, 18 Nov 2026 18:25:44 +0100 Subject: [PATCH 13/16] Report namespace --- internal/deployer/deployer.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/deployer/deployer.go b/internal/deployer/deployer.go index b628b510..b2fcc995 100644 --- a/internal/deployer/deployer.go +++ b/internal/deployer/deployer.go @@ -938,6 +938,7 @@ func (d *Deployer) PrintCentralDeploymentSummary() { // Deployment details log.Info(cyan.Sprint("│") + createRow("Component", component)) log.Info(cyan.Sprint("│") + createRow("Cluster Type", d.config.Roxie.ClusterType.String())) + log.Info(cyan.Sprint("│") + createRow("Namespace", d.config.Central.Namespace)) log.Info(cyan.Sprint("│") + createRow("Image Tag", imageTag.String())) log.Info(cyan.Sprint("│") + createRow("Kubernetes Context", kubeContext)) @@ -1116,6 +1117,7 @@ func (d *Deployer) PrintSecuredClusterDeploymentSummary() { // Deployment details log.Info(cyan.Sprint("│") + createRow("Component", component)) log.Info(cyan.Sprint("│") + createRow("Cluster Type", d.config.Roxie.ClusterType.String())) + log.Info(cyan.Sprint("│") + createRow("Namespace", d.config.SecuredCluster.Namespace)) log.Info(cyan.Sprint("│") + createRow("Image Tag", imageTag.String())) log.Info(cyan.Sprint("│") + createRow("Kubernetes Context", kubeContext)) From d00608261590f1809c5c996c7908ffedac35971c Mon Sep 17 00:00:00 2001 From: Moritz Clasmeier Date: Thu, 19 Nov 2026 10:05:22 +0100 Subject: [PATCH 14/16] New test --- tests/e2e/multiple_secured_clusters_test.go | 102 ++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 tests/e2e/multiple_secured_clusters_test.go diff --git a/tests/e2e/multiple_secured_clusters_test.go b/tests/e2e/multiple_secured_clusters_test.go new file mode 100644 index 00000000..e24d2d7e --- /dev/null +++ b/tests/e2e/multiple_secured_clusters_test.go @@ -0,0 +1,102 @@ +//go:build e2e + +package e2e + +import ( + "fmt" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestMultipleSecuredClusters verifies that several SecuredClusters can be deployed side by side, +// that teardown with an explicit namespace only removes the SecuredCluster in that namespace, and +// that teardown without configured namespaces discovers and removes everything. +func TestMultipleSecuredClusters(t *testing.T) { + modifiedCommonDeployArgs := append(commonDeployArgs, "--resources=tiny") + const nsA, nsB = "acs-sensor-a", "acs-sensor-b" + // roxie teardown removes the SecuredCluster resources but not the namespaces they + // lived in; delete the test-specific namespaces so they don't pile up on the cluster. + // Registered before registerClusterDumpAndCleanup, so that (cleanups running in reverse + // order) it only runs after the failure dump and roxie teardown have completed. + t.Cleanup(func() { + _ = exec.Command("kubectl", "delete", "namespace", nsA, nsB, "--ignore-not-found").Run() + }) + registerClusterDumpAndCleanup(t) + + // TODO: Drop the pinned tag (and use the default MAIN_IMAGE_TAG like other tests) once + // ACS 5.1.0 is released. Deploying multiple SecuredClusters into one cluster requires the + // operator to support the platform.stackrox.io/namespace-prefix-global-resources annotation + // (stackrox/stackrox#23099), which avoids name clashes of cluster-scoped resources. No released + // version supports it yet; the e2e suite otherwise defaults to the latest release. + const nightlyTag = "5.1.x-nightly-20261117" + + // Deploy Central. + envrcPath := filepath.Join(t.TempDir(), ".envrc") + runCommand(t, deployTimeout, nil, append([]string{roxieBinary, "deploy", "--early-readiness", "central", + "--tag", nightlyTag, "--envrc", envrcPath}, modifiedCommonDeployArgs...)...) + + // Deploy two SecuredClusters. + // TODO, using async waitgroup. + envrcEnv := loadEnvrcFile(t, envrcPath) + securedClusterCounter := 0 + deploySecuredCluster := func(ns string) { + t.Logf("=== Deploying SecuredCluster into namespace %s ===", ns) + securedClusterCounter++ + clusterName := fmt.Sprintf("sc-%d", securedClusterCounter) // For redeployment test we need fresh cluster names. + roxieCmd := append([]string{roxieBinary, "deploy", "--early-readiness", "secured-cluster", "--tag", nightlyTag, + "--set", `securedCluster.metadata.annotations.platform\.stackrox\.io/namespace-prefix-global-resources=true`, + "--set", "securedCluster.namespace=" + ns, "--set", "securedCluster.spec.clusterName=" + clusterName}, + modifiedCommonDeployArgs...) + runCommandPrefixed(t, ns+" ", deployTimeout, envrcEnv, roxieCmd...) + } + var wg sync.WaitGroup + for _, ns := range []string{nsA, nsB} { + wg.Go(func() { + deploySecuredCluster(ns) + }) + } + wg.Wait() + assert.ElementsMatch(t, []string{nsA, nsB}, crNamespaces(t, "securedcluster")) + uidB := securedClusterUID(t, nsB) + + // Redeploying A must not touch B: B's CR keeps its UID (i.e. was not deleted and recreated). + deploySecuredCluster(nsA) + assert.ElementsMatch(t, []string{nsA, nsB}, crNamespaces(t, "securedcluster")) + assert.Equal(t, uidB, securedClusterUID(t, nsB), "SecuredCluster in %s was replaced by redeploying %s", nsB, nsA) + + // Tear down SC in nsA. + t.Logf("=== Tearing down SecuredCluster in namespace %s only ===", nsA) + runCommand(t, teardownTimeout, nil, roxieBinary, "teardown", "--skip-user-config", "secured-cluster", "--set", "securedCluster.namespace="+nsA) + assert.Equal(t, []string{nsB}, crNamespaces(t, "securedcluster")) + + // Tear down everything. + t.Log("=== Tearing down everything without configured namespaces ===") + runCommand(t, teardownTimeout, nil, roxieBinary, "teardown", "--skip-user-config", "all") + assert.Empty(t, crNamespaces(t, "securedcluster")) + assert.Empty(t, crNamespaces(t, "central")) +} + +// crNamespaces returns the namespaces containing a CR of the given type. +func crNamespaces(t *testing.T, crType string) []string { + t.Helper() + + output, err := exec.Command("kubectl", "get", crType, "-A", "-o", "jsonpath={.items[*].metadata.namespace}").Output() + require.NoError(t, err, "Failed to list %s CRs", crType) + return strings.Fields(string(output)) +} + +// securedClusterUID returns the UID of the SecuredCluster CR in the given namespace. +func securedClusterUID(t *testing.T, namespace string) string { + t.Helper() + + output, err := exec.Command("kubectl", "get", "securedcluster", "stackrox-secured-cluster-services", + "-n", namespace, "-o", "jsonpath={.metadata.uid}").Output() + require.NoError(t, err, "Failed to get SecuredCluster UID in namespace %s", namespace) + return string(output) +} From fd35ff15a0dd4683a6ccd4730eee75f81b315d74 Mon Sep 17 00:00:00 2001 From: Moritz Clasmeier Date: Thu, 19 Nov 2026 10:02:08 +0100 Subject: [PATCH 15/16] New helper for interleaving roxie execution --- tests/e2e/helpers.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/e2e/helpers.go b/tests/e2e/helpers.go index 4a296745..68b186b4 100644 --- a/tests/e2e/helpers.go +++ b/tests/e2e/helpers.go @@ -138,7 +138,7 @@ func requireKubeContext() (string, error) { return ctx, nil } -func runCommand(t *testing.T, timeout time.Duration, env map[string]string, args ...string) { +func runCommandPrefixed(t *testing.T, prefix string, timeout time.Duration, env map[string]string, args ...string) { t.Helper() ctx, cancel := context.WithTimeout(context.Background(), timeout) @@ -156,8 +156,8 @@ func runCommand(t *testing.T, timeout time.Duration, env map[string]string, args // Stream output to both test log and buffers var stdout, stderr bytes.Buffer - cmd.Stdout = &testWriter{t: t, prefix: "[stdout] ", buf: &stdout} - cmd.Stderr = &testWriter{t: t, prefix: "[stderr] ", buf: &stderr} + cmd.Stdout = &testWriter{t: t, prefix: fmt.Sprintf("[%sstdout] ", prefix), buf: &stdout} + cmd.Stderr = &testWriter{t: t, prefix: fmt.Sprintf("[%sstderr] ", prefix), buf: &stderr} t.Logf("Running: %s", strings.Join(args, " ")) @@ -168,6 +168,11 @@ func runCommand(t *testing.T, timeout time.Duration, env map[string]string, args t.Logf("Command completed successfully") } +func runCommand(t *testing.T, timeout time.Duration, env map[string]string, args ...string) { + t.Helper() + runCommandPrefixed(t, "", timeout, env, args...) +} + // testWriter writes to both test log and a buffer type testWriter struct { t *testing.T From 3ca57adf5135293aacc0ee06a6e0d58aa82774d1 Mon Sep 17 00:00:00 2001 From: Moritz Clasmeier Date: Thu, 19 Nov 2026 10:03:22 +0100 Subject: [PATCH 16/16] Makefile: increase e2e runtime timeout --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ac3099f5..0095338d 100644 --- a/Makefile +++ b/Makefile @@ -141,7 +141,7 @@ run-test-e2e: ## Run end-to-end tests (requires roxie to be available in PATH, k @echo ""; \ echo "Invoking go test..." - $(GOTEST) -v -tags=e2e -timeout=120m -parallel=1 ./tests/e2e/... + $(GOTEST) -v -tags=e2e -timeout=150m -parallel=1 ./tests/e2e/... .PHONY: test-e2e test-e2e: build ## Run end-to-end tests (requires kubectl context and cluster access)