diff --git a/Makefile b/Makefile index ac3099f5..0095338d 100644 --- a/Makefile +++ b/Makefile @@ -141,7 +141,7 @@ run-test-e2e: ## Run end-to-end tests (requires roxie to be available in PATH, k @echo ""; \ echo "Invoking go test..." - $(GOTEST) -v -tags=e2e -timeout=120m -parallel=1 ./tests/e2e/... + $(GOTEST) -v -tags=e2e -timeout=150m -parallel=1 ./tests/e2e/... .PHONY: test-e2e test-e2e: build ## Run end-to-end tests (requires kubectl context and cluster access) diff --git a/cmd/teardown.go b/cmd/teardown.go index 37f01b94..c4424bf0 100644 --- a/cmd/teardown.go +++ b/cmd/teardown.go @@ -2,7 +2,6 @@ package main import ( "context" - "errors" "fmt" "time" @@ -10,7 +9,6 @@ import ( "github.com/stackrox/roxie/internal/component" "github.com/stackrox/roxie/internal/deployer" "github.com/stackrox/roxie/internal/env" - "github.com/stackrox/roxie/internal/k8s" log "github.com/stackrox/roxie/internal/logger" "github.com/stackrox/roxie/internal/manifest" ) @@ -60,16 +58,12 @@ func runTeardown(cmd *cobra.Command, args []string) error { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() - var clusterConfig *deployer.Config - clusterManifest, err := manifest.LoadManifestSecret(ctx) - if errors.Is(err, k8s.ErrResourceNotFound) { - log.Infof("No roxie manifest found in cluster, proceeding without it") - } else if err != nil { - log.Warningf("Failed to load manifest from cluster: %v", err) - } else { - clusterConfig = &clusterManifest.Config - } - + clusterConfig := new(deployer.DefaultConfig()) + // A configured namespace restricts teardown to that namespace. Drop the built-in + // default namespaces, so that only explicit user configuration (user config, --set, ...) + // restricts teardown and roxie deployments are otherwise discovered cluster-wide. + clusterConfig.Central.Namespace = "" + clusterConfig.SecuredCluster.Namespace = "" deploySettings, err := assembleConfigForCommand(clusterConfig, deploySettingsFromArgs, skipUserConfig) if err != nil { return err diff --git a/go.mod b/go.mod index 1a0133a7..f4071d50 100644 --- a/go.mod +++ b/go.mod @@ -144,7 +144,7 @@ require ( go.opentelemetry.io/otel/trace v1.45.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect golang.org/x/net v0.60.0 // indirect - golang.org/x/sync v0.23.0 // indirect + golang.org/x/sync v0.23.0 golang.org/x/sys v0.49.0 // indirect golang.org/x/text v0.42.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/internal/deployer/addons.go b/internal/deployer/addons.go index 8123d8f0..b61ba074 100644 --- a/internal/deployer/addons.go +++ b/internal/deployer/addons.go @@ -7,7 +7,9 @@ import ( "fmt" "regexp" "slices" + "strconv" + "github.com/stackrox/roxie/internal/helm" log "github.com/stackrox/roxie/internal/logger" ) @@ -52,24 +54,56 @@ func (d *Deployer) deployAddOns(ctx context.Context, addOns []AddOn) error { return nil } -func (d *Deployer) teardownAddOns(ctx context.Context, addOns []AddOn) { - if len(addOns) == 0 { - return +// teardownDiscoveredAddOns uninstalls add-on Helm releases in the given +// namespace. Add-ons are lifecycle-aligned with Central, so the caller +// passes the Central namespace. +func (d *Deployer) teardownDiscoveredAddOns(ctx context.Context, namespace string) error { + releases, err := helm.ListByPrefix(ctx, addOnReleasePrefix, namespace) + if err != nil { + return fmt.Errorf("listing add-on Helm releases in namespace %s: %w", namespace, err) + } + if len(releases) == 0 { + return nil } - log.Infof("Tearing down %d add-on(s)...", len(addOns)) + sortReleasesForTeardown(releases) - for _, addon := range addOns { - if err := addon.Teardown(ctx); err != nil { - if !addon.IsOptional() { - log.Errorf("Failed to tear down non-optional add-on %q: %v", addon.Name(), err) - } else { - log.Warningf("Failed to tear down optional add-on %q: %v", addon.Name(), err) - } - } else { - log.Successf("Add-on %q torn down", addon.Name()) + log.Infof("Tearing down %d add-on(s)...", len(releases)) + var errs []error + for _, release := range releases { + if err := ctx.Err(); err != nil { + errs = append(errs, err) + break } + if err := helm.Uninstall(ctx, release.Name, namespace); err != nil { + errs = append(errs, fmt.Errorf("tearing down add-on release %q: %w", release.Name, err)) + continue + } + log.Successf("Add-on release %q torn down", release.Name) + } + return errors.Join(errs...) +} + +// sortReleasesForTeardown orders add-on releases in reverse installation order +// (see sortAddOns): ascending priority, ties broken by descending name. +func sortReleasesForTeardown(releases []helm.Release) { + slices.SortFunc(releases, func(a, b helm.Release) int { + return cmp.Or( + cmp.Compare(releasePriority(a), releasePriority(b)), + cmp.Compare(b.Name, a.Name), // ties: descending name + ) + }) +} + +// releasePriority returns the priority stored on the release, falling back to the +// lowest priority if the label is missing or invalid (e.g. releases installed by +// older roxie versions). +func releasePriority(release helm.Release) uint64 { + prio, err := strconv.ParseUint(release.Labels[addOnPriorityLabel], 10, 0) + if err != nil { + return 0 } + return prio } // ResolveEnabledAddOns returns the enabled add-ons sorted by descending priority (name for ties at zero). diff --git a/internal/deployer/addons_helm_chart.go b/internal/deployer/addons_helm_chart.go index 3943fca3..b9a647f4 100644 --- a/internal/deployer/addons_helm_chart.go +++ b/internal/deployer/addons_helm_chart.go @@ -3,6 +3,7 @@ package deployer import ( "context" "fmt" + "strconv" "github.com/stackrox/roxie/internal/helm" log "github.com/stackrox/roxie/internal/logger" @@ -11,6 +12,9 @@ import ( const ( addOnReleasePrefix = "roxie-addon-" maxHelmReleaseNameLen = 53 + // addOnPriorityLabel is stored on the add-on's Helm release, so that teardown can + // restore the reverse installation order without consulting local config. + addOnPriorityLabel = "roxie.stackrox.io/addon-priority" ) type helmAddOn struct { @@ -54,6 +58,9 @@ func newHelmAddOn( } opts.ReleaseName = releaseName opts.Namespace = namespace + opts.Labels = map[string]string{ + addOnPriorityLabel: strconv.FormatUint(uint64(commonProperties.Priority), 10), + } return &helmAddOn{ name: name, releaseName: releaseName, diff --git a/internal/deployer/addons_test.go b/internal/deployer/addons_test.go index d749efcc..53b307cc 100644 --- a/internal/deployer/addons_test.go +++ b/internal/deployer/addons_test.go @@ -4,6 +4,7 @@ import ( "strings" "testing" + "github.com/stackrox/roxie/internal/helm" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -86,6 +87,7 @@ func TestResolveAddOns(t *testing.T) { assert.Equal(t, "example", ha.installOpts.ChartName) assert.Equal(t, "1.0.0", ha.installOpts.ChartVersion) assert.Equal(t, "acs-central", ha.installOpts.Namespace) + assert.Equal(t, map[string]string{addOnPriorityLabel: "0"}, ha.installOpts.Labels) }, }, { @@ -244,3 +246,37 @@ func TestResolveAddOns(t *testing.T) { }) } } + +func TestSortReleasesForTeardown(t *testing.T) { + withPrio := func(name, prio string) helm.Release { + return helm.Release{Name: name, Labels: map[string]string{"owner": "helm", addOnPriorityLabel: prio}} + } + releases := []helm.Release{ + {Name: "roxie-addon-legacy-b"}, + withPrio("roxie-addon-hi-prio", "10"), + withPrio("roxie-addon-alpha", "0"), + withPrio("roxie-addon-garbage", "not-a-number"), + withPrio("roxie-addon-mid-prio", "5"), + withPrio("roxie-addon-hi-prio-2", "10"), + {Name: "roxie-addon-legacy-a"}, + withPrio("roxie-addon-zebra", "0"), + } + + sortReleasesForTeardown(releases) + + names := make([]string, len(releases)) + for i, r := range releases { + names[i] = r.Name + } + // Exact reverse of the installation order; unlabeled/invalid releases count as lowest priority. + assert.Equal(t, []string{ + "roxie-addon-zebra", + "roxie-addon-legacy-b", + "roxie-addon-legacy-a", + "roxie-addon-garbage", + "roxie-addon-alpha", + "roxie-addon-mid-prio", + "roxie-addon-hi-prio-2", + "roxie-addon-hi-prio", + }, names) +} diff --git a/internal/deployer/deployer.go b/internal/deployer/deployer.go index e5b75310..b2fcc995 100644 --- a/internal/deployer/deployer.go +++ b/internal/deployer/deployer.go @@ -3,10 +3,10 @@ package deployer import ( "context" "crypto/rand" + "errors" "fmt" "os" "os/exec" - "slices" "strconv" "strings" "sync" @@ -14,6 +14,7 @@ import ( "time" "github.com/fatih/color" + "golang.org/x/sync/errgroup" "github.com/stackrox/roxie/internal/component" "github.com/stackrox/roxie/internal/containerrt" @@ -89,11 +90,11 @@ func (d *Deployer) deleteResources(ctx context.Context, namespace string, resour } // Expects that reconciliation for the RHACS operator is paused. -func (d *Deployer) deleteCentralResources(ctx context.Context) error { +func (d *Deployer) deleteCentralResources(ctx context.Context, namespace string) error { log.Info("Deleting Central resources") crExists := true - if _, err := k8s.RetrieveResourceFromCluster(ctx, d.config.Central.Namespace, "central", "stackrox-central-services"); err != nil { + if _, err := k8s.RetrieveResourceFromCluster(ctx, namespace, "central", "stackrox-central-services"); err != nil { if !k8s.IsResourceNotFound(err) { return fmt.Errorf("retrieving Central CR: %w", err) } @@ -102,12 +103,12 @@ func (d *Deployer) deleteCentralResources(ctx context.Context) error { if crExists { log.Info("Removing any pause-reconcile annotation from Central") - if err := d.removePauseReconcileAnnotation(ctx, "central", "stackrox-central-services", d.config.Central.Namespace); err != nil { + if err := d.removePauseReconcileAnnotation(ctx, "central", "stackrox-central-services", namespace); err != nil { return err } log.Debug("Removed any pause-reconcile annotation from Central") - err := d.deleteResource(ctx, d.config.Central.Namespace, "central", "stackrox-central-services", "--wait") + err := d.deleteResource(ctx, namespace, "central", "stackrox-central-services", "--wait") if err != nil { return err } @@ -116,6 +117,7 @@ func (d *Deployer) deleteCentralResources(ctx context.Context) error { log.Info("Deletion of Central resources requested, but Central CR is not present anymore") } + var group errgroup.Group for _, resource := range []ResourceToDelete{ {Name: "central-db", Kind: "pvc"}, {Name: "central-db-backup", Kind: "pvc"}, @@ -127,36 +129,42 @@ func (d *Deployer) deleteCentralResources(ctx context.Context) error { {Name: "scanner-db-password", Kind: "secret", OwnerName: centralCrName}, {Name: "stackrox-central-helm", Kind: "configmap"}, } { - log.Dimf("Attempting to delete %s/%s", resource.Kind, resource.Name) - if resource.OwnerName != "" { - // Avoid deletion if the resource does not have the expected owner. - // (e.g. in case central and secured cluster are deployed into the same namespace). - obj, err := k8s.RetrieveResourceFromCluster(ctx, d.config.Central.Namespace, resource.Kind, resource.Name) - if err != nil { - if !k8s.IsResourceNotFound(err) { - log.Warningf("Failed to retrieve %s/%s for owner checking: %v. Skipping deletion. Deployment might be affected.", resource.Kind, resource.Name, err) + group.Go(func() error { + log.Dimf("%s: Attempting to delete %s/%s", namespace, resource.Kind, resource.Name) + if resource.OwnerName != "" { + // Avoid deletion if the resource does not have the expected owner. + // (e.g. in case central and secured cluster are deployed into the same namespace). + obj, err := k8s.RetrieveResourceFromCluster(ctx, namespace, resource.Kind, resource.Name) + if err != nil { + if !k8s.IsResourceNotFound(err) { + log.Warningf("%s: Failed to retrieve %s/%s for owner checking: %v. Skipping deletion. Deployment might be affected.", namespace, resource.Kind, resource.Name, err) + return fmt.Errorf("retrieving resource %s/%s from namespace %s: %w", resource.Kind, resource.Name, namespace, err) + } + return nil // resource not found + } + if k8s.ResourceNotOwnedByName(obj, resource.OwnerName) { + log.Dimf("%s: Skipping deletion of %s/%s: not owned by %s", namespace, resource.Kind, resource.Name, resource.OwnerName) + return nil } - continue } - if k8s.ResourceNotOwnedByName(obj, resource.OwnerName) { - log.Dimf("Skipping deletion of %s/%s: not owned by %s", resource.Kind, resource.Name, resource.OwnerName) - continue + + if err := d.deleteResource(ctx, namespace, resource.Kind, resource.Name); err != nil { + return fmt.Errorf("failed to delete %s/%s in namespace %s: %w", resource.Kind, resource.Name, namespace, err) } - } - if err := d.deleteResource(ctx, d.config.Central.Namespace, resource.Kind, resource.Name); err != nil { - return fmt.Errorf("failed to delete %s/%s: %w", resource.Kind, resource.Name, err) - } + log.Dimf("%s: Deleted %s/%s", namespace, resource.Kind, resource.Name) + return nil + }) } - return nil + return group.Wait() } -func (d *Deployer) deleteSecuredClusterResources(ctx context.Context) error { +func (d *Deployer) deleteSecuredClusterResources(ctx context.Context, namespace string) error { log.Info("Deleting SecuredCluster resources") crExists := true - if _, err := k8s.RetrieveResourceFromCluster(ctx, d.config.SecuredCluster.Namespace, "securedcluster", "stackrox-secured-cluster-services"); err != nil { + if _, err := k8s.RetrieveResourceFromCluster(ctx, namespace, "securedcluster", securedClusterCrName); err != nil { if !k8s.IsResourceNotFound(err) { return fmt.Errorf("retrieving SecuredCluster CR: %w", err) } @@ -165,12 +173,12 @@ func (d *Deployer) deleteSecuredClusterResources(ctx context.Context) error { if crExists { log.Info("Removing any pause-reconcile annotation from SecuredCluster") - if err := d.removePauseReconcileAnnotation(ctx, "securedcluster", "stackrox-secured-cluster-services", d.config.SecuredCluster.Namespace); err != nil { + if err := d.removePauseReconcileAnnotation(ctx, "securedcluster", securedClusterCrName, namespace); err != nil { return err } log.Debug("Removed any pause-reconcile annotation from SecuredCluster") - err := d.deleteResource(ctx, d.config.SecuredCluster.Namespace, "securedcluster", "stackrox-secured-cluster-services", "--wait") + err := d.deleteResource(ctx, namespace, "securedcluster", securedClusterCrName, "--wait") if err != nil { return err } @@ -180,6 +188,7 @@ func (d *Deployer) deleteSecuredClusterResources(ctx context.Context) error { } // Delete resources, which are treated special. + group, ctx := errgroup.WithContext(ctx) for _, resource := range []ResourceToDelete{ {Name: "cluster-registration-secret", Kind: "secret"}, // scanner-v4-db is not managed by the operator's ReconcilePVCExtension, so its ownerReference @@ -190,28 +199,35 @@ func (d *Deployer) deleteSecuredClusterResources(ctx context.Context) error { // when both are deployed into the same namespace. {Name: "scanner-db-password", Kind: "secret", OwnerName: securedClusterCrName}, } { - log.Dimf("Attempting to delete %s/%s", resource.Kind, resource.Name) - if resource.OwnerName != "" { - // Avoid deletion if the resource does not have the expected owner. - // (e.g. in case central and secured cluster are deployed into the same namespace). - obj, err := k8s.RetrieveResourceFromCluster(ctx, d.config.SecuredCluster.Namespace, resource.Kind, resource.Name) - if err != nil { - if !k8s.IsResourceNotFound(err) { - log.Warningf("Failed to retrieve %s/%s for owner checking: %v. Skipping deletion. Deployment might be affected.", resource.Kind, resource.Name, err) + group.Go(func() error { + log.Dimf("%s: Attempting to delete %s/%s", namespace, resource.Kind, resource.Name) + if resource.OwnerName != "" { + // Avoid deletion if the resource does not have the expected owner. + // (e.g. in case central and secured cluster are deployed into the same namespace). + obj, err := k8s.RetrieveResourceFromCluster(ctx, namespace, resource.Kind, resource.Name) + if err != nil { + if !k8s.IsResourceNotFound(err) { + log.Warningf("%s: Failed to retrieve %s/%s for owner checking: %v. Skipping deletion. Deployment might be affected.", namespace, resource.Kind, resource.Name, err) + return fmt.Errorf("retrieving resource %s/%s from namespace %s: %w", resource.Kind, resource.Name, namespace, err) + } + return nil // resource not found + } + if k8s.ResourceNotOwnedByName(obj, resource.OwnerName) { + log.Dimf("%s: Skipping deletion of %s/%s: not owned by %s", namespace, resource.Kind, resource.Name, resource.OwnerName) + return nil } - continue } - if k8s.ResourceNotOwnedByName(obj, resource.OwnerName) { - log.Dimf("Skipping deletion of %s/%s: not owned by %s", resource.Kind, resource.Name, resource.OwnerName) - continue + + if err := d.deleteResource(ctx, namespace, resource.Kind, resource.Name); err != nil { + return fmt.Errorf("failed to delete %s/%s in namespace %s: %w", resource.Kind, resource.Name, namespace, err) } - } - if err := d.deleteResource(ctx, d.config.SecuredCluster.Namespace, resource.Kind, resource.Name); err != nil { - return fmt.Errorf("failed to delete %s/%s: %w", resource.Kind, resource.Name, err) - } + + log.Dimf("%s: Deleted %s/%s", namespace, resource.Kind, resource.Name) + return nil + }) } - return nil + return group.Wait() } func (d *Deployer) SetConfig(config Config) { @@ -418,9 +434,13 @@ func (d *Deployer) prepareCredentials(ctx context.Context) error { func (d *Deployer) deployCentral(ctx context.Context) error { log.Infof("Deploying Central to namespace %s", d.config.Central.Namespace) log.Infof("Central resource profile: %s", d.config.Central.ResourceProfile) - if d.namespaceExists(d.config.Central.Namespace) { + if ns := d.config.Central.Namespace; d.namespaceExists(ns) { log.Info("Existing Central deployment found, tearing down...") - if err := d.teardownCentral(ctx); err != nil { + // Only clean up the namespace we are about to deploy into. Unlike explicit + // `roxie teardown`, a redeployment must not use cluster-wide discovery, which + // could find (and delete) a Central, including its PVCs and add-ons, living in + // a different namespace. + if err := d.teardownCentralWithAddOnsInNamespace(ctx, ns); err != nil { log.Warningf("Error during teardown: %v", err) } } @@ -442,9 +462,13 @@ func (d *Deployer) deployCentral(ctx context.Context) error { func (d *Deployer) deploySecuredCluster(ctx context.Context) error { log.Infof("Deploying SecuredCluster to namespace %s", d.config.SecuredCluster.Namespace) log.Infof("SecuredCluster resource profile: %s", d.config.SecuredCluster.ResourceProfile) - if d.namespaceExists(d.config.SecuredCluster.Namespace) { + if ns := d.config.SecuredCluster.Namespace; d.namespaceExists(ns) { log.Info("Existing SecuredCluster deployment found, tearing down...") - if err := d.teardownSecuredCluster(ctx); err != nil { + // Only clean up the namespace we are about to deploy into. Unlike explicit + // `roxie teardown`, a redeployment must not use cluster-wide discovery, which + // would tear down every SecuredCluster on the cluster, not just the one being + // replaced. + if err := d.teardownSecuredClusterInNamespace(ctx, ns); err != nil { log.Warningf("Error during teardown: %v", err) } } @@ -455,109 +479,173 @@ func (d *Deployer) deploySecuredCluster(ctx context.Context) error { func (d *Deployer) Teardown(ctx context.Context, components component.Component) error { log.Infof("Starting teardown of %s", components) - if components.IncludesAddOns() { - enabledAddOns, err := d.ResolveEnabledAddOns() - if err != nil { - return fmt.Errorf("resolving enabled add-ons: %w", err) - } - slices.Reverse(enabledAddOns) // Tearing down in opposite order as deploying. - d.teardownAddOns(ctx, enabledAddOns) - } - switch components { case component.Central: - return d.teardownCentral(ctx) + return d.teardownCentralWithAddOns(ctx) + case component.AddOns: + return d.teardownAddOns(ctx) case component.SecuredCluster: return d.teardownSecuredCluster(ctx) case component.Operator: return d.teardownOperator(ctx) case component.Both, component.All: - // Tear down components in parallel for better performance + // Each component is torn down independently; errors are collected rather than + // aborting early, so that one failing component does not prevent cleanup of the + // others. They are returned to the caller so that a partial failure is not + // reported as a successful teardown. + var securedClusterErr, centralErr, operatorErr error var wg sync.WaitGroup - - // Always tear down central and sensor - wg.Add(2) - - go func() { - defer wg.Done() + wg.Go(func() { if err := d.teardownSecuredCluster(ctx); err != nil { - log.Warningf("Error tearing down secured cluster: %v", err) + securedClusterErr = fmt.Errorf("tearing down secured cluster: %w", err) } - }() - - go func() { - defer wg.Done() - if err := d.teardownCentral(ctx); err != nil { - log.Warningf("Error tearing down central: %v", err) + }) + wg.Go(func() { + if err := d.teardownCentralWithAddOns(ctx); err != nil { + centralErr = fmt.Errorf("tearing down central: %w", err) } - }() - + }) wg.Wait() // Tear down the operator strictly after Central/SecuredCluster are gone, // because the operator manages finalizers on their custom resources. if components == component.All { if err := d.teardownOperator(ctx); err != nil { - log.Warningf("Error tearing down operator: %v", err) + operatorErr = fmt.Errorf("tearing down operator: %w", err) } } - return nil - case component.AddOns: - return nil + return errors.Join(securedClusterErr, centralErr, operatorErr) default: return fmt.Errorf("unknown component: %s", components) } } -func (d *Deployer) teardownCentral(ctx context.Context) error { - log.Infof("🗑️ Tearing down central in namespace %s", d.config.Central.Namespace) +// teardownCentralWithAddOns tears down add-ons and Central resources. If a Central +// namespace is configured (only explicit user configuration, see runTeardown), only that +// namespace is cleaned up; otherwise the Central CR namespace is discovered from the cluster. +func (d *Deployer) teardownCentralWithAddOns(ctx context.Context) error { + if ns := d.config.Central.Namespace; ns != "" { + if !d.namespaceExists(ns) { + log.Infof("Namespace %s doesn't exist, skipping Central teardown", ns) + return nil + } + return d.teardownCentralWithAddOnsInNamespace(ctx, ns) + } - if !d.namespaceExists(d.config.Central.Namespace) { - log.Infof("Namespace %s doesn't exist, skipping", d.config.Central.Namespace) + namespace, err := k8s.DiscoverCRNamespace(ctx, "central", centralCrName) + if err != nil { + return fmt.Errorf("discovering Central namespace: %w", err) + } + if namespace == "" { + log.Infof("No Central CR found on cluster, skipping Central teardown") + return nil + } + return d.teardownCentralWithAddOnsInNamespace(ctx, namespace) +} + +func (d *Deployer) teardownCentralWithAddOnsInNamespace(ctx context.Context, namespace string) error { + addOnsErr := d.teardownDiscoveredAddOns(ctx, namespace) + if addOnsErr != nil { + addOnsErr = fmt.Errorf("tearing down add-ons: %w", addOnsErr) + } + + centralErr := d.teardownCentral(ctx, namespace) + + return errors.Join(addOnsErr, centralErr) +} + +func (d *Deployer) teardownAddOns(ctx context.Context) error { + if ns := d.config.Central.Namespace; ns != "" { + if !d.namespaceExists(ns) { + log.Infof("Namespace %s doesn't exist, skipping add-on teardown", ns) + return nil + } + return d.teardownDiscoveredAddOns(ctx, ns) + } + + namespace, err := k8s.DiscoverCRNamespace(ctx, "central", centralCrName) + if err != nil { + return fmt.Errorf("discovering Central namespace: %w", err) + } + if namespace == "" { + log.Infof("No Central CR found and no namespace configured, skipping add-on teardown") return nil } + return d.teardownDiscoveredAddOns(ctx, namespace) +} + +func (d *Deployer) teardownCentral(ctx context.Context, namespace string) error { + log.Infof("🗑️ Tearing down central in namespace %s", namespace) d.portForward.Stop() d.stopDetachedPortForward() // Add pause-reconcile annotation to not have the operator interfere during resource deletion. - if d.doesResourceExist(ctx, "central", "stackrox-central-services", d.config.Central.Namespace) { - if err := d.addPauseReconcileAnnotation(ctx, "central", "stackrox-central-services", d.config.Central.Namespace); err != nil { + if d.doesResourceExist(ctx, "central", centralCrName, namespace) { + if err := d.addPauseReconcileAnnotation(ctx, "central", centralCrName, namespace); err != nil { log.Warningf("Error adding pause-reconcile annotation: %v", err) } } log.Info("⏳ Waiting for Central resources to be fully deleted...") - if err := d.deleteCentralResources(ctx); err != nil { + if err := d.deleteCentralResources(ctx, namespace); err != nil { return fmt.Errorf("failed to delete Central resources: %w", err) } - log.Successf("✓ Central resources in namespace %s have been deleted", d.config.Central.Namespace) + log.Successf("✓ Central resources in namespace %s have been deleted", namespace) return nil } +// teardownSecuredCluster tears down SecuredCluster resources. If a SecuredCluster +// namespace is configured (only explicit user configuration, see runTeardown), only that +// namespace is cleaned up; otherwise all SecuredCluster CRs on the cluster are discovered +// and torn down. func (d *Deployer) teardownSecuredCluster(ctx context.Context) error { - log.Infof("🗑️ Tearing down secured cluster in namespace %s", d.config.SecuredCluster.Namespace) + if ns := d.config.SecuredCluster.Namespace; ns != "" { + if !d.namespaceExists(ns) { + log.Infof("Namespace %s doesn't exist, skipping SecuredCluster teardown", ns) + return nil + } + return d.teardownSecuredClusterInNamespace(ctx, ns) + } - if !d.namespaceExists(d.config.SecuredCluster.Namespace) { - log.Infof("Namespace %s doesn't exist, skipping", d.config.SecuredCluster.Namespace) + namespaces, err := k8s.DiscoverCRNamespaces(ctx, "securedcluster", securedClusterCrName) + if err != nil { + return fmt.Errorf("discovering SecuredCluster namespaces: %w", err) + } + if len(namespaces) == 0 { + log.Infof("No SecuredCluster CRs found on cluster, skipping SecuredCluster teardown") return nil } - if d.doesResourceExist(ctx, "securedcluster", "stackrox-secured-cluster-services", d.config.SecuredCluster.Namespace) { - // Add pause-reconcile annotation to not have the operator interfere during resource deletion. - if err := d.addPauseReconcileAnnotation(ctx, "securedcluster", "stackrox-secured-cluster-services", d.config.SecuredCluster.Namespace); err != nil { + var group errgroup.Group + for _, ns := range namespaces { + group.Go(func() error { + if err := d.teardownSecuredClusterInNamespace(ctx, ns); err != nil { + log.Warningf("Error tearing down SecuredCluster in namespace %s: %v", ns, err) + return fmt.Errorf("tearing down SecuredCluster in namespace %s: %w", ns, err) + } + return nil + }) + } + return group.Wait() +} + +func (d *Deployer) teardownSecuredClusterInNamespace(ctx context.Context, namespace string) error { + log.Infof("🗑️ Tearing down secured cluster in namespace %s", namespace) + + if d.doesResourceExist(ctx, "securedcluster", securedClusterCrName, namespace) { + if err := d.addPauseReconcileAnnotation(ctx, "securedcluster", securedClusterCrName, namespace); err != nil { log.Warningf("Error adding pause-reconcile annotation: %v", err) } } log.Info("⏳ Waiting for SecuredCluster resources to be fully deleted...") - err := d.deleteSecuredClusterResources(ctx) - if err != nil { - return fmt.Errorf("failed to delete SecuredCluster resources: %w", err) + if err := d.deleteSecuredClusterResources(ctx, namespace); err != nil { + return fmt.Errorf("failed to delete SecuredCluster resources in namespace %s: %w", namespace, err) } - log.Successf("✓ SecuredCluster resources in namespace %s have been deleted", d.config.SecuredCluster.Namespace) + log.Successf("✓ SecuredCluster resources in namespace %s have been deleted", namespace) return nil } @@ -850,6 +938,7 @@ func (d *Deployer) PrintCentralDeploymentSummary() { // Deployment details log.Info(cyan.Sprint("│") + createRow("Component", component)) log.Info(cyan.Sprint("│") + createRow("Cluster Type", d.config.Roxie.ClusterType.String())) + log.Info(cyan.Sprint("│") + createRow("Namespace", d.config.Central.Namespace)) log.Info(cyan.Sprint("│") + createRow("Image Tag", imageTag.String())) log.Info(cyan.Sprint("│") + createRow("Kubernetes Context", kubeContext)) @@ -1028,6 +1117,7 @@ func (d *Deployer) PrintSecuredClusterDeploymentSummary() { // Deployment details log.Info(cyan.Sprint("│") + createRow("Component", component)) log.Info(cyan.Sprint("│") + createRow("Cluster Type", d.config.Roxie.ClusterType.String())) + log.Info(cyan.Sprint("│") + createRow("Namespace", d.config.SecuredCluster.Namespace)) log.Info(cyan.Sprint("│") + createRow("Image Tag", imageTag.String())) log.Info(cyan.Sprint("│") + createRow("Kubernetes Context", kubeContext)) diff --git a/internal/helm/helm.go b/internal/helm/helm.go index b9b0359d..2165493b 100644 --- a/internal/helm/helm.go +++ b/internal/helm/helm.go @@ -26,6 +26,14 @@ type InstallOptions struct { ChartVersion string Namespace string Values map[string]any + // Labels are attached to the Helm release (stored on the release's storage Secret). + Labels map[string]string +} + +// Release is a minimal view of an installed Helm release. +type Release struct { + Name string + Labels map[string]string } const ( @@ -72,9 +80,9 @@ func Uninstall(ctx context.Context, releaseName, namespace string) error { }) } -// ListByPrefix returns the names of Helm releases whose name starts with the given prefix. -func ListByPrefix(ctx context.Context, prefix, namespace string) ([]string, error) { - var releases []string +// ListByPrefix returns the Helm releases whose name starts with the given prefix. +func ListByPrefix(ctx context.Context, prefix, namespace string) ([]Release, error) { + var releases []Release err := executeHelmActionWithRetries(ctx, "list", func(ctx context.Context) error { @@ -154,6 +162,7 @@ func doInstall(ctx context.Context, opts InstallOptions) error { install.ReleaseName = opts.ReleaseName install.Namespace = opts.Namespace install.Wait = false + install.Labels = opts.Labels chart, err := loader.Load(chartPath) if err != nil { @@ -170,6 +179,7 @@ func doUpgrade(ctx context.Context, cfg *action.Configuration, opts InstallOptio upgrade := action.NewUpgrade(cfg) upgrade.Namespace = opts.Namespace upgrade.Wait = false + upgrade.Labels = opts.Labels chart, err := loader.Load(chartPath) if err != nil { @@ -193,7 +203,7 @@ func doUninstall(ctx context.Context, releaseName, namespace string) error { return err } -func doListByPrefix(ctx context.Context, prefix, namespace string) ([]string, error) { +func doListByPrefix(ctx context.Context, prefix, namespace string) ([]Release, error) { cfg, err := newActionConfig(ctx, namespace) if err != nil { return nil, err @@ -201,15 +211,18 @@ func doListByPrefix(ctx context.Context, prefix, namespace string) ([]string, er list := action.NewList(cfg) list.Filter = "^" + regexp.QuoteMeta(prefix) + list.StateMask = action.ListDeployed | action.ListFailed | + action.ListPendingInstall | action.ListPendingUpgrade | action.ListPendingRollback | + action.ListUninstalling results, err := list.Run() if err != nil { return nil, err } - var releases []string + var releases []Release for _, r := range results { - releases = append(releases, r.Name) + releases = append(releases, Release{Name: r.Name, Labels: r.Labels}) } return releases, nil } diff --git a/internal/k8s/resource.go b/internal/k8s/resource.go index 67a04ae5..5974ebd2 100644 --- a/internal/k8s/resource.go +++ b/internal/k8s/resource.go @@ -62,6 +62,55 @@ func IsResourceNotFound(err error) bool { return errors.Is(err, ErrResourceNotFound) } +// DiscoverCRNamespaces returns the namespaces that contain a custom resource of +// the given type and name. It returns (nil, nil) when the CR type is not +// registered on the cluster or no matching instances are found. +func DiscoverCRNamespaces(ctx context.Context, crType, crName string) ([]string, error) { + jsonpathExpr := fmt.Sprintf(`{.items[?(@.metadata.name=="%s")].metadata.namespace}`, crName) + result, err := RunKubectl(ctx, KubectlOptions{ + Args: []string{"get", crType, "-A", "-o", "jsonpath=" + jsonpathExpr}, + }) + if err != nil { + stderr := strings.ToLower(result.Stderr) + if strings.Contains(stderr, "the server doesn't have a resource type") { + // kubectl also reports a missing resource type when API discovery is only + // partially successful (e.g. an unavailable aggregated API server). Only + // trust the message if full API discovery succeeds; otherwise teardown + // would silently skip cleanup of CRs that do exist. + if _, discoveryErr := RunKubectl(ctx, KubectlOptions{Args: []string{"api-resources", "-o", "name"}}); discoveryErr != nil { + return nil, fmt.Errorf("discovering %s/%s: resource type reported as missing, but API discovery is incomplete: %w", + crType, crName, errors.Join(err, discoveryErr)) + } + return nil, nil + } + return nil, fmt.Errorf("discovering %s/%s: %w", crType, crName, err) + } + output := strings.TrimSpace(result.Stdout) + if output == "" { + return nil, nil + } + namespaces := strings.Fields(output) + log.Infof("Discovered %s/%s in namespace(s): %s", crType, crName, strings.Join(namespaces, ", ")) + return namespaces, nil +} + +// DiscoverCRNamespace discovers the single namespace containing the named custom +// resource. It returns ("", nil) when no instance is found, and an error when +// the CR exists in more than one namespace. +func DiscoverCRNamespace(ctx context.Context, crType, crName string) (string, error) { + namespaces, err := DiscoverCRNamespaces(ctx, crType, crName) + if err != nil { + return "", err + } + if len(namespaces) > 1 { + return "", fmt.Errorf("found %s/%s in multiple namespaces: %s", crType, crName, strings.Join(namespaces, ", ")) + } + if len(namespaces) == 0 { + return "", nil + } + return namespaces[0], nil +} + // ResourceNotOwnedByName checks if the given unstructured object does not have an owner reference with the specified owner name. // The negative version of this function is better suited for our needs, because there can be multiple owner references and // we are only interested in checking if a given resource is NOT owned by a specific owner. diff --git a/internal/k8s/resource_integration_test.go b/internal/k8s/resource_integration_test.go index 0f335aec..eb4fa848 100644 --- a/internal/k8s/resource_integration_test.go +++ b/internal/k8s/resource_integration_test.go @@ -28,3 +28,9 @@ func TestRetrieveResourceFromCluster_Found(t *testing.T) { assert.Nil(t, err, "Expected no error to be returned when resource is found") assert.Equal(t, "default", obj.GetName(), "Expected resource name to match") } + +func TestDiscoverCRNamespaces_UnknownResourceType(t *testing.T) { + namespaces, err := DiscoverCRNamespaces(context.Background(), "roxienonexistentkind", "whatever") + assert.NoError(t, err, "Expected an unregistered resource type to be reported as absent, not as an error") + assert.Empty(t, namespaces) +} diff --git a/tests/e2e/helpers.go b/tests/e2e/helpers.go index 193943ef..68b186b4 100644 --- a/tests/e2e/helpers.go +++ b/tests/e2e/helpers.go @@ -138,7 +138,7 @@ func requireKubeContext() (string, error) { return ctx, nil } -func runCommand(t *testing.T, timeout time.Duration, env map[string]string, args ...string) { +func runCommandPrefixed(t *testing.T, prefix string, timeout time.Duration, env map[string]string, args ...string) { t.Helper() ctx, cancel := context.WithTimeout(context.Background(), timeout) @@ -156,8 +156,8 @@ func runCommand(t *testing.T, timeout time.Duration, env map[string]string, args // Stream output to both test log and buffers var stdout, stderr bytes.Buffer - cmd.Stdout = &testWriter{t: t, prefix: "[stdout] ", buf: &stdout} - cmd.Stderr = &testWriter{t: t, prefix: "[stderr] ", buf: &stderr} + cmd.Stdout = &testWriter{t: t, prefix: fmt.Sprintf("[%sstdout] ", prefix), buf: &stdout} + cmd.Stderr = &testWriter{t: t, prefix: fmt.Sprintf("[%sstderr] ", prefix), buf: &stderr} t.Logf("Running: %s", strings.Join(args, " ")) @@ -168,6 +168,11 @@ func runCommand(t *testing.T, timeout time.Duration, env map[string]string, args t.Logf("Command completed successfully") } +func runCommand(t *testing.T, timeout time.Duration, env map[string]string, args ...string) { + t.Helper() + runCommandPrefixed(t, "", timeout, env, args...) +} + // testWriter writes to both test log and a buffer type testWriter struct { t *testing.T @@ -407,7 +412,7 @@ func verifyHelmReleaseExists(t *testing.T, ctx context.Context, releaseName, nam releases, err := helm.ListByPrefix(ctx, releaseName, namespace) require.NoError(t, err, "listing Helm releases in namespace %s", namespace) - if !slices.Contains(releases, releaseName) { + if !slices.ContainsFunc(releases, func(r helm.Release) bool { return r.Name == releaseName }) { t.Fatalf("Helm release %s does not exist in namespace %s", releaseName, namespace) } } @@ -417,7 +422,7 @@ func verifyHelmReleaseNotExists(t *testing.T, ctx context.Context, releaseName, releases, err := helm.ListByPrefix(ctx, releaseName, namespace) require.NoError(t, err, "listing Helm releases in namespace %s", namespace) - if slices.Contains(releases, releaseName) { + if slices.ContainsFunc(releases, func(r helm.Release) bool { return r.Name == releaseName }) { t.Fatalf("Helm release %s already exists in namespace %s", releaseName, namespace) } } diff --git a/tests/e2e/multiple_secured_clusters_test.go b/tests/e2e/multiple_secured_clusters_test.go new file mode 100644 index 00000000..e24d2d7e --- /dev/null +++ b/tests/e2e/multiple_secured_clusters_test.go @@ -0,0 +1,102 @@ +//go:build e2e + +package e2e + +import ( + "fmt" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestMultipleSecuredClusters verifies that several SecuredClusters can be deployed side by side, +// that teardown with an explicit namespace only removes the SecuredCluster in that namespace, and +// that teardown without configured namespaces discovers and removes everything. +func TestMultipleSecuredClusters(t *testing.T) { + modifiedCommonDeployArgs := append(commonDeployArgs, "--resources=tiny") + const nsA, nsB = "acs-sensor-a", "acs-sensor-b" + // roxie teardown removes the SecuredCluster resources but not the namespaces they + // lived in; delete the test-specific namespaces so they don't pile up on the cluster. + // Registered before registerClusterDumpAndCleanup, so that (cleanups running in reverse + // order) it only runs after the failure dump and roxie teardown have completed. + t.Cleanup(func() { + _ = exec.Command("kubectl", "delete", "namespace", nsA, nsB, "--ignore-not-found").Run() + }) + registerClusterDumpAndCleanup(t) + + // TODO: Drop the pinned tag (and use the default MAIN_IMAGE_TAG like other tests) once + // ACS 5.1.0 is released. Deploying multiple SecuredClusters into one cluster requires the + // operator to support the platform.stackrox.io/namespace-prefix-global-resources annotation + // (stackrox/stackrox#23099), which avoids name clashes of cluster-scoped resources. No released + // version supports it yet; the e2e suite otherwise defaults to the latest release. + const nightlyTag = "5.1.x-nightly-20261117" + + // Deploy Central. + envrcPath := filepath.Join(t.TempDir(), ".envrc") + runCommand(t, deployTimeout, nil, append([]string{roxieBinary, "deploy", "--early-readiness", "central", + "--tag", nightlyTag, "--envrc", envrcPath}, modifiedCommonDeployArgs...)...) + + // Deploy two SecuredClusters. + // TODO, using async waitgroup. + envrcEnv := loadEnvrcFile(t, envrcPath) + securedClusterCounter := 0 + deploySecuredCluster := func(ns string) { + t.Logf("=== Deploying SecuredCluster into namespace %s ===", ns) + securedClusterCounter++ + clusterName := fmt.Sprintf("sc-%d", securedClusterCounter) // For redeployment test we need fresh cluster names. + roxieCmd := append([]string{roxieBinary, "deploy", "--early-readiness", "secured-cluster", "--tag", nightlyTag, + "--set", `securedCluster.metadata.annotations.platform\.stackrox\.io/namespace-prefix-global-resources=true`, + "--set", "securedCluster.namespace=" + ns, "--set", "securedCluster.spec.clusterName=" + clusterName}, + modifiedCommonDeployArgs...) + runCommandPrefixed(t, ns+" ", deployTimeout, envrcEnv, roxieCmd...) + } + var wg sync.WaitGroup + for _, ns := range []string{nsA, nsB} { + wg.Go(func() { + deploySecuredCluster(ns) + }) + } + wg.Wait() + assert.ElementsMatch(t, []string{nsA, nsB}, crNamespaces(t, "securedcluster")) + uidB := securedClusterUID(t, nsB) + + // Redeploying A must not touch B: B's CR keeps its UID (i.e. was not deleted and recreated). + deploySecuredCluster(nsA) + assert.ElementsMatch(t, []string{nsA, nsB}, crNamespaces(t, "securedcluster")) + assert.Equal(t, uidB, securedClusterUID(t, nsB), "SecuredCluster in %s was replaced by redeploying %s", nsB, nsA) + + // Tear down SC in nsA. + t.Logf("=== Tearing down SecuredCluster in namespace %s only ===", nsA) + runCommand(t, teardownTimeout, nil, roxieBinary, "teardown", "--skip-user-config", "secured-cluster", "--set", "securedCluster.namespace="+nsA) + assert.Equal(t, []string{nsB}, crNamespaces(t, "securedcluster")) + + // Tear down everything. + t.Log("=== Tearing down everything without configured namespaces ===") + runCommand(t, teardownTimeout, nil, roxieBinary, "teardown", "--skip-user-config", "all") + assert.Empty(t, crNamespaces(t, "securedcluster")) + assert.Empty(t, crNamespaces(t, "central")) +} + +// crNamespaces returns the namespaces containing a CR of the given type. +func crNamespaces(t *testing.T, crType string) []string { + t.Helper() + + output, err := exec.Command("kubectl", "get", crType, "-A", "-o", "jsonpath={.items[*].metadata.namespace}").Output() + require.NoError(t, err, "Failed to list %s CRs", crType) + return strings.Fields(string(output)) +} + +// securedClusterUID returns the UID of the SecuredCluster CR in the given namespace. +func securedClusterUID(t *testing.T, namespace string) string { + t.Helper() + + output, err := exec.Command("kubectl", "get", "securedcluster", "stackrox-secured-cluster-services", + "-n", namespace, "-o", "jsonpath={.metadata.uid}").Output() + require.NoError(t, err, "Failed to get SecuredCluster UID in namespace %s", namespace) + return string(output) +}