From 091f9f8ddc0972138521f15c99e413fceb60eb75 Mon Sep 17 00:00:00 2001 From: Gianluca Mardente Date: Fri, 20 Nov 2026 09:13:45 +0100 Subject: [PATCH] Requeue ClusterSummaries waiting on dependsOn instead of waiting for the poll A ClusterSummary blocked on a dependsOn prerequisite (deploy), or a prerequisite waiting for its dependents to be removed (delete), only noticed the change at its next 10 second poll, adding up to 10 seconds per dependency level. When a ClusterSummary becomes Provisioned, its dependents (found through the dependency manager, new GetDependents) are requeued: the in-memory cooldown is expired and the retrigger annotation is updated, so no new watch is needed. When a ClusterSummary is removed, the ClusterSummaries of the profiles it depends on are requeued if they are being deleted: their DeletedInstances entry is cleared and the retrigger annotation is updated. Both are done after the status change has been persisted, so the requeued ClusterSummaries do not read stale state from the cache. With --auto-deploy-dependencies=false the dependency manager tracks nothing and dependents fall back to the poll on deploy; the delete side reads dependsOn directly and is not affected. Measured on 6 fake clusters: a dependent finishes 10s after its prerequisite instead of 19-20s; on delete the prerequisite is removed 15s after its dependent instead of 19-21s, and the whole delete takes 30s instead of 55s. --- controllers/clustersummary_controller.go | 185 ++++++++++++++++++ controllers/clustersummary_controller_test.go | 162 +++++++++++++++ controllers/dependencymanager/manager.go | 19 ++ controllers/dependencymanager/manager_test.go | 27 +++ controllers/export_test.go | 4 + 5 files changed, 397 insertions(+) diff --git a/controllers/clustersummary_controller.go b/controllers/clustersummary_controller.go index ad0a35d7..e5d9bbee 100644 --- a/controllers/clustersummary_controller.go +++ b/controllers/clustersummary_controller.go @@ -52,6 +52,7 @@ import ( configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers/chartmanager" + "github.com/projectsveltos/addon-controller/controllers/dependencymanager" "github.com/projectsveltos/addon-controller/lib/clusterops" "github.com/projectsveltos/addon-controller/pkg/scope" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" @@ -233,6 +234,7 @@ func (r *ClusterSummaryReconciler) Reconcile(ctx context.Context, req ctrl.Reque // bypassing the intended NextReconcileTime backoff). We cannot rely solely on a watch event // to re-enqueue because the conflict may be caused by the controller's own logic (e.g. a // tier change), in which case no further watch event will arrive. + wasProvisioned := isCluterSummaryProvisioned(clusterSummary) defer func() { if err = clusterSummaryScope.Close(ctx); err != nil { if apierrors.IsConflict(err) { @@ -242,6 +244,8 @@ func (r *ClusterSummaryReconciler) Reconcile(ctx context.Context, req ctrl.Reque return } reterr = err + } else { + r.requeueWaitingClusterSummaries(ctx, clusterSummary, wasProvisioned, logger) } }() @@ -1906,6 +1910,187 @@ func (r *ClusterSummaryReconciler) areDependentsRemoved(ctx context.Context, clu return true, "", nil } +// requeueWaitingClusterSummaries requeues the ClusterSummaries that were waiting for clusterSummary to +// change. It is called once the change has been persisted: those ClusterSummaries read this one from the +// cache, and might otherwise still see it as it was. +func (r *ClusterSummaryReconciler) requeueWaitingClusterSummaries(ctx context.Context, + clusterSummary *configv1beta1.ClusterSummary, wasProvisioned bool, logger logr.Logger) { + + if !clusterSummary.DeletionTimestamp.IsZero() { + // Finalizer removed: this ClusterSummary is gone. Its prerequisites, if being deleted, were waiting for it + if !controllerutil.ContainsFinalizer(clusterSummary, configv1beta1.ClusterSummaryFinalizer) { + r.requeuePrerequisites(ctx, clusterSummary, logger) + } + return + } + + if !wasProvisioned && isCluterSummaryProvisioned(clusterSummary) { + // Dependents were waiting for this ClusterSummary to be provisioned + r.requeueDependents(ctx, clusterSummary, logger) + } +} + +// requeueDependents is called once a ClusterSummary has become Provisioned. The ClusterSummaries of +// the same cluster that list this profile in DependsOn are waiting for it: without this call they +// only notice at their next poll (normalRequeueAfter). The dependency manager knows which profiles +// depend on this one for this cluster. When auto-deploy-dependencies is disabled the manager does +// not track dependents, and they fall back to polling. +func (r *ClusterSummaryReconciler) requeueDependents(ctx context.Context, + clusterSummary *configv1beta1.ClusterSummary, logger logr.Logger) { + + depManager, err := dependencymanager.GetManagerInstance() + if err != nil { + return + } + + profileOwner, err := configv1beta1.GetProfileOwnerReference(clusterSummary) + if err != nil { + return + } + + prerequisite := &corev1.ObjectReference{ + Name: profileOwner.Name, + Kind: profileOwner.Kind, + APIVersion: configv1beta1.GroupVersion.String(), + } + if profileOwner.Kind == configv1beta1.ProfileKind { + prerequisite.Namespace = clusterSummary.Namespace + } + + dependents := depManager.GetDependents(prerequisite, getClusterObjectReferenceFromClusterSummary(clusterSummary)) + r.retriggerClusterSummaries(ctx, getDependentClusterSummaryKeys(clusterSummary, dependents), logger) +} + +// getDependentClusterSummaryKeys returns the keys of the ClusterSummaries, for the same cluster as +// clusterSummary, owned by the dependents profiles. +func getDependentClusterSummaryKeys(clusterSummary *configv1beta1.ClusterSummary, + dependents []corev1.ObjectReference) []types.NamespacedName { + + keys := make([]types.NamespacedName, len(dependents)) + for i := range dependents { + keys[i] = getClusterSummaryKey(clusterSummary, dependents[i].Kind, dependents[i].Name) + } + + return keys +} + +// getPrerequisiteClusterSummaryKeys returns the keys of the ClusterSummaries, for the same cluster as +// clusterSummary, owned by the profiles clusterSummary depends on. +func getPrerequisiteClusterSummaryKeys(clusterSummary *configv1beta1.ClusterSummary) []types.NamespacedName { + profileOwner, err := configv1beta1.GetProfileOwnerReference(clusterSummary) + if err != nil { + return nil + } + + dependsOn := clusterSummary.Spec.ClusterProfileSpec.DependsOn + keys := make([]types.NamespacedName, len(dependsOn)) + for i := range dependsOn { + keys[i] = getClusterSummaryKey(clusterSummary, profileOwner.Kind, dependsOn[i]) + } + + return keys +} + +// getClusterSummaryKey returns the key of the ClusterSummary for the profile, of the given kind and +// name, and the same cluster as clusterSummary +func getClusterSummaryKey(clusterSummary *configv1beta1.ClusterSummary, profileKind, profileName string, +) types.NamespacedName { + + isSveltosCluster := clusterSummary.Spec.ClusterType == libsveltosv1beta1.ClusterTypeSveltos + + return types.NamespacedName{ + Namespace: clusterSummary.Namespace, + Name: clusterops.GetClusterSummaryName(profileKind, profileName, clusterSummary.Spec.ClusterName, isSveltosCluster), + } +} + +// retriggerClusterSummaries causes the given ClusterSummaries to be reconciled right away: a blocked +// ClusterSummary has a cooldown set, which is expired here, and the retrigger annotation is updated. +func (r *ClusterSummaryReconciler) retriggerClusterSummaries(ctx context.Context, + keys []types.NamespacedName, logger logr.Logger) { + + for i := range keys { + // A ClusterSummary waiting for its prerequisites has a cooldown: expire it, otherwise + // the reconciliation triggered below would be skipped + r.expireCooldown(keys[i]) + + clusterSummary := &configv1beta1.ClusterSummary{} + if err := r.Get(ctx, keys[i], clusterSummary); err != nil { + if !apierrors.IsNotFound(err) { + logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to get ClusterSummary %s: %v", keys[i], err)) + } + continue + } + + logger.V(logs.LogDebug).Info(fmt.Sprintf("prerequisite provisioned. Requeuing ClusterSummary %s", keys[i])) + r.setRetriggerAnnotation(ctx, clusterSummary, logger) + } +} + +// requeuePrerequisites is called once a ClusterSummary has been removed. The ClusterSummaries of the +// same cluster for the profiles it depends on cannot be removed until their dependents are gone: if they +// are being deleted they are waiting for this one, and are reconciled right away instead of at their next poll. +func (r *ClusterSummaryReconciler) requeuePrerequisites(ctx context.Context, + clusterSummary *configv1beta1.ClusterSummary, logger logr.Logger) { + + keys := getPrerequisiteClusterSummaryKeys(clusterSummary) + for i := range keys { + prerequisite := &configv1beta1.ClusterSummary{} + if err := r.Get(ctx, keys[i], prerequisite); err != nil { + if !apierrors.IsNotFound(err) { + logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to get ClusterSummary %s: %v", keys[i], err)) + } + continue + } + + // Nothing is waiting for a ClusterSummary that is not being deleted + if prerequisite.DeletionTimestamp.IsZero() { + continue + } + + logger.V(logs.LogDebug).Info(fmt.Sprintf("dependent removed. Requeuing ClusterSummary %s", keys[i])) + r.expireDeletedInstance(keys[i]) + r.setRetriggerAnnotation(ctx, prerequisite, logger) + } +} + +// setRetriggerAnnotation updates the retrigger annotation, which causes a new reconciliation +func (r *ClusterSummaryReconciler) setRetriggerAnnotation(ctx context.Context, + clusterSummary *configv1beta1.ClusterSummary, logger logr.Logger) { + + patch := client.MergeFrom(clusterSummary.DeepCopy()) + if clusterSummary.Annotations == nil { + clusterSummary.Annotations = map[string]string{} + } + clusterSummary.Annotations[retriggerAnnotation] = time.Now().UTC().Format(time.RFC3339Nano) + if err := r.Patch(ctx, clusterSummary, patch); err != nil { + // The ClusterSummary will still be reconciled at its next poll + logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to requeue ClusterSummary %s/%s: %v", + clusterSummary.Namespace, clusterSummary.Name, err)) + } +} + +// expireDeletedInstance allows a ClusterSummary being deleted to be reconciled right away: +// skipReconciliation otherwise ignores it for deleteRequeueAfter after its last reconciliation. +func (r *ClusterSummaryReconciler) expireDeletedInstance(key types.NamespacedName) { + r.PolicyMux.Lock() + defer r.PolicyMux.Unlock() + + delete(r.DeletedInstances, key) +} + +// expireCooldown makes the in-memory cooldown of a ClusterSummary, if any, expire now. The entry is +// kept (not deleted): skipReconciliation would otherwise rebuild it from the persisted status. +func (r *ClusterSummaryReconciler) expireCooldown(key types.NamespacedName) { + r.PolicyMux.Lock() + defer r.PolicyMux.Unlock() + + if cooldown, ok := r.NextReconcileTimes[key]; ok { + cooldown.Until = time.Now() + r.NextReconcileTimes[key] = cooldown + } +} + func (r *ClusterSummaryReconciler) setFailureMessage(clusterSummaryScope *scope.ClusterSummaryScope, failureMessage string) { if clusterSummaryScope.ClusterSummary.Spec.ClusterProfileSpec.HelmCharts != nil { clusterSummaryScope.SetFailureMessage(libsveltosv1beta1.FeatureHelm, &failureMessage) diff --git a/controllers/clustersummary_controller_test.go b/controllers/clustersummary_controller_test.go index 895deb34..94e11d4b 100644 --- a/controllers/clustersummary_controller_test.go +++ b/controllers/clustersummary_controller_test.go @@ -647,6 +647,168 @@ var _ = Describe("ClustersummaryController", func() { Expect(featureResourcesVerified).To(BeTrue()) }) + It("getDependentClusterSummaryKeys returns the ClusterSummaries of the dependents for the same cluster", func() { + dependents := []corev1.ObjectReference{ + {Kind: configv1beta1.ClusterProfileKind, Name: randomString(), APIVersion: configv1beta1.GroupVersion.String()}, + {Kind: configv1beta1.ProfileKind, Name: randomString(), Namespace: namespace, + APIVersion: configv1beta1.GroupVersion.String()}, + } + + keys := controllers.GetDependentClusterSummaryKeys(clusterSummary, dependents) + Expect(keys).To(HaveLen(len(dependents))) + for i := range dependents { + Expect(keys[i].Namespace).To(Equal(clusterSummary.Namespace)) + Expect(keys[i].Name).To(Equal(clusterops.GetClusterSummaryName(dependents[i].Kind, dependents[i].Name, + clusterName, false))) + } + }) + + It("retriggerClusterSummaries expires the cooldown and updates the retrigger annotation", func() { + dependent := &configv1beta1.ClusterSummary{ + ObjectMeta: metav1.ObjectMeta{ + Name: randomString(), + Namespace: namespace, + }, + Spec: configv1beta1.ClusterSummarySpec{ + ClusterNamespace: cluster.Namespace, + ClusterName: cluster.Name, + ClusterType: libsveltosv1beta1.ClusterTypeCapi, + }, + } + Expect(testEnv.Create(context.TODO(), dependent)).To(Succeed()) + Expect(waitForObject(context.TODO(), testEnv.Client, dependent)).To(Succeed()) + + dependentKey := types.NamespacedName{Namespace: dependent.Namespace, Name: dependent.Name} + missingKey := types.NamespacedName{Namespace: namespace, Name: randomString()} + + reconciler := &controllers.ClusterSummaryReconciler{ + Client: testEnv.Client, + Scheme: scheme, + PolicyMux: sync.Mutex{}, + ClusterMap: make(map[corev1.ObjectReference]*libsveltosset.Set), + ReferenceMap: make(map[corev1.ObjectReference]*libsveltosset.Set), + NextReconcileTimes: map[types.NamespacedName]controllers.ReconcileCooldown{ + dependentKey: {Until: time.Now().Add(time.Hour), Generation: dependent.Generation}, + }, + } + + // A ClusterSummary that does not exist anymore is skipped + controllers.RetriggerClusterSummaries(reconciler, context.TODO(), []types.NamespacedName{missingKey, dependentKey}, + textlogger.NewLogger(textlogger.NewConfig())) + + reconciler.PolicyMux.Lock() + cooldown, ok := reconciler.NextReconcileTimes[dependentKey] + reconciler.PolicyMux.Unlock() + // The entry is kept, but it is not in the future anymore + Expect(ok).To(BeTrue()) + Expect(cooldown.Until.After(time.Now())).To(BeFalse()) + + Eventually(func() string { + currentDependent := &configv1beta1.ClusterSummary{} + err := testEnv.Get(context.TODO(), dependentKey, currentDependent) + if err != nil { + return "" + } + return currentDependent.Annotations["projectsveltos.io/retrigger"] + }, timeout, pollingInterval).ShouldNot(BeEmpty()) + }) + + It("getPrerequisiteClusterSummaryKeys returns the ClusterSummaries of the profiles it depends on", func() { + prerequisites := []string{randomString(), randomString()} + clusterSummary.Spec.ClusterProfileSpec.DependsOn = prerequisites + + keys := controllers.GetPrerequisiteClusterSummaryKeys(clusterSummary) + Expect(keys).To(HaveLen(len(prerequisites))) + for i := range prerequisites { + Expect(keys[i].Namespace).To(Equal(clusterSummary.Namespace)) + Expect(keys[i].Name).To(Equal(clusterops.GetClusterSummaryName(configv1beta1.ClusterProfileKind, + prerequisites[i], clusterName, false))) + } + }) + + It("requeuePrerequisites retriggers only the prerequisites being deleted", func() { + deletingProfile := randomString() + activeProfile := randomString() + deletingName := clusterops.GetClusterSummaryName(configv1beta1.ClusterProfileKind, deletingProfile, clusterName, false) + activeName := clusterops.GetClusterSummaryName(configv1beta1.ClusterProfileKind, activeProfile, clusterName, false) + + // The finalizer keeps the deleting ClusterSummary around with a DeletionTimestamp + deleting := &configv1beta1.ClusterSummary{ + ObjectMeta: metav1.ObjectMeta{ + Name: deletingName, + Namespace: namespace, + Finalizers: []string{configv1beta1.ClusterSummaryFinalizer}, + }, + Spec: configv1beta1.ClusterSummarySpec{ + ClusterNamespace: cluster.Namespace, + ClusterName: cluster.Name, + ClusterType: libsveltosv1beta1.ClusterTypeCapi, + }, + } + active := &configv1beta1.ClusterSummary{ + ObjectMeta: metav1.ObjectMeta{ + Name: activeName, + Namespace: namespace, + }, + Spec: configv1beta1.ClusterSummarySpec{ + ClusterNamespace: cluster.Namespace, + ClusterName: cluster.Name, + ClusterType: libsveltosv1beta1.ClusterTypeCapi, + }, + } + Expect(testEnv.Create(context.TODO(), deleting)).To(Succeed()) + Expect(testEnv.Create(context.TODO(), active)).To(Succeed()) + Expect(testEnv.Delete(context.TODO(), deleting)).To(Succeed()) + Expect(waitForObject(context.TODO(), testEnv.Client, active)).To(Succeed()) + Eventually(func() bool { + current := &configv1beta1.ClusterSummary{} + err := testEnv.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: deletingName}, current) + return err == nil && !current.DeletionTimestamp.IsZero() + }, timeout, pollingInterval).Should(BeTrue()) + + deletingKey := types.NamespacedName{Namespace: namespace, Name: deletingName} + activeKey := types.NamespacedName{Namespace: namespace, Name: activeName} + + reconciler := &controllers.ClusterSummaryReconciler{ + Client: testEnv.Client, + Scheme: scheme, + PolicyMux: sync.Mutex{}, + ClusterMap: make(map[corev1.ObjectReference]*libsveltosset.Set), + ReferenceMap: make(map[corev1.ObjectReference]*libsveltosset.Set), + DeletedInstances: map[types.NamespacedName]time.Time{ + deletingKey: time.Now(), + }, + } + + // clusterSummary depends on the two ClusterSummaries above and on one that does not exist + clusterSummary.Spec.ClusterProfileSpec.DependsOn = []string{deletingProfile, activeProfile, randomString()} + controllers.RequeuePrerequisites(reconciler, context.TODO(), clusterSummary, + textlogger.NewLogger(textlogger.NewConfig())) + + reconciler.PolicyMux.Lock() + _, ok := reconciler.DeletedInstances[deletingKey] + reconciler.PolicyMux.Unlock() + Expect(ok).To(BeFalse(), "the deleting ClusterSummary can be reconciled right away") + + Eventually(func() string { + current := &configv1beta1.ClusterSummary{} + err := testEnv.Get(context.TODO(), deletingKey, current) + if err != nil { + return "" + } + return current.Annotations["projectsveltos.io/retrigger"] + }, timeout, pollingInterval).ShouldNot(BeEmpty()) + + Consistently(func() string { + current := &configv1beta1.ClusterSummary{} + err := testEnv.Get(context.TODO(), activeKey, current) + if err != nil { + return "error" + } + return current.Annotations["projectsveltos.io/retrigger"] + }, "2s", pollingInterval).Should(BeEmpty()) + }) + It("shouldReconcile returns true when mode is OneTime but not all helm charts are deployed", func() { clusterSummary.Spec.ClusterProfileSpec.SyncMode = configv1beta1.SyncModeOneTime clusterSummary.Spec.ClusterProfileSpec.HelmCharts = []configv1beta1.HelmChart{ diff --git a/controllers/dependencymanager/manager.go b/controllers/dependencymanager/manager.go index 43cd8f04..f63b55f3 100644 --- a/controllers/dependencymanager/manager.go +++ b/controllers/dependencymanager/manager.go @@ -240,6 +240,25 @@ func (m *instance) GetClusterDeployments(profile *corev1.ObjectReference) []core return results } +// GetDependents returns the profiles that list prerequisite in their DependsOn and are deployed to cluster. +func (m *instance) GetDependents(prerequisite, cluster *corev1.ObjectReference) []corev1.ObjectReference { + if !m.enabled { + return nil + } + + m.chartMux.RLock() + defer m.chartMux.RUnlock() + + clusters := m.profileClusterRequests.getClusterDeployments(prerequisite) + requestingProfiles := clusters.getRequestingProfiles(cluster) + results := make([]corev1.ObjectReference, 0, len(requestingProfiles.Dependents)) + for d := range requestingProfiles.Dependents { + results = append(results, d) + } + + return results +} + // UpdatePrerequisites registers the prerequisites of a dependent (i.e a profile) as needed on the // dependent's matching clusters. // diff --git a/controllers/dependencymanager/manager_test.go b/controllers/dependencymanager/manager_test.go index 2e7a1e8d..491e93b5 100644 --- a/controllers/dependencymanager/manager_test.go +++ b/controllers/dependencymanager/manager_test.go @@ -89,6 +89,33 @@ var _ = Describe("Dependency manager", func() { } }) + It("GetDependents returns the profiles depending on a prerequisite for a given cluster", func() { + manager, err := dependencymanager.GetManagerInstance() + Expect(err).To(BeNil()) + + dependencies := getDependencies(profile) + manager.UpdateDependencies(objRef, profile.Status.MatchingClusterRefs, dependencies, logger) + + for i := range dependencies { + for j := range profile.Status.MatchingClusterRefs { + dependents := manager.GetDependents(&dependencies[i], &profile.Status.MatchingClusterRefs[j]) + Expect(dependents).To(Equal([]corev1.ObjectReference{*objRef})) + } + } + + By("A cluster the dependent is not deployed to has no dependents") + otherCluster := &corev1.ObjectReference{ + Namespace: randomString(), + Name: randomString(), + Kind: libsveltosv1beta1.SveltosClusterKind, + APIVersion: libsveltosv1beta1.GroupVersion.String(), + } + Expect(manager.GetDependents(&dependencies[0], otherCluster)).To(BeEmpty()) + + By("A profile that is not a prerequisite has no dependents") + Expect(manager.GetDependents(objRef, &profile.Status.MatchingClusterRefs[0])).To(BeEmpty()) + }) + It("updateConfigMap same prerequisite for multiple dependents", func() { manager, err := dependencymanager.GetManagerInstance() Expect(err).To(BeNil()) diff --git a/controllers/export_test.go b/controllers/export_test.go index fe7d32a5..bb0024e9 100644 --- a/controllers/export_test.go +++ b/controllers/export_test.go @@ -104,6 +104,10 @@ var ( SetFailureMessage = (*ClusterSummaryReconciler).setFailureMessage ResetFeatureStatus = (*ClusterSummaryReconciler).resetFeatureStatus PrepareForDeployment = (*ClusterSummaryReconciler).prepareForDeployment + GetDependentClusterSummaryKeys = getDependentClusterSummaryKeys + RetriggerClusterSummaries = (*ClusterSummaryReconciler).retriggerClusterSummaries + GetPrerequisiteClusterSummaryKeys = getPrerequisiteClusterSummaryKeys + RequeuePrerequisites = (*ClusterSummaryReconciler).requeuePrerequisites ConvertResultStatus = (*ClusterSummaryReconciler).convertResultStatus RequeueClusterSummaryForReference = (*ClusterSummaryReconciler).requeueClusterSummaryForReference