diff --git a/controllers/clustersummary_controller.go b/controllers/clustersummary_controller.go index ad0a35d7..e5d9bbee 100644 --- a/controllers/clustersummary_controller.go +++ b/controllers/clustersummary_controller.go @@ -52,6 +52,7 @@ import ( configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers/chartmanager" + "github.com/projectsveltos/addon-controller/controllers/dependencymanager" "github.com/projectsveltos/addon-controller/lib/clusterops" "github.com/projectsveltos/addon-controller/pkg/scope" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" @@ -233,6 +234,7 @@ func (r *ClusterSummaryReconciler) Reconcile(ctx context.Context, req ctrl.Reque // bypassing the intended NextReconcileTime backoff). We cannot rely solely on a watch event // to re-enqueue because the conflict may be caused by the controller's own logic (e.g. a // tier change), in which case no further watch event will arrive. + wasProvisioned := isCluterSummaryProvisioned(clusterSummary) defer func() { if err = clusterSummaryScope.Close(ctx); err != nil { if apierrors.IsConflict(err) { @@ -242,6 +244,8 @@ func (r *ClusterSummaryReconciler) Reconcile(ctx context.Context, req ctrl.Reque return } reterr = err + } else { + r.requeueWaitingClusterSummaries(ctx, clusterSummary, wasProvisioned, logger) } }() @@ -1906,6 +1910,187 @@ func (r *ClusterSummaryReconciler) areDependentsRemoved(ctx context.Context, clu return true, "", nil } +// requeueWaitingClusterSummaries requeues the ClusterSummaries that were waiting for clusterSummary to +// change. It is called once the change has been persisted: those ClusterSummaries read this one from the +// cache, and might otherwise still see it as it was. +func (r *ClusterSummaryReconciler) requeueWaitingClusterSummaries(ctx context.Context, + clusterSummary *configv1beta1.ClusterSummary, wasProvisioned bool, logger logr.Logger) { + + if !clusterSummary.DeletionTimestamp.IsZero() { + // Finalizer removed: this ClusterSummary is gone. Its prerequisites, if being deleted, were waiting for it + if !controllerutil.ContainsFinalizer(clusterSummary, configv1beta1.ClusterSummaryFinalizer) { + r.requeuePrerequisites(ctx, clusterSummary, logger) + } + return + } + + if !wasProvisioned && isCluterSummaryProvisioned(clusterSummary) { + // Dependents were waiting for this ClusterSummary to be provisioned + r.requeueDependents(ctx, clusterSummary, logger) + } +} + +// requeueDependents is called once a ClusterSummary has become Provisioned. The ClusterSummaries of +// the same cluster that list this profile in DependsOn are waiting for it: without this call they +// only notice at their next poll (normalRequeueAfter). The dependency manager knows which profiles +// depend on this one for this cluster. When auto-deploy-dependencies is disabled the manager does +// not track dependents, and they fall back to polling. +func (r *ClusterSummaryReconciler) requeueDependents(ctx context.Context, + clusterSummary *configv1beta1.ClusterSummary, logger logr.Logger) { + + depManager, err := dependencymanager.GetManagerInstance() + if err != nil { + return + } + + profileOwner, err := configv1beta1.GetProfileOwnerReference(clusterSummary) + if err != nil { + return + } + + prerequisite := &corev1.ObjectReference{ + Name: profileOwner.Name, + Kind: profileOwner.Kind, + APIVersion: configv1beta1.GroupVersion.String(), + } + if profileOwner.Kind == configv1beta1.ProfileKind { + prerequisite.Namespace = clusterSummary.Namespace + } + + dependents := depManager.GetDependents(prerequisite, getClusterObjectReferenceFromClusterSummary(clusterSummary)) + r.retriggerClusterSummaries(ctx, getDependentClusterSummaryKeys(clusterSummary, dependents), logger) +} + +// getDependentClusterSummaryKeys returns the keys of the ClusterSummaries, for the same cluster as +// clusterSummary, owned by the dependents profiles. +func getDependentClusterSummaryKeys(clusterSummary *configv1beta1.ClusterSummary, + dependents []corev1.ObjectReference) []types.NamespacedName { + + keys := make([]types.NamespacedName, len(dependents)) + for i := range dependents { + keys[i] = getClusterSummaryKey(clusterSummary, dependents[i].Kind, dependents[i].Name) + } + + return keys +} + +// getPrerequisiteClusterSummaryKeys returns the keys of the ClusterSummaries, for the same cluster as +// clusterSummary, owned by the profiles clusterSummary depends on. +func getPrerequisiteClusterSummaryKeys(clusterSummary *configv1beta1.ClusterSummary) []types.NamespacedName { + profileOwner, err := configv1beta1.GetProfileOwnerReference(clusterSummary) + if err != nil { + return nil + } + + dependsOn := clusterSummary.Spec.ClusterProfileSpec.DependsOn + keys := make([]types.NamespacedName, len(dependsOn)) + for i := range dependsOn { + keys[i] = getClusterSummaryKey(clusterSummary, profileOwner.Kind, dependsOn[i]) + } + + return keys +} + +// getClusterSummaryKey returns the key of the ClusterSummary for the profile, of the given kind and +// name, and the same cluster as clusterSummary +func getClusterSummaryKey(clusterSummary *configv1beta1.ClusterSummary, profileKind, profileName string, +) types.NamespacedName { + + isSveltosCluster := clusterSummary.Spec.ClusterType == libsveltosv1beta1.ClusterTypeSveltos + + return types.NamespacedName{ + Namespace: clusterSummary.Namespace, + Name: clusterops.GetClusterSummaryName(profileKind, profileName, clusterSummary.Spec.ClusterName, isSveltosCluster), + } +} + +// retriggerClusterSummaries causes the given ClusterSummaries to be reconciled right away: a blocked +// ClusterSummary has a cooldown set, which is expired here, and the retrigger annotation is updated. +func (r *ClusterSummaryReconciler) retriggerClusterSummaries(ctx context.Context, + keys []types.NamespacedName, logger logr.Logger) { + + for i := range keys { + // A ClusterSummary waiting for its prerequisites has a cooldown: expire it, otherwise + // the reconciliation triggered below would be skipped + r.expireCooldown(keys[i]) + + clusterSummary := &configv1beta1.ClusterSummary{} + if err := r.Get(ctx, keys[i], clusterSummary); err != nil { + if !apierrors.IsNotFound(err) { + logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to get ClusterSummary %s: %v", keys[i], err)) + } + continue + } + + logger.V(logs.LogDebug).Info(fmt.Sprintf("prerequisite provisioned. Requeuing ClusterSummary %s", keys[i])) + r.setRetriggerAnnotation(ctx, clusterSummary, logger) + } +} + +// requeuePrerequisites is called once a ClusterSummary has been removed. The ClusterSummaries of the +// same cluster for the profiles it depends on cannot be removed until their dependents are gone: if they +// are being deleted they are waiting for this one, and are reconciled right away instead of at their next poll. +func (r *ClusterSummaryReconciler) requeuePrerequisites(ctx context.Context, + clusterSummary *configv1beta1.ClusterSummary, logger logr.Logger) { + + keys := getPrerequisiteClusterSummaryKeys(clusterSummary) + for i := range keys { + prerequisite := &configv1beta1.ClusterSummary{} + if err := r.Get(ctx, keys[i], prerequisite); err != nil { + if !apierrors.IsNotFound(err) { + logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to get ClusterSummary %s: %v", keys[i], err)) + } + continue + } + + // Nothing is waiting for a ClusterSummary that is not being deleted + if prerequisite.DeletionTimestamp.IsZero() { + continue + } + + logger.V(logs.LogDebug).Info(fmt.Sprintf("dependent removed. Requeuing ClusterSummary %s", keys[i])) + r.expireDeletedInstance(keys[i]) + r.setRetriggerAnnotation(ctx, prerequisite, logger) + } +} + +// setRetriggerAnnotation updates the retrigger annotation, which causes a new reconciliation +func (r *ClusterSummaryReconciler) setRetriggerAnnotation(ctx context.Context, + clusterSummary *configv1beta1.ClusterSummary, logger logr.Logger) { + + patch := client.MergeFrom(clusterSummary.DeepCopy()) + if clusterSummary.Annotations == nil { + clusterSummary.Annotations = map[string]string{} + } + clusterSummary.Annotations[retriggerAnnotation] = time.Now().UTC().Format(time.RFC3339Nano) + if err := r.Patch(ctx, clusterSummary, patch); err != nil { + // The ClusterSummary will still be reconciled at its next poll + logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to requeue ClusterSummary %s/%s: %v", + clusterSummary.Namespace, clusterSummary.Name, err)) + } +} + +// expireDeletedInstance allows a ClusterSummary being deleted to be reconciled right away: +// skipReconciliation otherwise ignores it for deleteRequeueAfter after its last reconciliation. +func (r *ClusterSummaryReconciler) expireDeletedInstance(key types.NamespacedName) { + r.PolicyMux.Lock() + defer r.PolicyMux.Unlock() + + delete(r.DeletedInstances, key) +} + +// expireCooldown makes the in-memory cooldown of a ClusterSummary, if any, expire now. The entry is +// kept (not deleted): skipReconciliation would otherwise rebuild it from the persisted status. +func (r *ClusterSummaryReconciler) expireCooldown(key types.NamespacedName) { + r.PolicyMux.Lock() + defer r.PolicyMux.Unlock() + + if cooldown, ok := r.NextReconcileTimes[key]; ok { + cooldown.Until = time.Now() + r.NextReconcileTimes[key] = cooldown + } +} + func (r *ClusterSummaryReconciler) setFailureMessage(clusterSummaryScope *scope.ClusterSummaryScope, failureMessage string) { if clusterSummaryScope.ClusterSummary.Spec.ClusterProfileSpec.HelmCharts != nil { clusterSummaryScope.SetFailureMessage(libsveltosv1beta1.FeatureHelm, &failureMessage) diff --git a/controllers/clustersummary_controller_test.go b/controllers/clustersummary_controller_test.go index 895deb34..94e11d4b 100644 --- a/controllers/clustersummary_controller_test.go +++ b/controllers/clustersummary_controller_test.go @@ -647,6 +647,168 @@ var _ = Describe("ClustersummaryController", func() { Expect(featureResourcesVerified).To(BeTrue()) }) + It("getDependentClusterSummaryKeys returns the ClusterSummaries of the dependents for the same cluster", func() { + dependents := []corev1.ObjectReference{ + {Kind: configv1beta1.ClusterProfileKind, Name: randomString(), APIVersion: configv1beta1.GroupVersion.String()}, + {Kind: configv1beta1.ProfileKind, Name: randomString(), Namespace: namespace, + APIVersion: configv1beta1.GroupVersion.String()}, + } + + keys := controllers.GetDependentClusterSummaryKeys(clusterSummary, dependents) + Expect(keys).To(HaveLen(len(dependents))) + for i := range dependents { + Expect(keys[i].Namespace).To(Equal(clusterSummary.Namespace)) + Expect(keys[i].Name).To(Equal(clusterops.GetClusterSummaryName(dependents[i].Kind, dependents[i].Name, + clusterName, false))) + } + }) + + It("retriggerClusterSummaries expires the cooldown and updates the retrigger annotation", func() { + dependent := &configv1beta1.ClusterSummary{ + ObjectMeta: metav1.ObjectMeta{ + Name: randomString(), + Namespace: namespace, + }, + Spec: configv1beta1.ClusterSummarySpec{ + ClusterNamespace: cluster.Namespace, + ClusterName: cluster.Name, + ClusterType: libsveltosv1beta1.ClusterTypeCapi, + }, + } + Expect(testEnv.Create(context.TODO(), dependent)).To(Succeed()) + Expect(waitForObject(context.TODO(), testEnv.Client, dependent)).To(Succeed()) + + dependentKey := types.NamespacedName{Namespace: dependent.Namespace, Name: dependent.Name} + missingKey := types.NamespacedName{Namespace: namespace, Name: randomString()} + + reconciler := &controllers.ClusterSummaryReconciler{ + Client: testEnv.Client, + Scheme: scheme, + PolicyMux: sync.Mutex{}, + ClusterMap: make(map[corev1.ObjectReference]*libsveltosset.Set), + ReferenceMap: make(map[corev1.ObjectReference]*libsveltosset.Set), + NextReconcileTimes: map[types.NamespacedName]controllers.ReconcileCooldown{ + dependentKey: {Until: time.Now().Add(time.Hour), Generation: dependent.Generation}, + }, + } + + // A ClusterSummary that does not exist anymore is skipped + controllers.RetriggerClusterSummaries(reconciler, context.TODO(), []types.NamespacedName{missingKey, dependentKey}, + textlogger.NewLogger(textlogger.NewConfig())) + + reconciler.PolicyMux.Lock() + cooldown, ok := reconciler.NextReconcileTimes[dependentKey] + reconciler.PolicyMux.Unlock() + // The entry is kept, but it is not in the future anymore + Expect(ok).To(BeTrue()) + Expect(cooldown.Until.After(time.Now())).To(BeFalse()) + + Eventually(func() string { + currentDependent := &configv1beta1.ClusterSummary{} + err := testEnv.Get(context.TODO(), dependentKey, currentDependent) + if err != nil { + return "" + } + return currentDependent.Annotations["projectsveltos.io/retrigger"] + }, timeout, pollingInterval).ShouldNot(BeEmpty()) + }) + + It("getPrerequisiteClusterSummaryKeys returns the ClusterSummaries of the profiles it depends on", func() { + prerequisites := []string{randomString(), randomString()} + clusterSummary.Spec.ClusterProfileSpec.DependsOn = prerequisites + + keys := controllers.GetPrerequisiteClusterSummaryKeys(clusterSummary) + Expect(keys).To(HaveLen(len(prerequisites))) + for i := range prerequisites { + Expect(keys[i].Namespace).To(Equal(clusterSummary.Namespace)) + Expect(keys[i].Name).To(Equal(clusterops.GetClusterSummaryName(configv1beta1.ClusterProfileKind, + prerequisites[i], clusterName, false))) + } + }) + + It("requeuePrerequisites retriggers only the prerequisites being deleted", func() { + deletingProfile := randomString() + activeProfile := randomString() + deletingName := clusterops.GetClusterSummaryName(configv1beta1.ClusterProfileKind, deletingProfile, clusterName, false) + activeName := clusterops.GetClusterSummaryName(configv1beta1.ClusterProfileKind, activeProfile, clusterName, false) + + // The finalizer keeps the deleting ClusterSummary around with a DeletionTimestamp + deleting := &configv1beta1.ClusterSummary{ + ObjectMeta: metav1.ObjectMeta{ + Name: deletingName, + Namespace: namespace, + Finalizers: []string{configv1beta1.ClusterSummaryFinalizer}, + }, + Spec: configv1beta1.ClusterSummarySpec{ + ClusterNamespace: cluster.Namespace, + ClusterName: cluster.Name, + ClusterType: libsveltosv1beta1.ClusterTypeCapi, + }, + } + active := &configv1beta1.ClusterSummary{ + ObjectMeta: metav1.ObjectMeta{ + Name: activeName, + Namespace: namespace, + }, + Spec: configv1beta1.ClusterSummarySpec{ + ClusterNamespace: cluster.Namespace, + ClusterName: cluster.Name, + ClusterType: libsveltosv1beta1.ClusterTypeCapi, + }, + } + Expect(testEnv.Create(context.TODO(), deleting)).To(Succeed()) + Expect(testEnv.Create(context.TODO(), active)).To(Succeed()) + Expect(testEnv.Delete(context.TODO(), deleting)).To(Succeed()) + Expect(waitForObject(context.TODO(), testEnv.Client, active)).To(Succeed()) + Eventually(func() bool { + current := &configv1beta1.ClusterSummary{} + err := testEnv.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: deletingName}, current) + return err == nil && !current.DeletionTimestamp.IsZero() + }, timeout, pollingInterval).Should(BeTrue()) + + deletingKey := types.NamespacedName{Namespace: namespace, Name: deletingName} + activeKey := types.NamespacedName{Namespace: namespace, Name: activeName} + + reconciler := &controllers.ClusterSummaryReconciler{ + Client: testEnv.Client, + Scheme: scheme, + PolicyMux: sync.Mutex{}, + ClusterMap: make(map[corev1.ObjectReference]*libsveltosset.Set), + ReferenceMap: make(map[corev1.ObjectReference]*libsveltosset.Set), + DeletedInstances: map[types.NamespacedName]time.Time{ + deletingKey: time.Now(), + }, + } + + // clusterSummary depends on the two ClusterSummaries above and on one that does not exist + clusterSummary.Spec.ClusterProfileSpec.DependsOn = []string{deletingProfile, activeProfile, randomString()} + controllers.RequeuePrerequisites(reconciler, context.TODO(), clusterSummary, + textlogger.NewLogger(textlogger.NewConfig())) + + reconciler.PolicyMux.Lock() + _, ok := reconciler.DeletedInstances[deletingKey] + reconciler.PolicyMux.Unlock() + Expect(ok).To(BeFalse(), "the deleting ClusterSummary can be reconciled right away") + + Eventually(func() string { + current := &configv1beta1.ClusterSummary{} + err := testEnv.Get(context.TODO(), deletingKey, current) + if err != nil { + return "" + } + return current.Annotations["projectsveltos.io/retrigger"] + }, timeout, pollingInterval).ShouldNot(BeEmpty()) + + Consistently(func() string { + current := &configv1beta1.ClusterSummary{} + err := testEnv.Get(context.TODO(), activeKey, current) + if err != nil { + return "error" + } + return current.Annotations["projectsveltos.io/retrigger"] + }, "2s", pollingInterval).Should(BeEmpty()) + }) + It("shouldReconcile returns true when mode is OneTime but not all helm charts are deployed", func() { clusterSummary.Spec.ClusterProfileSpec.SyncMode = configv1beta1.SyncModeOneTime clusterSummary.Spec.ClusterProfileSpec.HelmCharts = []configv1beta1.HelmChart{ diff --git a/controllers/dependencymanager/manager.go b/controllers/dependencymanager/manager.go index 43cd8f04..f63b55f3 100644 --- a/controllers/dependencymanager/manager.go +++ b/controllers/dependencymanager/manager.go @@ -240,6 +240,25 @@ func (m *instance) GetClusterDeployments(profile *corev1.ObjectReference) []core return results } +// GetDependents returns the profiles that list prerequisite in their DependsOn and are deployed to cluster. +func (m *instance) GetDependents(prerequisite, cluster *corev1.ObjectReference) []corev1.ObjectReference { + if !m.enabled { + return nil + } + + m.chartMux.RLock() + defer m.chartMux.RUnlock() + + clusters := m.profileClusterRequests.getClusterDeployments(prerequisite) + requestingProfiles := clusters.getRequestingProfiles(cluster) + results := make([]corev1.ObjectReference, 0, len(requestingProfiles.Dependents)) + for d := range requestingProfiles.Dependents { + results = append(results, d) + } + + return results +} + // UpdatePrerequisites registers the prerequisites of a dependent (i.e a profile) as needed on the // dependent's matching clusters. // diff --git a/controllers/dependencymanager/manager_test.go b/controllers/dependencymanager/manager_test.go index 2e7a1e8d..491e93b5 100644 --- a/controllers/dependencymanager/manager_test.go +++ b/controllers/dependencymanager/manager_test.go @@ -89,6 +89,33 @@ var _ = Describe("Dependency manager", func() { } }) + It("GetDependents returns the profiles depending on a prerequisite for a given cluster", func() { + manager, err := dependencymanager.GetManagerInstance() + Expect(err).To(BeNil()) + + dependencies := getDependencies(profile) + manager.UpdateDependencies(objRef, profile.Status.MatchingClusterRefs, dependencies, logger) + + for i := range dependencies { + for j := range profile.Status.MatchingClusterRefs { + dependents := manager.GetDependents(&dependencies[i], &profile.Status.MatchingClusterRefs[j]) + Expect(dependents).To(Equal([]corev1.ObjectReference{*objRef})) + } + } + + By("A cluster the dependent is not deployed to has no dependents") + otherCluster := &corev1.ObjectReference{ + Namespace: randomString(), + Name: randomString(), + Kind: libsveltosv1beta1.SveltosClusterKind, + APIVersion: libsveltosv1beta1.GroupVersion.String(), + } + Expect(manager.GetDependents(&dependencies[0], otherCluster)).To(BeEmpty()) + + By("A profile that is not a prerequisite has no dependents") + Expect(manager.GetDependents(objRef, &profile.Status.MatchingClusterRefs[0])).To(BeEmpty()) + }) + It("updateConfigMap same prerequisite for multiple dependents", func() { manager, err := dependencymanager.GetManagerInstance() Expect(err).To(BeNil()) diff --git a/controllers/export_test.go b/controllers/export_test.go index fe7d32a5..bb0024e9 100644 --- a/controllers/export_test.go +++ b/controllers/export_test.go @@ -104,6 +104,10 @@ var ( SetFailureMessage = (*ClusterSummaryReconciler).setFailureMessage ResetFeatureStatus = (*ClusterSummaryReconciler).resetFeatureStatus PrepareForDeployment = (*ClusterSummaryReconciler).prepareForDeployment + GetDependentClusterSummaryKeys = getDependentClusterSummaryKeys + RetriggerClusterSummaries = (*ClusterSummaryReconciler).retriggerClusterSummaries + GetPrerequisiteClusterSummaryKeys = getPrerequisiteClusterSummaryKeys + RequeuePrerequisites = (*ClusterSummaryReconciler).requeuePrerequisites ConvertResultStatus = (*ClusterSummaryReconciler).convertResultStatus RequeueClusterSummaryForReference = (*ClusterSummaryReconciler).requeueClusterSummaryForReference