Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
185 changes: 185 additions & 0 deletions controllers/clustersummary_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ import (

configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1"
"github.com/projectsveltos/addon-controller/controllers/chartmanager"
"github.com/projectsveltos/addon-controller/controllers/dependencymanager"
"github.com/projectsveltos/addon-controller/lib/clusterops"
"github.com/projectsveltos/addon-controller/pkg/scope"
libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1"
Expand Down Expand Up @@ -233,6 +234,7 @@ func (r *ClusterSummaryReconciler) Reconcile(ctx context.Context, req ctrl.Reque
// bypassing the intended NextReconcileTime backoff). We cannot rely solely on a watch event
// to re-enqueue because the conflict may be caused by the controller's own logic (e.g. a
// tier change), in which case no further watch event will arrive.
wasProvisioned := isCluterSummaryProvisioned(clusterSummary)
defer func() {
if err = clusterSummaryScope.Close(ctx); err != nil {
if apierrors.IsConflict(err) {
Expand All @@ -242,6 +244,8 @@ func (r *ClusterSummaryReconciler) Reconcile(ctx context.Context, req ctrl.Reque
return
}
reterr = err
} else {
r.requeueWaitingClusterSummaries(ctx, clusterSummary, wasProvisioned, logger)
}
}()

Expand Down Expand Up @@ -1906,6 +1910,187 @@ func (r *ClusterSummaryReconciler) areDependentsRemoved(ctx context.Context, clu
return true, "", nil
}

// requeueWaitingClusterSummaries requeues the ClusterSummaries that were waiting for clusterSummary to
// change. It is called once the change has been persisted: those ClusterSummaries read this one from the
// cache, and might otherwise still see it as it was.
func (r *ClusterSummaryReconciler) requeueWaitingClusterSummaries(ctx context.Context,
clusterSummary *configv1beta1.ClusterSummary, wasProvisioned bool, logger logr.Logger) {

if !clusterSummary.DeletionTimestamp.IsZero() {
// Finalizer removed: this ClusterSummary is gone. Its prerequisites, if being deleted, were waiting for it
if !controllerutil.ContainsFinalizer(clusterSummary, configv1beta1.ClusterSummaryFinalizer) {
r.requeuePrerequisites(ctx, clusterSummary, logger)
}
return
}

if !wasProvisioned && isCluterSummaryProvisioned(clusterSummary) {
// Dependents were waiting for this ClusterSummary to be provisioned
r.requeueDependents(ctx, clusterSummary, logger)
}
}

// requeueDependents is called once a ClusterSummary has become Provisioned. The ClusterSummaries of
// the same cluster that list this profile in DependsOn are waiting for it: without this call they
// only notice at their next poll (normalRequeueAfter). The dependency manager knows which profiles
// depend on this one for this cluster. When auto-deploy-dependencies is disabled the manager does
// not track dependents, and they fall back to polling.
func (r *ClusterSummaryReconciler) requeueDependents(ctx context.Context,
clusterSummary *configv1beta1.ClusterSummary, logger logr.Logger) {

depManager, err := dependencymanager.GetManagerInstance()
if err != nil {
return
}

profileOwner, err := configv1beta1.GetProfileOwnerReference(clusterSummary)
if err != nil {
return
}

prerequisite := &corev1.ObjectReference{
Name: profileOwner.Name,
Kind: profileOwner.Kind,
APIVersion: configv1beta1.GroupVersion.String(),
}
if profileOwner.Kind == configv1beta1.ProfileKind {
prerequisite.Namespace = clusterSummary.Namespace
}

dependents := depManager.GetDependents(prerequisite, getClusterObjectReferenceFromClusterSummary(clusterSummary))
r.retriggerClusterSummaries(ctx, getDependentClusterSummaryKeys(clusterSummary, dependents), logger)
}

// getDependentClusterSummaryKeys returns the keys of the ClusterSummaries, for the same cluster as
// clusterSummary, owned by the dependents profiles.
func getDependentClusterSummaryKeys(clusterSummary *configv1beta1.ClusterSummary,
dependents []corev1.ObjectReference) []types.NamespacedName {

keys := make([]types.NamespacedName, len(dependents))
for i := range dependents {
keys[i] = getClusterSummaryKey(clusterSummary, dependents[i].Kind, dependents[i].Name)
}

return keys
}

// getPrerequisiteClusterSummaryKeys returns the keys of the ClusterSummaries, for the same cluster as
// clusterSummary, owned by the profiles clusterSummary depends on.
func getPrerequisiteClusterSummaryKeys(clusterSummary *configv1beta1.ClusterSummary) []types.NamespacedName {
profileOwner, err := configv1beta1.GetProfileOwnerReference(clusterSummary)
if err != nil {
return nil
}

dependsOn := clusterSummary.Spec.ClusterProfileSpec.DependsOn
keys := make([]types.NamespacedName, len(dependsOn))
for i := range dependsOn {
keys[i] = getClusterSummaryKey(clusterSummary, profileOwner.Kind, dependsOn[i])
}

return keys
}

// getClusterSummaryKey returns the key of the ClusterSummary for the profile, of the given kind and
// name, and the same cluster as clusterSummary
func getClusterSummaryKey(clusterSummary *configv1beta1.ClusterSummary, profileKind, profileName string,
) types.NamespacedName {

isSveltosCluster := clusterSummary.Spec.ClusterType == libsveltosv1beta1.ClusterTypeSveltos

return types.NamespacedName{
Namespace: clusterSummary.Namespace,
Name: clusterops.GetClusterSummaryName(profileKind, profileName, clusterSummary.Spec.ClusterName, isSveltosCluster),
}
}

// retriggerClusterSummaries causes the given ClusterSummaries to be reconciled right away: a blocked
// ClusterSummary has a cooldown set, which is expired here, and the retrigger annotation is updated.
func (r *ClusterSummaryReconciler) retriggerClusterSummaries(ctx context.Context,
keys []types.NamespacedName, logger logr.Logger) {

for i := range keys {
// A ClusterSummary waiting for its prerequisites has a cooldown: expire it, otherwise
// the reconciliation triggered below would be skipped
r.expireCooldown(keys[i])

clusterSummary := &configv1beta1.ClusterSummary{}
if err := r.Get(ctx, keys[i], clusterSummary); err != nil {
if !apierrors.IsNotFound(err) {
logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to get ClusterSummary %s: %v", keys[i], err))
}
continue
}

logger.V(logs.LogDebug).Info(fmt.Sprintf("prerequisite provisioned. Requeuing ClusterSummary %s", keys[i]))
r.setRetriggerAnnotation(ctx, clusterSummary, logger)
}
}

// requeuePrerequisites is called once a ClusterSummary has been removed. The ClusterSummaries of the
// same cluster for the profiles it depends on cannot be removed until their dependents are gone: if they
// are being deleted they are waiting for this one, and are reconciled right away instead of at their next poll.
func (r *ClusterSummaryReconciler) requeuePrerequisites(ctx context.Context,
clusterSummary *configv1beta1.ClusterSummary, logger logr.Logger) {

keys := getPrerequisiteClusterSummaryKeys(clusterSummary)
for i := range keys {
prerequisite := &configv1beta1.ClusterSummary{}
if err := r.Get(ctx, keys[i], prerequisite); err != nil {
if !apierrors.IsNotFound(err) {
logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to get ClusterSummary %s: %v", keys[i], err))
}
continue
}

// Nothing is waiting for a ClusterSummary that is not being deleted
if prerequisite.DeletionTimestamp.IsZero() {
continue
}

logger.V(logs.LogDebug).Info(fmt.Sprintf("dependent removed. Requeuing ClusterSummary %s", keys[i]))
r.expireDeletedInstance(keys[i])
r.setRetriggerAnnotation(ctx, prerequisite, logger)
}
}

// setRetriggerAnnotation updates the retrigger annotation, which causes a new reconciliation
func (r *ClusterSummaryReconciler) setRetriggerAnnotation(ctx context.Context,
clusterSummary *configv1beta1.ClusterSummary, logger logr.Logger) {

patch := client.MergeFrom(clusterSummary.DeepCopy())
if clusterSummary.Annotations == nil {
clusterSummary.Annotations = map[string]string{}
}
clusterSummary.Annotations[retriggerAnnotation] = time.Now().UTC().Format(time.RFC3339Nano)
if err := r.Patch(ctx, clusterSummary, patch); err != nil {
// The ClusterSummary will still be reconciled at its next poll
logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to requeue ClusterSummary %s/%s: %v",
clusterSummary.Namespace, clusterSummary.Name, err))
}
}

// expireDeletedInstance allows a ClusterSummary being deleted to be reconciled right away:
// skipReconciliation otherwise ignores it for deleteRequeueAfter after its last reconciliation.
func (r *ClusterSummaryReconciler) expireDeletedInstance(key types.NamespacedName) {
r.PolicyMux.Lock()
defer r.PolicyMux.Unlock()

delete(r.DeletedInstances, key)
}

// expireCooldown makes the in-memory cooldown of a ClusterSummary, if any, expire now. The entry is
// kept (not deleted): skipReconciliation would otherwise rebuild it from the persisted status.
func (r *ClusterSummaryReconciler) expireCooldown(key types.NamespacedName) {
r.PolicyMux.Lock()
defer r.PolicyMux.Unlock()

if cooldown, ok := r.NextReconcileTimes[key]; ok {
cooldown.Until = time.Now()
r.NextReconcileTimes[key] = cooldown
}
}

func (r *ClusterSummaryReconciler) setFailureMessage(clusterSummaryScope *scope.ClusterSummaryScope, failureMessage string) {
if clusterSummaryScope.ClusterSummary.Spec.ClusterProfileSpec.HelmCharts != nil {
clusterSummaryScope.SetFailureMessage(libsveltosv1beta1.FeatureHelm, &failureMessage)
Expand Down
162 changes: 162 additions & 0 deletions controllers/clustersummary_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -647,6 +647,168 @@ var _ = Describe("ClustersummaryController", func() {
Expect(featureResourcesVerified).To(BeTrue())
})

It("getDependentClusterSummaryKeys returns the ClusterSummaries of the dependents for the same cluster", func() {
dependents := []corev1.ObjectReference{
{Kind: configv1beta1.ClusterProfileKind, Name: randomString(), APIVersion: configv1beta1.GroupVersion.String()},
{Kind: configv1beta1.ProfileKind, Name: randomString(), Namespace: namespace,
APIVersion: configv1beta1.GroupVersion.String()},
}

keys := controllers.GetDependentClusterSummaryKeys(clusterSummary, dependents)
Expect(keys).To(HaveLen(len(dependents)))
for i := range dependents {
Expect(keys[i].Namespace).To(Equal(clusterSummary.Namespace))
Expect(keys[i].Name).To(Equal(clusterops.GetClusterSummaryName(dependents[i].Kind, dependents[i].Name,
clusterName, false)))
}
})

It("retriggerClusterSummaries expires the cooldown and updates the retrigger annotation", func() {
dependent := &configv1beta1.ClusterSummary{
ObjectMeta: metav1.ObjectMeta{
Name: randomString(),
Namespace: namespace,
},
Spec: configv1beta1.ClusterSummarySpec{
ClusterNamespace: cluster.Namespace,
ClusterName: cluster.Name,
ClusterType: libsveltosv1beta1.ClusterTypeCapi,
},
}
Expect(testEnv.Create(context.TODO(), dependent)).To(Succeed())
Expect(waitForObject(context.TODO(), testEnv.Client, dependent)).To(Succeed())

dependentKey := types.NamespacedName{Namespace: dependent.Namespace, Name: dependent.Name}
missingKey := types.NamespacedName{Namespace: namespace, Name: randomString()}

reconciler := &controllers.ClusterSummaryReconciler{
Client: testEnv.Client,
Scheme: scheme,
PolicyMux: sync.Mutex{},
ClusterMap: make(map[corev1.ObjectReference]*libsveltosset.Set),
ReferenceMap: make(map[corev1.ObjectReference]*libsveltosset.Set),
NextReconcileTimes: map[types.NamespacedName]controllers.ReconcileCooldown{
dependentKey: {Until: time.Now().Add(time.Hour), Generation: dependent.Generation},
},
}

// A ClusterSummary that does not exist anymore is skipped
controllers.RetriggerClusterSummaries(reconciler, context.TODO(), []types.NamespacedName{missingKey, dependentKey},
textlogger.NewLogger(textlogger.NewConfig()))

reconciler.PolicyMux.Lock()
cooldown, ok := reconciler.NextReconcileTimes[dependentKey]
reconciler.PolicyMux.Unlock()
// The entry is kept, but it is not in the future anymore
Expect(ok).To(BeTrue())
Expect(cooldown.Until.After(time.Now())).To(BeFalse())

Eventually(func() string {
currentDependent := &configv1beta1.ClusterSummary{}
err := testEnv.Get(context.TODO(), dependentKey, currentDependent)
if err != nil {
return ""
}
return currentDependent.Annotations["projectsveltos.io/retrigger"]
}, timeout, pollingInterval).ShouldNot(BeEmpty())
})

It("getPrerequisiteClusterSummaryKeys returns the ClusterSummaries of the profiles it depends on", func() {
prerequisites := []string{randomString(), randomString()}
clusterSummary.Spec.ClusterProfileSpec.DependsOn = prerequisites

keys := controllers.GetPrerequisiteClusterSummaryKeys(clusterSummary)
Expect(keys).To(HaveLen(len(prerequisites)))
for i := range prerequisites {
Expect(keys[i].Namespace).To(Equal(clusterSummary.Namespace))
Expect(keys[i].Name).To(Equal(clusterops.GetClusterSummaryName(configv1beta1.ClusterProfileKind,
prerequisites[i], clusterName, false)))
}
})

It("requeuePrerequisites retriggers only the prerequisites being deleted", func() {
deletingProfile := randomString()
activeProfile := randomString()
deletingName := clusterops.GetClusterSummaryName(configv1beta1.ClusterProfileKind, deletingProfile, clusterName, false)
activeName := clusterops.GetClusterSummaryName(configv1beta1.ClusterProfileKind, activeProfile, clusterName, false)

// The finalizer keeps the deleting ClusterSummary around with a DeletionTimestamp
deleting := &configv1beta1.ClusterSummary{
ObjectMeta: metav1.ObjectMeta{
Name: deletingName,
Namespace: namespace,
Finalizers: []string{configv1beta1.ClusterSummaryFinalizer},
},
Spec: configv1beta1.ClusterSummarySpec{
ClusterNamespace: cluster.Namespace,
ClusterName: cluster.Name,
ClusterType: libsveltosv1beta1.ClusterTypeCapi,
},
}
active := &configv1beta1.ClusterSummary{
ObjectMeta: metav1.ObjectMeta{
Name: activeName,
Namespace: namespace,
},
Spec: configv1beta1.ClusterSummarySpec{
ClusterNamespace: cluster.Namespace,
ClusterName: cluster.Name,
ClusterType: libsveltosv1beta1.ClusterTypeCapi,
},
}
Expect(testEnv.Create(context.TODO(), deleting)).To(Succeed())
Expect(testEnv.Create(context.TODO(), active)).To(Succeed())
Expect(testEnv.Delete(context.TODO(), deleting)).To(Succeed())
Expect(waitForObject(context.TODO(), testEnv.Client, active)).To(Succeed())
Eventually(func() bool {
current := &configv1beta1.ClusterSummary{}
err := testEnv.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: deletingName}, current)
return err == nil && !current.DeletionTimestamp.IsZero()
}, timeout, pollingInterval).Should(BeTrue())

deletingKey := types.NamespacedName{Namespace: namespace, Name: deletingName}
activeKey := types.NamespacedName{Namespace: namespace, Name: activeName}

reconciler := &controllers.ClusterSummaryReconciler{
Client: testEnv.Client,
Scheme: scheme,
PolicyMux: sync.Mutex{},
ClusterMap: make(map[corev1.ObjectReference]*libsveltosset.Set),
ReferenceMap: make(map[corev1.ObjectReference]*libsveltosset.Set),
DeletedInstances: map[types.NamespacedName]time.Time{
deletingKey: time.Now(),
},
}

// clusterSummary depends on the two ClusterSummaries above and on one that does not exist
clusterSummary.Spec.ClusterProfileSpec.DependsOn = []string{deletingProfile, activeProfile, randomString()}
controllers.RequeuePrerequisites(reconciler, context.TODO(), clusterSummary,
textlogger.NewLogger(textlogger.NewConfig()))

reconciler.PolicyMux.Lock()
_, ok := reconciler.DeletedInstances[deletingKey]
reconciler.PolicyMux.Unlock()
Expect(ok).To(BeFalse(), "the deleting ClusterSummary can be reconciled right away")

Eventually(func() string {
current := &configv1beta1.ClusterSummary{}
err := testEnv.Get(context.TODO(), deletingKey, current)
if err != nil {
return ""
}
return current.Annotations["projectsveltos.io/retrigger"]
}, timeout, pollingInterval).ShouldNot(BeEmpty())

Consistently(func() string {
current := &configv1beta1.ClusterSummary{}
err := testEnv.Get(context.TODO(), activeKey, current)
if err != nil {
return "error"
}
return current.Annotations["projectsveltos.io/retrigger"]
}, "2s", pollingInterval).Should(BeEmpty())
})

It("shouldReconcile returns true when mode is OneTime but not all helm charts are deployed", func() {
clusterSummary.Spec.ClusterProfileSpec.SyncMode = configv1beta1.SyncModeOneTime
clusterSummary.Spec.ClusterProfileSpec.HelmCharts = []configv1beta1.HelmChart{
Expand Down
Loading