From 928f22df5c9a63703906c04e9bec3ebfa738c5db Mon Sep 17 00:00:00 2001
From: Stefan Prodan <stefan.prodan@gmail.com>
Date: Fri, 2 Sep 2022 11:31:54 +0300
Subject: [PATCH] Reconcile Kubernetes class type objects in a dedicated stage

Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>
---
 controllers/kustomization_controller.go   | 61 +++++++++++++++++------
 controllers/kustomization_fetcher_test.go |  5 +-
 2 files changed, 49 insertions(+), 17 deletions(-)

diff --git a/controllers/kustomization_controller.go b/controllers/kustomization_controller.go
index 5c14527c..3ff43101 100644
--- a/controllers/kustomization_controller.go
+++ b/controllers/kustomization_controller.go
@@ -715,10 +715,14 @@ func (r *KustomizationReconciler) apply(ctx context.Context, manager *ssa.Resour
 	}
 
 	// contains only CRDs and Namespaces
-	var stageOne []*unstructured.Unstructured
+	var defStage []*unstructured.Unstructured
 
-	// contains all objects except for CRDs and Namespaces
-	var stageTwo []*unstructured.Unstructured
+	// contains only Kubernetes Class types e.g.: RuntimeClass, PriorityClass,
+	// StorageClas, VolumeSnapshotClass, IngressClass, GatewayClass, ClusterClass, etc
+	var classStage []*unstructured.Unstructured
+
+	// contains all objects except for CRDs, Namespaces and Class type objects
+	var resStage []*unstructured.Unstructured
 
 	// contains the objects' metadata after apply
 	resultSet := ssa.NewChangeSet()
@@ -730,25 +734,54 @@ func (r *KustomizationReconciler) apply(ctx context.Context, manager *ssa.Resour
 					ssa.FmtUnstructured(u))
 		}
 
-		if ssa.IsClusterDefinition(u) {
-			stageOne = append(stageOne, u)
-		} else {
-			stageTwo = append(stageTwo, u)
+		switch {
+		case ssa.IsClusterDefinition(u):
+			defStage = append(defStage, u)
+		case strings.HasSuffix(u.GetKind(), "Class"):
+			classStage = append(classStage, u)
+		default:
+			resStage = append(resStage, u)
 		}
+
 	}
 
 	var changeSetLog strings.Builder
 
 	// validate, apply and wait for CRDs and Namespaces to register
-	if len(stageOne) > 0 {
-		changeSet, err := manager.ApplyAll(ctx, stageOne, applyOpts)
+	if len(defStage) > 0 {
+		changeSet, err := manager.ApplyAll(ctx, defStage, applyOpts)
 		if err != nil {
 			return false, nil, err
 		}
 		resultSet.Append(changeSet.Entries)
 
 		if changeSet != nil && len(changeSet.Entries) > 0 {
-			log.Info("server-side apply completed", "output", changeSet.ToMap())
+			log.Info("server-side apply for cluster definitions completed", "output", changeSet.ToMap())
+			for _, change := range changeSet.Entries {
+				if change.Action != string(ssa.UnchangedAction) {
+					changeSetLog.WriteString(change.String() + "\n")
+				}
+			}
+		}
+
+		if err := manager.Wait(defStage, ssa.WaitOptions{
+			Interval: 2 * time.Second,
+			Timeout:  kustomization.GetTimeout(),
+		}); err != nil {
+			return false, nil, err
+		}
+	}
+
+	// validate, apply and wait for Class type objects to register
+	if len(classStage) > 0 {
+		changeSet, err := manager.ApplyAll(ctx, classStage, applyOpts)
+		if err != nil {
+			return false, nil, err
+		}
+		resultSet.Append(changeSet.Entries)
+
+		if changeSet != nil && len(changeSet.Entries) > 0 {
+			log.Info("server-side apply for cluster class types completed", "output", changeSet.ToMap())
 			for _, change := range changeSet.Entries {
 				if change.Action != string(ssa.UnchangedAction) {
 					changeSetLog.WriteString(change.String() + "\n")
@@ -756,7 +789,7 @@ func (r *KustomizationReconciler) apply(ctx context.Context, manager *ssa.Resour
 			}
 		}
 
-		if err := manager.Wait(stageOne, ssa.WaitOptions{
+		if err := manager.Wait(classStage, ssa.WaitOptions{
 			Interval: 2 * time.Second,
 			Timeout:  kustomization.GetTimeout(),
 		}); err != nil {
@@ -765,9 +798,9 @@ func (r *KustomizationReconciler) apply(ctx context.Context, manager *ssa.Resour
 	}
 
 	// sort by kind, validate and apply all the others objects
-	sort.Sort(ssa.SortableUnstructureds(stageTwo))
-	if len(stageTwo) > 0 {
-		changeSet, err := manager.ApplyAll(ctx, stageTwo, applyOpts)
+	sort.Sort(ssa.SortableUnstructureds(resStage))
+	if len(resStage) > 0 {
+		changeSet, err := manager.ApplyAll(ctx, resStage, applyOpts)
 		if err != nil {
 			return false, nil, fmt.Errorf("%w\n%s", err, changeSetLog.String())
 		}
diff --git a/controllers/kustomization_fetcher_test.go b/controllers/kustomization_fetcher_test.go
index a9d06525..83ade915 100644
--- a/controllers/kustomization_fetcher_test.go
+++ b/controllers/kustomization_fetcher_test.go
@@ -19,6 +19,7 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 
@@ -135,10 +136,8 @@ stringData:
 		g.Eventually(func() bool {
 			_ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), resultK)
 			ready := apimeta.FindStatusCondition(resultK.Status.Conditions, meta.ReadyCondition)
-			return ready.Reason == meta.ProgressingReason
+			return strings.Contains(ready.Message, "artifact not found")
 		}, timeout, time.Second).Should(BeTrue())
-
-		g.Expect(apimeta.FindStatusCondition(resultK.Status.Conditions, meta.ReadyCondition).Message).To(ContainSubstring("artifact not found"))
 	})
 
 	t.Run("recovers after not found errors", func(t *testing.T) {