cc_cluster lifecycle test - upgrade

Test overview:

Testebed deployed with a management cluster
Deploy classy cluster (class based).
Check addon resources
Upgrade classy cluster to next available TKR
Check addon resources
Delete classy cluster
Verify classy cluster resources have been deleted.

pkg/v1/tkg/test/tkgctl/shared/e2e_common.go

@@ -10,6 +10,7 @@ import (
 	"math/rand"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"sigs.k8s.io/cluster-api/util"
@@ -18,9 +19,12 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	runv1alpha3 "github.com/vmware-tanzu/tanzu-framework/apis/run/v1alpha3"
+	tkgclient "github.com/vmware-tanzu/tanzu-framework/pkg/v1/tkg/client"
 	"github.com/vmware-tanzu/tanzu-framework/pkg/v1/tkg/constants"
 	"github.com/vmware-tanzu/tanzu-framework/pkg/v1/tkg/test/framework"
 	"github.com/vmware-tanzu/tanzu-framework/pkg/v1/tkg/tkgctl"
+	"github.com/vmware-tanzu/tanzu-framework/pkg/v1/tkg/utils"
 )
 
 type E2ECommonSpecInput struct {
@@ -199,6 +203,49 @@ func E2ECommonSpec(ctx context.Context, inputGetter func() E2ECommonSpecInput) {
 				By(fmt.Sprintf("Get management cluster resources created by addons-manager for workload cluster %q on management cluster %q", clusterName, input.E2EConfig.ManagementClusterName))
 				clusterResources, err = getManagementClusterResources(ctx, mngclient, mngDynamicClient, mngAggregatedAPIResourcesClient, mngDiscoveryClient, namespace, clusterName, infrastructureName)
 				Expect(err).NotTo(HaveOccurred())
+
+				By(fmt.Sprintf("Upgrade workload cluster %q", clusterName))
+				err, currentTkrVersion := getCurrentTkrVersion(tkgCtlClient, clusterName, namespace)
+				Expect(err).ToNot(HaveOccurred())
+				err, nextAvailableTkrVersion := getNextAvailableTkrVersion(ctx, currentTkrVersion, mngtempFilePath)
+				Expect(err).ToNot(HaveOccurred())
+				err = tkgCtlClient.UpgradeCluster(tkgctl.UpgradeClusterOptions{
+					ClusterName: clusterName,
+					Namespace:   namespace,
+					TkrVersion:  nextAvailableTkrVersion,
+					SkipPrompt:  true,
+				})
+				Expect(err).ToNot(HaveOccurred())
+
+				Eventually(func() bool {
+					// Check should pass when the cluster's current tkr version is the same as next available version
+					// and the cluster is in running phase.
+					err, currentTkrVersion := getCurrentTkrVersion(tkgCtlClient, clusterName, namespace)
+					if err == nil && currentTkrVersion == nextAvailableTkrVersion {
+						describeClusterOptions := tkgctl.DescribeTKGClustersOptions{
+							ClusterName: clusterName,
+							Namespace:   namespace,
+							ShowDetails: true,
+						}
+						results, err := tkgCtlClient.DescribeCluster(describeClusterOptions)
+						if err == nil && results.Cluster.Status.Phase == string(tkgclient.TKGClusterPhaseRunning) {
+							return true
+						}
+					}
+					return false
+				})
+
+				By(fmt.Sprintf("Get k8s client for upgraded workload cluster %q", clusterName))
+				wlcClient, _, _, _, err = getClients(ctx, tempFilePath)
+				Expect(err).NotTo(HaveOccurred())
+
+				By(fmt.Sprintf("Verify addon packages on workload cluster %q matches clusterBootstrap info on management cluster %q", clusterName, input.E2EConfig.ManagementClusterName))
+				err = checkClusterCB(ctx, mngClient, wlcClient, input.E2EConfig.ManagementClusterName, constants.TkgNamespace, clusterName, namespace, infrastructureName, false)
+				Expect(err).To(BeNil())
+
+				By(fmt.Sprintf("Get management cluster resources created by addons-manager for workload cluster %q on management cluster %q", clusterName, input.E2EConfig.ManagementClusterName))
+				clusterResources, err = getManagementClusterResources(ctx, mngClient, mngDynamicClient, mngAggregatedAPIResourcesClient, mngDiscoveryClient, namespace, clusterName, infrastructureName)
+				Expect(err).NotTo(HaveOccurred())
 			}
 		}
 
@@ -223,3 +270,73 @@ func E2ECommonSpec(ctx context.Context, inputGetter func() E2ECommonSpecInput) {
 		By("Test successful !")
 	})
 }
+
+func getNextAvailableTkrVersion(ctx context.Context, currentTkrVersion, mngKubeConfigFilePath string) (error, string) {
+	mngClient, _, _, _, err := getClients(ctx, mngKubeConfigFilePath)
+	if err != nil {
+		return err, ""
+	}
+	tkrList := runv1alpha3.TanzuKubernetesReleaseList{}
+	mngClient.List(ctx, &tkrList)
+	tkrs := tkrList.Items
+	availableTkrs := make(map[string]string)
+	for i := range tkrs {
+		compatible := ""
+		for _, condition := range tkrs[i].Status.Conditions {
+			if condition.Type == runv1alpha3.ConditionCompatible {
+				compatible = string(condition.Status)
+			}
+		}
+		labels := tkrs[i].Labels
+		activeStatus := "True"
+		if labels != nil {
+			if _, exists := labels[runv1alpha3.LabelDeactivated]; exists {
+				activeStatus = "False"
+			}
+		}
+		if !strings.EqualFold(compatible, "true") || !strings.EqualFold(activeStatus, "true") {
+			continue
+		}
+
+		compareResult, err := utils.CompareVMwareVersionStrings(currentTkrVersion, tkrs[i].Spec.Version)
+		if err != nil {
+			return err, ""
+		}
+
+		if compareResult < 0 {
+			availableTkrs[tkrs[i].Spec.Version] = tkrs[i].Name
+		}
+
+	}
+	var tkrVersions []string
+	for version := range availableTkrs {
+		tkrVersions = append(tkrVersions, version)
+	}
+	if len(tkrVersions) < 1 {
+		return fmt.Errorf("no TKR version available for upgrade"), ""
+	}
+	nextAvailableVersion := tkrVersions[0]
+	for _, newVersion := range tkrVersions {
+		compareResult, err := utils.CompareVMwareVersionStrings(nextAvailableVersion, newVersion)
+		if err != nil {
+			return err, ""
+		}
+		if compareResult > 0 {
+			//newVersion is lower
+			nextAvailableVersion = newVersion
+		}
+	}
+	return nil, nextAvailableVersion
+}
+func getCurrentTkrVersion(tkgCtlClient tkgctl.TKGClient, clusterName, clusterNamespace string) (error, string) {
+	describeClusterOptions := tkgctl.DescribeTKGClustersOptions{
+		ClusterName: clusterName,
+		Namespace:   clusterNamespace,
+		ShowDetails: true,
+	}
+	results, err := tkgCtlClient.DescribeCluster(describeClusterOptions)
+	if err != nil {
+		return err, ""
+	}
+	return nil, results.ClusterInfo.TKR
+}