test: Expand KOWKNodeClass to Support Generic NodeClass for E2E tests (…

…#1963)

go.mod

@@ -79,7 +79,7 @@ require (
 	k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect
 	sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect
-	sigs.k8s.io/yaml v1.4.0 // indirect
+	sigs.k8s.io/yaml v1.4.0
 )
 
 require (

test/pkg/environment/common/default_kowknodeclass.yaml

@@ -0,0 +1,4 @@
+apiVersion: karpenter.kwok.sh/v1alpha1
+kind: KWOKNodeClass
+metadata:
+  name: default

test/pkg/environment/common/default_nodepool.yaml

@@ -0,0 +1,27 @@
+apiVersion: karpenter.sh/v1
+kind: NodePool
+metadata:
+ name: default
+spec:
+ disruption:
+   consolidationPolicy: WhenEmptyOrUnderutilized
+   consolidateAfter: Never
+   budgets:
+     - nodes: 100%
+ limits:
+   cpu: 1000
+   memory: 1000Gi
+ template:
+   spec:
+     expireAfter: Never
+     requirements:
+       - key: kubernetes.io/os
+         operator: In
+         values: ["linux"]
+       - key: karpenter.sh/capacity-type
+         operator: In
+         values: ["on-demand"]
+     nodeClassRef:
+      group: karpenter.kwok.sh
+      kind: KWOKNodeClass
+      name: default

test/pkg/environment/common/environment.go

@@ -18,7 +18,10 @@ package common
 
 import (
 	"context"
+	_ "embed"
+	"flag"
 	"fmt"
+	"io"
 	"log"
 	"os"
 	"strconv"
@@ -30,27 +33,35 @@ import (
 	"github.com/onsi/gomega"
 	"github.com/samber/lo"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/resource"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	serializeryaml "k8s.io/apimachinery/pkg/runtime/serializer/yaml"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	controllerruntime "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/yaml"
 
-	"sigs.k8s.io/karpenter/kwok/apis/v1alpha1"
 	v1 "sigs.k8s.io/karpenter/pkg/apis/v1"
 	"sigs.k8s.io/karpenter/pkg/operator"
-	"sigs.k8s.io/karpenter/pkg/test"
 	. "sigs.k8s.io/karpenter/pkg/utils/testing" //nolint:stylecheck
 	"sigs.k8s.io/karpenter/test/pkg/debug"
 )
 
 type ContextKey string
 
-const (
-	GitRefContextKey = ContextKey("gitRef")
+const GitRefContextKey = ContextKey("gitRef")
+
+// I need to add the the default kwok nodeclass path
+// That way it's not defined in code but we use it when we initialize the nodeclass
+var (
+	//go:embed default_kowknodeclass.yaml
+	defaultNodeClass []byte
+	//go:embed default_nodepool.yaml
+	defaultNodePool []byte
+	nodeClassPath   = flag.String("default-nodeclass", "", "Pass in a default cloud specific node class")
+	nodePoolPath    = flag.String("default-nodepool", "", "Pass in a default karpenter nodepool")
 )
 
 type Environment struct {
@@ -62,6 +73,7 @@ type Environment struct {
 	Config                *rest.Config
 	KubeClient            kubernetes.Interface
 	Monitor               *Monitor
+	DefaultNodeClass      *unstructured.Unstructured
 
 	OutputDir         string
 	StartingNodeCount int
@@ -90,6 +102,7 @@ func NewEnvironment(t *testing.T) *Environment {
 		Monitor:               NewMonitor(ctx, client),
 		TimeIntervalCollector: debug.NewTimestampCollector(),
 		OutputDir:             outputDir,
+		DefaultNodeClass:      decodeNodeClass(),
 	}
 }
 
@@ -144,42 +157,38 @@ func NewClient(ctx context.Context, config *rest.Config) client.Client {
 	return c
 }
 
-func (env *Environment) DefaultNodeClass() *v1alpha1.KWOKNodeClass {
-	return &v1alpha1.KWOKNodeClass{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: test.RandomName(),
-		},
+func (env *Environment) DefaultNodePool(nodeClass client.Object) *v1.NodePool {
+	nodePool := &v1.NodePool{}
+	if lo.FromPtr(nodePoolPath) == "" {
+		nodePool = object.Unmarshal[v1.NodePool](defaultNodePool)
+	} else {
+		file := lo.Must1(os.ReadFile(lo.FromPtr(nodePoolPath)))
+		lo.Must0(yaml.Unmarshal(file, nodePool))
 	}
-}
 
-func (env *Environment) DefaultNodePool(nodeClass *v1alpha1.KWOKNodeClass) *v1.NodePool {
-	nodePool := test.NodePool()
+	// Update to use the provided default nodeclass
 	nodePool.Spec.Template.Spec.NodeClassRef = &v1.NodeClassReference{
-		Name:  nodeClass.Name,
-		Kind:  object.GVK(nodeClass).Kind,
-		Group: object.GVK(nodeClass).Group,
-	}
-	nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirementWithMinValues{
-		{
-			NodeSelectorRequirement: corev1.NodeSelectorRequirement{
-				Key:      corev1.LabelOSStable,
-				Operator: corev1.NodeSelectorOpIn,
-				Values:   []string{string(corev1.Linux)},
-			},
-		},
-		{
-			NodeSelectorRequirement: corev1.NodeSelectorRequirement{
-				Key:      v1.CapacityTypeLabelKey,
-				Operator: corev1.NodeSelectorOpIn,
-				Values:   []string{v1.CapacityTypeOnDemand},
-			},
-		},
+		Kind:  env.DefaultNodeClass.GetObjectKind().GroupVersionKind().Kind,
+		Group: env.DefaultNodeClass.GetObjectKind().GroupVersionKind().Group,
+		Name:  env.DefaultNodeClass.GetName(),
 	}
-	nodePool.Spec.Disruption.ConsolidateAfter = v1.MustParseNillableDuration("Never")
-	nodePool.Spec.Template.Spec.ExpireAfter.Duration = nil
-	nodePool.Spec.Limits = v1.Limits(corev1.ResourceList{
-		corev1.ResourceCPU:    resource.MustParse("1000"),
-		corev1.ResourceMemory: resource.MustParse("1000Gi"),
-	})
+
 	return nodePool
 }
+
+func decodeNodeClass() *unstructured.Unstructured {
+	// Open the file
+	if lo.FromPtr(nodeClassPath) == "" {
+		return object.Unmarshal[unstructured.Unstructured](defaultNodeClass)
+	}
+
+	file := lo.Must1(os.Open(lo.FromPtr(nodeClassPath)))
+	content := lo.Must1(io.ReadAll(file))
+
+	decoder := serializeryaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme)
+	u := &unstructured.Unstructured{}
+	_, gvk, _ := decoder.Decode(content, nil, u)
+	u.SetGroupVersionKind(lo.FromPtr(gvk))
+
+	return u
+}

test/pkg/environment/common/setup.go

@@ -36,7 +36,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 
-	"sigs.k8s.io/karpenter/kwok/apis/v1alpha1"
 	v1 "sigs.k8s.io/karpenter/pkg/apis/v1"
 	"sigs.k8s.io/karpenter/pkg/test"
 	"sigs.k8s.io/karpenter/pkg/utils/pod"
@@ -52,7 +51,6 @@ var (
 		&v1.NodePoolList{},
 		&corev1.NodeList{},
 		&v1.NodeClaimList{},
-		&v1alpha1.KWOKNodeClassList{},
 	}
 	CleanableObjects = []client.Object{
 		&corev1.Pod{},
@@ -67,7 +65,6 @@ var (
 		&schedulingv1.PriorityClass{},
 		&corev1.Node{},
 		&v1.NodeClaim{},
-		&v1alpha1.KWOKNodeClass{},
 	}
 )
 
@@ -98,7 +95,7 @@ func (env *Environment) ExpectCleanCluster() {
 		Expect(pods.Items[i].Namespace).ToNot(Equal("default"),
 			fmt.Sprintf("expected no pods in the `default` namespace, found %s/%s", pods.Items[i].Namespace, pods.Items[i].Name))
 	}
-	for _, obj := range []client.Object{&v1.NodePool{}, &v1alpha1.KWOKNodeClass{}} {
+	for _, obj := range []client.Object{&v1.NodePool{}, env.DefaultNodeClass.DeepCopy()} {
 		metaList := &metav1.PartialObjectMetadataList{}
 		gvk := lo.Must(apiutil.GVKForObject(obj, env.Client.Scheme()))
 		metaList.SetGroupVersionKind(gvk)
@@ -125,7 +122,9 @@ func (env *Environment) AfterEach() {
 }
 
 func (env *Environment) PrintCluster() {
-	for _, obj := range ObjectListsToPrint {
+	nodeClassList := unstructured.UnstructuredList{}
+	nodeClassList.SetGroupVersionKind(env.DefaultNodeClass.GroupVersionKind())
+	for _, obj := range append(ObjectListsToPrint, nodeClassList.DeepCopy()) {
 		gvk := lo.Must(apiutil.GVKForObject(obj, env.Client.Scheme()))
 		By(fmt.Sprintf("printing %s(s)", gvk.Kind))
 		list := &unstructured.UnstructuredList{}
@@ -142,7 +141,7 @@ func (env *Environment) PrintCluster() {
 func (env *Environment) CleanupObjects(cleanableObjects ...client.Object) {
 	time.Sleep(time.Second) // wait one second to let the caches get up-to-date for deletion
 	wg := sync.WaitGroup{}
-	for _, obj := range cleanableObjects {
+	for _, obj := range append(cleanableObjects, env.DefaultNodeClass.DeepCopy()) {
 		wg.Add(1)
 		go func(obj client.Object) {
 			defer wg.Done()
@@ -163,7 +162,7 @@ func (env *Environment) CleanupObjects(cleanableObjects ...client.Object) {
 				// If the deletes eventually succeed, we should have no elements here at the end of the test
 				g.Expect(env.Client.List(env, metaList, client.HasLabels([]string{test.DiscoveryLabel}), client.Limit(1))).To(Succeed())
 				g.Expect(metaList.Items).To(HaveLen(0))
-			}).Should(Succeed())
+			}).WithTimeout(10 * time.Minute).Should(Succeed())
 		}(obj)
 	}
 	wg.Wait()

test/suites/perf/scheduling_test.go

@@ -86,7 +86,7 @@ var _ = Describe("Performance", func() {
 				nodeClaims := &v1.NodeClaimList{}
 				g.Expect(env.Client.List(env, nodeClaims, client.MatchingFields{"status.conditions[*].type": v1.ConditionTypeDrifted})).To(Succeed())
 				g.Expect(len(nodeClaims.Items)).To(Equal(0))
-			}).WithTimeout(3 * time.Minute).Should(Succeed())
+			}).WithTimeout(10 * time.Minute).Should(Succeed())
 			env.TimeIntervalCollector.End("Drift")
 		})
 		It("should do complex provisioning", func() {
@@ -148,7 +148,7 @@ var _ = Describe("Performance", func() {
 				nodeClaims := &v1.NodeClaimList{}
 				g.Expect(env.Client.List(env, nodeClaims, client.MatchingFields{"status.conditions[*].type": v1.ConditionTypeDrifted})).To(Succeed())
 				g.Expect(len(nodeClaims.Items)).To(Equal(0))
-			}).WithTimeout(3 * time.Minute).Should(Succeed())
+			}).WithTimeout(10 * time.Minute).Should(Succeed())
 			env.TimeIntervalCollector.End("Drift")
 		})
 	})

test/suites/perf/suite_test.go

@@ -24,6 +24,7 @@ import (
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	"sigs.k8s.io/karpenter/kwok/apis/v1alpha1"
@@ -34,7 +35,7 @@ import (
 )
 
 var nodePool *v1.NodePool
-var nodeClass *v1alpha1.KWOKNodeClass
+var nodeClass client.Object
 var env *common.Environment
 
 var testLabels = map[string]string{
@@ -61,7 +62,7 @@ func TestPerf(t *testing.T) {
 
 var _ = BeforeEach(func() {
 	env.BeforeEach()
-	nodeClass = env.DefaultNodeClass()
+	nodeClass = env.DefaultNodeClass.DeepCopy()
 	nodePool = env.DefaultNodePool(nodeClass)
 	test.ReplaceRequirements(nodePool, v1.NodeSelectorRequirementWithMinValues{
 		NodeSelectorRequirement: corev1.NodeSelectorRequirement{