From c3e7098d91e607fd51a2f3796209d1571ad1deef Mon Sep 17 00:00:00 2001 From: Aidan Rowe Date: Fri, 22 Nov 2024 07:09:50 +1000 Subject: [PATCH] feat: add cpu sustained clock speed label to instance metadata (#7043) --- .../templates/karpenter.sh_nodeclaims.yaml | 2 +- .../templates/karpenter.sh_nodepools.yaml | 4 +- hack/code/instancetype_testdata_gen/main.go | 1 + hack/validation/labels.sh | 2 +- hack/validation/requirements.sh | 4 +- pkg/apis/crds/karpenter.sh_nodeclaims.yaml | 2 +- pkg/apis/crds/karpenter.sh_nodepools.yaml | 4 +- pkg/apis/v1/labels.go | 2 + .../zz_generated.describe_instance_types.go | 80 +++++++++++-------- pkg/providers/instancetype/suite_test.go | 3 + pkg/providers/instancetype/types.go | 6 ++ test/suites/scheduling/suite_test.go | 21 ++--- 12 files changed, 80 insertions(+), 51 deletions(-) diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml index 7976cfe0f45a..e85d305d9aa7 100644 --- a/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml +++ b/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml @@ -121,7 +121,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self != "kubernetes.io/hostname" - message: label domain "karpenter.k8s.aws" is restricted - rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") minValues: description: |- This field is ALPHA and can be dropped or replaced at any time diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml index f20e73cd0d55..ffe4932aad54 100644 --- a/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml +++ b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml @@ -209,7 +209,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self.all(x, x != "kubernetes.io/hostname") - message: label domain "karpenter.k8s.aws" is restricted - rule: self.all(x, x in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) + rule: self.all(x, x in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) type: object spec: description: |- @@ -268,7 +268,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self != "kubernetes.io/hostname" - message: label domain "karpenter.k8s.aws" is restricted - rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") minValues: description: |- This field is ALPHA and can be dropped or replaced at any time diff --git a/hack/code/instancetype_testdata_gen/main.go b/hack/code/instancetype_testdata_gen/main.go index 0aefceee2e3a..a852a4b79d31 100644 --- a/hack/code/instancetype_testdata_gen/main.go +++ b/hack/code/instancetype_testdata_gen/main.go @@ -129,6 +129,7 @@ func getInstanceTypeInfo(info ec2types.InstanceTypeInfo) string { fmt.Fprintf(src, "ProcessorInfo: &ec2types.ProcessorInfo{\n") fmt.Fprintf(src, "Manufacturer: aws.String(\"%s\"),\n", lo.FromPtr(info.ProcessorInfo.Manufacturer)) fmt.Fprintf(src, "SupportedArchitectures: []ec2types.ArchitectureType{%s},\n", getStringSliceData(info.ProcessorInfo.SupportedArchitectures)) + fmt.Fprintf(src, "SustainedClockSpeedInGhz: aws.Float64(%f),\n", lo.FromPtr(info.ProcessorInfo.SustainedClockSpeedInGhz)) fmt.Fprintf(src, "},\n") fmt.Fprintf(src, "VCpuInfo: &ec2types.VCpuInfo{\n") fmt.Fprintf(src, "DefaultCores: aws.Int32(%d),\n", lo.FromPtr(info.VCpuInfo.DefaultCores)) diff --git a/hack/validation/labels.sh b/hack/validation/labels.sh index e2b891780686..3c837d3dcbb3 100755 --- a/hack/validation/labels.sh +++ b/hack/validation/labels.sh @@ -4,4 +4,4 @@ # ## checking for restricted labels while filtering out well known labels yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.template.properties.metadata.properties.labels.x-kubernetes-validations += [ - {"message": "label domain \"karpenter.k8s.aws\" is restricted", "rule": "self.all(x, x in [\"karpenter.k8s.aws/instance-encryption-in-transit-supported\", \"karpenter.k8s.aws/instance-category\", \"karpenter.k8s.aws/instance-hypervisor\", \"karpenter.k8s.aws/instance-family\", \"karpenter.k8s.aws/instance-generation\", \"karpenter.k8s.aws/instance-local-nvme\", \"karpenter.k8s.aws/instance-size\", \"karpenter.k8s.aws/instance-cpu\",\"karpenter.k8s.aws/instance-cpu-manufacturer\",\"karpenter.k8s.aws/instance-memory\", \"karpenter.k8s.aws/instance-ebs-bandwidth\", \"karpenter.k8s.aws/instance-network-bandwidth\", \"karpenter.k8s.aws/instance-gpu-name\", \"karpenter.k8s.aws/instance-gpu-manufacturer\", \"karpenter.k8s.aws/instance-gpu-count\", \"karpenter.k8s.aws/instance-gpu-memory\", \"karpenter.k8s.aws/instance-accelerator-name\", \"karpenter.k8s.aws/instance-accelerator-manufacturer\", \"karpenter.k8s.aws/instance-accelerator-count\"] || !x.find(\"^([^/]+)\").endsWith(\"karpenter.k8s.aws\"))"}]' -i pkg/apis/crds/karpenter.sh_nodepools.yaml \ No newline at end of file + {"message": "label domain \"karpenter.k8s.aws\" is restricted", "rule": "self.all(x, x in [\"karpenter.k8s.aws/instance-encryption-in-transit-supported\", \"karpenter.k8s.aws/instance-category\", \"karpenter.k8s.aws/instance-hypervisor\", \"karpenter.k8s.aws/instance-family\", \"karpenter.k8s.aws/instance-generation\", \"karpenter.k8s.aws/instance-local-nvme\", \"karpenter.k8s.aws/instance-size\", \"karpenter.k8s.aws/instance-cpu\",\"karpenter.k8s.aws/instance-cpu-manufacturer\",\"karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz\",\"karpenter.k8s.aws/instance-memory\", \"karpenter.k8s.aws/instance-ebs-bandwidth\", \"karpenter.k8s.aws/instance-network-bandwidth\", \"karpenter.k8s.aws/instance-gpu-name\", \"karpenter.k8s.aws/instance-gpu-manufacturer\", \"karpenter.k8s.aws/instance-gpu-count\", \"karpenter.k8s.aws/instance-gpu-memory\", \"karpenter.k8s.aws/instance-accelerator-name\", \"karpenter.k8s.aws/instance-accelerator-manufacturer\", \"karpenter.k8s.aws/instance-accelerator-count\"] || !x.find(\"^([^/]+)\").endsWith(\"karpenter.k8s.aws\"))"}]' -i pkg/apis/crds/karpenter.sh_nodepools.yaml \ No newline at end of file diff --git a/hack/validation/requirements.sh b/hack/validation/requirements.sh index 66ede1e622bd..d6b157fe2880 100755 --- a/hack/validation/requirements.sh +++ b/hack/validation/requirements.sh @@ -5,9 +5,9 @@ # v1 ## checking for restricted labels while filtering out well known labels yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.requirements.items.properties.key.x-kubernetes-validations += [ - {"message": "label domain \"karpenter.k8s.aws\" is restricted", "rule": "self in [\"karpenter.k8s.aws/instance-encryption-in-transit-supported\", \"karpenter.k8s.aws/instance-category\", \"karpenter.k8s.aws/instance-hypervisor\", \"karpenter.k8s.aws/instance-family\", \"karpenter.k8s.aws/instance-generation\", \"karpenter.k8s.aws/instance-local-nvme\", \"karpenter.k8s.aws/instance-size\", \"karpenter.k8s.aws/instance-cpu\",\"karpenter.k8s.aws/instance-cpu-manufacturer\",\"karpenter.k8s.aws/instance-memory\", \"karpenter.k8s.aws/instance-ebs-bandwidth\", \"karpenter.k8s.aws/instance-network-bandwidth\", \"karpenter.k8s.aws/instance-gpu-name\", \"karpenter.k8s.aws/instance-gpu-manufacturer\", \"karpenter.k8s.aws/instance-gpu-count\", \"karpenter.k8s.aws/instance-gpu-memory\", \"karpenter.k8s.aws/instance-accelerator-name\", \"karpenter.k8s.aws/instance-accelerator-manufacturer\", \"karpenter.k8s.aws/instance-accelerator-count\"] || !self.find(\"^([^/]+)\").endsWith(\"karpenter.k8s.aws\")"}]' -i pkg/apis/crds/karpenter.sh_nodeclaims.yaml + {"message": "label domain \"karpenter.k8s.aws\" is restricted", "rule": "self in [\"karpenter.k8s.aws/instance-encryption-in-transit-supported\", \"karpenter.k8s.aws/instance-category\", \"karpenter.k8s.aws/instance-hypervisor\", \"karpenter.k8s.aws/instance-family\", \"karpenter.k8s.aws/instance-generation\", \"karpenter.k8s.aws/instance-local-nvme\", \"karpenter.k8s.aws/instance-size\", \"karpenter.k8s.aws/instance-cpu\",\"karpenter.k8s.aws/instance-cpu-manufacturer\",\"karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz\",\"karpenter.k8s.aws/instance-memory\", \"karpenter.k8s.aws/instance-ebs-bandwidth\", \"karpenter.k8s.aws/instance-network-bandwidth\", \"karpenter.k8s.aws/instance-gpu-name\", \"karpenter.k8s.aws/instance-gpu-manufacturer\", \"karpenter.k8s.aws/instance-gpu-count\", \"karpenter.k8s.aws/instance-gpu-memory\", \"karpenter.k8s.aws/instance-accelerator-name\", \"karpenter.k8s.aws/instance-accelerator-manufacturer\", \"karpenter.k8s.aws/instance-accelerator-count\"] || !self.find(\"^([^/]+)\").endsWith(\"karpenter.k8s.aws\")"}]' -i pkg/apis/crds/karpenter.sh_nodeclaims.yaml # # Adding validation for nodepool # ## checking for restricted labels while filtering out well known labels yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.template.properties.spec.properties.requirements.items.properties.key.x-kubernetes-validations += [ - {"message": "label domain \"karpenter.k8s.aws\" is restricted", "rule": "self in [\"karpenter.k8s.aws/instance-encryption-in-transit-supported\", \"karpenter.k8s.aws/instance-category\", \"karpenter.k8s.aws/instance-hypervisor\", \"karpenter.k8s.aws/instance-family\", \"karpenter.k8s.aws/instance-generation\", \"karpenter.k8s.aws/instance-local-nvme\", \"karpenter.k8s.aws/instance-size\", \"karpenter.k8s.aws/instance-cpu\",\"karpenter.k8s.aws/instance-cpu-manufacturer\",\"karpenter.k8s.aws/instance-memory\", \"karpenter.k8s.aws/instance-ebs-bandwidth\", \"karpenter.k8s.aws/instance-network-bandwidth\", \"karpenter.k8s.aws/instance-gpu-name\", \"karpenter.k8s.aws/instance-gpu-manufacturer\", \"karpenter.k8s.aws/instance-gpu-count\", \"karpenter.k8s.aws/instance-gpu-memory\", \"karpenter.k8s.aws/instance-accelerator-name\", \"karpenter.k8s.aws/instance-accelerator-manufacturer\", \"karpenter.k8s.aws/instance-accelerator-count\"] || !self.find(\"^([^/]+)\").endsWith(\"karpenter.k8s.aws\")"}]' -i pkg/apis/crds/karpenter.sh_nodepools.yaml + {"message": "label domain \"karpenter.k8s.aws\" is restricted", "rule": "self in [\"karpenter.k8s.aws/instance-encryption-in-transit-supported\", \"karpenter.k8s.aws/instance-category\", \"karpenter.k8s.aws/instance-hypervisor\", \"karpenter.k8s.aws/instance-family\", \"karpenter.k8s.aws/instance-generation\", \"karpenter.k8s.aws/instance-local-nvme\", \"karpenter.k8s.aws/instance-size\", \"karpenter.k8s.aws/instance-cpu\",\"karpenter.k8s.aws/instance-cpu-manufacturer\",\"karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz\",\"karpenter.k8s.aws/instance-memory\", \"karpenter.k8s.aws/instance-ebs-bandwidth\", \"karpenter.k8s.aws/instance-network-bandwidth\", \"karpenter.k8s.aws/instance-gpu-name\", \"karpenter.k8s.aws/instance-gpu-manufacturer\", \"karpenter.k8s.aws/instance-gpu-count\", \"karpenter.k8s.aws/instance-gpu-memory\", \"karpenter.k8s.aws/instance-accelerator-name\", \"karpenter.k8s.aws/instance-accelerator-manufacturer\", \"karpenter.k8s.aws/instance-accelerator-count\"] || !self.find(\"^([^/]+)\").endsWith(\"karpenter.k8s.aws\")"}]' -i pkg/apis/crds/karpenter.sh_nodepools.yaml diff --git a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml index 7976cfe0f45a..e85d305d9aa7 100644 --- a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml +++ b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml @@ -121,7 +121,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self != "kubernetes.io/hostname" - message: label domain "karpenter.k8s.aws" is restricted - rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") minValues: description: |- This field is ALPHA and can be dropped or replaced at any time diff --git a/pkg/apis/crds/karpenter.sh_nodepools.yaml b/pkg/apis/crds/karpenter.sh_nodepools.yaml index f20e73cd0d55..ffe4932aad54 100644 --- a/pkg/apis/crds/karpenter.sh_nodepools.yaml +++ b/pkg/apis/crds/karpenter.sh_nodepools.yaml @@ -209,7 +209,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self.all(x, x != "kubernetes.io/hostname") - message: label domain "karpenter.k8s.aws" is restricted - rule: self.all(x, x in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) + rule: self.all(x, x in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) type: object spec: description: |- @@ -268,7 +268,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self != "kubernetes.io/hostname" - message: label domain "karpenter.k8s.aws" is restricted - rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") minValues: description: |- This field is ALPHA and can be dropped or replaced at any time diff --git a/pkg/apis/v1/labels.go b/pkg/apis/v1/labels.go index f87eea423b5c..a6f45815b50e 100644 --- a/pkg/apis/v1/labels.go +++ b/pkg/apis/v1/labels.go @@ -38,6 +38,7 @@ func init() { LabelInstanceLocalNVME, LabelInstanceCPU, LabelInstanceCPUManufacturer, + LabelInstanceCPUSustainedClockSpeedMhz, LabelInstanceMemory, LabelInstanceEBSBandwidth, LabelInstanceNetworkBandwidth, @@ -109,6 +110,7 @@ var ( LabelInstanceSize = apis.Group + "/instance-size" LabelInstanceCPU = apis.Group + "/instance-cpu" LabelInstanceCPUManufacturer = apis.Group + "/instance-cpu-manufacturer" + LabelInstanceCPUSustainedClockSpeedMhz = apis.Group + "/instance-cpu-sustained-clock-speed-mhz" LabelInstanceMemory = apis.Group + "/instance-memory" LabelInstanceEBSBandwidth = apis.Group + "/instance-ebs-bandwidth" LabelInstanceNetworkBandwidth = apis.Group + "/instance-network-bandwidth" diff --git a/pkg/fake/zz_generated.describe_instance_types.go b/pkg/fake/zz_generated.describe_instance_types.go index 4a3b89a232cd..68a9664f3616 100644 --- a/pkg/fake/zz_generated.describe_instance_types.go +++ b/pkg/fake/zz_generated.describe_instance_types.go @@ -36,8 +36,9 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: "nitro", ProcessorInfo: &ec2types.ProcessorInfo{ - Manufacturer: aws.String("AWS"), - SupportedArchitectures: []ec2types.ArchitectureType{"arm64"}, + Manufacturer: aws.String("AWS"), + SupportedArchitectures: []ec2types.ArchitectureType{"arm64"}, + SustainedClockSpeedInGhz: aws.Float64(2.500000), }, VCpuInfo: &ec2types.VCpuInfo{ DefaultCores: aws.Int32(2), @@ -80,8 +81,9 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: "nitro", ProcessorInfo: &ec2types.ProcessorInfo{ - Manufacturer: aws.String("Intel"), - SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + Manufacturer: aws.String("Intel"), + SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + SustainedClockSpeedInGhz: aws.Float64(3.000000), }, VCpuInfo: &ec2types.VCpuInfo{ DefaultCores: aws.Int32(48), @@ -154,8 +156,9 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: "nitro", ProcessorInfo: &ec2types.ProcessorInfo{ - Manufacturer: aws.String("AMD"), - SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + Manufacturer: aws.String("AMD"), + SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + SustainedClockSpeedInGhz: aws.Float64(3.000000), }, VCpuInfo: &ec2types.VCpuInfo{ DefaultCores: aws.Int32(32), @@ -213,8 +216,9 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: "nitro", ProcessorInfo: &ec2types.ProcessorInfo{ - Manufacturer: aws.String("Intel"), - SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + Manufacturer: aws.String("Intel"), + SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + SustainedClockSpeedInGhz: aws.Float64(2.500000), }, VCpuInfo: &ec2types.VCpuInfo{ DefaultCores: aws.Int32(16), @@ -275,8 +279,9 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: "nitro", ProcessorInfo: &ec2types.ProcessorInfo{ - Manufacturer: aws.String("AMD"), - SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + Manufacturer: aws.String("AMD"), + SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + SustainedClockSpeedInGhz: aws.Float64(3.600000), }, VCpuInfo: &ec2types.VCpuInfo{ DefaultCores: aws.Int32(48), @@ -334,8 +339,9 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: "nitro", ProcessorInfo: &ec2types.ProcessorInfo{ - Manufacturer: aws.String("AMD"), - SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + Manufacturer: aws.String("AMD"), + SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + SustainedClockSpeedInGhz: aws.Float64(3.600000), }, VCpuInfo: &ec2types.VCpuInfo{ DefaultCores: aws.Int32(2), @@ -393,8 +399,9 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: "nitro", ProcessorInfo: &ec2types.ProcessorInfo{ - Manufacturer: aws.String("Intel"), - SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + Manufacturer: aws.String("Intel"), + SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + SustainedClockSpeedInGhz: aws.Float64(3.100000), }, VCpuInfo: &ec2types.VCpuInfo{ DefaultCores: aws.Int32(1), @@ -437,8 +444,9 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(true), Hypervisor: "", ProcessorInfo: &ec2types.ProcessorInfo{ - Manufacturer: aws.String("Intel"), - SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + Manufacturer: aws.String("Intel"), + SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + SustainedClockSpeedInGhz: aws.Float64(3.100000), }, VCpuInfo: &ec2types.VCpuInfo{ DefaultCores: aws.Int32(48), @@ -481,8 +489,9 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: "nitro", ProcessorInfo: &ec2types.ProcessorInfo{ - Manufacturer: aws.String("Intel"), - SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + Manufacturer: aws.String("Intel"), + SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + SustainedClockSpeedInGhz: aws.Float64(3.100000), }, VCpuInfo: &ec2types.VCpuInfo{ DefaultCores: aws.Int32(2), @@ -525,8 +534,9 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: "nitro", ProcessorInfo: &ec2types.ProcessorInfo{ - Manufacturer: aws.String("Intel"), - SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + Manufacturer: aws.String("Intel"), + SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + SustainedClockSpeedInGhz: aws.Float64(3.500000), }, VCpuInfo: &ec2types.VCpuInfo{ DefaultCores: aws.Int32(64), @@ -579,8 +589,9 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: "xen", ProcessorInfo: &ec2types.ProcessorInfo{ - Manufacturer: aws.String("Intel"), - SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + Manufacturer: aws.String("Intel"), + SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + SustainedClockSpeedInGhz: aws.Float64(2.700000), }, VCpuInfo: &ec2types.VCpuInfo{ DefaultCores: aws.Int32(16), @@ -635,8 +646,9 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: "nitro", ProcessorInfo: &ec2types.ProcessorInfo{ - Manufacturer: aws.String("Intel"), - SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + Manufacturer: aws.String("Intel"), + SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + SustainedClockSpeedInGhz: aws.Float64(2.500000), }, VCpuInfo: &ec2types.VCpuInfo{ DefaultCores: aws.Int32(1), @@ -679,8 +691,9 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: "nitro", ProcessorInfo: &ec2types.ProcessorInfo{ - Manufacturer: aws.String("AWS"), - SupportedArchitectures: []ec2types.ArchitectureType{"arm64"}, + Manufacturer: aws.String("AWS"), + SupportedArchitectures: []ec2types.ArchitectureType{"arm64"}, + SustainedClockSpeedInGhz: aws.Float64(2.500000), }, VCpuInfo: &ec2types.VCpuInfo{ DefaultCores: aws.Int32(2), @@ -723,8 +736,9 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: "nitro", ProcessorInfo: &ec2types.ProcessorInfo{ - Manufacturer: aws.String("AWS"), - SupportedArchitectures: []ec2types.ArchitectureType{"arm64"}, + Manufacturer: aws.String("AWS"), + SupportedArchitectures: []ec2types.ArchitectureType{"arm64"}, + SustainedClockSpeedInGhz: aws.Float64(2.500000), }, VCpuInfo: &ec2types.VCpuInfo{ DefaultCores: aws.Int32(2), @@ -767,8 +781,9 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: "nitro", ProcessorInfo: &ec2types.ProcessorInfo{ - Manufacturer: aws.String("AWS"), - SupportedArchitectures: []ec2types.ArchitectureType{"arm64"}, + Manufacturer: aws.String("AWS"), + SupportedArchitectures: []ec2types.ArchitectureType{"arm64"}, + SustainedClockSpeedInGhz: aws.Float64(2.500000), }, VCpuInfo: &ec2types.VCpuInfo{ DefaultCores: aws.Int32(4), @@ -811,8 +826,9 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: "nitro", ProcessorInfo: &ec2types.ProcessorInfo{ - Manufacturer: aws.String("Intel"), - SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + Manufacturer: aws.String("Intel"), + SupportedArchitectures: []ec2types.ArchitectureType{"x86_64"}, + SustainedClockSpeedInGhz: aws.Float64(3.500000), }, VCpuInfo: &ec2types.VCpuInfo{ DefaultCores: aws.Int32(4), diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index 76e1abfa114f..2b2bc4c0b892 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -236,6 +236,7 @@ var _ = Describe("InstanceTypeProvider", func() { v1.LabelInstanceSize: "8xlarge", v1.LabelInstanceCPU: "32", v1.LabelInstanceCPUManufacturer: "intel", + v1.LabelInstanceCPUSustainedClockSpeedMhz: "2500", v1.LabelInstanceMemory: "131072", v1.LabelInstanceEBSBandwidth: "9500", v1.LabelInstanceNetworkBandwidth: "50000", @@ -292,6 +293,7 @@ var _ = Describe("InstanceTypeProvider", func() { v1.LabelInstanceSize: "8xlarge", v1.LabelInstanceCPU: "32", v1.LabelInstanceCPUManufacturer: "intel", + v1.LabelInstanceCPUSustainedClockSpeedMhz: "2500", v1.LabelInstanceMemory: "131072", v1.LabelInstanceEBSBandwidth: "9500", v1.LabelInstanceNetworkBandwidth: "50000", @@ -344,6 +346,7 @@ var _ = Describe("InstanceTypeProvider", func() { v1.LabelInstanceFamily: "inf2", v1.LabelInstanceSize: "xlarge", v1.LabelInstanceCPU: "4", + v1.LabelInstanceCPUSustainedClockSpeedMhz: "3600", v1.LabelInstanceCPUManufacturer: "amd", v1.LabelInstanceMemory: "16384", v1.LabelInstanceEBSBandwidth: "10000", diff --git a/pkg/providers/instancetype/types.go b/pkg/providers/instancetype/types.go index 1151cecf81ef..8f178418e690 100644 --- a/pkg/providers/instancetype/types.go +++ b/pkg/providers/instancetype/types.go @@ -198,6 +198,7 @@ func computeRequirements(info ec2types.InstanceTypeInfo, offerings cloudprovider // Well Known to AWS scheduling.NewRequirement(v1.LabelInstanceCPU, corev1.NodeSelectorOpIn, fmt.Sprint(lo.FromPtr(info.VCpuInfo.DefaultVCpus))), scheduling.NewRequirement(v1.LabelInstanceCPUManufacturer, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1.LabelInstanceCPUSustainedClockSpeedMhz, corev1.NodeSelectorOpDoesNotExist), scheduling.NewRequirement(v1.LabelInstanceMemory, corev1.NodeSelectorOpIn, fmt.Sprint(lo.FromPtr(info.MemoryInfo.SizeInMiB))), scheduling.NewRequirement(v1.LabelInstanceEBSBandwidth, corev1.NodeSelectorOpDoesNotExist), scheduling.NewRequirement(v1.LabelInstanceNetworkBandwidth, corev1.NodeSelectorOpDoesNotExist), @@ -272,6 +273,11 @@ func computeRequirements(info ec2types.InstanceTypeInfo, offerings cloudprovider if info.ProcessorInfo != nil { requirements.Get(v1.LabelInstanceCPUManufacturer).Insert(lowerKabobCase(aws.ToString(info.ProcessorInfo.Manufacturer))) } + // CPU Sustained Clock Speed + if info.ProcessorInfo != nil { + // Convert from Ghz to Mhz and round to nearest whole number - converting from float64 to int to support Gt and Lt operators + requirements.Get(v1.LabelInstanceCPUSustainedClockSpeedMhz).Insert(fmt.Sprint(int(math.Round(aws.ToFloat64(info.ProcessorInfo.SustainedClockSpeedInGhz) * 1000)))) + } // EBS Max Bandwidth if info.EbsInfo != nil && info.EbsInfo.EbsOptimizedInfo != nil && info.EbsInfo.EbsOptimizedSupport == ec2types.EbsOptimizedSupportDefault { requirements.Get(v1.LabelInstanceEBSBandwidth).Insert(fmt.Sprint(lo.FromPtr(info.EbsInfo.EbsOptimizedInfo.MaximumBandwidthInMbps))) diff --git a/test/suites/scheduling/suite_test.go b/test/suites/scheduling/suite_test.go index fb720222f842..724bbfdb3ee5 100644 --- a/test/suites/scheduling/suite_test.go +++ b/test/suites/scheduling/suite_test.go @@ -109,16 +109,17 @@ var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() { karpv1.NodePoolLabelKey: nodePool.Name, corev1.LabelInstanceTypeStable: "c5.large", // Well Known to AWS - v1.LabelInstanceHypervisor: "nitro", - v1.LabelInstanceCategory: "c", - v1.LabelInstanceGeneration: "5", - v1.LabelInstanceFamily: "c5", - v1.LabelInstanceSize: "large", - v1.LabelInstanceCPU: "2", - v1.LabelInstanceCPUManufacturer: "intel", - v1.LabelInstanceMemory: "4096", - v1.LabelInstanceEBSBandwidth: "4750", - v1.LabelInstanceNetworkBandwidth: "750", + v1.LabelInstanceHypervisor: "nitro", + v1.LabelInstanceCategory: "c", + v1.LabelInstanceGeneration: "5", + v1.LabelInstanceFamily: "c5", + v1.LabelInstanceSize: "large", + v1.LabelInstanceCPU: "2", + v1.LabelInstanceCPUManufacturer: "intel", + v1.LabelInstanceCPUSustainedClockSpeedMhz: "3400", + v1.LabelInstanceMemory: "4096", + v1.LabelInstanceEBSBandwidth: "4750", + v1.LabelInstanceNetworkBandwidth: "750", } selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement {