Skip to content

Commit

Permalink
Merge pull request kubernetes-sigs#6176 from fabriziopandini/handle-k…
Browse files Browse the repository at this point in the history 
…ubeadm1.24-kubelet-ConfigMap-name-change

🌱 handle kubeadm 1.24 kubelet ConfigMap name change
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored Feb 24, 2022
2 parents 61e7dcc + 9705e43 commit 38467d2
Show file tree
Hide file tree
Showing 4 changed files with 178 additions and 36 deletions.
17 changes: 15 additions & 2 deletions controlplane/kubeadm/internal/workload_cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,12 @@ var (
// NOTE: The following assumes that kubeadm version equals to Kubernetes version.
minVerKubeletSystemdDriver = semver.MustParse("1.21.0")

// Starting from v1.24.0 kubeadm uses "kubelet-config" a ConfigMap name for KubeletConfiguration,
// Dropping the X-Y suffix.
//
// NOTE: The following assumes that kubeadm version equals to Kubernetes version.
minVerUnversionedKubeletConfig = semver.MustParse("1.24.0")

// ErrControlPlaneMinNodes signals that a cluster doesn't meet the minimum required nodes
// to remove an etcd member.
ErrControlPlaneMinNodes = errors.New("cluster has fewer than 2 control plane nodes; removing an etcd member is not supported")
Expand Down Expand Up @@ -179,7 +185,7 @@ func (w *Workload) UpdateKubernetesVersionInKubeadmConfigMap(ctx context.Context
// This is a necessary process for upgrades.
func (w *Workload) UpdateKubeletConfigMap(ctx context.Context, version semver.Version) error {
// Check if the desired configmap already exists
desiredKubeletConfigMapName := fmt.Sprintf("kubelet-config-%d.%d", version.Major, version.Minor)
desiredKubeletConfigMapName := generateKubeletConfigName(version)
configMapKey := ctrlclient.ObjectKey{Name: desiredKubeletConfigMapName, Namespace: metav1.NamespaceSystem}
_, err := w.getConfigMap(ctx, configMapKey)
if err == nil {
Expand All @@ -190,7 +196,14 @@ func (w *Workload) UpdateKubeletConfigMap(ctx context.Context, version semver.Ve
return errors.Wrapf(err, "error determining if kubelet configmap %s exists", desiredKubeletConfigMapName)
}

previousMinorVersionKubeletConfigMapName := fmt.Sprintf("kubelet-config-%d.%d", version.Major, version.Minor-1)
previousMinorVersionKubeletConfigMapName := generateKubeletConfigName(semver.Version{Major: version.Major, Minor: version.Minor - 1})

// If desired and previous ConfigMap name are the same it means we already completed the transition
// to the unified KubeletConfigMap name in the previous upgrade; no additional operations are required.
if desiredKubeletConfigMapName == previousMinorVersionKubeletConfigMapName {
return nil
}

configMapKey = ctrlclient.ObjectKey{Name: previousMinorVersionKubeletConfigMapName, Namespace: metav1.NamespaceSystem}
// Returns a copy
cm, err := w.getConfigMap(ctx, configMapKey)
Expand Down
9 changes: 8 additions & 1 deletion controlplane/kubeadm/internal/workload_cluster_rbac.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,11 @@ const (
// KubeletConfigMapRolePrefix defines base kubelet configuration ConfigMap role prefix.
KubeletConfigMapRolePrefix = "kubeadm:"

// KubeletConfigMapName defines base kubelet configuration ConfigMap name.
// KubeletConfigMapName defines base kubelet configuration ConfigMap name for kubeadm < 1.24.
KubeletConfigMapName = "kubelet-config-%d.%d"

// UnversionedKubeletConfigMapName defines base kubelet configuration ConfigMap for kubeadm >= 1.24.
UnversionedKubeletConfigMapName = "kubelet-config"
)

// EnsureResource creates a resoutce if the target resource doesn't exist. If the resource exists already, this function will ignore the resource instead.
Expand Down Expand Up @@ -101,6 +104,10 @@ func (w *Workload) AllowBootstrapTokensToGetNodes(ctx context.Context) error {
}

func generateKubeletConfigName(version semver.Version) string {
majorMinor := semver.Version{Major: version.Major, Minor: version.Minor}
if majorMinor.GTE(minVerUnversionedKubeletConfig) {
return UnversionedKubeletConfigMapName
}
return fmt.Sprintf(KubeletConfigMapName, version.Major, version.Minor)
}

Expand Down
103 changes: 88 additions & 15 deletions controlplane/kubeadm/internal/workload_cluster_rbac_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,32 +24,86 @@ import (
. "github.com/onsi/gomega"
rbacv1 "k8s.io/api/rbac/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
)

func TestCluster_ReconcileKubeletRBACBinding_NoError(t *testing.T) {
type wantRBAC struct {
role ctrlclient.ObjectKey
roleBinding ctrlclient.ObjectKey
}
tests := []struct {
name string
client ctrlclient.Client
name string
client ctrlclient.Client
version semver.Version
want *wantRBAC
}{
{
name: "role binding and role already exist",
client: &fakeClient{
get: map[string]interface{}{
"kube-system/kubeadm:kubelet-config-1.12": &rbacv1.RoleBinding{},
"kube-system/kubeadm:kubelet-config-1.13": &rbacv1.Role{},
},
name: "creates role and role binding for Kubernetes/kubeadm < v1.24",
client: fake.NewClientBuilder().Build(),
version: semver.MustParse("1.23.3"),
want: &wantRBAC{
role: ctrlclient.ObjectKey{Namespace: metav1.NamespaceSystem, Name: "kubeadm:kubelet-config-1.23"},
roleBinding: ctrlclient.ObjectKey{Namespace: metav1.NamespaceSystem, Name: "kubeadm:kubelet-config-1.23"},
},
},
{
name: "role binding and role don't exist",
client: &fakeClient{},
name: "tolerates existing role binding for Kubernetes/kubeadm < v1.24",
client: fake.NewClientBuilder().WithObjects(
&rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "kubeadm:kubelet-config-1.23"}, RoleRef: rbacv1.RoleRef{
Name: "kubeadm:kubelet-config-1.23",
}},
&rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "kubeadm:kubelet-config-1.23"}, Rules: []rbacv1.PolicyRule{{
Verbs: []string{"get"},
APIGroups: []string{""},
Resources: []string{"configmaps"},
ResourceNames: []string{"kubelet-config-1.23"},
}}},
).Build(),
version: semver.MustParse("1.23.3"),
want: &wantRBAC{
role: ctrlclient.ObjectKey{Namespace: metav1.NamespaceSystem, Name: "kubeadm:kubelet-config-1.23"},
roleBinding: ctrlclient.ObjectKey{Namespace: metav1.NamespaceSystem, Name: "kubeadm:kubelet-config-1.23"},
},
},
{
name: "create returns an already exists error",
client: &fakeClient{
createErr: apierrors.NewAlreadyExists(schema.GroupResource{}, ""),
name: "creates role and role binding for Kubernetes/kubeadm >= v1.24",
client: fake.NewClientBuilder().Build(),
version: semver.MustParse("1.24.0"),
want: &wantRBAC{
role: ctrlclient.ObjectKey{Namespace: metav1.NamespaceSystem, Name: "kubeadm:kubelet-config"},
roleBinding: ctrlclient.ObjectKey{Namespace: metav1.NamespaceSystem, Name: "kubeadm:kubelet-config"},
},
},
{
name: "creates role and role binding for Kubernetes/kubeadm >= v1.24 ignoring pre-release and build tags",
client: fake.NewClientBuilder().Build(),
version: semver.MustParse("1.24.0-alpha.1+xyz.1"),
want: &wantRBAC{
role: ctrlclient.ObjectKey{Namespace: metav1.NamespaceSystem, Name: "kubeadm:kubelet-config"},
roleBinding: ctrlclient.ObjectKey{Namespace: metav1.NamespaceSystem, Name: "kubeadm:kubelet-config"},
},
},
{
name: "tolerates existing role binding for Kubernetes/kubeadm >= v1.24",
client: fake.NewClientBuilder().WithObjects(
&rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "kubeadm:kubelet-config"}, RoleRef: rbacv1.RoleRef{
Name: "kubeadm:kubelet-config",
}},
&rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "kubeadm:kubelet-config"}, Rules: []rbacv1.PolicyRule{{
Verbs: []string{"get"},
APIGroups: []string{""},
Resources: []string{"configmaps"},
ResourceNames: []string{"kubelet-config"},
}}},
).Build(),
version: semver.MustParse("1.24.1"),
want: &wantRBAC{
role: ctrlclient.ObjectKey{Namespace: metav1.NamespaceSystem, Name: "kubeadm:kubelet-config"},
roleBinding: ctrlclient.ObjectKey{Namespace: metav1.NamespaceSystem, Name: "kubeadm:kubelet-config"},
},
},
}
Expand All @@ -61,8 +115,27 @@ func TestCluster_ReconcileKubeletRBACBinding_NoError(t *testing.T) {
c := &Workload{
Client: tt.client,
}
g.Expect(c.ReconcileKubeletRBACBinding(ctx, semver.MustParse("1.12.3"))).To(Succeed())
g.Expect(c.ReconcileKubeletRBACRole(ctx, semver.MustParse("1.13.3"))).To(Succeed())
g.Expect(c.ReconcileKubeletRBACBinding(ctx, tt.version)).To(Succeed())
g.Expect(c.ReconcileKubeletRBACRole(ctx, tt.version)).To(Succeed())
if tt.want != nil {
r := &rbacv1.Role{}
// Role exists
g.Expect(tt.client.Get(ctx, tt.want.role, r)).To(Succeed())
// Role ensure grants for the KubeletConfig config map
g.Expect(r.Rules).To(Equal([]rbacv1.PolicyRule{
{
Verbs: []string{"get"},
APIGroups: []string{""},
Resources: []string{"configmaps"},
ResourceNames: []string{generateKubeletConfigName(tt.version)},
},
}))
// RoleBinding exists
b := &rbacv1.RoleBinding{}
// RoleBinding refers to the role
g.Expect(tt.client.Get(ctx, tt.want.roleBinding, b)).To(Succeed())
g.Expect(b.RoleRef.Name).To(Equal(tt.want.role.Name))
}
})
}
}
Expand Down
85 changes: 67 additions & 18 deletions controlplane/kubeadm/internal/workload_cluster_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ package internal
import (
"context"
"errors"
"fmt"
"testing"

"github.com/blang/semver"
Expand Down Expand Up @@ -365,9 +364,10 @@ func TestUpdateKubeletConfigMap(t *testing.T) {
objs []client.Object
expectErr bool
expectCgroupDriver string
expectNewConfigMap bool
}{
{
name: "create new config map",
name: "create new config map for 1.19 --> 1.20 (anything < 1.24); config map for previous version is copied",
version: semver.Version{Major: 1, Minor: 20},
objs: []client.Object{&corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -379,14 +379,51 @@ func TestUpdateKubeletConfigMap(t *testing.T) {
kubeletConfigKey: yaml.Raw(`
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
foo: bar
`),
},
}},
expectErr: false,
expectCgroupDriver: "",
expectNewConfigMap: true,
},
{
name: "KubeletConfig 1.21 gets the cgroupDriver set if empty",
name: "create new config map 1.23 --> 1.24; config map for previous version is copied",
version: semver.Version{Major: 1, Minor: 24},
objs: []client.Object{&corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "kubelet-config-1.23",
Namespace: metav1.NamespaceSystem,
ResourceVersion: "some-resource-version",
},
Data: map[string]string{
kubeletConfigKey: yaml.Raw(`
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
foo: bar
`),
},
}},
expectNewConfigMap: true,
},
{
name: "create new config map >=1.24 --> next; no op",
version: semver.Version{Major: 1, Minor: 25},
objs: []client.Object{&corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "kubelet-config",
Namespace: metav1.NamespaceSystem,
ResourceVersion: "some-resource-version",
},
Data: map[string]string{
kubeletConfigKey: yaml.Raw(`
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
foo: bar
`),
},
}},
},
{
name: "1.20 --> 1.21 sets the cgroupDriver if empty",
version: semver.Version{Major: 1, Minor: 21},
objs: []client.Object{&corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -397,14 +434,16 @@ func TestUpdateKubeletConfigMap(t *testing.T) {
Data: map[string]string{
kubeletConfigKey: yaml.Raw(`
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration`),
kind: KubeletConfiguration
foo: bar
`),
},
}},
expectErr: false,
expectCgroupDriver: "systemd",
expectNewConfigMap: true,
},
{
name: "KubeletConfig 1.21 preserves cgroupDriver if already set",
name: "1.20 --> 1.21 preserves cgroupDriver if already set",
version: semver.Version{Major: 1, Minor: 21},
objs: []client.Object{&corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -416,18 +455,18 @@ func TestUpdateKubeletConfigMap(t *testing.T) {
kubeletConfigKey: yaml.Raw(`
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: foo`),
cgroupDriver: cgroupfs
foo: bar
`),
},
}},
expectErr: false,
expectCgroupDriver: "foo",
expectCgroupDriver: "cgroupfs",
expectNewConfigMap: true,
},
{
name: "returns error if cannot find previous config map",
version: semver.Version{Major: 1, Minor: 21},
objs: nil,
expectErr: true,
expectCgroupDriver: "",
name: "returns error if cannot find previous config map",
version: semver.Version{Major: 1, Minor: 21},
expectErr: true,
},
}

Expand All @@ -444,14 +483,24 @@ func TestUpdateKubeletConfigMap(t *testing.T) {
return
}
g.Expect(err).ToNot(HaveOccurred())

// Check if the resulting ConfigMap exists
var actualConfig corev1.ConfigMap
g.Expect(w.Client.Get(
ctx,
client.ObjectKey{Name: fmt.Sprintf("kubelet-config-%d.%d", tt.version.Major, tt.version.Minor), Namespace: metav1.NamespaceSystem},
client.ObjectKey{Name: generateKubeletConfigName(tt.version), Namespace: metav1.NamespaceSystem},
&actualConfig,
)).To(Succeed())
g.Expect(actualConfig.ResourceVersion).ToNot(Equal("some-resource-version"))
// Check other values are carried over for previous config map
g.Expect(actualConfig.Data[kubeletConfigKey]).To(ContainSubstring("foo"))
// Check the cgroupvalue has the expected value
g.Expect(actualConfig.Data[kubeletConfigKey]).To(ContainSubstring(tt.expectCgroupDriver))
// check if the config map is new
if tt.expectNewConfigMap {
g.Expect(actualConfig.ResourceVersion).ToNot(Equal("some-resource-version"))
} else {
g.Expect(actualConfig.ResourceVersion).To(Equal("some-resource-version"))
}
})
}
}
Expand Down

0 comments on commit 38467d2

Please # to comment.