From 1f94d7e97550197f29e640d6ffe7de6111226bdc Mon Sep 17 00:00:00 2001 From: Andy Liu Date: Sat, 5 Oct 2019 15:45:40 +0800 Subject: [PATCH] etcd: expose more configs for the etcd pods. --- pkg/apis/etcd/v1beta2/cluster.go | 36 ++++++++++++++ pkg/util/k8sutil/k8sutil.go | 85 +++++++++++++++++++++++++++++--- pkg/util/k8sutil/pod_util.go | 4 -- 3 files changed, 115 insertions(+), 10 deletions(-) diff --git a/pkg/apis/etcd/v1beta2/cluster.go b/pkg/apis/etcd/v1beta2/cluster.go index 889534b6b..9ab0277b5 100644 --- a/pkg/apis/etcd/v1beta2/cluster.go +++ b/pkg/apis/etcd/v1beta2/cluster.go @@ -64,6 +64,39 @@ func (c *EtcdCluster) AsOwner() metav1.OwnerReference { } } +type ProbeConfig struct { + InitialDelaySeconds int `json:"initialDelaySeconds,omitempty"` + TimeoutSeconds int `json:"timeoutSeconds,omitempty"` + PeriodSeconds int `json:"periodSeconds,omitempty"` + FailureThreshold int `json:"failureThreshold,omitempty"` +} + +type EtcdConfig struct { + // Heartbeat timeout setting for etcd pod + HeartbeatTimeout int `json:"heartbeatTimeout,omitempty"` + + // Election timeout setting for etcd pod + ElectionTimeout int `json:"electionTimeout,omitempty"` + + // Snapshot count setting for etcd pod + SnapshotCount int `json:"snapshotCount,omitempty"` + + // AutoCompactionMode, https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/maintenance.md + AutoCompactionMode string `json:"autoCompactionMode,omitempty"` + + // AutoCompactionRetention, https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/maintenance.md + AutoCompactionRetention string `json:"autoCompactionRetention,omitempty"` + + // ExperimentalPeerSkipClientSANVerification indicates whether the peer client san verification will be skipped. + ExperimentalPeerSkipClientSANVerification bool `json:"experimentalPeerSkipClientSANVerification,omitempty"` + + // ReadinessProbeConfig is for the container's readiness probe. + ReadinessProbeConfig ProbeConfig `json:"readinessProbe,omitempty"` + + // LivenessProbeConfig is for the container's readiness probe. + LivenessProbeConfig ProbeConfig `json:"livenessProbe,omitempty"` +} + type ClusterSpec struct { // Size is the expected size of the etcd cluster. // The etcd-operator will eventually make the size of the running @@ -92,6 +125,9 @@ type ClusterSpec struct { // Paused is to pause the control of the operator for the etcd cluster. Paused bool `json:"paused,omitempty"` + // EtcdConfig contains the more configs for the etcd pods. + EtcdConfig `json:",inline"` + // Pod defines the policy to create pod for the etcd pod. // // Updating Pod does not take effect on any existing etcd pods. diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index 8db36f18a..dfe9d0b6e 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -302,6 +302,30 @@ func newEtcdPod(m *etcdutil.Member, initialCluster []string, clusterName, state, "--listen-peer-urls=%s --listen-client-urls=%s --advertise-client-urls=%s "+ "--initial-cluster=%s --initial-cluster-state=%s", dataDir, m.Name, m.PeerURL(), m.ListenPeerURL(), m.ListenClientURL(), m.ClientURL(), strings.Join(initialCluster, ","), state) + if cs.HeartbeatTimeout > 0 { + commands += fmt.Sprintf(" --heartbeat-interval=%d", cs.HeartbeatTimeout) + } + + if cs.ElectionTimeout > 0 { + commands += fmt.Sprintf(" --election-timeout=%d", cs.ElectionTimeout) + } + + if cs.SnapshotCount > 0 { + commands += fmt.Sprintf(" --snapshot-count=%d", cs.SnapshotCount) + } + + if cs.AutoCompactionMode != "" { + commands += fmt.Sprintf(" --auto-compaction-mode=%s", cs.AutoCompactionMode) + } + + if cs.AutoCompactionRetention != "" { + commands += fmt.Sprintf(" --auto-compaction-retention=%s", cs.AutoCompactionRetention) + } + + if cs.ExperimentalPeerSkipClientSANVerification { + commands += fmt.Sprintf(" --experimental-peer-skip-client-san-verification") + } + if m.SecurePeer { commands += fmt.Sprintf(" --peer-client-cert-auth=true --peer-trusted-ca-file=%[1]s/peer-ca.crt --peer-cert-file=%[1]s/peer.crt --peer-key-file=%[1]s/peer.key", peerTLSDir) } @@ -318,12 +342,7 @@ func newEtcdPod(m *etcdutil.Member, initialCluster []string, clusterName, state, "etcd_cluster": clusterName, } - livenessProbe := newEtcdProbe(cs.TLS.IsSecureClient()) - readinessProbe := newEtcdProbe(cs.TLS.IsSecureClient()) - readinessProbe.InitialDelaySeconds = 1 - readinessProbe.TimeoutSeconds = 5 - readinessProbe.PeriodSeconds = 5 - readinessProbe.FailureThreshold = 3 + livenessProbe, readinessProbe := provisionProbeConfigs(cs) container := containerWithProbes( etcdContainer(strings.Split(commands, " "), cs.Repository, cs.Version), @@ -406,6 +425,60 @@ func newEtcdPod(m *etcdutil.Member, initialCluster []string, clusterName, state, return pod } +func provisionProbeConfigs(cs api.ClusterSpec) (livenessProbe *v1.Probe, readinessProbe *v1.Probe) { + livenessProbe = newEtcdProbe(cs.TLS.IsSecureClient()) + + if cs.LivenessProbeConfig.InitialDelaySeconds != 0 { + livenessProbe.InitialDelaySeconds = cs.LivenessProbeConfig.InitialDelaySeconds + } else { + livenessProbe.InitialDelaySeconds = 10 + } + + if cs.LivenessProbeConfig.TimeoutSeconds != 0 { + livenessProbe.TimeoutSeconds = cs.LivenessProbeConfig.TimeoutSeconds + } else { + livenessProbe.TimeoutSeconds = 10 + } + + if cs.LivenessProbeConfig.PeriodSeconds != 0 { + livenessProbe.PeriodSeconds = cs.LivenessProbeConfig.PeriodSeconds + } else { + livenessProbe.PeriodSeconds = 60 + } + + if cs.LivenessProbeConfig.FailureThreshold != 0 { + livenessProbe.FailureThreshold = cs.LivenessProbeConfig.FailureThreshold + } else { + livenessProbe.FailureThreshold = 3 + } + + readinessProbe = newEtcdProbe(cs.TLS.IsSecureClient()) + + if cs.ReadinessProbeConfig.InitialDelaySeconds != 0 { + livenessProbe.InitialDelaySeconds = cs.LivenessProbeConfig.InitialDelaySeconds + } else { + livenessProbe.InitialDelaySeconds = 1 + } + + if cs.ReadinessProbeConfig.TimeoutSeconds != 0 { + livenessProbe.TimeoutSeconds = cs.LivenessProbeConfig.TimeoutSeconds + } else { + livenessProbe.TimeoutSeconds = 5 + } + + if cs.ReadinessProbeConfig.PeriodSeconds != 0 { + livenessProbe.PeriodSeconds = cs.LivenessProbeConfig.PeriodSeconds + } else { + livenessProbe.PeriodSeconds = 5 + } + + if cs.ReadinessProbeConfig.FailureThreshold != 0 { + livenessProbe.FailureThreshold = cs.LivenessProbeConfig.FailureThreshold + } else { + livenessProbe.FailureThreshold = 3 + } +} + func podSecurityContext(podPolicy *api.PodPolicy) *v1.PodSecurityContext { if podPolicy == nil { return nil diff --git a/pkg/util/k8sutil/pod_util.go b/pkg/util/k8sutil/pod_util.go index c9f085a32..1d05ba1dc 100644 --- a/pkg/util/k8sutil/pod_util.go +++ b/pkg/util/k8sutil/pod_util.go @@ -81,10 +81,6 @@ func newEtcdProbe(isSecure bool) *v1.Probe { Command: []string{"/bin/sh", "-ec", cmd}, }, }, - InitialDelaySeconds: 10, - TimeoutSeconds: 10, - PeriodSeconds: 60, - FailureThreshold: 3, } }