refactor yurtadm init/join and support to create high-availability OpenYurt cluster

config/yurtadm/samples/openyurt-v0.7.0/Clusterfile

@@ -0,0 +1,28 @@
+apiVersion: sealer.cloud/v2
+kind: Cluster
+metadata:
+  name: my-cluster
+spec:
+  hosts:
+  - ips: [ 1.2.3.4 ]
+    roles: [ master ]
+  image: test:latest
+  ssh:
+    passwd: xxx
+    pk: /root/.ssh/id_rsa
+    user: root
+  env:
+  - PodCIDR=10.244.0.0/16
+  cmd_args:
+  - BindAddress=0.0.0.0
+  - ClusterCIDR=10.244.0.0/16
+---
+
+## Custom configurations must specify kind, will be merged to default kubeadm configs
+kind: ClusterConfiguration
+networking:
+  podSubnet: 10.244.0.0/16
+  serviceSubnet: 10.96.0.0/12
+controllerManager:
+  extraArgs:
+    controllers: -nodelifecycle,*,bootstrapsigner,tokencleaner

config/yurtadm/samples/openyurt-v0.7.0/README.md

@@ -0,0 +1,38 @@
+# OpenYurt ClusterImage
+
+Currently, `yurtadm init` is implemented by sealer v0.8.5 to create kubernetes master nodes.
+
+## Install sealer
+
+```bash
+wget https://github.com/sealerio/sealer/releases/download/v0.8.5/sealer-v0.8.5-linux-amd64.tar.gz
+tar -zxvf sealer-v0.8.5-linux-amd64.tar.gz -C /usr/bin
+```
+
+## Build your own OpenYurt Cluster
+
+Modify the Kubefile to build your own OpenYurt cluster image.
+
+### 1. Build OpenYurt Cluster Image
+
+```bash
+cd ./cluster-image
+
+# build openyurt ClusterImage
+sealer build -t registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1.21.14 -f Kubefile .
+
+# push to dockerhub
+sealer push registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1.21.14
+```
+
+### 2. Make a Clusterfile
+
+A sample Clusterfile in ./Clusterfile
+
+### 3. Run OpenYurt Cluster
+
+```bash
+sealer apply -f Clusterfile
+```
+
+Note: `yurtadm init` only creates master nodes. For worker nodes, you should use `yurtadm join`.

config/yurtadm/samples/openyurt-v0.7.0/cluster-image/Kubefile

@@ -0,0 +1,18 @@
+FROM kubernetes:v1.21.14-alpine
+
+# flannel: https://github.com/sealerio/applications/tree/main/flannel
+# cni has been replaced by the content in openyurt-cni-0.8.7-0.x86_64.rpm in https://github.com/openyurtio/openyurt/releases
+# Reason see: https://openyurt.io/docs/user-manuals/network/edge-pod-network/#ipam-pod-ip-address-kept
+
+COPY cni .
+COPY init-kube.sh /scripts/
+COPY kube-flannel.yaml.tmpl manifests/
+
+COPY shell-plugin.yaml plugins
+
+# openyurt
+COPY yamls/* manifests
+COPY install.sh .
+RUN chmod 777 install.sh
+
+CMD bind_address=${BindAddress} cluster_cidr=${ClusterCIDR} ./install.sh

config/yurtadm/samples/openyurt-v0.7.0/cluster-image/init-kube.sh

@@ -0,0 +1,175 @@
+#!/bin/bash
+
+# Copyright © 2022 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Open ipvs
+modprobe -- ip_vs
+modprobe -- ip_vs_rr
+modprobe -- ip_vs_wrr
+modprobe -- ip_vs_sh
+modprobe -- br_netfilter
+## version_ge 4.19 4.19 true ;
+## version_ge 5.4 4.19 true ;
+## version_ge 3.10 4.19 false ;
+
+version_ge(){
+    test "$(echo "$@" | tr ' ' '\n' | sort -rV | head -n 1)" == "$1"
+}
+
+disable_selinux(){
+    if [ -s /etc/selinux/config ] && grep 'SELINUX=enforcing' /etc/selinux/config; then
+        sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
+        setenforce 0
+    fi
+}
+
+get_distribution() {
+	lsb_dist=""
+	# Every system that we officially support has /etc/os-release
+	if [ -r /etc/os-release ]; then
+		lsb_dist="$(. /etc/os-release && echo "$ID")"
+	fi
+	# Returning an empty string here should be alright since the
+	# case statements don't act unless you provide an actual value
+	echo "$lsb_dist"
+}
+
+disable_firewalld() {
+  lsb_dist=$( get_distribution )
+	lsb_dist="$(echo "$lsb_dist" | tr '[:upper:]' '[:lower:]')"
+	case "$lsb_dist" in
+		ubuntu|deepin|debian|raspbian)
+			command -v ufw &> /dev/null && ufw disable
+		;;
+		centos|rhel|ol|sles|kylin|neokylin)
+			systemctl stop firewalld && systemctl disable firewalld
+		;;
+		*)
+		 	systemctl stop firewalld && systemctl disable firewalld
+			echo "unknown system, use default to stop firewalld"
+		;;
+	esac
+}
+
+kernel_version=$(uname -r | cut -d- -f1)
+if version_ge "${kernel_version}" 4.19; then
+  modprobe -- nf_conntrack
+else
+  modprobe -- nf_conntrack_ipv4
+fi
+
+cat <<EOF >  /etc/sysctl.d/k8s.conf
+net.bridge.bridge-nf-call-ip6tables = 1
+net.bridge.bridge-nf-call-iptables = 1
+net.ipv4.conf.all.rp_filter=0
+EOF
+sysctl --system
+sysctl -w net.ipv4.ip_forward=1
+disable_firewalld
+swapoff -a || true
+disable_selinux
+
+chmod -R 755 ../bin/*
+chmod 644 ../bin
+cp ../bin/* /usr/bin
+cp ../scripts/kubelet-pre-start.sh /usr/bin
+#cni
+mkdir /opt/cni/bin -p
+chmod -R 755 ../cni/*
+chmod 644 ../cni
+cp ../cni/* /opt/cni/bin
+
+# Cgroup driver
+mkdir -p /etc/systemd/system
+cp ../etc/kubelet.service /etc/systemd/system/
+[ -d /etc/systemd/system/kubelet.service.d ] || mkdir /etc/systemd/system/kubelet.service.d
+cp ../etc/10-kubeadm.conf /etc/systemd/system/kubelet.service.d/
+
+[ -d /var/lib/kubelet ] || mkdir -p /var/lib/kubelet/
+
+cat <<EOF > /var/lib/kubelet/config.yaml
+address: 0.0.0.0
+apiVersion: kubelet.config.k8s.io/v1beta1
+authentication:
+  anonymous:
+    enabled: false
+  webhook:
+    cacheTTL: 2m0s
+    enabled: true
+  x509:
+    clientCAFile: /etc/kubernetes/pki/ca.crt
+authorization:
+  mode: Webhook
+  webhook:
+    cacheAuthorizedTTL: 5m0s
+    cacheUnauthorizedTTL: 30s
+cgroupDriver: ${criDriver}
+cgroupsPerQOS: true
+clusterDomain: cluster.local
+configMapAndSecretChangeDetectionStrategy: Watch
+containerLogMaxFiles: 5
+containerLogMaxSize: 10Mi
+contentType: application/vnd.kubernetes.protobuf
+cpuCFSQuota: true
+cpuCFSQuotaPeriod: 100ms
+cpuManagerPolicy: none
+cpuManagerReconcilePeriod: 10s
+enableControllerAttachDetach: true
+enableDebuggingHandlers: true
+enforceNodeAllocatable:
+- pods
+eventBurst: 10
+eventRecordQPS: 5
+evictionHard:
+  imagefs.available: 15%
+  memory.available: 100Mi
+  nodefs.available: 10%
+  nodefs.inodesFree: 5%
+evictionPressureTransitionPeriod: 5m0s
+failSwapOn: true
+fileCheckFrequency: 20s
+hairpinMode: promiscuous-bridge
+healthzBindAddress: 127.0.0.1
+healthzPort: 10248
+httpCheckFrequency: 20s
+imageGCHighThresholdPercent: 85
+imageGCLowThresholdPercent: 80
+imageMinimumGCAge: 2m0s
+iptablesDropBit: 15
+iptablesMasqueradeBit: 14
+kind: KubeletConfiguration
+kubeAPIBurst: 10
+kubeAPIQPS: 5
+makeIPTablesUtilChains: true
+maxOpenFiles: 1000000
+maxPods: 110
+nodeLeaseDurationSeconds: 40
+nodeStatusUpdateFrequency: 10s
+oomScoreAdj: -999
+podPidsLimit: -1
+port: 10250
+registryBurst: 10
+registryPullQPS: 5
+resolvConf: /etc/resolv.conf
+rotateCertificates: true
+runtimeRequestTimeout: 2m0s
+serializeImagePulls: true
+staticPodPath: /etc/kubernetes/manifests
+streamingConnectionIdleTimeout: 4h0m0s
+syncFrequency: 1m0s
+volumeStatsAggPeriod: 1m0s
+EOF
+
+systemctl enable kubelet

config/yurtadm/samples/openyurt-v0.7.0/cluster-image/install.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Copyright 2022 The OpenYurt Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "[INFO] Start installing OpenYurt."
+
+kubectl apply -f manifests/kube-flannel.yaml
+
+## install openyurt components
+kubectl apply -f manifests/yurt-controller-manager.yaml
+kubectl apply -f manifests/yurt-tunnel-agent.yaml
+kubectl apply -f manifests/yurt-tunnel-server.yaml
+kubectl apply -f manifests/yurt-app-manager.yaml
+kubectl apply -f manifests/yurthub-cfg.yaml
+
+## configure coredns
+kubectl scale --replicas=0 deployment/coredns -n kube-system
+kubectl apply -f manifests/coredns.yaml
+kubectl annotate svc kube-dns -n kube-system openyurt.io/topologyKeys='openyurt.io/nodepool'
+
+## configure kube-proxy
+str_patch='{"data": {"config.conf": "apiVersion: kubeproxy.config.k8s.io/v1alpha1\nbindAddress: '${bind_address}'\nfeatureGates:\n  EndpointSliceProxying: true\nbindAddressHardFail: false\nclusterCIDR: '${cluster_cidr}'\nconfigSyncPeriod: 0s\nenableProfiling: false\nipvs:\n  minSyncPeriod: 0s\n  strictARP: false\nkind: KubeProxyConfiguration\nmode: ipvs\nudpIdleTimeout: 0s\nwinkernel:\n  enableDSR: false\nkubeconfig.conf:"}}'
+kubectl patch cm -n kube-system kube-proxy --patch "${str_patch}"  && kubectl delete pod --selector k8s-app=kube-proxy -n kube-system
+
+echo "[INFO] OpenYurt is successfully installed."