improve some details

in yurtadm, use an image with a fixed version number to prevent unavailability caused by image:latest changes

config/yurtadm/cluster-image/README.md

@@ -1,13 +1,75 @@
-# Build an OpenYurt ClusterImage
+# OpenYurt ClusterImage
 
-`yurtadm init` is implemented by sealer, you can modify the kubefile to make your own openyurt cluster image.
+Currently, `yurtadm init` is implemented by sealer v0.8.5 to create kubernetes master nodes.
+
+## install sealer
+
+```bash
+wget https://github.com/sealerio/sealer/releases/download/v0.8.5/sealer-v0.8.5-linux-amd64.tar.gz
+tar -zxvf sealer-v0.8.5-linux-amd64.tar.gz -C /usr/bin
+```
+
+## Build your own OpenYurt Cluster
+
+Modify the Kubefile to build your own OpenYurt cluster image.
+
+### 1. Build OpenYurt Cluster Image
 
 ```bash
 cd openyurt-latest
 
 # build openyurt ClusterImage
-sealer build -t registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1198 -f Kubefile .
+sealer build -t registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1.19.8 -f Kubefile .
 
 # push to dockerhub
-sealer push registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1198
-```
+sealer push registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1.19.8
+```
+
+### 2. Make a Clusterfile
+
+A sample Clusterfile:
+
+```yaml
+apiVersion: sealer.cloud/v2
+kind: Cluster
+metadata:
+  name: my-cluster
+spec:
+  hosts:
+  - ips: [ 1.2.3.4 ]
+    roles: [ master ]
+  image: registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1.19.8
+  ssh:
+    passwd: xxx
+    pk: /root/.ssh/id_rsa
+    user: root
+  env:
+  - PodCIDR=10.244.0.0/16
+---
+
+## Custom configurations must specify kind, will be merged to default kubeadm configs
+kind: ClusterConfiguration
+networking:
+  podSubnet: 10.244.0.0/16
+  serviceSubnet: 10.96.0.0/12
+controllerManager:
+  extraArgs:
+    controllers: -nodelifecycle,*,bootstrapsigner,tokencleaner
+
+---
+
+## Custom configurations must specify kind
+kind: KubeProxyConfiguration
+bindAddress: 0.0.0.0
+clusterCIDR: 10.244.0.0/16
+featuregates:
+  EndpointSliceProxying: true
+```
+
+### 3. Run OpenYurt Cluster
+
+```bash
+sealer apply -f Clusterfile
+```
+
+Note: `yurtadm init` only creates master nodes. For worker nodes, you should use `yurtadm join`.

config/yurtadm/cluster-image/openyurt-latest/Kubefile → config/yurtadm/cluster-image/openyurt-v0.7.0/Kubefile

@@ -1,6 +1,9 @@
 FROM kubernetes:v1.19.8-alpine
 
 # flannel: https://github.com/sealerio/applications/tree/main/flannel
+# cni has been replaced by the content in openyurt-cni-0.8.7-0.x86_64.rpm in https://github.com/openyurtio/openyurt/releases
+# Reason see: https://openyurt.io/docs/user-manuals/network/edge-pod-network/#ipam-pod-ip-address-kept
+
 COPY cni .
 COPY init-kube.sh /scripts/
 COPY kube-flannel.yaml.tmpl manifests/

config/yurtadm/cluster-image/openyurt-latest/install.sh → config/yurtadm/cluster-image/openyurt-v0.7.0/install.sh

@@ -18,9 +18,6 @@ echo "[INFO] Start installing OpenYurt."
 
 kubectl apply -f manifests/kube-flannel.yaml
 
-## label node
-kubectl label node $HOSTNAME openyurt.io/is-edge-worker=false
-
 ## install openyurt components
 kubectl apply -f manifests/yurt-controller-manager.yaml
 kubectl apply -f manifests/yurt-tunnel-agent.yaml
@@ -34,6 +31,6 @@ kubectl apply -f manifests/coredns.yaml
 kubectl annotate svc kube-dns -n kube-system openyurt.io/topologyKeys='openyurt.io/nodepool'
 
 ## configure kube-proxy
-kubectl patch cm -n kube-system kube-proxy --patch '{"data": {"config.conf": "apiVersion: kubeproxy.config.k8s.io/v1alpha1\nbindAddress: 0.0.0.0\nfeatureGates:\n  EndpointSliceProxying: true\nbindAddressHardFail: false\nclusterCIDR: 100.64.0.0/10\nconfigSyncPeriod: 0s\nenableProfiling: false\nipvs:\n  excludeCIDRs:\n  - 10.103.97.2/32\n  minSyncPeriod: 0s\n  strictARP: false\nkind: KubeProxyConfiguration\nmode: ipvs\nudpIdleTimeout: 0s\nwinkernel:\n  enableDSR: false\nkubeconfig.conf:"}}'  && kubectl delete pod --selector k8s-app=kube-proxy -n kube-system
+kubectl patch cm kube-proxy -n kube-system --patch '{"data":{"kubeconfig.conf":""}}' && kubectl delete pod -n kube-system --selector k8s-app=kube-proxy
 
 echo "[INFO] OpenYurt is successfully installed."

config/yurtadm/cluster-image/openyurt-latest/yamls/yurt-app-manager.yaml → config/yurtadm/cluster-image/openyurt-v0.7.0/yamls/yurt-app-manager.yaml

@@ -1103,7 +1103,7 @@ spec:
         - --v=4
         command:
         - /usr/local/bin/yurt-app-manager
-        image: openyurt/yurt-app-manager:latest
+        image: openyurt/yurt-app-manager:v0.6.0-beta.1
         imagePullPolicy: Always
         name: manager
         ports:

config/yurtadm/cluster-image/openyurt-latest/yamls/yurt-controller-manager.yaml → config/yurtadm/cluster-image/openyurt-v0.7.0/yamls/yurt-controller-manager.yaml

@@ -143,6 +143,6 @@ spec:
                 - "false"
       containers:
       - name: yurt-controller-manager
-        image: openyurt/yurt-controller-manager:latest
+        image: openyurt/yurt-controller-manager:v0.7.0
         command:
         - yurt-controller-manager

config/yurtadm/cluster-image/openyurt-latest/yamls/yurt-tunnel-agent.yaml → config/yurtadm/cluster-image/openyurt-v0.7.0/yamls/yurt-tunnel-agent.yaml

@@ -24,7 +24,7 @@ spec:
         - --node-name=$(NODE_NAME)
         - --node-ip=$(POD_IP)
         - --v=2
-        image: openyurt/yurt-tunnel-agent:latest
+        image: openyurt/yurt-tunnel-agent:v0.7.0
         imagePullPolicy: IfNotPresent
         name: yurt-tunnel-agent
         volumeMounts:

config/yurtadm/cluster-image/openyurt-latest/yamls/yurt-tunnel-server.yaml.tmpl → config/yurtadm/cluster-image/openyurt-v0.7.0/yamls/yurt-tunnel-server.yaml.tmpl

@@ -195,7 +195,7 @@ spec:
         openyurt.io/is-edge-worker: "false"
       containers:
       - name: yurt-tunnel-server
-        image: openyurt/yurt-tunnel-server:latest
+        image: openyurt/yurt-tunnel-server:v0.7.0
         imagePullPolicy: IfNotPresent
         command:
         - yurt-tunnel-server

pkg/yurtadm/README.md

@@ -26,13 +26,20 @@ Execute the following command as root account, no need to install container runt
 
 ```bash
 # Initialize an OpenYurt cluster.
-yurtadm init --apiserver-advertise-address 1.2.3.4 --openyurt-version latest --passwd xxx
+yurtadm init --apiserver-advertise-address 1.2.3.4 --openyurt-version v0.7.0 --passwd xxx
 
 # Initialize an OpenYurt high availability cluster.
 yurtadm init --apiserver-advertise-address 1.2.3.4,1.2.3.5,1.2.3.6 --openyurt-version v0.7.0 --passwd xxx
 ```
 `yurtadm init` will use sealer to create a K8s cluster. And kubeadm, kubectl, docker, etc. will all be installed during this process.
 
+Note: The following components are installed during `yurtadm init` :
+- kubeadm
+- kubectl
+- kubelet
+- kube-proxy
+- docker
+
 ### 3.3 Joining node to cluster
 Currently, you can use kubeadm token create to get bootstrap token.
 Get bootstrap token from the master:
@@ -43,7 +50,11 @@ W0720 20:46:19.782354   31205 configset.go:348] WARNING: kubeadm cannot validate
 abcdef.0123456789abcdef
 ```
 
-Before `yurtadm join` you need to install a runtime (like docker) on the worker node. Then execute the `yurtadm join` command in the worker node:
+Before `yurtadm join` you need to:
+- install a runtime (like docker) on the worker node
+- copy the yurtadm command to the node to be joined
+
+Then execute the `yurtadm join` command in the worker node:
 
 ```bash
 # Join the edge node to cluster.
@@ -59,5 +70,34 @@ yurtadm join 1.2.3.4:6443 --token=abcdef.0123456789abcdef --node-type=cloud --di
 yurtadm join 1.2.3.4:6443,1.2.3.5:6443,1.2.3.6:6443 --token=abcdef.0123456789abcdef --node-type=cloud --discovery-token-unsafe-skip-ca-verification --v=5
 ```
 
+Note: The following components are installed during `yurtadm init` :
+- kubeadm
+- kubectl
+- kubelet
+- kube-proxy
+
+### 3.3 Delete node
+When you need to delete a node joined using `yurtadm join`, the steps are as follows:
+
+In master:
+
+```bash
+kubectl drain {NodeName} --delete-local-data --force --ignore-daemonsets
+kubectl delete node {NodeName}
+```
+
+In your joined node:
+
+```bash
+yurtadm reset
+```
+
+Then delete the `/etc/cni/net.d` dir:
+
+```bash
+rm -rf /etc/cni/net.d
+```
+
+
 ## Other Problems
-Temporarily yurtadm only supports openyurt v0.7.0 and latest, k8s v1.19.8 version.
+Temporarily yurtadm only supports openyurt v0.7.0, k8s v1.19.8 version.

pkg/yurtadm/cmd/yurtinit/init.go

@@ -43,17 +43,21 @@ const (
 	APIServerAdvertiseAddress = "apiserver-advertise-address"
 	//YurttunnelServerAddress flag sets the IP address of Yurttunnel Server.
 	YurttunnelServerAddress = "yurt-tunnel-server-address"
-	// NetworkingServiceSubnet flag sets the range of IP address for service VIPs.
-	NetworkingServiceSubnet = "service-cidr"
-	// NetworkingPodSubnet flag sets the range of IP addresses for the pod network. If set, the control plane will automatically allocate CIDRs for every node.
-	NetworkingPodSubnet = "pod-network-cidr"
+	// NetworkingServiceSubnet flag sets the subnet used by kubernetes Services.
+	NetworkingServiceSubnet = "service-subnet"
+	// NetworkingPodSubnet flag sets the subnet used by Pods.
+	NetworkingPodSubnet = "pod-subnet"
+	// ClusterCIDR flag sets the CIDR range of the pods in the cluster. It is used to bridge traffic coming from outside of the cluster.
+	ClusterCIDR = "cluster-cidr"
+	// KubeProxyBindAddress flag sets the IP address for the proxy server to serve on (set to 0.0.0.0 for all interfaces)
+	KubeProxyBindAddress = "kube-proxy-bind-address"
 	// OpenYurtVersion flag sets the OpenYurt version for the control plane.
 	OpenYurtVersion = "openyurt-version"
 	// K8sVersion flag sets the Kubernetes version for the control plane.
 	K8sVersion = "k8s-version"
 	// ImageRepository flag sets the container registry to pull control plane images from.
 	ImageRepository = "image-repository"
-	// PassWd flag is the password of master server.
+	// PassWd flag sets the password of master server.
 	PassWd = "passwd"
 
 	TmpDownloadDir = "/tmp"
@@ -91,16 +95,25 @@ networking:
 controllerManager:
   extraArgs:
     controllers: -nodelifecycle,*,bootstrapsigner,tokencleaner
+
+---
+
+## Custom configurations must specify kind
+kind: KubeProxyConfiguration
+bindAddress: {{.bind_address}}
+clusterCIDR: {{.cluster_cidr}}
+featuregates:
+  EndpointSliceProxying: true
 `
 )
 
 var (
 	initExample = templates.Examples(i18n.T(`
 		# Initialize an OpenYurt cluster.
-		yurtadm init --apiserver-advertise-address 1.2.3.4 --openyurt-version latest --passwd xxx
+		yurtadm init --apiserver-advertise-address 1.2.3.4 --openyurt-version v0.7.0 --passwd xxx
 		
 		# Initialize an OpenYurt high availability cluster.
-		yurtadm init --apiserver-advertise-address 1.2.3.4,1.2.3.5,1.2.3.6 --openyurt-version latest --passwd xxx
+		yurtadm init --apiserver-advertise-address 1.2.3.4,1.2.3.5,1.2.3.6 --openyurt-version v0.7.0 --passwd xxx
 	`))
 
 	ValidSealerVersions = []string{
@@ -150,11 +163,11 @@ func addFlags(flagset *flag.FlagSet, o *InitOptions) {
 		"The yurt-tunnel-server address.")
 	flagset.StringVarP(
 		&o.ServiceSubnet, NetworkingServiceSubnet, "", o.ServiceSubnet,
-		"Use alternative range of IP address for service VIPs.",
+		"ServiceSubnet is the subnet used by kubernetes Services.",
 	)
 	flagset.StringVarP(
 		&o.PodSubnet, NetworkingPodSubnet, "", o.PodSubnet,
-		"Specify range of IP addresses for the pod network. If set, the control plane will automatically allocate CIDRs for every node.",
+		"PodSubnet is the subnet used by Pods.",
 	)
 	flagset.StringVarP(&o.Password, PassWd, "p", o.Password,
 		"set master server ssh password",
@@ -170,6 +183,12 @@ func addFlags(flagset *flag.FlagSet, o *InitOptions) {
 	flagset.StringVarP(&o.ImageRepository, ImageRepository, "", o.ImageRepository,
 		"Choose a registry to pull cluster images from",
 	)
+	flagset.StringVarP(&o.ClusterCIDR, ClusterCIDR, "", o.ClusterCIDR,
+		"Choose a CIDR range of the pods in the cluster",
+	)
+	flagset.StringVarP(&o.KubeProxyBindAddress, KubeProxyBindAddress, "", o.KubeProxyBindAddress,
+		"Choose an IP address for the proxy server to serve on",
+	)
 }
 
 func NewInitializerWithOptions(o *InitOptions) *clusterInitializer {
@@ -257,6 +276,8 @@ func (ci *clusterInitializer) PrepareClusterfile() error {
 		"pod_subnet":                ci.PodSubnet,
 		"service_subnet":            ci.ServiceSubnet,
 		"yurttunnel_server_address": ci.YurttunnelServerAddress,
+		"cluster_cidr":              ci.ClusterCIDR,
+		"bind_address":              ci.KubeProxyBindAddress,
 	})
 	if err != nil {
 		return err

pkg/yurtadm/cmd/yurtinit/options.go

@@ -32,19 +32,23 @@ type InitOptions struct {
 	YurttunnelServerAddress string
 	ServiceSubnet           string
 	PodSubnet               string
+	ClusterCIDR             string
 	Password                string
 	ImageRepository         string
 	OpenYurtVersion         string
 	K8sVersion              string
+	KubeProxyBindAddress    string
 }
 
 func NewInitOptions() *InitOptions {
 	return &InitOptions{
-		ImageRepository: constants.DefaultOpenYurtImageRegistry,
-		OpenYurtVersion: constants.DefaultOpenYurtVersion,
-		K8sVersion:      constants.DefaultK8sVersion,
-		PodSubnet:       constants.DefaultPodSubnet,
-		ServiceSubnet:   constants.DefaultServiceSubnet,
+		ImageRepository:      constants.DefaultOpenYurtImageRegistry,
+		OpenYurtVersion:      constants.DefaultOpenYurtVersion,
+		K8sVersion:           constants.DefaultK8sVersion,
+		PodSubnet:            constants.DefaultPodSubnet,
+		ServiceSubnet:        constants.DefaultServiceSubnet,
+		ClusterCIDR:          constants.DefaultClusterCIDR,
+		KubeProxyBindAddress: constants.DefaultKubeProxyBindAddress,
 	}
 }
 

pkg/yurtadm/constants/constants.go

@@ -37,6 +37,7 @@ const (
 	YurthubStaticPodFileName = "yurthub.yaml"
 	PauseImagePath           = "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2"
 
+	OpenYurtCniUrl                  = "https://github.com/openyurtio/openyurt/releases/download/v0.7.0/openyurt-cni-0.8.7-0.x86_64.rpm"
 	CniUrlFormat                    = "https://aliacs-edge-k8s-cn-hangzhou.oss-cn-hangzhou.aliyuncs.com/public/pkg/openyurt/cni/%s/cni-plugins-linux-%s-%s.tgz"
 	DefaultKubernetesResourceServer = "dl.k8s.io"
 	KubeUrlFormat                   = "https://%s/%s/kubernetes-node-linux-%s.tar.gz"
@@ -47,19 +48,22 @@ const (
 	CloudNode = "cloud"
 
 	DefaultOpenYurtImageRegistry = "registry.cn-hangzhou.aliyuncs.com/openyurt"
-	DefaultOpenYurtVersion       = "latest"
-	DefaultK8sVersion            = "1198" // default kubernetes 1.19.8
+	DefaultOpenYurtVersion       = "v0.7.0"
+	DefaultK8sVersion            = "1.19.8" // default kubernetes 1.19.8
 	DefaultPodSubnet             = "10.244.0.0/16"
 	DefaultServiceSubnet         = "10.96.0.0/12"
-	YurtControllerManager        = "yurt-controller-manager"
-	YurtTunnelServer             = "yurt-tunnel-server"
-	YurtTunnelAgent              = "yurt-tunnel-agent"
-	Yurthub                      = "yurthub"
-	YurtAppManager               = "yurt-app-manager"
-	YurtAppManagerNamespace      = "kube-system"
-	DirMode                      = 0755
-	FileMode                     = 0666
-	KubeletServiceContent        = `
+	DefaultClusterCIDR           = "10.244.0.0/16"
+	DefaultKubeProxyBindAddress  = "0.0.0.0"
+
+	YurtControllerManager   = "yurt-controller-manager"
+	YurtTunnelServer        = "yurt-tunnel-server"
+	YurtTunnelAgent         = "yurt-tunnel-agent"
+	Yurthub                 = "yurthub"
+	YurtAppManager          = "yurt-app-manager"
+	YurtAppManagerNamespace = "kube-system"
+	DirMode                 = 0755
+	FileMode                = 0666
+	KubeletServiceContent   = `
 [Unit]
 Description=kubelet: The Kubernetes Node Agent
 Documentation=http://kubernetes.io/docs/