improve yurtadm init/join and support to create high-availability Ope…

…nYurt cluster

config/yurtadm/cluster-image/README.md

@@ -0,0 +1,13 @@
+# Build an OpenYurt ClusterImage
+
+`yurtadm init` is implemented by sealer, you can modify the kubefile to make your own openyurt cluster image.
+
+```bash
+cd openyurt-latest
+
+# build openyurt ClusterImage
+sealer build -t registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1198 -f Kubefile .
+
+# push to dockerhub
+sealer push registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1198
+```

config/yurtadm/cluster-image/openyurt-latest/Kubefile

@@ -0,0 +1,15 @@
+FROM kubernetes:v1.19.8-alpine
+
+# flannel: https://github.com/sealerio/applications/tree/main/flannel
+COPY cni .
+COPY init-kube.sh /scripts/
+COPY kube-flannel.yaml.tmpl manifests/
+
+COPY shell-plugin.yaml plugins
+
+# openyurt
+COPY yamls/* manifests
+COPY install.sh .
+RUN chmod 777 install.sh
+
+CMD ./install.sh

config/yurtadm/cloudimage/openyurt-latest/flannel/init-kube.sh → config/yurtadm/cluster-image/openyurt-latest/init-kube.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright © 2021 Alibaba Group Holding Ltd.
+# Copyright © 2022 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -117,8 +117,6 @@ authorization:
     cacheUnauthorizedTTL: 30s
 cgroupDriver: ${criDriver}
 cgroupsPerQOS: true
-clusterDNS:
-- 10.96.0.10
 clusterDomain: cluster.local
 configMapAndSecretChangeDetectionStrategy: Watch
 containerLogMaxFiles: 5

config/yurtadm/cloudimage/openyurt-latest/install-openyurt.sh → config/yurtadm/cluster-image/openyurt-latest/install.sh

@@ -16,6 +16,8 @@
 
 echo "[INFO] Start installing OpenYurt."
 
+kubectl apply -f manifests/kube-flannel.yaml
+
 ## label node
 kubectl label node $HOSTNAME openyurt.io/is-edge-worker=false
 
@@ -27,9 +29,9 @@ kubectl apply -f manifests/yurt-app-manager.yaml
 kubectl apply -f manifests/yurthub-cfg.yaml
 
 ## configure coredns
+kubectl scale --replicas=0 deployment/coredns -n kube-system
 kubectl apply -f manifests/coredns.yaml
 kubectl annotate svc kube-dns -n kube-system openyurt.io/topologyKeys='openyurt.io/nodepool'
-kubectl scale --replicas=0 deployment/coredns -n kube-system
 
 ## configure kube-proxy
 kubectl patch cm -n kube-system kube-proxy --patch '{"data": {"config.conf": "apiVersion: kubeproxy.config.k8s.io/v1alpha1\nbindAddress: 0.0.0.0\nfeatureGates:\n  EndpointSliceProxying: true\nbindAddressHardFail: false\nclusterCIDR: 100.64.0.0/10\nconfigSyncPeriod: 0s\nenableProfiling: false\nipvs:\n  excludeCIDRs:\n  - 10.103.97.2/32\n  minSyncPeriod: 0s\n  strictARP: false\nkind: KubeProxyConfiguration\nmode: ipvs\nudpIdleTimeout: 0s\nwinkernel:\n  enableDSR: false\nkubeconfig.conf:"}}'  && kubectl delete pod --selector k8s-app=kube-proxy -n kube-system

config/yurtadm/cloudimage/openyurt-latest/flannel/kube-flannel.yml → config/yurtadm/cluster-image/openyurt-latest/kube-flannel.yaml.tmpl

@@ -66,6 +66,7 @@ rules:
   resources:
   - nodes
   verbs:
+  - get
   - list
   - watch
 - apiGroups:
@@ -125,7 +126,7 @@ data:
     }
   net-conf.json: |
     {
-      "Network": "10.244.0.0/16",
+      "Network": "{{ .PodCIDR }}",
       "Backend": {
         "Type": "vxlan"
       }
@@ -166,7 +167,7 @@ spec:
       serviceAccountName: flannel
       initContainers:
       - name: install-cni
-        image: quay.io/coreos/flannel:v0.14.0
+        image: registry.cn-hangzhou.aliyuncs.com/openyurt/flannel-edge:v0.14.0-1
         command:
         - cp
         args:
@@ -180,7 +181,7 @@ spec:
           mountPath: /etc/kube-flannel/
       containers:
       - name: kube-flannel
-        image: quay.io/coreos/flannel:v0.14.0
+        image: registry.cn-hangzhou.aliyuncs.com/openyurt/flannel-edge:v0.14.0-1
         command:
         - /opt/bin/flanneld
         args:

config/yurtadm/cloudimage/openyurt-latest/yamls/yurt-tunnel-server.yaml → config/yurtadm/cluster-image/openyurt-latest/yamls/yurt-tunnel-server.yaml.tmpl

@@ -202,6 +202,7 @@ spec:
         args:
         - --bind-address=$(NODE_IP)
         - --insecure-bind-address=$(NODE_IP)
+        - --cert-ips={{ .YurttunnelServerAddress }}
         - --proxy-strategy=destHost
         - --v=2
         env:

go.mod

@@ -22,8 +22,6 @@ require (
 	github.com/openyurtio/yurt-app-manager-api v0.18.8
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.11.0
-	github.com/sealyun/lvscare v1.1.2-alpha.2 // indirect
-	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spf13/cobra v1.2.1
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.7.0

go.sum

@@ -477,8 +477,6 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/moby/ipvs v1.0.1 h1:aoZ7fhLTXgDbzVrAnvV+XbKOU8kOET7B3+xULDF/1o0=
-github.com/moby/ipvs v1.0.1/go.mod h1:2pngiyseZbIKXNv7hsKj3O9UEz30c53MT9005gt2hxQ=
 github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
@@ -568,8 +566,6 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
-github.com/sealyun/lvscare v1.1.2-alpha.2 h1:SlnEAXOPn5gC6l8tgvbn9fApyJaQ0ecQHSV3cEfvQYY=
-github.com/sealyun/lvscare v1.1.2-alpha.2/go.mod h1:FtOEdsXuYtw9Jwd/Jct25K+PcpUFSDemvF4VgNygjj0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
@@ -596,7 +592,6 @@ github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
 github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw=
@@ -631,13 +626,10 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
-github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
 github.com/vishvananda/netlink v1.1.1-0.20200603190939-5a869a71f0cb h1:MY3XXjEi7+I9L6iwK4x0KWNL9OaWMQ5CntP06o+8zZc=
 github.com/vishvananda/netlink v1.1.1-0.20200603190939-5a869a71f0cb/go.mod h1:FSQhuTO7eHT34mPzX+B04SUAjiqLxtXs1et0S6l9k4k=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
-github.com/wonderivan/logger v1.0.0 h1:Z6Nz+3SNcizolx3ARH11axdD4DXjFpb2J+ziGUVlv/U=
-github.com/wonderivan/logger v1.0.0/go.mod h1:NObMfQ3WOLKfYEZuGeZQfuQfSPE5+QNgRddVMzsAT/k=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
@@ -864,7 +856,6 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

pkg/yurtadm/README.md

@@ -7,7 +7,7 @@ To expand the cluster later, users can use the `yurtadm join` command to add edg
 ## 2.Ability
 Using yurtadm, you can do:
 - Create a simple openyurt cluster with just one command.
-- Create a High-Availability OpenYurt cluster.
+- Create a High Availability OpenYurt cluster.
 
 ## 3.Process
 ### 3.1 Compile Yurtadm
@@ -26,10 +26,10 @@ Execute the following command as root account, no need to install container runt
 
 ```bash
 # Initialize an OpenYurt cluster.
-yurtadm init --apiserver-advertise-address 192.168.152.131 --openyurt-version latest --passwd 1234
+yurtadm init --apiserver-advertise-address 1.2.3.4 --openyurt-version latest --passwd xxx
 
-# Initialize an OpenYurt cluster with multiple masters.
-yurtadm init --apiserver-advertise-address 192.168.152.131,192.168.152.132 --openyurt-version v0.7.0 --passwd 1234
+# Initialize an OpenYurt high availability cluster.
+yurtadm init --apiserver-advertise-address 1.2.3.4,1.2.3.5,1.2.3.6 --openyurt-version v0.7.0 --passwd xxx
 ```
 `yurtadm init` will use sealer to create a K8s cluster. And kubeadm, kubectl, docker, etc. will all be installed during this process.
 
@@ -40,23 +40,23 @@ Get bootstrap token from the master:
 ```bash
 kubeadm token create
 W0720 20:46:19.782354   31205 configset.go:348] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io]
-zffaj3.a5vjzf09qn9ft3gt
+abcdef.0123456789abcdef
 ```
 
 Before `yurtadm join` you need to install a runtime (like docker) on the worker node. Then execute the `yurtadm join` command in the worker node:
 
 ```bash
 # Join the edge node to cluster.
-yurtadm join 192.168.152.131:6443 --token=zffaj3.a5vjzf09qn9ft3gt --node-type=edge --discovery-token-unsafe-skip-ca-verification --v=5
+yurtadm join 1.2.3.4:6443 --token=abcdef.0123456789abcdef --node-type=edge --discovery-token-unsafe-skip-ca-verification --v=5
 
-# Join the edge node to cluster with multiple masters.
-yurtadm join 192.168.152.131:6443,192.168.152.132:6443 --token=zffaj3.a5vjzf09qn9ft3gt --node-type=edge --discovery-token-unsafe-skip-ca-verification --v=5
+# Join the edge node to a high availability cluster.
+yurtadm join 1.2.3.4:6443,1.2.3.5:6443,1.2.3.6:6443 --token=abcdef.0123456789abcdef --node-type=edge --discovery-token-unsafe-skip-ca-verification --v=5
 
 # Join the cloud node to cluster.
-yurtadm join 192.168.152.131:6443 --token=zffaj3.a5vjzf09qn9ft3gt --node-type=cloud --discovery-token-unsafe-skip-ca-verification --v=5
+yurtadm join 1.2.3.4:6443 --token=abcdef.0123456789abcdef --node-type=cloud --discovery-token-unsafe-skip-ca-verification --v=5
 
-# Join the cloud node to cluster with multiple masters.
-yurtadm join 192.168.152.131:6443,192.168.152.132:6443 --token=zffaj3.a5vjzf09qn9ft3gt --node-type=cloud --discovery-token-unsafe-skip-ca-verification --v=5
+# Join the cloud node to a high availability cluster.
+yurtadm join 1.2.3.4:6443,1.2.3.5:6443,1.2.3.6:6443 --token=abcdef.0123456789abcdef --node-type=cloud --discovery-token-unsafe-skip-ca-verification --v=5
 ```
 
 ## Other Problems

pkg/yurtadm/cmd/join/join.go

@@ -48,16 +48,16 @@ import (
 var (
 	joinExample = templates.Examples(i18n.T(`
 		# Join the edge node to cluster.
-		yurtadm join 192.168.152.131:6443 --token=zffaj3.a5vjzf09qn9ft3gt --node-type=edge --discovery-token-unsafe-skip-ca-verification --v=5
+		yurtadm join 1.2.3.4:6443 --token=abcdef.0123456789abcdef --node-type=edge --discovery-token-unsafe-skip-ca-verification --v=5
 
-		# Join the edge node to cluster with multiple masters.
-		yurtadm join 192.168.152.131:6443,192.168.152.132:6443 --token=zffaj3.a5vjzf09qn9ft3gt --node-type=edge --discovery-token-unsafe-skip-ca-verification --v=5
+		# Join the edge node to a high availability cluster.
+		yurtadm join 1.2.3.4:6443,1.2.3.5:6443,1.2.3.6:6443 --token=abcdef.0123456789abcdef --node-type=edge --discovery-token-unsafe-skip-ca-verification --v=5
 
 		# Join the cloud node to cluster.
-		yurtadm join 192.168.152.131:6443 --token=zffaj3.a5vjzf09qn9ft3gt --node-type=cloud --discovery-token-unsafe-skip-ca-verification --v=5
+		yurtadm join 1.2.3.4:6443 --token=abcdef.0123456789abcdef --node-type=cloud --discovery-token-unsafe-skip-ca-verification --v=5
 
-		# Join the cloud node to cluster with multiple masters.
-		yurtadm join 192.168.152.131:6443,192.168.152.132:6443 --token=zffaj3.a5vjzf09qn9ft3gt --node-type=cloud --discovery-token-unsafe-skip-ca-verification --v=5
+		# Join the cloud node to a high availability cluster.
+		yurtadm join 1.2.3.4:6443,1.2.3.5:6443,1.2.3.6:6443 --token=abcdef.0123456789abcdef --node-type=cloud --discovery-token-unsafe-skip-ca-verification --v=5
 	`))
 
 	joinWorkerNodeDoneMsg = dedent.Dedent(`