From 2878046434e814962ef892f57080cbc80a132925 Mon Sep 17 00:00:00 2001 From: zengfeng Date: Wed, 10 Aug 2022 14:10:24 +0800 Subject: [PATCH] improve some details in yurtadm, use an image with a fixed version number to prevent unavailability caused by image:latest changes --- config/yurtadm/cluster-image/README.md | 72 ++++++++++++++++-- .../Kubefile | 3 + .../cni/bridge | Bin .../cni/dhcp | Bin .../cni/flannel | Bin .../cni/host-device | Bin .../cni/host-local | Bin .../cni/ipvlan | Bin .../cni/loopback | Bin .../cni/macvlan | Bin .../cni/portmap | Bin .../cni/ptp | Bin .../cni/sample | Bin .../cni/tuning | Bin .../cni/vlan | Bin .../init-kube.sh | 0 .../install.sh | 5 +- .../kube-flannel.yaml.tmpl | 0 .../shell-plugin.yaml | 0 .../yamls/coredns.yaml | 0 .../yamls/yurt-app-manager.yaml | 2 +- .../yamls/yurt-controller-manager.yaml | 2 +- .../yamls/yurt-tunnel-agent.yaml | 2 +- .../yamls/yurt-tunnel-server.yaml.tmpl | 2 +- .../yamls/yurthub-cfg.yaml | 0 pkg/yurtadm/README.md | 46 ++++++++++- pkg/yurtadm/cmd/yurtinit/init.go | 39 +++++++--- pkg/yurtadm/cmd/yurtinit/options.go | 14 ++-- pkg/yurtadm/constants/constants.go | 26 ++++--- pkg/yurtadm/util/kubernetes/util.go | 21 +---- 30 files changed, 175 insertions(+), 59 deletions(-) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/Kubefile (57%) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/cni/bridge (100%) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/cni/dhcp (100%) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/cni/flannel (100%) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/cni/host-device (100%) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/cni/host-local (100%) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/cni/ipvlan (100%) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/cni/loopback (100%) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/cni/macvlan (100%) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/cni/portmap (100%) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/cni/ptp (100%) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/cni/sample (100%) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/cni/tuning (100%) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/cni/vlan (100%) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/init-kube.sh (100%) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/install.sh (66%) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/kube-flannel.yaml.tmpl (100%) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/shell-plugin.yaml (100%) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/yamls/coredns.yaml (100%) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/yamls/yurt-app-manager.yaml (99%) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/yamls/yurt-controller-manager.yaml (98%) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/yamls/yurt-tunnel-agent.yaml (97%) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/yamls/yurt-tunnel-server.yaml.tmpl (98%) rename config/yurtadm/cluster-image/{openyurt-latest => openyurt-v0.7.0}/yamls/yurthub-cfg.yaml (100%) diff --git a/config/yurtadm/cluster-image/README.md b/config/yurtadm/cluster-image/README.md index 23cc0822b7f..25c1c9ce1ec 100644 --- a/config/yurtadm/cluster-image/README.md +++ b/config/yurtadm/cluster-image/README.md @@ -1,13 +1,75 @@ -# Build an OpenYurt ClusterImage +# OpenYurt ClusterImage -`yurtadm init` is implemented by sealer, you can modify the kubefile to make your own openyurt cluster image. +Currently, `yurtadm init` is implemented by sealer v0.8.5 to create kubernetes master nodes. + +## install sealer + +```bash +wget https://github.com/sealerio/sealer/releases/download/v0.8.5/sealer-v0.8.5-linux-amd64.tar.gz +tar -zxvf sealer-v0.8.5-linux-amd64.tar.gz -C /usr/bin +``` + +## Build your own OpenYurt Cluster + +Modify the Kubefile to build your own OpenYurt cluster image. + +### 1. Build OpenYurt Cluster Image ```bash cd openyurt-latest # build openyurt ClusterImage -sealer build -t registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1198 -f Kubefile . +sealer build -t registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1.19.8 -f Kubefile . # push to dockerhub -sealer push registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1198 -``` \ No newline at end of file +sealer push registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1.19.8 +``` + +### 2. Make a Clusterfile + +A sample Clusterfile: + +```yaml +apiVersion: sealer.cloud/v2 +kind: Cluster +metadata: + name: my-cluster +spec: + hosts: + - ips: [ 1.2.3.4 ] + roles: [ master ] + image: registry-1.docker.io/your_dockerhub_username/openyurt-cluster:latest-k8s-1.19.8 + ssh: + passwd: xxx + pk: /root/.ssh/id_rsa + user: root + env: + - PodCIDR=10.244.0.0/16 +--- + +## Custom configurations must specify kind, will be merged to default kubeadm configs +kind: ClusterConfiguration +networking: + podSubnet: 10.244.0.0/16 + serviceSubnet: 10.96.0.0/12 +controllerManager: + extraArgs: + controllers: -nodelifecycle,*,bootstrapsigner,tokencleaner + +--- + +## Custom configurations must specify kind +kind: KubeProxyConfiguration +bindAddress: 0.0.0.0 +clusterCIDR: 10.244.0.0/16 +featuregates: + EndpointSliceProxying: true +``` + +### 3. Run OpenYurt Cluster + +```bash +sealer apply -f Clusterfile +``` + +Note: `yurtadm init` only creates master nodes. For worker nodes, you should use `yurtadm join`. \ No newline at end of file diff --git a/config/yurtadm/cluster-image/openyurt-latest/Kubefile b/config/yurtadm/cluster-image/openyurt-v0.7.0/Kubefile similarity index 57% rename from config/yurtadm/cluster-image/openyurt-latest/Kubefile rename to config/yurtadm/cluster-image/openyurt-v0.7.0/Kubefile index 6076cb2cd88..666bdeee2b7 100644 --- a/config/yurtadm/cluster-image/openyurt-latest/Kubefile +++ b/config/yurtadm/cluster-image/openyurt-v0.7.0/Kubefile @@ -1,6 +1,9 @@ FROM kubernetes:v1.19.8-alpine # flannel: https://github.com/sealerio/applications/tree/main/flannel +# cni has been replaced by the content in openyurt-cni-0.8.7-0.x86_64.rpm in https://github.com/openyurtio/openyurt/releases +# Reason see: https://openyurt.io/docs/user-manuals/network/edge-pod-network/#ipam-pod-ip-address-kept + COPY cni . COPY init-kube.sh /scripts/ COPY kube-flannel.yaml.tmpl manifests/ diff --git a/config/yurtadm/cluster-image/openyurt-latest/cni/bridge b/config/yurtadm/cluster-image/openyurt-v0.7.0/cni/bridge similarity index 100% rename from config/yurtadm/cluster-image/openyurt-latest/cni/bridge rename to config/yurtadm/cluster-image/openyurt-v0.7.0/cni/bridge diff --git a/config/yurtadm/cluster-image/openyurt-latest/cni/dhcp b/config/yurtadm/cluster-image/openyurt-v0.7.0/cni/dhcp similarity index 100% rename from config/yurtadm/cluster-image/openyurt-latest/cni/dhcp rename to config/yurtadm/cluster-image/openyurt-v0.7.0/cni/dhcp diff --git a/config/yurtadm/cluster-image/openyurt-latest/cni/flannel b/config/yurtadm/cluster-image/openyurt-v0.7.0/cni/flannel similarity index 100% rename from config/yurtadm/cluster-image/openyurt-latest/cni/flannel rename to config/yurtadm/cluster-image/openyurt-v0.7.0/cni/flannel diff --git a/config/yurtadm/cluster-image/openyurt-latest/cni/host-device b/config/yurtadm/cluster-image/openyurt-v0.7.0/cni/host-device similarity index 100% rename from config/yurtadm/cluster-image/openyurt-latest/cni/host-device rename to config/yurtadm/cluster-image/openyurt-v0.7.0/cni/host-device diff --git a/config/yurtadm/cluster-image/openyurt-latest/cni/host-local b/config/yurtadm/cluster-image/openyurt-v0.7.0/cni/host-local similarity index 100% rename from config/yurtadm/cluster-image/openyurt-latest/cni/host-local rename to config/yurtadm/cluster-image/openyurt-v0.7.0/cni/host-local diff --git a/config/yurtadm/cluster-image/openyurt-latest/cni/ipvlan b/config/yurtadm/cluster-image/openyurt-v0.7.0/cni/ipvlan similarity index 100% rename from config/yurtadm/cluster-image/openyurt-latest/cni/ipvlan rename to config/yurtadm/cluster-image/openyurt-v0.7.0/cni/ipvlan diff --git a/config/yurtadm/cluster-image/openyurt-latest/cni/loopback b/config/yurtadm/cluster-image/openyurt-v0.7.0/cni/loopback similarity index 100% rename from config/yurtadm/cluster-image/openyurt-latest/cni/loopback rename to config/yurtadm/cluster-image/openyurt-v0.7.0/cni/loopback diff --git a/config/yurtadm/cluster-image/openyurt-latest/cni/macvlan b/config/yurtadm/cluster-image/openyurt-v0.7.0/cni/macvlan similarity index 100% rename from config/yurtadm/cluster-image/openyurt-latest/cni/macvlan rename to config/yurtadm/cluster-image/openyurt-v0.7.0/cni/macvlan diff --git a/config/yurtadm/cluster-image/openyurt-latest/cni/portmap b/config/yurtadm/cluster-image/openyurt-v0.7.0/cni/portmap similarity index 100% rename from config/yurtadm/cluster-image/openyurt-latest/cni/portmap rename to config/yurtadm/cluster-image/openyurt-v0.7.0/cni/portmap diff --git a/config/yurtadm/cluster-image/openyurt-latest/cni/ptp b/config/yurtadm/cluster-image/openyurt-v0.7.0/cni/ptp similarity index 100% rename from config/yurtadm/cluster-image/openyurt-latest/cni/ptp rename to config/yurtadm/cluster-image/openyurt-v0.7.0/cni/ptp diff --git a/config/yurtadm/cluster-image/openyurt-latest/cni/sample b/config/yurtadm/cluster-image/openyurt-v0.7.0/cni/sample similarity index 100% rename from config/yurtadm/cluster-image/openyurt-latest/cni/sample rename to config/yurtadm/cluster-image/openyurt-v0.7.0/cni/sample diff --git a/config/yurtadm/cluster-image/openyurt-latest/cni/tuning b/config/yurtadm/cluster-image/openyurt-v0.7.0/cni/tuning similarity index 100% rename from config/yurtadm/cluster-image/openyurt-latest/cni/tuning rename to config/yurtadm/cluster-image/openyurt-v0.7.0/cni/tuning diff --git a/config/yurtadm/cluster-image/openyurt-latest/cni/vlan b/config/yurtadm/cluster-image/openyurt-v0.7.0/cni/vlan similarity index 100% rename from config/yurtadm/cluster-image/openyurt-latest/cni/vlan rename to config/yurtadm/cluster-image/openyurt-v0.7.0/cni/vlan diff --git a/config/yurtadm/cluster-image/openyurt-latest/init-kube.sh b/config/yurtadm/cluster-image/openyurt-v0.7.0/init-kube.sh similarity index 100% rename from config/yurtadm/cluster-image/openyurt-latest/init-kube.sh rename to config/yurtadm/cluster-image/openyurt-v0.7.0/init-kube.sh diff --git a/config/yurtadm/cluster-image/openyurt-latest/install.sh b/config/yurtadm/cluster-image/openyurt-v0.7.0/install.sh similarity index 66% rename from config/yurtadm/cluster-image/openyurt-latest/install.sh rename to config/yurtadm/cluster-image/openyurt-v0.7.0/install.sh index 9914fd09f00..ba53a7695ef 100644 --- a/config/yurtadm/cluster-image/openyurt-latest/install.sh +++ b/config/yurtadm/cluster-image/openyurt-v0.7.0/install.sh @@ -18,9 +18,6 @@ echo "[INFO] Start installing OpenYurt." kubectl apply -f manifests/kube-flannel.yaml -## label node -kubectl label node $HOSTNAME openyurt.io/is-edge-worker=false - ## install openyurt components kubectl apply -f manifests/yurt-controller-manager.yaml kubectl apply -f manifests/yurt-tunnel-agent.yaml @@ -34,6 +31,6 @@ kubectl apply -f manifests/coredns.yaml kubectl annotate svc kube-dns -n kube-system openyurt.io/topologyKeys='openyurt.io/nodepool' ## configure kube-proxy -kubectl patch cm -n kube-system kube-proxy --patch '{"data": {"config.conf": "apiVersion: kubeproxy.config.k8s.io/v1alpha1\nbindAddress: 0.0.0.0\nfeatureGates:\n EndpointSliceProxying: true\nbindAddressHardFail: false\nclusterCIDR: 100.64.0.0/10\nconfigSyncPeriod: 0s\nenableProfiling: false\nipvs:\n excludeCIDRs:\n - 10.103.97.2/32\n minSyncPeriod: 0s\n strictARP: false\nkind: KubeProxyConfiguration\nmode: ipvs\nudpIdleTimeout: 0s\nwinkernel:\n enableDSR: false\nkubeconfig.conf:"}}' && kubectl delete pod --selector k8s-app=kube-proxy -n kube-system +kubectl patch cm kube-proxy -n kube-system --patch '{"data":{"kubeconfig.conf":""}}' && kubectl delete pod -n kube-system --selector k8s-app=kube-proxy echo "[INFO] OpenYurt is successfully installed." \ No newline at end of file diff --git a/config/yurtadm/cluster-image/openyurt-latest/kube-flannel.yaml.tmpl b/config/yurtadm/cluster-image/openyurt-v0.7.0/kube-flannel.yaml.tmpl similarity index 100% rename from config/yurtadm/cluster-image/openyurt-latest/kube-flannel.yaml.tmpl rename to config/yurtadm/cluster-image/openyurt-v0.7.0/kube-flannel.yaml.tmpl diff --git a/config/yurtadm/cluster-image/openyurt-latest/shell-plugin.yaml b/config/yurtadm/cluster-image/openyurt-v0.7.0/shell-plugin.yaml similarity index 100% rename from config/yurtadm/cluster-image/openyurt-latest/shell-plugin.yaml rename to config/yurtadm/cluster-image/openyurt-v0.7.0/shell-plugin.yaml diff --git a/config/yurtadm/cluster-image/openyurt-latest/yamls/coredns.yaml b/config/yurtadm/cluster-image/openyurt-v0.7.0/yamls/coredns.yaml similarity index 100% rename from config/yurtadm/cluster-image/openyurt-latest/yamls/coredns.yaml rename to config/yurtadm/cluster-image/openyurt-v0.7.0/yamls/coredns.yaml diff --git a/config/yurtadm/cluster-image/openyurt-latest/yamls/yurt-app-manager.yaml b/config/yurtadm/cluster-image/openyurt-v0.7.0/yamls/yurt-app-manager.yaml similarity index 99% rename from config/yurtadm/cluster-image/openyurt-latest/yamls/yurt-app-manager.yaml rename to config/yurtadm/cluster-image/openyurt-v0.7.0/yamls/yurt-app-manager.yaml index d24f7ae3146..af55aac74c8 100644 --- a/config/yurtadm/cluster-image/openyurt-latest/yamls/yurt-app-manager.yaml +++ b/config/yurtadm/cluster-image/openyurt-v0.7.0/yamls/yurt-app-manager.yaml @@ -1103,7 +1103,7 @@ spec: - --v=4 command: - /usr/local/bin/yurt-app-manager - image: openyurt/yurt-app-manager:latest + image: openyurt/yurt-app-manager:v0.6.0-beta.1 imagePullPolicy: Always name: manager ports: diff --git a/config/yurtadm/cluster-image/openyurt-latest/yamls/yurt-controller-manager.yaml b/config/yurtadm/cluster-image/openyurt-v0.7.0/yamls/yurt-controller-manager.yaml similarity index 98% rename from config/yurtadm/cluster-image/openyurt-latest/yamls/yurt-controller-manager.yaml rename to config/yurtadm/cluster-image/openyurt-v0.7.0/yamls/yurt-controller-manager.yaml index c1330fcf463..8dd9a50f8c2 100644 --- a/config/yurtadm/cluster-image/openyurt-latest/yamls/yurt-controller-manager.yaml +++ b/config/yurtadm/cluster-image/openyurt-v0.7.0/yamls/yurt-controller-manager.yaml @@ -143,6 +143,6 @@ spec: - "false" containers: - name: yurt-controller-manager - image: openyurt/yurt-controller-manager:latest + image: openyurt/yurt-controller-manager:v0.7.0 command: - yurt-controller-manager \ No newline at end of file diff --git a/config/yurtadm/cluster-image/openyurt-latest/yamls/yurt-tunnel-agent.yaml b/config/yurtadm/cluster-image/openyurt-v0.7.0/yamls/yurt-tunnel-agent.yaml similarity index 97% rename from config/yurtadm/cluster-image/openyurt-latest/yamls/yurt-tunnel-agent.yaml rename to config/yurtadm/cluster-image/openyurt-v0.7.0/yamls/yurt-tunnel-agent.yaml index acb2b35341f..10c2945ce6c 100644 --- a/config/yurtadm/cluster-image/openyurt-latest/yamls/yurt-tunnel-agent.yaml +++ b/config/yurtadm/cluster-image/openyurt-v0.7.0/yamls/yurt-tunnel-agent.yaml @@ -24,7 +24,7 @@ spec: - --node-name=$(NODE_NAME) - --node-ip=$(POD_IP) - --v=2 - image: openyurt/yurt-tunnel-agent:latest + image: openyurt/yurt-tunnel-agent:v0.7.0 imagePullPolicy: IfNotPresent name: yurt-tunnel-agent volumeMounts: diff --git a/config/yurtadm/cluster-image/openyurt-latest/yamls/yurt-tunnel-server.yaml.tmpl b/config/yurtadm/cluster-image/openyurt-v0.7.0/yamls/yurt-tunnel-server.yaml.tmpl similarity index 98% rename from config/yurtadm/cluster-image/openyurt-latest/yamls/yurt-tunnel-server.yaml.tmpl rename to config/yurtadm/cluster-image/openyurt-v0.7.0/yamls/yurt-tunnel-server.yaml.tmpl index 361e66ea1e7..0eb7629892d 100644 --- a/config/yurtadm/cluster-image/openyurt-latest/yamls/yurt-tunnel-server.yaml.tmpl +++ b/config/yurtadm/cluster-image/openyurt-v0.7.0/yamls/yurt-tunnel-server.yaml.tmpl @@ -195,7 +195,7 @@ spec: openyurt.io/is-edge-worker: "false" containers: - name: yurt-tunnel-server - image: openyurt/yurt-tunnel-server:latest + image: openyurt/yurt-tunnel-server:v0.7.0 imagePullPolicy: IfNotPresent command: - yurt-tunnel-server diff --git a/config/yurtadm/cluster-image/openyurt-latest/yamls/yurthub-cfg.yaml b/config/yurtadm/cluster-image/openyurt-v0.7.0/yamls/yurthub-cfg.yaml similarity index 100% rename from config/yurtadm/cluster-image/openyurt-latest/yamls/yurthub-cfg.yaml rename to config/yurtadm/cluster-image/openyurt-v0.7.0/yamls/yurthub-cfg.yaml diff --git a/pkg/yurtadm/README.md b/pkg/yurtadm/README.md index 2207597e261..651a5383081 100644 --- a/pkg/yurtadm/README.md +++ b/pkg/yurtadm/README.md @@ -26,13 +26,20 @@ Execute the following command as root account, no need to install container runt ```bash # Initialize an OpenYurt cluster. -yurtadm init --apiserver-advertise-address 1.2.3.4 --openyurt-version latest --passwd xxx +yurtadm init --apiserver-advertise-address 1.2.3.4 --openyurt-version v0.7.0 --passwd xxx # Initialize an OpenYurt high availability cluster. yurtadm init --apiserver-advertise-address 1.2.3.4,1.2.3.5,1.2.3.6 --openyurt-version v0.7.0 --passwd xxx ``` `yurtadm init` will use sealer to create a K8s cluster. And kubeadm, kubectl, docker, etc. will all be installed during this process. +Note: The following components are installed during `yurtadm init` : +- kubeadm +- kubectl +- kubelet +- kube-proxy +- docker + ### 3.3 Joining node to cluster Currently, you can use kubeadm token create to get bootstrap token. Get bootstrap token from the master: @@ -43,7 +50,11 @@ W0720 20:46:19.782354 31205 configset.go:348] WARNING: kubeadm cannot validate abcdef.0123456789abcdef ``` -Before `yurtadm join` you need to install a runtime (like docker) on the worker node. Then execute the `yurtadm join` command in the worker node: +Before `yurtadm join` you need to: +- install a runtime (like docker) on the worker node +- copy the yurtadm command to the node to be joined + +Then execute the `yurtadm join` command in the worker node: ```bash # Join the edge node to cluster. @@ -59,5 +70,34 @@ yurtadm join 1.2.3.4:6443 --token=abcdef.0123456789abcdef --node-type=cloud --di yurtadm join 1.2.3.4:6443,1.2.3.5:6443,1.2.3.6:6443 --token=abcdef.0123456789abcdef --node-type=cloud --discovery-token-unsafe-skip-ca-verification --v=5 ``` +Note: The following components are installed during `yurtadm init` : +- kubeadm +- kubectl +- kubelet +- kube-proxy + +### 3.3 Delete node +When you need to delete a node joined using `yurtadm join`, the steps are as follows: + +In master: + +```bash +kubectl drain {NodeName} --delete-local-data --force --ignore-daemonsets +kubectl delete node {NodeName} +``` + +In your joined node: + +```bash +yurtadm reset +``` + +Then delete the `/etc/cni/net.d` dir: + +```bash +rm -rf /etc/cni/net.d +``` + + ## Other Problems -Temporarily yurtadm only supports openyurt v0.7.0 and latest, k8s v1.19.8 version. \ No newline at end of file +Temporarily yurtadm only supports openyurt v0.7.0, k8s v1.19.8 version. \ No newline at end of file diff --git a/pkg/yurtadm/cmd/yurtinit/init.go b/pkg/yurtadm/cmd/yurtinit/init.go index 03387d8b493..84480bc0795 100644 --- a/pkg/yurtadm/cmd/yurtinit/init.go +++ b/pkg/yurtadm/cmd/yurtinit/init.go @@ -43,17 +43,21 @@ const ( APIServerAdvertiseAddress = "apiserver-advertise-address" //YurttunnelServerAddress flag sets the IP address of Yurttunnel Server. YurttunnelServerAddress = "yurt-tunnel-server-address" - // NetworkingServiceSubnet flag sets the range of IP address for service VIPs. - NetworkingServiceSubnet = "service-cidr" - // NetworkingPodSubnet flag sets the range of IP addresses for the pod network. If set, the control plane will automatically allocate CIDRs for every node. - NetworkingPodSubnet = "pod-network-cidr" + // NetworkingServiceSubnet flag sets the subnet used by kubernetes Services. + NetworkingServiceSubnet = "service-subnet" + // NetworkingPodSubnet flag sets the subnet used by Pods. + NetworkingPodSubnet = "pod-subnet" + // ClusterCIDR flag sets the CIDR range of the pods in the cluster. It is used to bridge traffic coming from outside of the cluster. + ClusterCIDR = "cluster-cidr" + // KubeProxyBindAddress flag sets the IP address for the proxy server to serve on (set to 0.0.0.0 for all interfaces) + KubeProxyBindAddress = "kube-proxy-bind-address" // OpenYurtVersion flag sets the OpenYurt version for the control plane. OpenYurtVersion = "openyurt-version" // K8sVersion flag sets the Kubernetes version for the control plane. K8sVersion = "k8s-version" // ImageRepository flag sets the container registry to pull control plane images from. ImageRepository = "image-repository" - // PassWd flag is the password of master server. + // PassWd flag sets the password of master server. PassWd = "passwd" TmpDownloadDir = "/tmp" @@ -91,16 +95,25 @@ networking: controllerManager: extraArgs: controllers: -nodelifecycle,*,bootstrapsigner,tokencleaner + +--- + +## Custom configurations must specify kind +kind: KubeProxyConfiguration +bindAddress: {{.bind_address}} +clusterCIDR: {{.cluster_cidr}} +featuregates: + EndpointSliceProxying: true ` ) var ( initExample = templates.Examples(i18n.T(` # Initialize an OpenYurt cluster. - yurtadm init --apiserver-advertise-address 1.2.3.4 --openyurt-version latest --passwd xxx + yurtadm init --apiserver-advertise-address 1.2.3.4 --openyurt-version v0.7.0 --passwd xxx # Initialize an OpenYurt high availability cluster. - yurtadm init --apiserver-advertise-address 1.2.3.4,1.2.3.5,1.2.3.6 --openyurt-version latest --passwd xxx + yurtadm init --apiserver-advertise-address 1.2.3.4,1.2.3.5,1.2.3.6 --openyurt-version v0.7.0 --passwd xxx `)) ValidSealerVersions = []string{ @@ -150,11 +163,11 @@ func addFlags(flagset *flag.FlagSet, o *InitOptions) { "The yurt-tunnel-server address.") flagset.StringVarP( &o.ServiceSubnet, NetworkingServiceSubnet, "", o.ServiceSubnet, - "Use alternative range of IP address for service VIPs.", + "ServiceSubnet is the subnet used by kubernetes Services.", ) flagset.StringVarP( &o.PodSubnet, NetworkingPodSubnet, "", o.PodSubnet, - "Specify range of IP addresses for the pod network. If set, the control plane will automatically allocate CIDRs for every node.", + "PodSubnet is the subnet used by Pods.", ) flagset.StringVarP(&o.Password, PassWd, "p", o.Password, "set master server ssh password", @@ -170,6 +183,12 @@ func addFlags(flagset *flag.FlagSet, o *InitOptions) { flagset.StringVarP(&o.ImageRepository, ImageRepository, "", o.ImageRepository, "Choose a registry to pull cluster images from", ) + flagset.StringVarP(&o.ClusterCIDR, ClusterCIDR, "", o.ClusterCIDR, + "Choose a CIDR range of the pods in the cluster", + ) + flagset.StringVarP(&o.KubeProxyBindAddress, KubeProxyBindAddress, "", o.KubeProxyBindAddress, + "Choose an IP address for the proxy server to serve on", + ) } func NewInitializerWithOptions(o *InitOptions) *clusterInitializer { @@ -257,6 +276,8 @@ func (ci *clusterInitializer) PrepareClusterfile() error { "pod_subnet": ci.PodSubnet, "service_subnet": ci.ServiceSubnet, "yurttunnel_server_address": ci.YurttunnelServerAddress, + "cluster_cidr": ci.ClusterCIDR, + "bind_address": ci.KubeProxyBindAddress, }) if err != nil { return err diff --git a/pkg/yurtadm/cmd/yurtinit/options.go b/pkg/yurtadm/cmd/yurtinit/options.go index 9ed603a08e3..b1cb66464e1 100644 --- a/pkg/yurtadm/cmd/yurtinit/options.go +++ b/pkg/yurtadm/cmd/yurtinit/options.go @@ -32,19 +32,23 @@ type InitOptions struct { YurttunnelServerAddress string ServiceSubnet string PodSubnet string + ClusterCIDR string Password string ImageRepository string OpenYurtVersion string K8sVersion string + KubeProxyBindAddress string } func NewInitOptions() *InitOptions { return &InitOptions{ - ImageRepository: constants.DefaultOpenYurtImageRegistry, - OpenYurtVersion: constants.DefaultOpenYurtVersion, - K8sVersion: constants.DefaultK8sVersion, - PodSubnet: constants.DefaultPodSubnet, - ServiceSubnet: constants.DefaultServiceSubnet, + ImageRepository: constants.DefaultOpenYurtImageRegistry, + OpenYurtVersion: constants.DefaultOpenYurtVersion, + K8sVersion: constants.DefaultK8sVersion, + PodSubnet: constants.DefaultPodSubnet, + ServiceSubnet: constants.DefaultServiceSubnet, + ClusterCIDR: constants.DefaultClusterCIDR, + KubeProxyBindAddress: constants.DefaultKubeProxyBindAddress, } } diff --git a/pkg/yurtadm/constants/constants.go b/pkg/yurtadm/constants/constants.go index 2819fa0bbb1..9853941841f 100644 --- a/pkg/yurtadm/constants/constants.go +++ b/pkg/yurtadm/constants/constants.go @@ -37,6 +37,7 @@ const ( YurthubStaticPodFileName = "yurthub.yaml" PauseImagePath = "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2" + OpenYurtCniUrl = "https://github.com/openyurtio/openyurt/releases/download/v0.7.0/openyurt-cni-0.8.7-0.x86_64.rpm" CniUrlFormat = "https://aliacs-edge-k8s-cn-hangzhou.oss-cn-hangzhou.aliyuncs.com/public/pkg/openyurt/cni/%s/cni-plugins-linux-%s-%s.tgz" DefaultKubernetesResourceServer = "dl.k8s.io" KubeUrlFormat = "https://%s/%s/kubernetes-node-linux-%s.tar.gz" @@ -47,19 +48,22 @@ const ( CloudNode = "cloud" DefaultOpenYurtImageRegistry = "registry.cn-hangzhou.aliyuncs.com/openyurt" - DefaultOpenYurtVersion = "latest" - DefaultK8sVersion = "1198" // default kubernetes 1.19.8 + DefaultOpenYurtVersion = "v0.7.0" + DefaultK8sVersion = "1.19.8" // default kubernetes 1.19.8 DefaultPodSubnet = "10.244.0.0/16" DefaultServiceSubnet = "10.96.0.0/12" - YurtControllerManager = "yurt-controller-manager" - YurtTunnelServer = "yurt-tunnel-server" - YurtTunnelAgent = "yurt-tunnel-agent" - Yurthub = "yurthub" - YurtAppManager = "yurt-app-manager" - YurtAppManagerNamespace = "kube-system" - DirMode = 0755 - FileMode = 0666 - KubeletServiceContent = ` + DefaultClusterCIDR = "10.244.0.0/16" + DefaultKubeProxyBindAddress = "0.0.0.0" + + YurtControllerManager = "yurt-controller-manager" + YurtTunnelServer = "yurt-tunnel-server" + YurtTunnelAgent = "yurt-tunnel-agent" + Yurthub = "yurthub" + YurtAppManager = "yurt-app-manager" + YurtAppManagerNamespace = "kube-system" + DirMode = 0755 + FileMode = 0666 + KubeletServiceContent = ` [Unit] Description=kubelet: The Kubernetes Node Agent Documentation=http://kubernetes.io/docs/ diff --git a/pkg/yurtadm/util/kubernetes/util.go b/pkg/yurtadm/util/kubernetes/util.go index de456060c3c..3d69b5a6051 100644 --- a/pkg/yurtadm/util/kubernetes/util.go +++ b/pkg/yurtadm/util/kubernetes/util.go @@ -127,7 +127,7 @@ func CheckAndInstallKubelet(kubernetesResourceServer, clusterVersion string) err savePath := fmt.Sprintf("%s/kubernetes-node-linux-%s.tar.gz", constants.TmpDownloadDir, runtime.GOARCH) klog.V(1).Infof("Download kubelet from: %s", packageUrl) if err := util.DownloadFile(packageUrl, savePath, 3); err != nil { - return fmt.Errorf("Download kuelet fail: %w", err) + return fmt.Errorf("Download kubelet fail: %w", err) } if err := util.Untar(savePath, constants.TmpDownloadDir); err != nil { return err @@ -145,23 +145,8 @@ func CheckAndInstallKubelet(kubernetesResourceServer, clusterVersion string) err return nil } - ////download and install kubernetes-cni - //cniUrl := fmt.Sprintf(constants.CniUrlFormat, constants.KubeCniVersion, runtime.GOARCH, constants.KubeCniVersion) - //savePath := fmt.Sprintf("%s/cni-plugins-linux-%s-%s.tgz", constants.TmpDownloadDir, runtime.GOARCH, constants.KubeCniVersion) - //klog.V(1).Infof("Download cni from: %s", cniUrl) - //if err := util.DownloadFile(cniUrl, savePath, 3); err != nil { - // return err - //} - // - //if err := os.MkdirAll(constants.KubeCniDir, 0600); err != nil { - // return err - //} - //if err := util.Untar(savePath, constants.KubeCniDir); err != nil { - // return err - //} - - //download and install kubernetes-cni - cniUrl := "https://github.com/openyurtio/openyurt/releases/download/v0.7.0/openyurt-cni-0.8.7-0.x86_64.rpm" + // download and install openyurt-cni + cniUrl := fmt.Sprintf("%s", constants.OpenYurtCniUrl) savePath := fmt.Sprintf("%s/openyurt-cni-0.8.7-0.x86_64.rpm", constants.TmpDownloadDir) klog.V(1).Infof("Download cni from: %s", cniUrl) if err := util.DownloadFile(cniUrl, savePath, 3); err != nil {