Skip to content

Commit

Permalink
feat(autok3s): compatible with etcd HA mode
Browse files Browse the repository at this point in the history 
Signed-off-by: Jason-ZW <zhenyang@rancher.com>
  • Loading branch information
@rancher-sy-bot
rancher-sy-bot committed Sep 25, 2020
1 parent bbf5483 commit c3a2444
Show file tree
Hide file tree
Showing 3 changed files with 42 additions and 39 deletions.
48 changes: 30 additions & 18 deletions pkg/cluster/cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ import (
)

var (
masterCommand = "curl -sLS %s | %s INSTALL_K3S_REGISTRIES='%s' K3S_TOKEN='%s' INSTALL_K3S_EXEC='--tls-san %s %s' sh -\n"
workerCommand = "curl -sLS %s | %s INSTALL_K3S_REGISTRIES='%s' K3S_URL='https://%s:6443' K3S_TOKEN='%s' INSTALL_K3S_EXEC='%s' sh -\n"
initCommand = "curl -sLS %s | %s INSTALL_K3S_CHANNEL=latest INSTALL_K3S_REGISTRIES='%s' K3S_TOKEN='%s' INSTALL_K3S_EXEC='server %s --tls-san %s %s' sh -\n"
joinCommand = "curl -sLS %s | %s INSTALL_K3S_CHANNEL=latest INSTALL_K3S_REGISTRIES='%s' K3S_URL='https://%s:6443' K3S_TOKEN='%s' INSTALL_K3S_EXEC='%s' sh -\n"
catCfgCommand = "cat /etc/rancher/k3s/k3s.yaml"
dockerCommand = "curl https://get.docker.com | VERSION=19.03 sh -s - %s\n"
deployUICommand = "echo \"%s\" > \"%s/ui.yaml\""
Expand Down Expand Up @@ -98,8 +98,8 @@ func InitK3sCluster(cluster *types.Cluster) error {
masterExtraArgs := cluster.MasterExtraArgs
workerExtraArgs := cluster.WorkerExtraArgs

if len(cluster.MasterNodes) > 1 || cluster.DataStore != "" {
masterExtraArgs += "--datastore-endpoint " + cluster.DataStore
if cluster.DataStore != "" {
masterExtraArgs += " --datastore-endpoint " + cluster.DataStore
}

if noFlannel {
Expand All @@ -114,8 +114,8 @@ func InitK3sCluster(cluster *types.Cluster) error {
masterExtraArgs += " --disable-cloud-controller --no-deploy servicelb --kubelet-arg=cloud-provider=external"
}

logrus.Infof("[%s] creating k3s master-%d ...\n", cluster.Provider, 1)
if err := initMaster0(k3sScript, k3sMirror, dockerMirror, publicIP, masterExtraArgs, cluster, cluster.MasterNodes[0], aliCCM); err != nil {
logrus.Infof("[%s] creating k3s master-%d...\n", cluster.Provider, 1)
if err := initMaster(k3sScript, k3sMirror, dockerMirror, publicIP, masterExtraArgs, cluster, cluster.MasterNodes[0], aliCCM); err != nil {
return err
}
logrus.Infof("[%s] successfully created k3s master-%d\n", cluster.Provider, 1)
Expand All @@ -131,7 +131,7 @@ func InitK3sCluster(cluster *types.Cluster) error {
continue
}
logrus.Infof("[%s] creating k3s master-%d...\n", cluster.Provider, i+1)
initMaster(masterWaitGroup, masterErrChan, k3sScript, k3sMirror, dockerMirror, publicIP, masterExtraArgs, cluster, master, aliCCM)
initAdditionalMaster(masterWaitGroup, masterErrChan, k3sScript, k3sMirror, dockerMirror, publicIP, masterExtraArgs, cluster, master, aliCCM)
logrus.Infof("[%s] successfully created k3s master-%d\n", cluster.Provider, i+1)
}

Expand Down Expand Up @@ -289,7 +289,7 @@ func JoinK3sNode(merged, added *types.Cluster) error {
if added.Status.MasterNodes[i].InstanceID == full.InstanceID {
go func(i int, full types.Node) {
logrus.Infof("[%s] joining k3s master-%d...\n", merged.Provider, i+1)
joinMaster(waitGroup, errChan, noFlannel, k3sScript, k3sMirror, dockerMirror, extraArgs, merged, added, full, aliCCM)
joinMaster(waitGroup, errChan, noFlannel, k3sScript, k3sMirror, dockerMirror, extraArgs, merged, full, aliCCM)
logrus.Infof("[%s] successfully joined k3s master-%d\n", merged.Provider, i+1)
}(i, full)
break
Expand Down Expand Up @@ -498,7 +498,7 @@ func OverwriteCfg(context string) error {
return clientcmd.WriteToFile(*c, fmt.Sprintf("%s/%s", common.CfgPath, common.KubeCfgFile))
}

func initMaster0(k3sScript, k3sMirror, dockerMirror, ip, extraArgs string, cluster *types.Cluster, master types.Node,
func initMaster(k3sScript, k3sMirror, dockerMirror, ip, extraArgs string, cluster *types.Cluster, master types.Node,
aliCCM *alibaba.CloudControllerManager) error {
if strings.Contains(extraArgs, "--docker") {
if _, err := execute(&hosts.Host{Node: master},
Expand All @@ -513,14 +513,15 @@ func initMaster0(k3sScript, k3sMirror, dockerMirror, ip, extraArgs string, clust
}

if _, err := execute(&hosts.Host{Node: master},
fmt.Sprintf(masterCommand, k3sScript, k3sMirror, cluster.Registries, cluster.Token, ip, strings.TrimSpace(extraArgs)), false); err != nil {
fmt.Sprintf(initCommand, k3sScript, k3sMirror, cluster.Registries, cluster.Token, "--cluster-init", ip,
strings.TrimSpace(extraArgs)), false); err != nil {
return err
}

return nil
}

func initMaster(wg *sync.WaitGroup, errChan chan error, k3sScript, k3sMirror, dockerMirror, ip, extraArgs string,
func initAdditionalMaster(wg *sync.WaitGroup, errChan chan error, k3sScript, k3sMirror, dockerMirror, ip, extraArgs string,
cluster *types.Cluster, master types.Node, aliCCM *alibaba.CloudControllerManager) {
defer wg.Done()
if strings.Contains(extraArgs, "--docker") {
Expand All @@ -530,13 +531,18 @@ func initMaster(wg *sync.WaitGroup, errChan chan error, k3sScript, k3sMirror, do
}
}

if !strings.Contains(extraArgs, "server --server") {
extraArgs += " server --server " + fmt.Sprintf("https://%s:6443", ip)
}

if aliCCM != nil {
extraArgs += fmt.Sprintf(" --kubelet-arg=provider-id=alicloud://%s.%s --node-name=%s.%s",
aliCCM.Region, master.InstanceID, aliCCM.Region, master.InstanceID)
}

if _, err := execute(&hosts.Host{Node: master},
fmt.Sprintf(masterCommand, k3sScript, k3sMirror, cluster.Registries, cluster.Token, ip, strings.TrimSpace(extraArgs)), false); err != nil {
fmt.Sprintf(joinCommand, k3sScript, k3sMirror, cluster.Registries, ip, cluster.Token,
strings.TrimSpace(extraArgs)), false); err != nil {
errChan <- err
}
}
Expand All @@ -557,16 +563,22 @@ func initWorker(wg *sync.WaitGroup, errChan chan error, k3sScript, k3sMirror, do
}

if _, err := execute(&hosts.Host{Node: worker},
fmt.Sprintf(workerCommand, k3sScript, k3sMirror, cluster.Registries, cluster.URL, cluster.Token, strings.TrimSpace(extraArgs)), false); err != nil {
fmt.Sprintf(joinCommand, k3sScript, k3sMirror, cluster.Registries, cluster.URL, cluster.Token,
strings.TrimSpace(extraArgs)), false); err != nil {
errChan <- err
}
}

func joinMaster(wg *sync.WaitGroup, errChan chan error, noFlannel bool, k3sScript, k3sMirror, dockerMirror,
extraArgs string, merged, added *types.Cluster, full types.Node, aliCCM *alibaba.CloudControllerManager) {
extraArgs string, merged *types.Cluster, full types.Node, aliCCM *alibaba.CloudControllerManager) {
defer wg.Done()
if len(added.MasterNodes) >= 1 || merged.DataStore != "" {
extraArgs += "server --datastore-endpoint " + merged.DataStore

if !strings.Contains(extraArgs, "server --server") {
extraArgs += " server --server " + fmt.Sprintf("https://%s:6443", merged.URL)
}

if merged.DataStore != "" {
extraArgs += " --datastore-endpoint " + merged.DataStore
}

if noFlannel {
Expand Down Expand Up @@ -595,7 +607,7 @@ func joinMaster(wg *sync.WaitGroup, errChan chan error, noFlannel bool, k3sScrip

// for now, use the workerCommand to join the additional master server node.
if _, err := execute(&hosts.Host{Node: full},
fmt.Sprintf(workerCommand, k3sScript, k3sMirror, merged.Registries, merged.URL, merged.Token,
fmt.Sprintf(joinCommand, k3sScript, k3sMirror, merged.Registries, merged.URL, merged.Token,
strings.TrimSpace(extraArgs)), false); err != nil {
errChan <- err
}
Expand All @@ -617,7 +629,7 @@ func joinWorker(wg *sync.WaitGroup, errChan chan error, k3sScript, k3sMirror, do
}

if _, err := execute(&hosts.Host{Node: full},
fmt.Sprintf(workerCommand, k3sScript, k3sMirror, merged.Registries, merged.URL, merged.Token, strings.TrimSpace(extraArgs)), false); err != nil {
fmt.Sprintf(joinCommand, k3sScript, k3sMirror, merged.Registries, merged.URL, merged.Token, strings.TrimSpace(extraArgs)), false); err != nil {
errChan <- err
}
}
Expand Down
30 changes: 12 additions & 18 deletions pkg/providers/alibaba/alibaba.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -768,12 +768,7 @@ func (p *Alibaba) createCheck() error {
p.GetProviderName())
}

if masterNum > 1 && p.DataStore == "" {
return fmt.Errorf("[%s] calling preflight error: HA mode `--datastore` can not be empty",
p.GetProviderName())
}

if masterNum > 1 && strings.Contains(p.MasterExtraArgs, "--datastore-endpoint") && p.DataStore != "" {
if strings.Contains(p.MasterExtraArgs, "--datastore-endpoint") && p.DataStore != "" {
return fmt.Errorf("[%s] calling preflight error: `--masterExtraArgs='--datastore-endpoint'` is duplicated with `--datastore`",
p.GetProviderName())
}
Expand Down Expand Up @@ -811,13 +806,7 @@ func (p *Alibaba) createCheck() error {
}

func (p *Alibaba) joinCheck() error {
masterNum, _ := strconv.Atoi(p.Master)
if masterNum >= 1 && p.DataStore == "" {
return fmt.Errorf("[%s] calling preflight error: HA mode `--datastore` can not be empty",
p.GetProviderName())
}

if masterNum > 1 && strings.Contains(p.MasterExtraArgs, "--datastore-endpoint") && p.DataStore != "" {
if strings.Contains(p.MasterExtraArgs, "--datastore-endpoint") && p.DataStore != "" {
return fmt.Errorf("[%s] calling preflight error: `--masterExtraArgs='--datastore-endpoint'` is duplicated with `--datastore`",
p.GetProviderName())
}
Expand Down Expand Up @@ -947,7 +936,8 @@ func (p *Alibaba) releaseEipAddress(allocationID string) error {

func (p *Alibaba) releaseEipAddresses(rollBack bool) {
var releaseEipIds []string
// unassociate master eip address

// unassociate master eip address.
for _, master := range p.MasterNodes {
if master.RollBack == rollBack {
for _, allocationID := range master.EipAllocationIds {
Expand All @@ -958,7 +948,8 @@ func (p *Alibaba) releaseEipAddresses(rollBack bool) {
}
}
}
// unassociate worker eip address

// unassociate worker eip address.
for _, worker := range p.WorkerNodes {
if worker.RollBack == rollBack {
for _, allocationID := range worker.EipAllocationIds {
Expand All @@ -970,17 +961,20 @@ func (p *Alibaba) releaseEipAddresses(rollBack bool) {
}
}

// release eips with tags
// list eips with tags.
tags := []vpc.ListTagResourcesTag{{Key: "autok3s", Value: "true"}, {Key: "cluster", Value: common.TagClusterPrefix + p.Name}}
allocationIds, err := p.listVpcTagResources(resourceTypeEip, releaseEipIds, tags)
if err != nil {
p.logger.Errorf("[%s] error when query eip address: %v\n", p.GetProviderName(), err)
}
// eip can be released only when status is `Available`
// wait eip to be `Available` status

// eip can be released only when status is `Available`.
// wait eip to be `Available` status.
if err := p.getEipStatus(allocationIds, eipStatusAvailable); err != nil {
p.logger.Errorf("[%s] error when query eip status: %v\n", p.GetProviderName(), err)
}

// release eips.
for _, allocationID := range allocationIds {
if err := p.releaseEipAddress(allocationID); err != nil {
p.logger.Errorf("[%s] error when release eip address %s: %v\n", p.GetProviderName(), allocationID, err)
Expand Down
3 changes: 0 additions & 3 deletions pkg/providers/alibaba/flag.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,9 +189,6 @@ func (p *Alibaba) GetJoinFlags(cmd *cobra.Command) *pflag.FlagSet {
}
}

// need reset master number.
p.Master = "0"

if matched != nil {
// join command need merge status & token value.
p.Status = matched.Status
Expand Down

0 comments on commit c3a2444

Please # to comment.