feat(alibaba): support multiple kubeconfig contexts

Signed-off-by: Jason-ZW <zhenyang@rancher.com>

cmd/cmd.go

@@ -26,9 +26,9 @@ const ascIIStr = `
 
 var (
 	cmd = &cobra.Command{
-		Use:   "autok3s",
-		Short: "autok3s is used to manage the lifecycle of K3s on multiple cloud providers",
-		Long:  `autok3s is used to manage the lifecycle of K3s on multiple cloud providers.`,
+		Use:              "autok3s",
+		Short:            "autok3s is used to manage the lifecycle of K3s on multiple cloud providers",
+		Long:             `autok3s is used to manage the lifecycle of K3s on multiple cloud providers.`,
 		TraverseChildren: true,
 	}
 )

go.mod

@@ -48,7 +48,9 @@ require (
 	golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
 	k8s.io/apimachinery v0.0.0
+	k8s.io/cli-runtime v0.0.0
 	k8s.io/client-go v0.0.0
 	k8s.io/component-base v0.0.0
+	k8s.io/kubectl v0.0.0
 	k8s.io/kubernetes v0.0.0-00010101000000-000000000000
 )

pkg/cli/kubectl/kubectl.go

@@ -9,8 +9,10 @@ import (
 
 	"github.com/Jason-ZW/autok3s/pkg/common"
 
+	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
+	"k8s.io/client-go/tools/clientcmd"
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/component-base/logs"
 	"k8s.io/kubernetes/pkg/kubectl/cmd"
@@ -33,8 +35,18 @@ func Main() {
 }
 
 func EmbedCommand() *cobra.Command {
+	cfg := fmt.Sprintf("%s/%s", common.CfgPath, common.KubeCfgFile)
+
 	c := cmd.NewDefaultKubectlCommand()
 	c.Short = "Kubectl controls the Kubernetes cluster manager"
-	c.PersistentFlags().Set("kubeconfig", fmt.Sprintf("%s/%s", common.CfgPath, common.KubeCfgFile))
+	_ = c.PersistentFlags().Set("kubeconfig", cfg)
+
+	f := c.PersistentFlags().Lookup("kubeconfig")
+	f.DefValue = cfg
+
+	if err := os.Setenv(clientcmd.RecommendedConfigPathEnvVar, cfg); err != nil {
+		logrus.Errorf("[kubectl] failed to set %s=%s env\n", clientcmd.RecommendedConfigPathEnvVar, cfg)
+	}
+
 	return c
 }

pkg/cluster/cluster.go

@@ -1,8 +1,11 @@
 package cluster
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
+	"io/ioutil"
+	"os"
 	"strings"
 
 	"github.com/Jason-ZW/autok3s/pkg/common"
@@ -11,6 +14,11 @@ import (
 	"github.com/Jason-ZW/autok3s/pkg/utils"
 
 	"github.com/ghodss/yaml"
+	"github.com/sirupsen/logrus"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/kubectl/pkg/cmd/config"
+	"k8s.io/kubectl/pkg/scheme"
 )
 
 var (
@@ -91,7 +99,7 @@ func ReadFromState(cluster *types.Cluster) ([]types.Cluster, error) {
 	r := make([]types.Cluster, 0)
 	v := common.CfgPath
 	if v == "" {
-		return r, errors.New("cfg path is empty\n")
+		return r, errors.New("[cluster] cfg path is empty\n")
 	}
 
 	clusters, err := utils.ReadYaml(v, common.StateFile)
@@ -101,7 +109,7 @@ func ReadFromState(cluster *types.Cluster) ([]types.Cluster, error) {
 
 	converts, err := ConvertToClusters(clusters)
 	if err != nil {
-		return r, errors.New(fmt.Sprintf("failed to unmarshal state file, msg: %s\n", err.Error()))
+		return r, fmt.Errorf("[cluster] failed to unmarshal state file, msg: %s\n", err.Error())
 	}
 
 	for _, c := range converts {
@@ -117,7 +125,7 @@ func AppendToState(cluster *types.Cluster) ([]types.Cluster, error) {
 	r := make([]types.Cluster, 0)
 	v := common.CfgPath
 	if v == "" {
-		return r, errors.New("cfg path is empty\n")
+		return r, errors.New("[cluster] cfg path is empty\n")
 	}
 
 	clusters, err := utils.ReadYaml(v, common.StateFile)
@@ -127,7 +135,7 @@ func AppendToState(cluster *types.Cluster) ([]types.Cluster, error) {
 
 	converts, err := ConvertToClusters(clusters)
 	if err != nil {
-		return r, errors.New(fmt.Sprintf("failed to unmarshal state file, msg: %s\n", err.Error()))
+		return r, fmt.Errorf("[cluster] failed to unmarshal state file, msg: %s\n", err.Error())
 	}
 
 	for _, c := range converts {
@@ -193,7 +201,7 @@ func saveState(cluster *types.Cluster) error {
 
 	v := common.CfgPath
 	if v == "" {
-		return errors.New("cfg path is empty\n")
+		return errors.New("[cluster] cfg path is empty\n")
 	}
 
 	return utils.WriteYaml(r, v, common.StateFile)
@@ -208,10 +216,83 @@ func saveCfg(cfg, ip, context string) error {
 
 	result := replacer.Replace(cfg)
 
-	err := utils.EnsureFileExist(common.CfgPath, common.KubeCfgFile)
+	tempPath := fmt.Sprintf("%s/.kube", common.CfgPath)
+
+	temp, err := ioutil.TempFile(tempPath, common.KubeCfgTempName)
+	if err != nil {
+		return fmt.Errorf("[cluster] generate kubecfg temp file error, msg=%s\n", err.Error())
+	}
+	defer func() {
+		_ = temp.Close()
+	}()
+
+	err = utils.WriteBytesToYaml([]byte(result), tempPath, temp.Name()[strings.Index(temp.Name(), common.KubeCfgTempName):])
+	if err != nil {
+		return fmt.Errorf("[cluster] write content to kubecfg temp file error, msg=%s\n", err.Error())
+	}
+
+	return mergeCfg(context, temp.Name())
+}
+
+func mergeCfg(context, right string) error {
+	defer func() {
+		if err := os.Remove(right); err != nil {
+			logrus.Errorf("[cluster] remove kubecfg temp file error, msg=%s\n", err)
+		}
+	}()
+
+	if err := utils.EnsureFileExist(common.CfgPath, common.KubeCfgFile); err != nil {
+		return fmt.Errorf("[cluster] ensure kubecfg exist error, msg=%s\n", err.Error())
+	}
+
+	if err := overwriteCfg(context); err != nil {
+		return fmt.Errorf("[cluster] overwrite kubecfg error, msg=%s\n", err.Error())
+	}
+
+	if err := os.Setenv(clientcmd.RecommendedConfigPathEnvVar, fmt.Sprintf("%s:%s",
+		fmt.Sprintf("%s/%s", common.CfgPath, common.KubeCfgFile), right)); err != nil {
+		return fmt.Errorf("[cluster] set env error when merging kubecfg, msg=%s\n", err.Error())
+	}
+
+	out := &bytes.Buffer{}
+	opt := config.ViewOptions{
+		Flatten:      true,
+		PrintFlags:   genericclioptions.NewPrintFlags("").WithTypeSetter(scheme.Scheme).WithDefaultOutput("yaml"),
+		ConfigAccess: clientcmd.NewDefaultPathOptions(),
+		IOStreams:    genericclioptions.IOStreams{Out: out},
+	}
+	_ = opt.Merge.Set("true")
+
+	printer, err := opt.PrintFlags.ToPrinter()
+	if err != nil {
+		return fmt.Errorf("[cluster] generate view options error, msg=%s\n", err.Error())
+	}
+	opt.PrintObject = printer.PrintObj
+
+	if err := opt.Run(); err != nil {
+		return fmt.Errorf("[cluster] merging kubecfg error, msg=%s\n", err.Error())
+	}
+
+	return utils.WriteBytesToYaml(out.Bytes(), common.CfgPath, common.KubeCfgFile)
+}
+
+func overwriteCfg(context string) error {
+	c, err := clientcmd.LoadFromFile(fmt.Sprintf("%s/%s", common.CfgPath, common.KubeCfgFile))
 	if err != nil {
 		return err
 	}
 
-	return utils.WriteBytesToYaml([]byte(result), common.CfgPath, common.KubeCfgFile)
+	if _, found := c.Clusters[context]; found {
+		delete(c.Clusters, context)
+	}
+
+	if _, found := c.Contexts[context]; found {
+		delete(c.Contexts, context)
+	}
+
+	if _, found := c.AuthInfos[context]; found {
+		delete(c.AuthInfos, context)
+	}
+
+	return clientcmd.WriteToFile(*c, fmt.Sprintf("%s/%s", common.CfgPath, common.KubeCfgFile))
 }

pkg/common/common.go

@@ -16,6 +16,7 @@ const (
 	ConfigFile           = "config.yaml"
 	StateFile            = ".state"
 	KubeCfgFile          = ".kube/config"
+	KubeCfgTempName      = "autok3s-temp"
 )
 
 var (

pkg/hosts/dialer.go

@@ -39,7 +39,7 @@ func SSHDialer(h *Host) (*dialer, error) {
 }
 
 func (d *dialer) OpenTunnel() (*Tunnel, error) {
-	wait.ErrWaitTimeout = errors.New(fmt.Sprintf("[dialer] calling openTunnel error. address=[%s]\n", d.sshAddress))
+	wait.ErrWaitTimeout = fmt.Errorf("[dialer] calling openTunnel error. address=[%s]\n", d.sshAddress)
 
 	var conn *ssh.Client
 	var err error

pkg/providers/alibaba/alibaba.go

@@ -1,7 +1,6 @@
 package alibaba
 
 import (
-	"errors"
 	"fmt"
 	"strconv"
 	"strings"
@@ -178,8 +177,8 @@ func (p *Alibaba) Rollback() error {
 	request.InstanceId = &ids
 	request.Force = requests.NewBoolean(true)
 
-	wait.ErrWaitTimeout = errors.New(fmt.Sprintf("[%s] calling rollback error, please remove the cloud provider instances manually. region=%s, "+
-		"instanceName=%s, message=the maximum number of attempts reached\n", p.GetProviderName(), p.Region, ids))
+	wait.ErrWaitTimeout = fmt.Errorf("[%s] calling rollback error, please remove the cloud provider instances manually. region=%s, "+
+		"instanceName=%s, message=the maximum number of attempts reached\n", p.GetProviderName(), p.Region, ids)
 
 	// retry 5 times, total 120 seconds.
 	backoff := wait.Backoff{
@@ -239,8 +238,8 @@ func (p *Alibaba) runInstances(num, startIndex int, master bool) error {
 
 	response, err := p.c.RunInstances(request)
 	if err != nil || len(response.InstanceIdSets.InstanceIdSet) != num {
-		return errors.New(fmt.Sprintf("[%s] calling runInstances error. region=%s, "+"instanceName=%s, message=[%s]\n",
-			p.GetProviderName(), p.Region, request.InstanceName, err.Error()))
+		return fmt.Errorf("[%s] calling runInstances error. region=%s, "+"instanceName=%s, message=[%s]\n",
+			p.GetProviderName(), p.Region, request.InstanceName, err.Error())
 	}
 	for _, id := range response.InstanceIdSets.InstanceIdSet {
 		if master {
@@ -264,8 +263,8 @@ func (p *Alibaba) getInstanceStatus() error {
 	request.Scheme = "https"
 	request.InstanceId = &ids
 
-	wait.ErrWaitTimeout = errors.New(fmt.Sprintf("[%s] calling getInstanceStatus error. region=%s, "+"instanceName=%s, message=not running status\n",
-		p.GetProviderName(), p.Region, ids))
+	wait.ErrWaitTimeout = fmt.Errorf("[%s] calling getInstanceStatus error. region=%s, "+"instanceName=%s, message=not running status\n",
+		p.GetProviderName(), p.Region, ids)
 
 	if err := wait.ExponentialBackoff(common.Backoff, func() (bool, error) {
 		response, err := p.c.DescribeInstanceStatus(request)
@@ -387,8 +386,8 @@ func (p *Alibaba) describeInstances() (*ecs.DescribeInstancesResponse, error) {
 
 	response, err := p.c.DescribeInstances(request)
 	if err == nil && len(response.Instances.Instance) == 0 {
-		return nil, errors.New(fmt.Sprintf("[%s] calling describeInstances error. region=%s, "+"instanceName=%s, message=[%s]\n",
-			p.GetProviderName(), p.Region, request.InstanceName, err.Error()))
+		return nil, fmt.Errorf("[%s] calling describeInstances error. region=%s, "+"instanceName=%s, message=[%s]\n",
+			p.GetProviderName(), p.Region, request.InstanceName, err.Error())
 	}
 
 	return response, nil
@@ -415,8 +414,8 @@ func (p *Alibaba) createCheck() error {
 	}
 
 	if exist {
-		return errors.New(fmt.Sprintf("[%s] calling preflight error: cluster name `%s` already exist\n",
-			p.GetProviderName(), p.Name))
+		return fmt.Errorf("[%s] calling preflight error: cluster name `%s` already exist\n",
+			p.GetProviderName(), p.Name)
 	}
 
 	return nil
@@ -430,8 +429,8 @@ func (p *Alibaba) joinCheck() error {
 	}
 
 	if !exist {
-		return errors.New(fmt.Sprintf("[%s] calling preflight error: cluster name `%s` do not exist\n",
-			p.GetProviderName(), p.Name))
+		return fmt.Errorf("[%s] calling preflight error: cluster name `%s` do not exist\n",
+			p.GetProviderName(), p.Name)
 	}
 
 	return nil

pkg/providers/alibaba/flag.go

@@ -1,7 +1,6 @@
 package alibaba
 
 import (
-	"errors"
 	"fmt"
 	"reflect"
 	"strings"
@@ -42,7 +41,7 @@ func (p *Alibaba) GetCreateFlags(cmd *cobra.Command) *pflag.FlagSet {
 			return nil
 		}
 
-		return errors.New(fmt.Sprintf("required flags(s) \"%s\" not set\n", errFlags))
+		return fmt.Errorf("required flags(s) \"%s\" not set\n", errFlags)
 	}
 
 	return cmd.Flags()
@@ -99,7 +98,7 @@ func (p *Alibaba) GetJoinFlags(cmd *cobra.Command) *pflag.FlagSet {
 			return nil
 		}
 
-		return errors.New(fmt.Sprintf("required flags(s) \"%s\" not set\n", errFlags))
+		return fmt.Errorf("required flags(s) \"%s\" not set\n", errFlags)
 	}
 
 	return cmd.Flags()
@@ -149,7 +148,7 @@ func (p *Alibaba) GetCredentialFlags(cmd *cobra.Command) *pflag.FlagSet {
 			return nil
 		}
 
-		return errors.New(fmt.Sprintf("required flags(s) \"%s\" not set\n", errFlags))
+		return fmt.Errorf("required flags(s) \"%s\" not set\n", errFlags)
 	}
 
 	return cmd.Flags()

pkg/utils/file.go

@@ -1,7 +1,6 @@
 package utils
 
 import (
-	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -19,7 +18,7 @@ const (
 
 func EnsureFileExist(path, file string) error {
 	if path == "" {
-		return errors.New(fmt.Sprintf("path %s cannot be empty\n", path))
+		return fmt.Errorf("path %s cannot be empty\n", path)
 	}
 
 	n := fmt.Sprintf("%s/%s", path, file)