Refactor configuration loading

Refactor configuration loading, eliminating the hard to follow
`ClientConfigLoadingRules`.

The Runtime config has been converted into a "first class citizen" by making a
dedicated struct for it.

It now stores not only the config, but also the "k0svars". This allows
commands like `token create` or `k0s status` to reuse all the parameters of
the controller, such as `--data-dir` and `--status-socket` without the user
having to supply them again. The runtime config also stores a pid, which is
used to roughly check if the file belongs to an active process or if it's a
leftover from a previous crash. The pid mechanism also prevents two k0s
controllers from running in the same directory.

The `k0svars`, aka `constant.CfgVars` has been moved to `config.CfgVars`
as it is not a constant to begin with.

The `k0sVars` now duplicates most of the `config.CLIOptions`, this is to
reduce the number of arguments that need to be passed around, as the
essentials are available from `k0svars`, which was passed around to almost
everywhere already.

The `config.GetCmdOptions()` now takes the `*cobra.Command` as an argument
and can return an error. The cobra command is used to build the "k0svars" by
looking at flag values.

Signed-off-by: Kimmo Lehto <klehto@mirantis.com>
Co-authored-by: Tom Wieczorek <twz123@users.noreply.github.com>
Signed-off-by: Kimmo Lehto <klehto@mirantis.com>

cmd/airgap/listimages.go

@@ -17,10 +17,14 @@ limitations under the License.
 package airgap
 
 import (
+	"context"
 	"fmt"
+	"time"
 
 	"github.com/k0sproject/k0s/pkg/airgap"
 	"github.com/k0sproject/k0s/pkg/config"
+	"github.com/k0sproject/k0s/pkg/kubernetes"
+	"github.com/sirupsen/logrus"
 
 	"github.com/spf13/cobra"
 )
@@ -33,13 +37,26 @@ func NewAirgapListImagesCmd() *cobra.Command {
 		Short:   "List image names and version needed for air-gap install",
 		Example: `k0s airgap list-images`,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			c := config.GetCmdOpts()
-			clusterConfig, err := config.LoadClusterConfig(c.K0sVars)
+			opts, err := config.GetCmdOpts(cmd)
 			if err != nil {
-				return fmt.Errorf("failed to load cluster config: %w", err)
+				return err
 			}
-			uris := airgap.GetImageURIs(clusterConfig.Spec, all)
-			for _, uri := range uris {
+
+			adminClientFactory := kubernetes.NewAdminClientFactory(opts.K0sVars.AdminKubeConfigPath)
+
+			ctx, cancel := context.WithTimeout(cmd.Context(), 2*time.Minute)
+			defer cancel()
+
+			clusterConfig, err := opts.K0sVars.FetchDynamicConfig(ctx, adminClientFactory)
+			if err != nil {
+				logrus.WithError(err).Warn("Failed to get cluster config, falling back to local config")
+				clusterConfig, err = opts.K0sVars.NodeConfig()
+				if err != nil {
+					return fmt.Errorf("failed to get local config: %w", err)
+				}
+			}
+
+			for _, uri := range airgap.GetImageURIs(clusterConfig.Spec, all) {
 				fmt.Fprintln(cmd.OutOrStdout(), uri)
 			}
 			return nil

cmd/api/api.go

@@ -21,6 +21,7 @@ import (
 	"crypto/tls"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 	"os"
@@ -47,7 +48,7 @@ import (
 )
 
 type command struct {
-	config.CLIOptions
+	*config.CLIOptions
 	client kubernetes.Interface
 }
 
@@ -71,7 +72,11 @@ func NewAPICmd() *cobra.Command {
 			return config.CallParentPersistentPreRun(cmd, args)
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return (&command{CLIOptions: config.GetCmdOpts()}).start()
+			opts, err := config.GetCmdOpts(cmd)
+			if err != nil {
+				return err
+			}
+			return (&command{CLIOptions: opts}).start()
 		},
 	}
 	cmd.SilenceUsage = true
@@ -88,7 +93,11 @@ func (c *command) start() (err error) {
 
 	prefix := "/v1beta1"
 	mux := http.NewServeMux()
-	storage := c.NodeConfig.Spec.Storage
+	nodeConfig, err := c.K0sVars.NodeConfig()
+	if err != nil {
+		return err
+	}
+	storage := nodeConfig.Spec.Storage
 
 	if storage.Type == v1beta1.EtcdStorageType && !storage.Etcd.IsExternalClusterUsed() {
 		// Only mount the etcd handler if we're running on internal etcd storage
@@ -102,11 +111,12 @@ func (c *command) start() (err error) {
 			c.controllerHandler(c.caHandler())))
 	}
 	mux.Handle(prefix+"/calico/kubeconfig", mw.AllowMethods(http.MethodGet)(
-		c.workerHandler(c.kubeConfigHandler())))
+		c.workerHandler(c.kubeConfigHandler(nodeConfig.Spec.API.APIAddressURL())),
+	))
 
 	srv := &http.Server{
 		Handler: mux,
-		Addr:    fmt.Sprintf(":%d", c.NodeConfig.Spec.API.K0sAPIPort),
+		Addr:    fmt.Sprintf(":%d", nodeConfig.Spec.API.K0sAPIPort),
 		TLSConfig: &tls.Config{
 			MinVersion:   tls.VersionTLS12,
 			CipherSuites: constant.AllowedTLS12CipherSuiteIDs,
@@ -177,7 +187,7 @@ func (c *command) etcdHandler() http.Handler {
 	})
 }
 
-func (c *command) kubeConfigHandler() http.Handler {
+func (c *command) kubeConfigHandler(apiAddress string) http.Handler {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
 		tpl := `apiVersion: v1
 kind: Config
@@ -214,7 +224,7 @@ users:
 			break
 		}
 		if !found {
-			sendError(fmt.Errorf("no calico-node-token secret found"), resp)
+			sendError(errors.New("no calico-node-token secret found"), resp)
 			return
 		}
 
@@ -227,7 +237,7 @@ users:
 				Token     string
 				Namespace string
 			}{
-				Server:    c.NodeConfig.Spec.API.APIAddressURL(),
+				Server:    apiAddress,
 				Ca:        base64.StdEncoding.EncodeToString(secretWithToken.Data["ca.crt"]),
 				Token:     string(secretWithToken.Data["token"]),
 				Namespace: string(secretWithToken.Data["namespace"]),

cmd/backup/backup.go

@@ -42,17 +42,25 @@ func NewBackupCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "backup",
 		Short: "Back-Up k0s configuration. Must be run as root (or with sudo)",
+		PreRun: func(cmd *cobra.Command, args []string) {
+			// ensure logs don't mess up output
+			logrus.SetOutput(cmd.ErrOrStderr())
+		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			c := command(config.GetCmdOpts())
-			if c.NodeConfig.Spec.Storage.Etcd.IsExternalClusterUsed() {
+			opts, err := config.GetCmdOpts(cmd)
+			if err != nil {
+				return err
+			}
+			c := (*command)(opts)
+			nodeConfig, err := c.K0sVars.NodeConfig()
+			if err != nil {
+				return err
+			}
+			if nodeConfig.Spec.Storage.Etcd.IsExternalClusterUsed() {
 				return fmt.Errorf("command 'k0s backup' does not support external etcd cluster")
 			}
 			return c.backup(savePath, cmd.OutOrStdout())
 		},
-		PreRunE: func(cmd *cobra.Command, args []string) error {
-			c := command(config.GetCmdOpts())
-			return config.PreRunValidateConfig(c.K0sVars)
-		},
 	}
 	cmd.Flags().StringVar(&savePath, "save-path", "", "destination directory path for backup assets, use '-' for stdout")
 	cmd.SilenceUsage = true
@@ -73,18 +81,23 @@ func (c *command) backup(savePath string, out io.Writer) error {
 		return fmt.Errorf("cannot find data-dir (%v). check your environment and/or command input and try again", c.K0sVars.DataDir)
 	}
 
-	status, err := status.GetStatusInfo(config.StatusSocket)
+	status, err := status.GetStatusInfo(c.K0sVars.StatusSocketPath)
 	if err != nil {
 		return fmt.Errorf("unable to detect cluster status %s", err)
 	}
 	logrus.Debugf("detected role for backup operations: %v", status.Role)
 
+	nodeConfig, err := c.K0sVars.NodeConfig()
+	if err != nil {
+		return err
+	}
+
 	if strings.Contains(status.Role, "controller") {
 		mgr, err := backup.NewBackupManager()
 		if err != nil {
 			return err
 		}
-		return mgr.RunBackup(c.NodeConfig.Spec, c.K0sVars, savePath, out)
+		return mgr.RunBackup(nodeConfig.Spec, c.K0sVars, savePath, out)
 	}
 	return fmt.Errorf("backup command must be run on the controller node, have `%s`", status.Role)
 }

cmd/config/edit.go

@@ -29,8 +29,11 @@ func NewEditCmd() *cobra.Command {
 		Use:   "edit",
 		Short: "Launch the editor configured in your shell to edit k0s configuration",
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			c := config.GetCmdOpts()
-			os.Args = []string{os.Args[0], "kubectl", "--data-dir", c.K0sVars.DataDir, "-n", "kube-system", "edit", "clusterconfig", "k0s"}
+			opts, err := config.GetCmdOpts(cmd)
+			if err != nil {
+				return err
+			}
+			os.Args = []string{os.Args[0], "kubectl", "--data-dir", opts.K0sVars.DataDir, "-n", "kube-system", "edit", "clusterconfig", "k0s"}
 			return cmd.Execute()
 		},
 	}

cmd/config/status.go

@@ -31,8 +31,11 @@ func NewStatusCmd() *cobra.Command {
 		Use:   "status",
 		Short: "Display dynamic configuration reconciliation status",
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			c := config.GetCmdOpts()
-			os.Args = []string{os.Args[0], "kubectl", "--data-dir", c.K0sVars.DataDir, "-n", "kube-system", "get", "event", "--field-selector", "involvedObject.name=k0s"}
+			opts, err := config.GetCmdOpts(cmd)
+			if err != nil {
+				return err
+			}
+			os.Args = []string{os.Args[0], "kubectl", "--data-dir", opts.K0sVars.DataDir, "-n", "kube-system", "get", "event", "--field-selector", "involvedObject.name=k0s"}
 			if outputFormat != "" {
 				os.Args = append(os.Args, "-o", outputFormat)
 			}

cmd/config/validate.go

@@ -17,6 +17,12 @@ limitations under the License.
 package config
 
 import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
 	"github.com/k0sproject/k0s/pkg/config"
 
 	"github.com/spf13/cobra"
@@ -29,17 +35,41 @@ func NewValidateCmd() *cobra.Command {
 		Long: `Example:
    k0s config validate --config path_to_config.yaml`,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			c := config.GetCmdOpts()
+			var reader io.Reader
+
+			// config.CfgFile is the global value holder for --config flag, set by cobra/pflag
+			switch config.CfgFile {
+			case "-":
+				reader = os.Stdin
+			case "":
+				return errors.New("--config can't be emmpty")
+			default:
+				f, err := os.Open(config.CfgFile)
+				if err != nil {
+					return err
+				}
+				defer f.Close()
+				reader = f
+			}
+
+			cfg, err := v1beta1.ConfigFromReader(reader)
+			if err != nil {
+				return fmt.Errorf("failed to read config: %w", err)
+			}
+
+			errs := cfg.Validate()
+			if len(errs) > 0 {
+				return fmt.Errorf("config validation failed: %w", errors.Join(errs...))
+			}
 
-			loadingRules := config.ClientConfigLoadingRules{K0sVars: c.K0sVars}
-			_, err := loadingRules.ParseRuntimeConfig()
-			return err
+			return nil
 		},
 		SilenceUsage:  true,
 		SilenceErrors: true,
 	}
 
 	cmd.PersistentFlags().AddFlagSet(config.GetPersistentFlagSet())
 	cmd.Flags().AddFlagSet(config.FileInputFlag())
+	_ = cmd.MarkFlagRequired("config")
 	return cmd
 }

cmd/controller/certificates.go

@@ -27,6 +27,7 @@ import (
 	"github.com/k0sproject/k0s/internal/pkg/file"
 	"github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
 	"github.com/k0sproject/k0s/pkg/certificate"
+	"github.com/k0sproject/k0s/pkg/config"
 	"github.com/k0sproject/k0s/pkg/constant"
 
 	"k8s.io/client-go/tools/clientcmd"
@@ -41,7 +42,7 @@ type Certificates struct {
 	CACert      string
 	CertManager certificate.Manager
 	ClusterSpec *v1beta1.ClusterSpec
-	K0sVars     constant.CfgVars
+	K0sVars     *config.CfgVars
 }
 
 // Init initializes the certificate component