diff --git a/cmd/kubeconfig/admin_test.go b/cmd/kubeconfig/admin_test.go index bfc0f92c9473..8bf76907270f 100644 --- a/cmd/kubeconfig/admin_test.go +++ b/cmd/kubeconfig/admin_test.go @@ -55,16 +55,15 @@ func TestAdmin(t *testing.T) { }, }, adminConfPath)) - rtConfigPath := filepath.Join(dataDir, "run", "k0s.yaml") - writeYAML(t, rtConfigPath, &config.RuntimeConfig{ - TypeMeta: metav1.TypeMeta{APIVersion: v1beta1.SchemeGroupVersion.String(), Kind: config.RuntimeConfigKind}, - Spec: &config.RuntimeConfigSpec{K0sVars: &config.CfgVars{ - AdminKubeConfigPath: adminConfPath, - DataDir: dataDir, - RuntimeConfigPath: rtConfigPath, - StartupConfigPath: configPath, - }}, - }) + k0sVars := &config.CfgVars{ + AdminKubeConfigPath: adminConfPath, + DataDir: dataDir, + RuntimeConfigPath: filepath.Join(dataDir, "run", "k0s.yaml"), + StartupConfigPath: configPath, + } + cfg, err := config.NewRuntimeConfig(k0sVars, nil) + require.NoError(t, err) + t.Cleanup(func() { assert.NoError(t, cfg.Spec.Cleanup()) }) var stdout bytes.Buffer var stderr strings.Builder @@ -89,18 +88,24 @@ func TestAdmin_NoAdminConfig(t *testing.T) { dataDir := t.TempDir() configPath := filepath.Join(dataDir, "k0s.yaml") - adminConfPath := filepath.Join(dataDir, "admin.conf") - rtConfigPath := filepath.Join(dataDir, "run", "k0s.yaml") - writeYAML(t, rtConfigPath, &config.RuntimeConfig{ - TypeMeta: metav1.TypeMeta{APIVersion: v1beta1.SchemeGroupVersion.String(), Kind: config.RuntimeConfigKind}, - Spec: &config.RuntimeConfigSpec{K0sVars: &config.CfgVars{ - AdminKubeConfigPath: adminConfPath, - DataDir: dataDir, - RuntimeConfigPath: rtConfigPath, - StartupConfigPath: configPath, + writeYAML(t, configPath, &v1beta1.ClusterConfig{ + TypeMeta: metav1.TypeMeta{APIVersion: v1beta1.SchemeGroupVersion.String(), Kind: v1beta1.ClusterConfigKind}, + Spec: &v1beta1.ClusterSpec{API: &v1beta1.APISpec{ + Port: 65432, ExternalAddress: "not-here.example.com", }}, }) + k0sVars := &config.CfgVars{ + AdminKubeConfigPath: filepath.Join(dataDir, "admin.conf"), + DataDir: dataDir, + RuntimeConfigPath: filepath.Join(dataDir, "run", "k0s.yaml"), + StartupConfigPath: filepath.Join(dataDir, "k0s.yaml"), + } + + cfg, err := config.NewRuntimeConfig(k0sVars, nil) + require.NoError(t, err) + t.Cleanup(func() { assert.NoError(t, cfg.Spec.Cleanup()) }) + var stdout, stderr strings.Builder underTest := cmd.NewRootCmd() underTest.SetArgs([]string{"kubeconfig", "--data-dir", dataDir, "admin"}) @@ -110,7 +115,7 @@ func TestAdmin_NoAdminConfig(t *testing.T) { assert.Error(t, underTest.Execute()) assert.Empty(t, stdout.String()) - msg := fmt.Sprintf("admin config %q not found, check if the control plane is initialized on this node", adminConfPath) + msg := fmt.Sprintf("admin config %q not found, check if the control plane is initialized on this node", k0sVars.AdminKubeConfigPath) assert.Equal(t, "Error: "+msg+"\n", stderr.String()) } diff --git a/pkg/config/flock_unix.go b/pkg/config/flock_unix.go new file mode 100644 index 000000000000..7d81ffdba201 --- /dev/null +++ b/pkg/config/flock_unix.go @@ -0,0 +1,57 @@ +//go:build unix + +/* +Copyright 2025 k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "golang.org/x/sys/unix" + "os" +) + +// tryLock attempts to acquire the lock. Returns *os.File if successful, nil otherwise. +func tryLock(path string) (*os.File, error) { + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return nil, err + } + + if err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil { + _ = file.Close() + if err == unix.EWOULDBLOCK { + return nil, ErrK0sAlreadyRunning // Lock is already held by another process + } + return nil, err + } + return file, nil +} + +// isLocked checks if the lock is currently held by another process. +func isLocked(path string) bool { + file, err := os.OpenFile(path, os.O_RDWR, 0600) + if err != nil { + return false + } + defer file.Close() + + // Attempt a non-blocking shared lock to test the lock state + if err := unix.Flock(int(file.Fd()), unix.LOCK_SH|unix.LOCK_NB); err != nil { + return err == unix.EWOULDBLOCK + } + + return false +} diff --git a/pkg/config/flock_windows.go b/pkg/config/flock_windows.go new file mode 100644 index 000000000000..56dc1aeffb76 --- /dev/null +++ b/pkg/config/flock_windows.go @@ -0,0 +1,79 @@ +//go:build windows + +/* +Copyright 2025 k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "golang.org/x/sys/windows" + "os" +) + +// tryLock attempts to acquire the lock. Returns true if successful, false otherwise. +func tryLock(path string) (*os.File, error) { + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return nil, err + } + + handle := windows.Handle(file.Fd()) + overlapped := new(windows.Overlapped) // The OVERLAPPED structure, required for asynchronous I/O operations + + // Attempt to lock the file exclusively and fail immediately if it's already locked + // https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-lockfileex + err = windows.LockFileEx( + handle, // 1. HANDLE hFile: The handle to the file (must have GENERIC_READ or GENERIC_WRITE access) + windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, // 2. DWORD dwFlags: Specifies the lock type and behavior + 0, // 3. DWORD dwReserved: Reserved, must be zero + 1, // 4. DWORD nNumberOfBytesToLockLow: Low-order part of the range of bytes to lock (1 byte in this case) + 0, // 5. DWORD nNumberOfBytesToLockHigh: High-order part of the range of bytes to lock (0 for single-byte lock) + overlapped, // 6. LPOVERLAPPED lpOverlapped: Pointer to an OVERLAPPED structure, required for this function + ) + if err != nil { + file.Close() + if err == windows.ERROR_LOCK_VIOLATION { + return nil, ErrK0sAlreadyRunning // Lock is already held by another process + } + return nil, err + } + + return file, nil +} + +// isLocked checks if the lock is currently held by another process. +func isLocked(path string) bool { + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return false + } + defer file.Close() + + handle := windows.Handle(file.Fd()) + overlapped := new(windows.Overlapped) + + // Try to acquire a shared lock without waiting + // https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-lockfileex + err = windows.LockFileEx( + handle, // 1. HANDLE hFile: The handle to the file (must have GENERIC_READ or GENERIC_WRITE access) + windows.LOCKFILE_FAIL_IMMEDIATELY, // Try without waiting + 0, // 3. DWORD dwReserved: Reserved, must be zero + 1, // 4. DWORD nNumberOfBytesToLockLow: Low-order part of the range of bytes to lock (1 byte in this case) + 0, // 5. DWORD nNumberOfBytesToLockHigh: High-order part of the range of bytes to lock (0 for single-byte lock) + overlapped, // 6. LPOVERLAPPED lpOverlapped: Pointer to an OVERLAPPED structure, required for this function + ) + return err != nil +} diff --git a/pkg/config/runtime.go b/pkg/config/runtime.go index fbba3fed17d1..43892d625c67 100644 --- a/pkg/config/runtime.go +++ b/pkg/config/runtime.go @@ -56,10 +56,14 @@ type RuntimeConfig struct { type RuntimeConfigSpec struct { NodeConfig *v1beta1.ClusterConfig `json:"nodeConfig"` K0sVars *CfgVars `json:"k0sVars"` - Pid int `json:"pid"` + lockFile *os.File } func LoadRuntimeConfig(path string) (*RuntimeConfigSpec, error) { + if !isLocked(path + ".lock") { + return nil, ErrK0sNotRunning + } + content, err := os.ReadFile(path) if err != nil { return nil, err @@ -69,20 +73,8 @@ func LoadRuntimeConfig(path string) (*RuntimeConfigSpec, error) { if err != nil { return nil, fmt.Errorf("failed to parse runtime configuration: %w", err) } - spec := config.Spec - - // If a pid is defined but there's no process found, the instance of k0s is - // expected to have died, in which case the existing config is removed and - // an error is returned, which allows the controller startup to proceed to - // initialize a new runtime config. - if spec.Pid != 0 { - if err := checkPid(spec.Pid); err != nil { - defer func() { _ = spec.Cleanup() }() - return nil, errors.Join(ErrK0sNotRunning, err) - } - } - return spec, nil + return config.Spec, nil } func ParseRuntimeConfig(content []byte) (*RuntimeConfig, error) { @@ -108,8 +100,27 @@ func ParseRuntimeConfig(content []byte) (*RuntimeConfig, error) { } func NewRuntimeConfig(k0sVars *CfgVars, nodeConfig *v1beta1.ClusterConfig) (*RuntimeConfig, error) { - if _, err := LoadRuntimeConfig(k0sVars.RuntimeConfigPath); err == nil { - return nil, ErrK0sAlreadyRunning + if err := dir.Init(filepath.Dir(k0sVars.RuntimeConfigPath), constant.RunDirMode); err != nil { + logrus.Warnf("failed to initialize runtime config dir: %v", err) + } + + // A file lock is acquired using `flock(2)` to ensure that only one + // instance of the `k0s` process can modify the runtime configuration + // at a time. The lock is tied to the lifetime of the `k0s` process, + // meaning that if the process terminates unexpectedly, the lock is + // automatically released by the operating system. This ensures that + // subsequent processes can acquire the lock without manual cleanup. + // https://man7.org/linux/man-pages/man2/flock.2.html + // + // It works similar on Windows, but with LockFileEx + + path, err := filepath.Abs(k0sVars.RuntimeConfigPath + ".lock") + if err != nil { + return nil, err + } + lockFile, err := tryLock(path) + if err != nil { + return nil, err } cfg := &RuntimeConfig{ @@ -123,7 +134,7 @@ func NewRuntimeConfig(k0sVars *CfgVars, nodeConfig *v1beta1.ClusterConfig) (*Run Spec: &RuntimeConfigSpec{ NodeConfig: nodeConfig, K0sVars: k0sVars, - Pid: os.Getpid(), + lockFile: lockFile, }, } @@ -132,10 +143,6 @@ func NewRuntimeConfig(k0sVars *CfgVars, nodeConfig *v1beta1.ClusterConfig) (*Run return nil, err } - if err := dir.Init(filepath.Dir(k0sVars.RuntimeConfigPath), constant.RunDirMode); err != nil { - logrus.Warnf("failed to initialize runtime config dir: %v", err) - } - if err := os.WriteFile(k0sVars.RuntimeConfigPath, content, 0600); err != nil { return nil, fmt.Errorf("failed to write runtime config: %w", err) } @@ -149,7 +156,15 @@ func (r *RuntimeConfigSpec) Cleanup() error { } if err := os.Remove(r.K0sVars.RuntimeConfigPath); err != nil { - return fmt.Errorf("failed to clean up runtime config file: %w", err) + logrus.Warnf("failed to clean up runtime config file: %v", err) + } + + if err := r.lockFile.Close(); err != nil { + return fmt.Errorf("failed to close the runtime config file: %w", err) + } + + if err := os.Remove(r.lockFile.Name()); err != nil { + return fmt.Errorf("failed to delete %s: %w", r.lockFile.Name(), err) } return nil } diff --git a/pkg/config/runtime_test.go b/pkg/config/runtime_test.go index e0f600383bed..a31688055ec5 100644 --- a/pkg/config/runtime_test.go +++ b/pkg/config/runtime_test.go @@ -27,7 +27,7 @@ import ( "sigs.k8s.io/yaml" ) -func TestLoadRuntimeConfig_K0sNotRunning(t *testing.T) { +func TestLoadRuntimeConfig(t *testing.T) { // write some content to the runtime config file rtConfigPath := filepath.Join(t.TempDir(), "runtime-config") content := []byte(`--- @@ -37,7 +37,6 @@ spec: nodeConfig: metadata: name: k0s - pid: -1 `) require.NoError(t, os.WriteFile(rtConfigPath, content, 0644)) @@ -78,9 +77,9 @@ func TestNewRuntimeConfig(t *testing.T) { // create a new runtime config and check if it's valid cfg, err := NewRuntimeConfig(k0sVars, nodeConfig) if assert.NoError(t, err) && assert.NotNil(t, cfg) && assert.NotNil(t, cfg.Spec) { + t.Cleanup(func() { assert.NoError(t, cfg.Spec.Cleanup()) }) assert.Same(t, k0sVars, cfg.Spec.K0sVars) assert.Same(t, nodeConfig, cfg.Spec.NodeConfig) - assert.Equal(t, os.Getpid(), cfg.Spec.Pid) } assert.FileExists(t, rtConfigPath) diff --git a/pkg/config/runtime_unix.go b/pkg/config/runtime_unix.go deleted file mode 100644 index 134e6a863ee6..000000000000 --- a/pkg/config/runtime_unix.go +++ /dev/null @@ -1,38 +0,0 @@ -//go:build unix - -/* -Copyright 2023 k0s authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "fmt" - "os" - "syscall" -) - -func checkPid(pid int) error { - proc, err := os.FindProcess(pid) - if err != nil { - return fmt.Errorf("failed to find process: %w", err) - } - - if err := proc.Signal(syscall.Signal(0)); err != nil { - return fmt.Errorf("failed to signal process: %w", err) - } - - return nil -} diff --git a/pkg/config/runtime_windows.go b/pkg/config/runtime_windows.go deleted file mode 100644 index dd8903c93c0d..000000000000 --- a/pkg/config/runtime_windows.go +++ /dev/null @@ -1,33 +0,0 @@ -/* -Copyright 2023 k0s authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "fmt" - - "golang.org/x/sys/windows" -) - -func checkPid(pid int) error { - procHandle, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) - if err != nil { - return fmt.Errorf("failed to find process: %w", err) - } - defer func() { _ = windows.CloseHandle(procHandle) }() - - return nil -}