From 96b9b5334819791fe86a821b22bb52c342730e67 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Mon, 10 Feb 2025 12:34:52 -0800 Subject: [PATCH] Migrate E2E SecretsEncryption to Docker test Remove secrets encryption E2E from Drone Signed-off-by: Derek Nola --- .drone.yml | 6 - .github/workflows/e2e.yaml | 6 +- tests/docker/resources/secrets.yaml | 49 +++++ .../secretsencryption_test.go | 184 ++++++++++++++++++ 4 files changed, 237 insertions(+), 8 deletions(-) create mode 100644 tests/docker/resources/secrets.yaml create mode 100644 tests/docker/secretsencryption/secretsencryption_test.go diff --git a/.drone.yml b/.drone.yml index 66a602db9a69..d1e5f4e852d9 100644 --- a/.drone.yml +++ b/.drone.yml @@ -617,11 +617,6 @@ steps: vagrant destroy -f go test -v -timeout=45m ./validatecluster_test.go -ci -local cp ./coverage.out /tmp/artifacts/validate-coverage.out - - | - cd ../secretsencryption - vagrant destroy -f - go test -v -timeout=30m ./secretsencryption_test.go -ci -local - cp ./coverage.out /tmp/artifacts/se-coverage.out - | cd ../splitserver vagrant destroy -f @@ -661,7 +656,6 @@ steps: from_secret: codecov_token files: - /tmp/artifacts/validate-coverage.out - - /tmp/artifacts/se-coverage.out - /tmp/artifacts/split-coverage.out - /tmp/artifacts/upgrade-coverage.out flags: diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index d748230b5e58..5391f578de19 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -160,11 +160,13 @@ jobs: strategy: fail-fast: false matrix: - dtest: [autoimport, basics, bootstraptoken, cacerts, etcd, hardened, lazypull, skew, snapshotrestore, token, upgrade] + dtest: [autoimport, basics, bootstraptoken, cacerts, etcd, hardened, lazypull, skew, secretsencryption, snapshotrestore, token, upgrade] arch: [amd64, arm64] exclude: - dtest: autoimport arch: arm64 + - dtest: secretsencryption + arch: arm64 - dtest: snapshotrestore arch: arm64 runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} @@ -207,7 +209,7 @@ jobs: cd ./tests/docker/${{ matrix.dtest }} # These tests use rancher/systemd-node and have different flags. - CI_TESTS="autoimport hardened snapshotrestore token" + CI_TESTS="autoimport hardened secretsencryption snapshotrestore token" if [ ${{ matrix.dtest }} = "upgrade" ] || [ ${{ matrix.dtest }} = "skew" ]; then ./${{ matrix.dtest }}.test -k3sImage=$K3S_IMAGE -channel=$CHANNEL elif [[ $CI_TESTS =~ ${{ matrix.dtest }} ]]; then diff --git a/tests/docker/resources/secrets.yaml b/tests/docker/resources/secrets.yaml new file mode 100644 index 000000000000..4605bcc7c621 --- /dev/null +++ b/tests/docker/resources/secrets.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Secret +metadata: + name: docker-secret1 +type: Opaque +stringData: + config.yaml: | + key: "hello" + val: "world" +--- +apiVersion: v1 +kind: Secret +metadata: + name: docker-secret2 +type: Opaque +stringData: + config.yaml: | + key: "good" + val: "day" +--- +apiVersion: v1 +kind: Secret +metadata: + name: docker-secret3 +type: Opaque +stringData: + config.yaml: | + key: "top-secret" + val: "information" +--- +apiVersion: v1 +kind: Secret +metadata: + name: docker-secret4 +type: Opaque +stringData: + config.yaml: | + key: "lock" + val: "key" +--- +apiVersion: v1 +kind: Secret +metadata: + name: docker-secret5 +type: Opaque +stringData: + config.yaml: | + key: "last" + val: "call" \ No newline at end of file diff --git a/tests/docker/secretsencryption/secretsencryption_test.go b/tests/docker/secretsencryption/secretsencryption_test.go new file mode 100644 index 000000000000..a28167291c02 --- /dev/null +++ b/tests/docker/secretsencryption/secretsencryption_test.go @@ -0,0 +1,184 @@ +package secretsencryption + +import ( + "flag" + "testing" + + "github.com/k3s-io/k3s/tests" + "github.com/k3s-io/k3s/tests/docker" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var serverCount = flag.Int("serverCount", 3, "number of server nodes") +var ci = flag.Bool("ci", false, "running on CI") + +func Test_DockerSecretsEncryption(t *testing.T) { + RegisterFailHandler(Fail) + flag.Parse() + suiteConfig, reporterConfig := GinkgoConfiguration() + RunSpecs(t, "Secrets Encryption Test Suite", suiteConfig, reporterConfig) +} + +var tc *docker.TestConfig + +var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() { + Context("Setup Cluster", func() { + It("should provision servers and agents", func() { + var err error + tc, err = docker.NewTestConfig("rancher/systemd-node") + Expect(err).NotTo(HaveOccurred()) + tc.ServerYaml = `secrets-encryption: true` + Expect(tc.ProvisionServers(*serverCount)).To(Succeed()) + Eventually(func() error { + return tests.CheckDefaultDeployments(tc.KubeconfigFile) + }, "60s", "5s").Should(Succeed()) + Eventually(func() error { + return tests.NodesReady(tc.KubeconfigFile, tc.GetNodeNames()) + }, "40s", "5s").Should(Succeed()) + }) + }) + Context("Secrets Keys are rotated:", func() { + It("Deploys several secrets", func() { + _, err := tc.DeployWorkload("secrets.yaml") + Expect(err).NotTo(HaveOccurred(), "Secrets not deployed") + }) + + It("Verifies encryption start stage", func() { + cmd := "k3s secrets-encrypt status" + for _, node := range tc.Servers { + res, err := node.RunCmdOnNode(cmd) + Expect(err).NotTo(HaveOccurred()) + Expect(res).Should(ContainSubstring("Encryption Status: Enabled")) + Expect(res).Should(ContainSubstring("Current Rotation Stage: start")) + Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match")) + } + }) + + It("Rotates the Secrets-Encryption Keys", func() { + cmd := "k3s secrets-encrypt rotate-keys" + res, err := tc.Servers[0].RunCmdOnNode(cmd) + Expect(err).NotTo(HaveOccurred(), res) + for i, node := range tc.Servers { + Eventually(func(g Gomega) { + cmd := "k3s secrets-encrypt status" + res, err := node.RunCmdOnNode(cmd) + g.Expect(err).NotTo(HaveOccurred(), res) + g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: hash does not match")) + if i == 0 { + g.Expect(res).Should(ContainSubstring("Current Rotation Stage: reencrypt_finished")) + } else { + g.Expect(res).Should(ContainSubstring("Current Rotation Stage: start")) + } + }, "420s", "10s").Should(Succeed()) + } + }) + + It("Restarts K3s servers", func() { + Expect(docker.RestartCluster(tc.Servers)).To(Succeed()) + }) + + It("Verifies reencryption_finished stage", func() { + cmd := "k3s secrets-encrypt status" + for _, node := range tc.Servers { + Eventually(func(g Gomega) { + res, err := node.RunCmdOnNode(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled")) + g.Expect(res).Should(ContainSubstring("Current Rotation Stage: reencrypt_finished")) + g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match")) + }, "420s", "2s").Should(Succeed()) + } + }) + + }) + + Context("Disabling Secrets-Encryption", func() { + It("Disables encryption", func() { + cmd := "k3s secrets-encrypt disable" + res, err := tc.Servers[0].RunCmdOnNode(cmd) + Expect(err).NotTo(HaveOccurred(), res) + + cmd = "k3s secrets-encrypt status" + Eventually(func() (string, error) { + return tc.Servers[0].RunCmdOnNode(cmd) + }, "240s", "10s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished")) + + for i, node := range tc.Servers { + Eventually(func(g Gomega) { + res, err := node.RunCmdOnNode(cmd) + g.Expect(err).NotTo(HaveOccurred(), res) + if i == 0 { + g.Expect(res).Should(ContainSubstring("Encryption Status: Disabled")) + } else { + g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled")) + } + }, "420s", "2s").Should(Succeed()) + } + }) + + It("Restarts K3s servers", func() { + Expect(docker.RestartCluster(tc.Servers)).To(Succeed()) + }) + + It("Verifies encryption disabled on all nodes", func() { + cmd := "k3s secrets-encrypt status" + for _, node := range tc.Servers { + Eventually(func(g Gomega) { + g.Expect(node.RunCmdOnNode(cmd)).Should(ContainSubstring("Encryption Status: Disabled")) + }, "420s", "2s").Should(Succeed()) + } + }) + + }) + + Context("Enabling Secrets-Encryption", func() { + It("Enables encryption", func() { + cmd := "k3s secrets-encrypt enable" + res, err := tc.Servers[0].RunCmdOnNode(cmd) + Expect(err).NotTo(HaveOccurred(), res) + + cmd = "k3s secrets-encrypt status" + Eventually(func() (string, error) { + return tc.Servers[0].RunCmdOnNode(cmd) + }, "180s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished")) + + for i, node := range tc.Servers { + Eventually(func(g Gomega) { + res, err := node.RunCmdOnNode(cmd) + g.Expect(err).NotTo(HaveOccurred(), res) + if i == 0 { + g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled")) + } else { + g.Expect(res).Should(ContainSubstring("Encryption Status: Disabled")) + } + }, "420s", "2s").Should(Succeed()) + } + }) + + It("Restarts K3s servers", func() { + Expect(docker.RestartCluster(tc.Servers)).To(Succeed()) + }) + + It("Verifies encryption enabled on all nodes", func() { + cmd := "k3s secrets-encrypt status" + for _, node := range tc.Servers { + Eventually(func(g Gomega) { + g.Expect(node.RunCmdOnNode(cmd)).Should(ContainSubstring("Encryption Status: Enabled")) + }, "420s", "2s").Should(Succeed()) + } + }) + }) + +}) + +var failed bool +var _ = AfterEach(func() { + failed = failed || CurrentSpecReport().Failed() +}) + +var _ = AfterSuite(func() { + if *ci || (tc != nil && !failed) { + Expect(tc.Cleanup()).To(Succeed()) + } +})