Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Add Framework for Conformance Tests #98

Merged
merged 2 commits into from
May 22, 2023
Merged

Add Framework for Conformance Tests #98

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2f547fe
Add modules required to support conformace tests
tssurya May 14, 2023
00bf1ff
Add Conformance Test Framework
tssurya May 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@ help: ## Display this help.

##@ Development

# Command-line flags passed to "go test" for the conformance
# test. These are passed after the "-args" flag.
CONFORMANCE_FLAGS ?=

.PHONY: fmt
fmt: ## Run go fmt against code.
go fmt ./...
Expand All @@ -47,6 +51,10 @@ verify:
crd-e2e:
hack/crd-e2e.sh -v

.PHONY: conformance
conformance:
go test -v ./conformance/... -args ${CONFORMANCE_FLAGS}

##@ Deployment
install: generate ## Install CRDs into the K8s cluster specified in ~/.kube/config.
kubectl kustomize config/crd | kubectl apply -f -
Expand Down
204 changes: 204 additions & 0 deletions conformance/base/manifests.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,204 @@
# This file contains the base resources that most conformance tests will rely
# on. This includes 4 namespaces along with Server and Client Deployments in
# each of them that can be used as backends and probes for testing traffic.
# Create 4 namespaces
apiVersion: v1
kind: Namespace
metadata:
name: network-policy-conformance-gryffindor
labels:
conformance-house: gryffindor
---
apiVersion: v1
kind: Namespace
metadata:
name: network-policy-conformance-slytherin
labels:
conformance-house: slytherin
---
apiVersion: v1
kind: Namespace
metadata:
name: network-policy-conformance-hufflepuff
labels:
conformance-house: hufflepuff
---
apiVersion: v1
kind: Namespace
metadata:
name: network-policy-conformance-ravenclaw
labels:
conformance-house: ravenclaw
# Create 4 deployments; 2 pods each under these namespaces
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: harry-potter
namespace: network-policy-conformance-gryffindor
spec:
selector:
matchLabels:
conformance-house: gryffindor
replicas: 2
template:
metadata:
labels:
conformance-house: gryffindor
spec:
containers:
- name: harry-potter-client
image: registry.k8s.io/e2e-test-images/agnhost:2.43
- name: harry-potter-80
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost serve-hostname --tcp --http=false --port 80"]
- name: harry-potter-8080
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost serve-hostname --tcp --http=false --port 8080"]
- name: harry-potter-5353
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost serve-hostname --udp --http=false --port 5353"]
- name: harry-potter-53
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost serve-hostname --udp --http=false --port 53"]
- name: harry-potter-9003
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost porter"]
env:
- name: SERVE_SCTP_PORT_9003
value: "foo"
- name: harry-potter-9005
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost porter"]
env:
- name: SERVE_SCTP_PORT_9005
value: "foo"
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: draco-malfoy
namespace: network-policy-conformance-slytherin
spec:
selector:
matchLabels:
conformance-house: slytherin
replicas: 2
template:
metadata:
labels:
conformance-house: slytherin
spec:
containers:
- name: draco-malfoy-client
image: registry.k8s.io/e2e-test-images/agnhost:2.43
- name: draco-malfoy-80
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost serve-hostname --tcp --http=false --port 80"]
- name: draco-malfoy-8080
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost serve-hostname --tcp --http=false --port 8080"]
- name: draco-malfoy-5353
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost serve-hostname --udp --http=false --port 5353"]
- name: draco-malfoy-53
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost serve-hostname --udp --http=false --port 53"]
- name: draco-malfoy-9003
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost porter"]
env:
- name: SERVE_SCTP_PORT_9003
value: "foo"
- name: draco-malfoy-9005
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost porter"]
env:
- name: SERVE_SCTP_PORT_9005
value: "foo"
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: cedric-diggory
namespace: network-policy-conformance-hufflepuff
spec:
selector:
matchLabels:
conformance-house: hufflepuff
replicas: 2
template:
metadata:
labels:
conformance-house: hufflepuff
spec:
containers:
- name: cedric-diggory-client
image: registry.k8s.io/e2e-test-images/agnhost:2.43
- name: cedric-diggory-80
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost serve-hostname --tcp --http=false --port 80"]
- name: cedric-diggory-8080
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost serve-hostname --tcp --http=false --port 8080"]
- name: cedric-diggory-5353
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost serve-hostname --udp --http=false --port 5353"]
- name: cedric-diggory-53
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost serve-hostname --udp --http=false --port 53"]
- name: cedric-diggory-9003
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost porter"]
env:
- name: SERVE_SCTP_PORT_9003
value: "foo"
- name: cedric-diggory-9005
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost porter"]
env:
- name: SERVE_SCTP_PORT_9005
value: "foo"
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: luna-lovegood
namespace: network-policy-conformance-ravenclaw
spec:
selector:
matchLabels:
conformance-house: ravenclaw
replicas: 2
template:
metadata:
labels:
conformance-house: ravenclaw
spec:
containers:
- name: luna-lovegood-client
image: registry.k8s.io/e2e-test-images/agnhost:2.43
- name: luna-lovegood-80
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost serve-hostname --tcp --http=false --port 80"]
- name: luna-lovegood-8080
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost serve-hostname --tcp --http=false --port 8080"]
- name: luna-lovegood-5353
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost serve-hostname --udp --http=false --port 5353"]
- name: luna-lovegood-53
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost serve-hostname --udp --http=false --port 53"]
- name: luna-lovegood-9003
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost porter"]
env:
- name: SERVE_SCTP_PORT_9003
value: "foo"
- name: luna-lovegood-9005
image: registry.k8s.io/e2e-test-images/agnhost:2.43
command: ["/bin/bash", "-c", "/agnhost porter"]
env:
- name: SERVE_SCTP_PORT_9005
value: "foo"
90 changes: 90 additions & 0 deletions conformance/conformance_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// conformance_test contains code to run the conformance tests. This is in its own package to avoid circular imports.
package conformance_test

import (
"flag"
"strings"
"testing"

"sigs.k8s.io/network-policy-api/apis/v1alpha1"
"sigs.k8s.io/network-policy-api/conformance/tests"
"sigs.k8s.io/network-policy-api/conformance/utils/flags"
"sigs.k8s.io/network-policy-api/conformance/utils/suite"

"k8s.io/apimachinery/pkg/util/sets"
_ "k8s.io/client-go/plugin/pkg/client/auth"
"k8s.io/kubernetes/test/e2e/framework"
e2econfig "k8s.io/kubernetes/test/e2e/framework/config"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/config"
)

func TestConformance(t *testing.T) {
cfg, err := config.GetConfig()
if err != nil {
t.Fatalf("Error loading Kubernetes config: %v", err)
}
client, err := client.New(cfg, client.Options{})
if err != nil {
t.Fatalf("Error initializing Kubernetes client: %v", err)
}

v1alpha1.AddToScheme(client.Scheme())

supportedFeatures := parseSupportedFeatures(*flags.SupportedFeatures)
exemptFeatures := parseSupportedFeatures(*flags.ExemptFeatures)

// Register test flags, then parse flags.
handleFlags()

t.Logf("Running conformance tests with cleanup: %t\n debug: %t\n enable all features: %t \n supported features: [%v]\n exempt features: [%v]",
*flags.CleanupBaseResources, *flags.ShowDebug, *flags.EnableAllSupportedFeatures, *flags.SupportedFeatures, *flags.ExemptFeatures)

cSuite := suite.New(suite.Options{
Copy link
Member

@astoycos astoycos May 22, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tssurya Would it be difficult to have a "--dry-run" flag here?

Then in our upstream CI we could run the tests against a vanilla KIND cluster without actually testing an implementation

It could help ensure the future stability of the tests

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm assuming it would just involve adding it to your "poke..." function

Copy link
Contributor Author

@tssurya tssurya May 22, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hmm great point! I think this should be doable.. let me open a card for this to enhance this further - #107
Also note that all implementations should be running these exact tests downstream, so if something breaks here we will know for sure :D because I will have trouble in OVN-K CI :D - let me take a stab at this issue once I have figured out the whole picture of how I will digest this downstream on the CI...

Client: client,
Debug: *flags.ShowDebug,
CleanupBaseResources: *flags.CleanupBaseResources,
SupportedFeatures: supportedFeatures,
ExemptFeatures: exemptFeatures,
EnableAllSupportedFeatures: *flags.EnableAllSupportedFeatures,
})
cSuite.Setup(t)

cSuite.Run(t, tests.ConformanceTests)
}

// parseSupportedFeatures parses flag arguments and converts the string to
// sets.Set[suite.SupportedFeature]
func parseSupportedFeatures(f string) sets.Set[suite.SupportedFeature] {
if f == "" {
return nil
}
res := sets.Set[suite.SupportedFeature]{}
for _, value := range strings.Split(f, ",") {
res.Insert(suite.SupportedFeature(value))
}
return res
}

// handleFlags sets up all flags and parses the command line.
func handleFlags() {
e2econfig.CopyFlags(e2econfig.Flags, flag.CommandLine)
framework.RegisterCommonFlags(flag.CommandLine)
flag.Parse()
}
22 changes: 22 additions & 0 deletions conformance/embed.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package conformance

import "embed"

//go:embed tests/* base/*
var Manifests embed.FS
21 changes: 21 additions & 0 deletions conformance/tests/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tests

import "sigs.k8s.io/network-policy-api/conformance/utils/suite"

var ConformanceTests []suite.ConformanceTest
Loading