From 0d3df83520dc16eb4e07f39cbf9cb54d56f60d2d Mon Sep 17 00:00:00 2001 From: Thor <1187526662@qq.com> Date: Mon, 29 Jun 2020 15:12:36 +0800 Subject: [PATCH 01/13] add e2e cases about preemption Signed-off-by: Thor <1187526662@qq.com> --- hack/e2e-kind-config.yaml | 3 +- test/e2e/job_scheduling.go | 91 -------------------------------------- test/e2e/preempt.go | 86 +++++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 93 deletions(-) create mode 100644 test/e2e/preempt.go diff --git a/hack/e2e-kind-config.yaml b/hack/e2e-kind-config.yaml index f7590b0b25..21714c6096 100644 --- a/hack/e2e-kind-config.yaml +++ b/hack/e2e-kind-config.yaml @@ -8,5 +8,4 @@ nodes: # the three workers - role: worker - role: worker - - role: worker - - role: worker + - role: worker \ No newline at end of file diff --git a/test/e2e/job_scheduling.go b/test/e2e/job_scheduling.go index d094efc5b6..1c8af54779 100644 --- a/test/e2e/job_scheduling.go +++ b/test/e2e/job_scheduling.go @@ -192,97 +192,6 @@ var _ = Describe("Job E2E Test", func() { Expect(err).NotTo(HaveOccurred()) }) - It("Preemption", func() { - ctx := initTestContext(options{ - priorityClasses: map[string]int32{ - masterPriority: masterPriorityValue, - workerPriority: workerPriorityValue, - }, - }) - defer cleanupTestContext(ctx) - - slot := oneCPU - rep := clusterSize(ctx, slot) - - job := &jobSpec{ - tasks: []taskSpec{ - { - img: defaultNginxImage, - req: slot, - min: 1, - rep: rep, - }, - }, - } - - job.name = "preemptee-qj" - job.pri = workerPriority - job1 := createJob(ctx, job) - err := waitTasksReady(ctx, job1, int(rep)) - Expect(err).NotTo(HaveOccurred()) - - job.name = "preemptor-qj" - job.pri = masterPriority - job.min = rep / 2 - job2 := createJob(ctx, job) - err = waitTasksReady(ctx, job1, int(rep)/2) - Expect(err).NotTo(HaveOccurred()) - - err = waitTasksReady(ctx, job2, int(rep)/2) - Expect(err).NotTo(HaveOccurred()) - }) - - It("Multiple Preemption", func() { - ctx := initTestContext(options{ - priorityClasses: map[string]int32{ - masterPriority: masterPriorityValue, - workerPriority: workerPriorityValue, - }, - }) - defer cleanupTestContext(ctx) - - slot := oneCPU - rep := clusterSize(ctx, slot) - - job := &jobSpec{ - tasks: []taskSpec{ - { - img: defaultNginxImage, - req: slot, - min: 1, - rep: rep, - }, - }, - } - - job.name = "multipreemptee-qj" - job.pri = workerPriority - job1 := createJob(ctx, job) - - err := waitTasksReady(ctx, job1, int(rep)) - Expect(err).NotTo(HaveOccurred()) - - job.name = "multipreemptor-qj1" - job.pri = masterPriority - job.min = rep / 3 - job2 := createJob(ctx, job) - Expect(err).NotTo(HaveOccurred()) - - job.name = "multipreemptor-qj2" - job.pri = masterPriority - job3 := createJob(ctx, job) - Expect(err).NotTo(HaveOccurred()) - - err = waitTasksReady(ctx, job1, int(rep)/3) - Expect(err).NotTo(HaveOccurred()) - - err = waitTasksReady(ctx, job2, int(rep)/3) - Expect(err).NotTo(HaveOccurred()) - - err = waitTasksReady(ctx, job3, int(rep)/3) - Expect(err).NotTo(HaveOccurred()) - }) - It("Schedule BestEffort Job", func() { ctx := initTestContext(options{}) defer cleanupTestContext(ctx) diff --git a/test/e2e/preempt.go b/test/e2e/preempt.go new file mode 100644 index 0000000000..492db9cd89 --- /dev/null +++ b/test/e2e/preempt.go @@ -0,0 +1,86 @@ +package e2e + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Job E2E Test", func() { + It("schedule high priority job without preemption when resource is enough", func() { + ctx := initTestContext(options{ + priorityClasses: map[string]int32{ + masterPriority: masterPriorityValue, + workerPriority: workerPriorityValue, + }, + }) + defer cleanupTestContext(ctx) + + slot := oneCPU + + job := &jobSpec{ + tasks: []taskSpec{ + { + img: defaultNginxImage, + req: slot, + min: 1, + rep: 1, + }, + }, + } + + job.name = "preemptee" + job.pri = workerPriority + preempteeJob := createJob(ctx, job) + err := waitTasksReady(ctx, preempteeJob, 1) + Expect(err).NotTo(HaveOccurred()) + + job.name = "preemptor" + job.pri = masterPriority + preemptorJob := createJob(ctx, job) + err = waitTasksReady(ctx, preempteeJob, 1) + Expect(err).NotTo(HaveOccurred()) + + err = waitTasksReady(ctx, preemptorJob, 1) + Expect(err).NotTo(HaveOccurred()) + }) + + It("schedule high priority job with preemption when resource is NOT enough", func() { + ctx := initTestContext(options{ + priorityClasses: map[string]int32{ + masterPriority: masterPriorityValue, + workerPriority: workerPriorityValue, + }, + }) + defer cleanupTestContext(ctx) + + slot := oneCPU + rep := clusterSize(ctx, slot) + + job := &jobSpec{ + tasks: []taskSpec{ + { + img: defaultNginxImage, + req: slot, + min: 1, + rep: rep, + }, + }, + } + + job.name = "preemptee" + job.pri = workerPriority + preempteeJob := createJob(ctx, job) + err := waitTasksReady(ctx, preempteeJob, int(rep)) + Expect(err).NotTo(HaveOccurred()) + + job.name = "preemptor" + job.pri = masterPriority + job.min = rep / 2 + preemptorJob := createJob(ctx, job) + err = waitTasksReady(ctx, preempteeJob, int(rep)/2) + Expect(err).NotTo(HaveOccurred()) + + err = waitTasksReady(ctx, preemptorJob, int(rep)/2) + Expect(err).NotTo(HaveOccurred()) + }) +}) From 2440818cb62e50255440a42e63e2b4f90d48da11 Mon Sep 17 00:00:00 2001 From: Thor <1187526662@qq.com> Date: Mon, 29 Jun 2020 16:09:27 +0800 Subject: [PATCH 02/13] increase wait time in job_scheduling e2e cases Signed-off-by: Thor <1187526662@qq.com> --- test/e2e/job_scheduling.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/job_scheduling.go b/test/e2e/job_scheduling.go index 1c8af54779..c623d5fe7d 100644 --- a/test/e2e/job_scheduling.go +++ b/test/e2e/job_scheduling.go @@ -513,7 +513,7 @@ var _ = Describe("Job E2E Test", func() { if expectPod%1 == 1 { expectPod-- } - err = wait.Poll(100*time.Millisecond, oneMinute, func() (bool, error) { + err = wait.Poll(100*time.Millisecond, twoMinute, func() (bool, error) { fsScheduledPod = 0 testScheduledPod = 0 @@ -609,7 +609,7 @@ var _ = Describe("Job E2E Test", func() { if expectPod%1 == 1 { expectPod-- } - err = wait.Poll(100*time.Millisecond, oneMinute, func() (bool, error) { + err = wait.Poll(100*time.Millisecond, twoMinute, func() (bool, error) { q1ScheduledPod = 0 q2ScheduledPod = 0 From 79a29736d5c317a21c4a9cda41930f5fa8b8dfef Mon Sep 17 00:00:00 2001 From: Thor <1187526662@qq.com> Date: Mon, 29 Jun 2020 17:06:53 +0800 Subject: [PATCH 03/13] separate reclaim case from queue Signed-off-by: Thor <1187526662@qq.com> --- test/e2e/queue.go | 82 --------------------------------------- test/e2e/reclaim.go | 94 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 82 deletions(-) create mode 100644 test/e2e/reclaim.go diff --git a/test/e2e/queue.go b/test/e2e/queue.go index a1c6f91403..4fda26d6b1 100644 --- a/test/e2e/queue.go +++ b/test/e2e/queue.go @@ -193,86 +193,4 @@ var _ = Describe("Queue E2E Test", func() { Expect(err).NotTo(HaveOccurred(), "Error waiting for queue unknown") }) }) - - It("Reclaim", func() { - q1, q2 := "reclaim-q1", "reclaim-q2" - ctx := initTestContext(options{ - queues: []string{q1, q2}, - priorityClasses: map[string]int32{ - "low-priority": 10, - "high-priority": 10000, - }, - }) - defer cleanupTestContext(ctx) - - slot := oneCPU - rep := clusterSize(ctx, slot) - - spec := &jobSpec{ - tasks: []taskSpec{ - { - img: defaultNginxImage, - req: slot, - min: 1, - rep: rep, - }, - }, - } - - spec.name = "q1-qj-1" - spec.queue = q1 - spec.pri = "low-priority" - job1 := createJob(ctx, spec) - err := waitJobReady(ctx, job1) - Expect(err).NotTo(HaveOccurred()) - - err = waitQueueStatus(func() (bool, error) { - queue, err := ctx.vcclient.SchedulingV1beta1().Queues().Get(context.TODO(), q1, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - return queue.Status.Running == 1, nil - }) - Expect(err).NotTo(HaveOccurred()) - - expected := int(rep) / 2 - // Reduce one pod to tolerate decimal fraction. - if expected > 1 { - expected-- - } else { - err := fmt.Errorf("expected replica <%d> is too small", expected) - Expect(err).NotTo(HaveOccurred()) - } - - spec.name = "q2-qj-2" - spec.queue = q2 - spec.pri = "high-priority" - job2 := createJob(ctx, spec) - err = waitTasksReady(ctx, job2, expected) - Expect(err).NotTo(HaveOccurred()) - - err = waitTasksReady(ctx, job1, expected) - Expect(err).NotTo(HaveOccurred()) - - // Test Queue status - spec = &jobSpec{ - name: "q1-qj-2", - queue: q1, - tasks: []taskSpec{ - { - img: defaultNginxImage, - req: slot, - min: rep * 2, - rep: rep * 2, - }, - }, - } - job3 := createJob(ctx, spec) - err = waitJobStatePending(ctx, job3) - Expect(err).NotTo(HaveOccurred()) - err = waitQueueStatus(func() (bool, error) { - queue, err := ctx.vcclient.SchedulingV1beta1().Queues().Get(context.TODO(), q1, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - return queue.Status.Pending == 1, nil - }) - Expect(err).NotTo(HaveOccurred()) - }) }) diff --git a/test/e2e/reclaim.go b/test/e2e/reclaim.go new file mode 100644 index 0000000000..8de54cdbfe --- /dev/null +++ b/test/e2e/reclaim.go @@ -0,0 +1,94 @@ +package e2e + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = Describe("Queue E2E Test", func() { + It("Reclaim", func() { + q1, q2 := "reclaim-q1", "reclaim-q2" + ctx := initTestContext(options{ + queues: []string{q1, q2}, + priorityClasses: map[string]int32{ + "low-priority": 10, + "high-priority": 10000, + }, + }) + defer cleanupTestContext(ctx) + + slot := oneCPU + rep := clusterSize(ctx, slot) + + spec := &jobSpec{ + tasks: []taskSpec{ + { + img: defaultNginxImage, + req: slot, + min: 1, + rep: rep, + }, + }, + } + + spec.name = "q1-qj-1" + spec.queue = q1 + spec.pri = "low-priority" + job1 := createJob(ctx, spec) + err := waitJobReady(ctx, job1) + Expect(err).NotTo(HaveOccurred()) + + err = waitQueueStatus(func() (bool, error) { + queue, err := ctx.vcclient.SchedulingV1beta1().Queues().Get(context.TODO(), q1, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + return queue.Status.Running == 1, nil + }) + Expect(err).NotTo(HaveOccurred()) + + expected := int(rep) / 2 + // Reduce one pod to tolerate decimal fraction. + if expected > 1 { + expected-- + } else { + err := fmt.Errorf("expected replica <%d> is too small", expected) + Expect(err).NotTo(HaveOccurred()) + } + + spec.name = "q2-qj-2" + spec.queue = q2 + spec.pri = "high-priority" + job2 := createJob(ctx, spec) + err = waitTasksReady(ctx, job2, expected) + Expect(err).NotTo(HaveOccurred()) + + err = waitTasksReady(ctx, job1, expected) + Expect(err).NotTo(HaveOccurred()) + + // Test Queue status + spec = &jobSpec{ + name: "q1-qj-2", + queue: q1, + tasks: []taskSpec{ + { + img: defaultNginxImage, + req: slot, + min: rep * 2, + rep: rep * 2, + }, + }, + } + job3 := createJob(ctx, spec) + err = waitJobStatePending(ctx, job3) + Expect(err).NotTo(HaveOccurred()) + err = waitQueueStatus(func() (bool, error) { + queue, err := ctx.vcclient.SchedulingV1beta1().Queues().Get(context.TODO(), q1, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + return queue.Status.Pending == 1, nil + }) + Expect(err).NotTo(HaveOccurred()) + }) +}) From 9daa1e695eb2e719fdcaa4b067288a7c642ee392 Mon Sep 17 00:00:00 2001 From: Thor <1187526662@qq.com> Date: Tue, 30 Jun 2020 11:11:36 +0800 Subject: [PATCH 04/13] add some e2e cases Signed-off-by: Thor <1187526662@qq.com> --- test/e2e/preempt.go | 123 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 121 insertions(+), 2 deletions(-) diff --git a/test/e2e/preempt.go b/test/e2e/preempt.go index 492db9cd89..9d699e4271 100644 --- a/test/e2e/preempt.go +++ b/test/e2e/preempt.go @@ -1,8 +1,13 @@ package e2e import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" + "context" + + "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + schedulingv1beta1 "volcano.sh/volcano/pkg/apis/scheduling/v1beta1" ) var _ = Describe("Job E2E Test", func() { @@ -83,4 +88,118 @@ var _ = Describe("Job E2E Test", func() { err = waitTasksReady(ctx, preemptorJob, int(rep)/2) Expect(err).NotTo(HaveOccurred()) }) + + It("preemption don't work when podgroup is pending", func() { + ctx := initTestContext(options{ + priorityClasses: map[string]int32{ + masterPriority: masterPriorityValue, + workerPriority: workerPriorityValue, + }, + }) + defer cleanupTestContext(ctx) + + pgName := "pending-pg" + pg := &schedulingv1beta1.PodGroup{ + ObjectMeta: v1.ObjectMeta{ + Namespace: ctx.namespace, + Name: pgName, + }, + Spec: schedulingv1beta1.PodGroupSpec{ + MinMember: 1, + MinResources: &thirtyCPU, + }, + Status: schedulingv1beta1.PodGroupStatus{ + Phase: schedulingv1beta1.PodGroupPending, + }, + } + _, err := ctx.vcclient.SchedulingV1beta1().PodGroups(ctx.namespace).Create(context.TODO(), pg, v1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + slot := oneCPU + rep := clusterSize(ctx, slot) + job := &jobSpec{ + tasks: []taskSpec{ + { + img: defaultNginxImage, + req: slot, + min: 1, + rep: rep, + }, + }, + } + job.name = "preemptee" + job.pri = workerPriority + preempteeJob := createJob(ctx, job) + err := waitTasksReady(ctx, preempteeJob, int(rep)) + Expect(err).NotTo(HaveOccurred()) + + pod := &corev1.Pod{ + TypeMeta: v1.TypeMeta{ + APIVersion: "v1", + Kind: "Pod", + }, + ObjectMeta: v1.ObjectMeta{ + Namespace: ctx.namespace, + Name: podName, + Annotations: map[string]string{schedulingv1beta1.KubeGroupNameAnnotationKey: pgName}, + }, + Spec: corev1.PodSpec{ + SchedulerName: "volcano", + Containers: createContainers(defaultNginxImage, "", "", oneCPU, oneCPU, 0), + PriorityClassName: masterPriority, + }, + } + _, err = ctx.kubeclient.CoreV1().Pods(ctx.namespace).Create(context.TODO(), pod, v1.CreateOptions{}) + Expect(err).To(HaveOccurred()) + }) + + It("preemption only works in the same queue", func() { + ctx := initTestContext(options{ + queues: []string{"q1, q2"} + priorityClasses: map[string]int32{ + masterPriority: masterPriorityValue, + workerPriority: workerPriorityValue, + }, + }) + defer cleanupTestContext(ctx) + + slot := oneCPU + rep := clusterSize(ctx, slot) + job := &jobSpec{ + tasks: []taskSpec{ + { + img: defaultNginxImage, + req: slot, + min: 1, + rep: int(rep)/2, + }, + }, + } + + job.name = "j1-q1" + job.pri = workerPriority + job.queue = "q1" + queue1Job := createJob(ctx, job) + err := waitTasksReady(ctx, queue1Job, int(rep)/2) + Expect(err).NotTo(HaveOccurred()) + + job.name = "j2-q2" + job.pri = workerPriority + job.queue = "q2" + queue2Job := createJob(ctx, job) + err = waitTasksReady(ctx, queue2Job, int(rep)/2) + Expect(err).NotTo(HaveOccurred()) + + job.name = "j3-q1" + job.pri = masterPriority + job.queue = "q1" + job.tasks[0].rep = int(rep) + queue1Job3 := createJob(ctx, job) + err = waitTasksReady(ctx, queue1Job3, int(rep)/2) + Expect(err).NotTo(HaveOccurred()) + err = waitTasksReady(ctx, queue1Job, int(rep)/2) + Expect(err).Should(ContainSubstring(`actual got`)) + err = waitTasksReady(ctx, queue2Job, int(rep)/2) + Expect(err).NotTo(HaveOccurred()) + }) }) From 246db1e8af4b76ef909a74957a49b482a69b8add Mon Sep 17 00:00:00 2001 From: Thor <1187526662@qq.com> Date: Tue, 30 Jun 2020 11:15:38 +0800 Subject: [PATCH 05/13] bug fix Signed-off-by: Thor <1187526662@qq.com> --- test/e2e/preempt.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/e2e/preempt.go b/test/e2e/preempt.go index 9d699e4271..3ec94464a8 100644 --- a/test/e2e/preempt.go +++ b/test/e2e/preempt.go @@ -155,7 +155,7 @@ var _ = Describe("Job E2E Test", func() { It("preemption only works in the same queue", func() { ctx := initTestContext(options{ - queues: []string{"q1, q2"} + queues: []string{"q1, q2"}, priorityClasses: map[string]int32{ masterPriority: masterPriorityValue, workerPriority: workerPriorityValue, @@ -171,7 +171,7 @@ var _ = Describe("Job E2E Test", func() { img: defaultNginxImage, req: slot, min: 1, - rep: int(rep)/2, + rep: int(rep) / 2, }, }, } @@ -179,14 +179,14 @@ var _ = Describe("Job E2E Test", func() { job.name = "j1-q1" job.pri = workerPriority job.queue = "q1" - queue1Job := createJob(ctx, job) + queue1Job := createJob(ctx, job) err := waitTasksReady(ctx, queue1Job, int(rep)/2) Expect(err).NotTo(HaveOccurred()) job.name = "j2-q2" job.pri = workerPriority job.queue = "q2" - queue2Job := createJob(ctx, job) + queue2Job := createJob(ctx, job) err = waitTasksReady(ctx, queue2Job, int(rep)/2) Expect(err).NotTo(HaveOccurred()) @@ -194,7 +194,7 @@ var _ = Describe("Job E2E Test", func() { job.pri = masterPriority job.queue = "q1" job.tasks[0].rep = int(rep) - queue1Job3 := createJob(ctx, job) + queue1Job3 := createJob(ctx, job) err = waitTasksReady(ctx, queue1Job3, int(rep)/2) Expect(err).NotTo(HaveOccurred()) err = waitTasksReady(ctx, queue1Job, int(rep)/2) From 0c874e05cd559af0718a215cf78b471ad19c7b42 Mon Sep 17 00:00:00 2001 From: Thor <1187526662@qq.com> Date: Tue, 30 Jun 2020 11:40:45 +0800 Subject: [PATCH 06/13] bug fix Signed-off-by: Thor <1187526662@qq.com> --- test/e2e/preempt.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/preempt.go b/test/e2e/preempt.go index 3ec94464a8..7bc62b1c04 100644 --- a/test/e2e/preempt.go +++ b/test/e2e/preempt.go @@ -4,10 +4,10 @@ import ( "context" "github.com/onsi/gomega" + schedulingv1beta1 "volcano.sh/volcano/pkg/apis/scheduling/v1beta1" corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - schedulingv1beta1 "volcano.sh/volcano/pkg/apis/scheduling/v1beta1" + v1 "k8s.io/api/core/v1" ) var _ = Describe("Job E2E Test", func() { From 77e3915315ff7db1c9d769d871982c6b85f01716 Mon Sep 17 00:00:00 2001 From: Thor <1187526662@qq.com> Date: Tue, 30 Jun 2020 11:57:54 +0800 Subject: [PATCH 07/13] bug fix Signed-off-by: Thor <1187526662@qq.com> --- test/e2e/preempt.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/e2e/preempt.go b/test/e2e/preempt.go index 7bc62b1c04..adf2d0be37 100644 --- a/test/e2e/preempt.go +++ b/test/e2e/preempt.go @@ -3,11 +3,12 @@ package e2e import ( "context" - "github.com/onsi/gomega" - schedulingv1beta1 "volcano.sh/volcano/pkg/apis/scheduling/v1beta1" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" + schedulingv1beta1 "volcano.sh/volcano/pkg/apis/scheduling/v1beta1" ) var _ = Describe("Job E2E Test", func() { @@ -113,7 +114,7 @@ var _ = Describe("Job E2E Test", func() { }, } _, err := ctx.vcclient.SchedulingV1beta1().PodGroups(ctx.namespace).Create(context.TODO(), pg, v1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) slot := oneCPU rep := clusterSize(ctx, slot) From 946c5e998a4be4f00c9cc141a05d0f109f3d9c5e Mon Sep 17 00:00:00 2001 From: Thor <1187526662@qq.com> Date: Tue, 30 Jun 2020 14:17:56 +0800 Subject: [PATCH 08/13] bug fix Signed-off-by: Thor <1187526662@qq.com> --- hack/e2e-kind-config.yaml | 5 +++-- test/e2e/preempt.go | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/hack/e2e-kind-config.yaml b/hack/e2e-kind-config.yaml index 21714c6096..323f852db9 100644 --- a/hack/e2e-kind-config.yaml +++ b/hack/e2e-kind-config.yaml @@ -1,11 +1,12 @@ # this config file contains all config fields with comments kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 -# 1 control plane node and 3 workers +# 1 control plane node and 4 workers nodes: # the control plane node config - role: control-plane - # the three workers + # the four workers + - role: worker - role: worker - role: worker - role: worker \ No newline at end of file diff --git a/test/e2e/preempt.go b/test/e2e/preempt.go index adf2d0be37..0b53972565 100644 --- a/test/e2e/preempt.go +++ b/test/e2e/preempt.go @@ -7,7 +7,7 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" schedulingv1beta1 "volcano.sh/volcano/pkg/apis/scheduling/v1beta1" ) @@ -131,7 +131,7 @@ var _ = Describe("Job E2E Test", func() { job.name = "preemptee" job.pri = workerPriority preempteeJob := createJob(ctx, job) - err := waitTasksReady(ctx, preempteeJob, int(rep)) + err = waitTasksReady(ctx, preempteeJob, int(rep)) Expect(err).NotTo(HaveOccurred()) pod := &corev1.Pod{ @@ -172,7 +172,7 @@ var _ = Describe("Job E2E Test", func() { img: defaultNginxImage, req: slot, min: 1, - rep: int(rep) / 2, + rep: rep / 2, }, }, } @@ -194,7 +194,7 @@ var _ = Describe("Job E2E Test", func() { job.name = "j3-q1" job.pri = masterPriority job.queue = "q1" - job.tasks[0].rep = int(rep) + job.tasks[0].rep = rep queue1Job3 := createJob(ctx, job) err = waitTasksReady(ctx, queue1Job3, int(rep)/2) Expect(err).NotTo(HaveOccurred()) From 417d358733bd82f757102179d5d50877d832e937 Mon Sep 17 00:00:00 2001 From: Thor <1187526662@qq.com> Date: Tue, 30 Jun 2020 14:38:50 +0800 Subject: [PATCH 09/13] bug fix Signed-off-by: Thor <1187526662@qq.com> --- test/e2e/preempt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/preempt.go b/test/e2e/preempt.go index 0b53972565..d609d9a411 100644 --- a/test/e2e/preempt.go +++ b/test/e2e/preempt.go @@ -141,7 +141,7 @@ var _ = Describe("Job E2E Test", func() { }, ObjectMeta: v1.ObjectMeta{ Namespace: ctx.namespace, - Name: podName, + Name: "preemptor-pod", Annotations: map[string]string{schedulingv1beta1.KubeGroupNameAnnotationKey: pgName}, }, Spec: corev1.PodSpec{ From 65bb8870fb9839e573e4cfd64847051c2ae573d9 Mon Sep 17 00:00:00 2001 From: Thor <1187526662@qq.com> Date: Tue, 30 Jun 2020 15:40:39 +0800 Subject: [PATCH 10/13] bug fix Signed-off-by: Thor <1187526662@qq.com> --- test/e2e/preempt.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/e2e/preempt.go b/test/e2e/preempt.go index d609d9a411..8f73f78941 100644 --- a/test/e2e/preempt.go +++ b/test/e2e/preempt.go @@ -156,7 +156,7 @@ var _ = Describe("Job E2E Test", func() { It("preemption only works in the same queue", func() { ctx := initTestContext(options{ - queues: []string{"q1, q2"}, + queues: []string{"q1-preemption", "q2-reference"}, priorityClasses: map[string]int32{ masterPriority: masterPriorityValue, workerPriority: workerPriorityValue, @@ -179,21 +179,21 @@ var _ = Describe("Job E2E Test", func() { job.name = "j1-q1" job.pri = workerPriority - job.queue = "q1" + job.queue = "q1-preemption" queue1Job := createJob(ctx, job) err := waitTasksReady(ctx, queue1Job, int(rep)/2) Expect(err).NotTo(HaveOccurred()) job.name = "j2-q2" job.pri = workerPriority - job.queue = "q2" + job.queue = "q2-reference" queue2Job := createJob(ctx, job) err = waitTasksReady(ctx, queue2Job, int(rep)/2) Expect(err).NotTo(HaveOccurred()) job.name = "j3-q1" job.pri = masterPriority - job.queue = "q1" + job.queue = "q1-preemption" job.tasks[0].rep = rep queue1Job3 := createJob(ctx, job) err = waitTasksReady(ctx, queue1Job3, int(rep)/2) From 1809b511b9f254095a858ef2de25c2e5566df98b Mon Sep 17 00:00:00 2001 From: Thor <1187526662@qq.com> Date: Tue, 30 Jun 2020 16:35:06 +0800 Subject: [PATCH 11/13] bug fix Signed-off-by: Thor <1187526662@qq.com> --- test/e2e/preempt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/preempt.go b/test/e2e/preempt.go index 8f73f78941..f548cd94d4 100644 --- a/test/e2e/preempt.go +++ b/test/e2e/preempt.go @@ -199,7 +199,7 @@ var _ = Describe("Job E2E Test", func() { err = waitTasksReady(ctx, queue1Job3, int(rep)/2) Expect(err).NotTo(HaveOccurred()) err = waitTasksReady(ctx, queue1Job, int(rep)/2) - Expect(err).Should(ContainSubstring(`actual got`)) + Expect(err).Should(ContainSubstring("actual got")) err = waitTasksReady(ctx, queue2Job, int(rep)/2) Expect(err).NotTo(HaveOccurred()) }) From 4f397e81cdb2ce1140aca40b871e5abd7f8bc635 Mon Sep 17 00:00:00 2001 From: Thor <1187526662@qq.com> Date: Tue, 30 Jun 2020 21:13:02 +0800 Subject: [PATCH 12/13] modify preempt expected result Signed-off-by: Thor <1187526662@qq.com> --- test/e2e/preempt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/preempt.go b/test/e2e/preempt.go index f548cd94d4..da0eb8bef3 100644 --- a/test/e2e/preempt.go +++ b/test/e2e/preempt.go @@ -196,7 +196,7 @@ var _ = Describe("Job E2E Test", func() { job.queue = "q1-preemption" job.tasks[0].rep = rep queue1Job3 := createJob(ctx, job) - err = waitTasksReady(ctx, queue1Job3, int(rep)/2) + err = waitTasksReady(ctx, queue1Job3, 1) Expect(err).NotTo(HaveOccurred()) err = waitTasksReady(ctx, queue1Job, int(rep)/2) Expect(err).Should(ContainSubstring("actual got")) From 08547bd9845bd67f120d8ac075bcb8d62825d7ef Mon Sep 17 00:00:00 2001 From: Thor <1187526662@qq.com> Date: Wed, 1 Jul 2020 09:09:58 +0800 Subject: [PATCH 13/13] bug fix Signed-off-by: Thor <1187526662@qq.com> --- test/e2e/preempt.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/preempt.go b/test/e2e/preempt.go index da0eb8bef3..a830283757 100644 --- a/test/e2e/preempt.go +++ b/test/e2e/preempt.go @@ -198,8 +198,8 @@ var _ = Describe("Job E2E Test", func() { queue1Job3 := createJob(ctx, job) err = waitTasksReady(ctx, queue1Job3, 1) Expect(err).NotTo(HaveOccurred()) - err = waitTasksReady(ctx, queue1Job, int(rep)/2) - Expect(err).Should(ContainSubstring("actual got")) + err = waitTasksReady(ctx, queue1Job, 0) + Expect(err).NotTo(HaveOccurred()) err = waitTasksReady(ctx, queue2Job, int(rep)/2) Expect(err).NotTo(HaveOccurred()) })