From f46d58124225ea7999b7266656c60ca673bd3082 Mon Sep 17 00:00:00 2001 From: kuritka Date: Thu, 16 Jun 2022 16:45:42 +0200 Subject: [PATCH] Weight Round Robin (3/4) - annotations related to #50 implemented code that adds an annotation to the local dnsEndpoint. - covered by unit-tests - the annotation is as follows: ```json { "eu":{ "weight":35, "targets":[ "10.10.0.1", "10.10.0.2" ] }, "us":{ "weight":50, "targets":[ "10.0.0.1", "10.0.0.2" ] }, "za":{ "weight":15, "targets":[ "10.22.0.1", "10.22.0.2", "10.22.1.1" ] } } ``` Signed-off-by: kuritka --- controllers/dnsupdate.go | 43 ++++- controllers/providers/assistant/gslb.go | 4 +- controllers/providers/assistant/target.go | 23 ++- controllers/weight_test.go | 166 ++++++++++++++++++ .../k8gb.absa.oss_v1beta1_gslb_cr_weight.yaml | 47 +++++ 5 files changed, 272 insertions(+), 11 deletions(-) create mode 100644 controllers/weight_test.go create mode 100644 deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_weight.yaml diff --git a/controllers/dnsupdate.go b/controllers/dnsupdate.go index 3fe70766b5..ef2d7c3e69 100644 --- a/controllers/dnsupdate.go +++ b/controllers/dnsupdate.go @@ -19,11 +19,13 @@ Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic */ import ( + "encoding/json" "fmt" "sort" "strings" "github.com/k8gb-io/k8gb/controllers/depresolver" + "github.com/k8gb-io/k8gb/controllers/providers/assistant" k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -41,6 +43,8 @@ func sortTargets(targets []string) []string { func (r *GslbReconciler) gslbDNSEndpoint(gslb *k8gbv1beta1.Gslb) (*externaldns.DNSEndpoint, error) { var gslbHosts []*externaldns.Endpoint var ttl = externaldns.TTL(gslb.Spec.Strategy.DNSTtlSeconds) + var targets = assistant.NewTargets() + var externalAnnotation map[string]string serviceHealth, err := r.getServiceHealthStatus(gslb) if err != nil { @@ -75,7 +79,9 @@ func (r *GslbReconciler) gslbDNSEndpoint(gslb *k8gbv1beta1.Gslb) (*externaldns.D } // Check if host is alive on external Gslb - externalTargets := r.DNSProvider.GetExternalTargets(host).GetIPs() + externalTargetsAndRegions := r.DNSProvider.GetExternalTargets(host) + externalTargets := externalTargetsAndRegions.GetIPs() + targets.Append(externalTargetsAndRegions) sortTargets(externalTargets) @@ -139,11 +145,16 @@ func (r *GslbReconciler) gslbDNSEndpoint(gslb *k8gbv1beta1.Gslb) (*externaldns.D Endpoints: gslbHosts, } + externalAnnotation, err = r.getLocalTargetAnnotations(gslb, targets) + if err != nil { + return nil, err + } + dnsEndpoint := &externaldns.DNSEndpoint{ ObjectMeta: metav1.ObjectMeta{ Name: gslb.Name, Namespace: gslb.Namespace, - Annotations: map[string]string{"k8gb.absa.oss/dnstype": "local"}, + Annotations: externalAnnotation, Labels: map[string]string{"k8gb.absa.oss/dnstype": "local"}, }, Spec: dnsEndpointSpec, @@ -166,3 +177,31 @@ func (r *GslbReconciler) updateRuntimeStatus(gslb *k8gbv1beta1.Gslb, isPrimary b m.UpdateFailoverStatus(gslb, isPrimary, isHealthy, finalTargets) } } + +// getLocalTargetAnnotations returns single annotation when strategy is RoundRobin and Weights are filled +// otherwise returns empty annotations +func (r *GslbReconciler) getLocalTargetAnnotations(gslb *k8gbv1beta1.Gslb, targets assistant.Targets) (annotations map[string]string, err error) { + type wrr struct { + Weight int `json:"weight"` + Targets []string `json:"targets"` + } + const rrAnnotation = "k8gb.absa.oss/weight-round-robin" + + annotations = map[string]string{"k8gb.absa.oss/dnstype": "local"} + + if gslb.Spec.Strategy.Type == depresolver.RoundRobinStrategy && !gslb.Spec.Strategy.Weight.IsEmpty() { + var bytes []byte + var mwrr = make(map[string]wrr, 0) + + for k, v := range gslb.Spec.Strategy.Weight { + mwrr[k] = wrr{Weight: v.Int(), Targets: targets[k].IPs} + } + + bytes, err = json.Marshal(mwrr) + if err != nil { + return annotations, err + } + annotations[rrAnnotation] = string(bytes) + } + return annotations, err +} diff --git a/controllers/providers/assistant/gslb.go b/controllers/providers/assistant/gslb.go index 4e87bd8e67..e8e17814fd 100644 --- a/controllers/providers/assistant/gslb.go +++ b/controllers/providers/assistant/gslb.go @@ -297,7 +297,7 @@ func dnsQuery(host string, nameservers utils.DNSList) (*dns.Msg, error) { } func (r *Gslb) GetExternalTargets(host string, extClusterNsNames map[string]string) (targets Targets) { - targets = Targets{} + targets = NewTargets() for _, cluster := range extClusterNsNames { // Use edgeDNSServer for resolution of NS names and fallback to local nameservers log.Info(). @@ -327,7 +327,7 @@ func (r *Gslb) GetExternalTargets(host string, extClusterNsNames map[string]stri } clusterTargets := getARecords(a) if len(clusterTargets) > 0 { - targets = append(targets, Target{cluster, clusterTargets}) + targets[cluster] = Target{clusterTargets} log.Info(). Strs("clusterTargets", clusterTargets). Str("cluster", cluster). diff --git a/controllers/providers/assistant/target.go b/controllers/providers/assistant/target.go index ca8d523ed2..4f9f3f16bc 100644 --- a/controllers/providers/assistant/target.go +++ b/controllers/providers/assistant/target.go @@ -19,17 +19,26 @@ Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic */ type Target struct { - Region string - IPs []string + IPs []string } -type Targets []Target +type Targets map[string]Target -func (t Targets) GetIPs() (targets []string) { +func NewTargets() Targets { + return make(map[string]Target, 0) +} + +func (t Targets) GetIPs() (ips []string) { // initializing targets to avoid possible nil reference errors (serialization etc.) - targets = []string{} + ips = []string{} for _, v := range t { - targets = append(targets, v.IPs...) + ips = append(ips, v.IPs...) + } + return ips +} + +func (t Targets) Append(targets Targets) { + for k, v := range targets { + t[k] = v } - return targets } diff --git a/controllers/weight_test.go b/controllers/weight_test.go new file mode 100644 index 0000000000..6064dc33f7 --- /dev/null +++ b/controllers/weight_test.go @@ -0,0 +1,166 @@ +package controllers + +/* +Copyright 2022 The k8gb Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic +*/ + +import ( + "context" + "fmt" + "testing" + + "github.com/k8gb-io/k8gb/controllers/depresolver" + "sigs.k8s.io/controller-runtime/pkg/client" + + k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" + + "github.com/golang/mock/gomock" + "github.com/k8gb-io/k8gb/controllers/providers/assistant" + "github.com/k8gb-io/k8gb/controllers/providers/dns" + "github.com/stretchr/testify/assert" + externaldns "sigs.k8s.io/external-dns/endpoint" +) + +func TestWeight(t *testing.T) { + // arrange + type wrr struct { + weight string + targets []string + } + var tests = []struct { + name string + data map[string]wrr + injectWeights bool + host string + annotation string + }{ + { + name: "eu35-us50-za15", + injectWeights: true, + host: "roundrobin.cloud.example.com", + data: map[string]wrr{ + "eu": {weight: "35%", targets: []string{"10.10.0.1", "10.10.0.2"}}, + "us": {weight: "50%", targets: []string{"10.0.0.1", "10.0.0.2"}}, + "za": {weight: "15%", targets: []string{"10.22.0.1", "10.22.0.2", "10.22.1.1"}}, + }, + annotation: `{"eu":{"weight":35,"targets":["10.10.0.1","10.10.0.2"]},` + + `"us":{"weight":50,"targets":["10.0.0.1","10.0.0.2"]},` + + `"za":{"weight":15,"targets":["10.22.0.1","10.22.0.2","10.22.1.1"]}}`, + }, + + { + name: "eu100-us0-za0", + injectWeights: true, + host: "roundrobin.cloud.example.com", + data: map[string]wrr{ + "eu": {weight: "100%", targets: []string{"10.10.0.1", "10.10.0.2"}}, + "us": {weight: "0%", targets: []string{"10.0.0.1", "10.0.0.2"}}, + "za": {weight: "0%", targets: []string{"10.22.0.1", "10.22.0.2", "10.22.1.1"}}, + }, + annotation: `{"eu":{"weight":100,"targets":["10.10.0.1","10.10.0.2"]},` + + `"us":{"weight":0,"targets":["10.0.0.1","10.0.0.2"]},` + + `"za":{"weight":0,"targets":["10.22.0.1","10.22.0.2","10.22.1.1"]}}`, + }, + + { + name: "no weights without external targets", + injectWeights: false, + host: "roundrobin.cloud.example.com", + data: map[string]wrr{}, + }, + + { + name: "no weights with external targets", + injectWeights: false, + host: "roundrobin.cloud.example.com", + data: map[string]wrr{ + "eu": {weight: "100%", targets: []string{"10.10.0.1", "10.10.0.2"}}, + "us": {weight: "0%", targets: []string{"10.0.0.1", "10.0.0.2"}}, + "za": {weight: "0%", targets: []string{"10.22.0.1", "10.22.0.2", "10.22.1.1"}}, + }, + }, + + { + name: "us100", + injectWeights: true, + host: "roundrobin.cloud.example.com", + data: map[string]wrr{ + "us": {weight: "0%", targets: []string{"10.0.0.1", "10.0.0.2"}}, + }, + annotation: `{"us":{"weight":0,"targets":["10.0.0.1","10.0.0.2"]}}`, + }, + + { + name: "weights0", + injectWeights: true, + host: "roundrobin.cloud.example.com", + data: map[string]wrr{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + + assertAnnotation := func(gslb *k8gbv1beta1.Gslb, ep *externaldns.DNSEndpoint) error { + assert.NotNil(t, ep) + assert.NotNil(t, gslb) + assert.Equal(t, ep.ObjectMeta.Annotations["k8gb.absa.oss/dnstype"], "local") + str, found := ep.ObjectMeta.Annotations["k8gb.absa.oss/weight-round-robin"] + // annotation doesnt exist + assert.Equal(t, len(test.annotation) != 0, found) + // annotation is equal to tested value + assert.Equal(t, test.annotation, str) + return nil + } + + injectWeight := func(ctx context.Context, gslb *k8gbv1beta1.Gslb, client client.Client) error { + if !test.injectWeights { + return nil + } + gslb.Spec.Strategy.Weight = make(map[string]k8gbv1beta1.Percentage, 0) + for k, w := range test.data { + gslb.Spec.Strategy.Weight[k] = k8gbv1beta1.Percentage(w.weight) + } + return nil + } + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + settings := provideSettings(t, predefinedConfig) + m := dns.NewMockProvider(ctrl) + r := depresolver.NewMockResolver(ctrl) + m.EXPECT().GslbIngressExposedIPs(gomock.Any()).Return([]string{}, nil).Times(1) + m.EXPECT().SaveDNSEndpoint(gomock.Any(), gomock.Any()).Do(assertAnnotation).Return(fmt.Errorf("save DNS error")).Times(1) + m.EXPECT().CreateZoneDelegationForExternalDNS(gomock.Any()).Return(nil).AnyTimes() + r.EXPECT().ResolveGslbSpec(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(injectWeight).AnyTimes() + + ts := assistant.Targets{} + for k, w := range test.data { + ts[k] = assistant.Target{IPs: w.targets} + } + m.EXPECT().GetExternalTargets("roundrobin.cloud.example.com").Return(ts).Times(1) + m.EXPECT().GetExternalTargets("notfound.cloud.example.com").Return(assistant.Targets{}).Times(1) + m.EXPECT().GetExternalTargets("unhealthy.cloud.example.com").Return(assistant.Targets{}).Times(1) + + settings.reconciler.DNSProvider = m + settings.reconciler.DepResolver = r + + // act, assert + _, _ = settings.reconciler.Reconcile(context.TODO(), settings.request) + }) + } +} diff --git a/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_weight.yaml b/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_weight.yaml new file mode 100644 index 0000000000..a7bbe34b5a --- /dev/null +++ b/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_weight.yaml @@ -0,0 +1,47 @@ +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: test-gslb + namespace: test-gslb +spec: + ingress: + ingressClassName: nginx + rules: + - host: notfound.cloud.example.com # This is the GSLB enabled host that clients would use + http: # This section mirrors the same structure as that of an Ingress resource and will be used verbatim when creating the corresponding Ingress resource that will match the GSLB host + paths: + - path: / + pathType: Prefix + backend: + service: + name: non-existing-app # Gslb should reflect NotFound status + port: + name: http + - host: unhealthy.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: unhealthy-app # Gslb should reflect Unhealthy status + port: + name: http + - host: roundrobin.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: frontend-podinfo # Gslb should reflect Healthy status and create associated DNS records + port: + name: http + strategy: + type: roundRobin # Use a round robin load balancing strategy, when deciding which downstream clusters to route clients too + splitBrainThresholdSeconds: 300 # Threshold after which external cluster is filtered out from delegated zone when it doesn't look alive + dnsTtlSeconds: 30 # TTL value for automatically created DNS records + weight: + eu: 35% + us: 50% + za: 15%