From 0fb88d7d891b7c32d222b5bda9d23f808eea16d9 Mon Sep 17 00:00:00 2001 From: paxbit Date: Fri, 7 May 2021 21:39:52 +0200 Subject: [PATCH 1/2] feat: skip by reason and skip pod updates... added: - NS annotation secunet.sentry/skip-event-reasons: allow configuration of skip event reasons per namespace - env SKIP_EVENT_REASONS: global skip by reason config if NS has no skip config annotation - Pod annotation secunet.sentry/ignore-pod-updates=true: allows for suppression of pod update event handling through the forwarder changed: - skip config declaration format now supports specifying a resource type before a criteria: [involved object type:]criteria[,...] E.g. normal,Pod:warning,Service:error See: parseSkipConfig(...) at skip.go:71 --- application.go | 48 +++++++++++----- application_test.go | 33 ++++++++--- main.go | 17 ++++-- skip.go | 132 ++++++++++++++++++++++++++++++++++++++------ test_pod.yaml | 2 + 5 files changed, 190 insertions(+), 42 deletions(-) diff --git a/application.go b/application.go index 7232e4b..6178709 100644 --- a/application.go +++ b/application.go @@ -26,6 +26,7 @@ import ( "log" "os" "strconv" + "strings" "time" "github.com/getsentry/sentry-go" @@ -44,14 +45,16 @@ type terminationKey struct { } type application struct { - clientset *kubernetes.Clientset - defaultEnvironment string - globalSkipEventLevels []string - nsSkipEventLevels map[string][]string - terminationsSeen *lru.Cache - namespaces []string + clientset *kubernetes.Clientset + defaultEnvironment string + globalSkipConfig map[string]struct{} + nsSkipEventCriteria map[string]map[string]struct{} + terminationsSeen *lru.Cache + namespaces []string } +const AnyNS = "__GLOBAL__" + func (app *application) Run() (chan struct{}, error) { terminationsSeen, err := lru.New(500) if err != nil { @@ -63,7 +66,9 @@ func (app *application) Run() (chan struct{}, error) { errGroup, ctx := errgroup.WithContext(context.Background()) - app.nsSkipEventLevels = make(map[string][]string) + app.nsSkipEventCriteria = make(map[string]map[string]struct{}) + app.nsSkipEventCriteria[AnyNS] = app.globalSkipConfig + errGroup.Go(func() error { return app.monitorNamespaces(stop, ctx, func() { for _, namespace := range app.namespaces { @@ -105,7 +110,7 @@ func (app application) monitorNamespaces(stop chan struct{}, ctx context.Context _, controller := cache.NewInformer( watchList, &v1.Namespace{}, - time.Second*60, + time.Second*120, cache.ResourceEventHandlerFuncs{ AddFunc: app.handleNsAdd, UpdateFunc: app.handleNsUpdate, @@ -177,10 +182,21 @@ func (app *application) handleNsUpdate(_, newNS interface{}) { ns, _ := newNS.(*v1.Namespace) - value, _ := ns.ObjectMeta.Annotations[SkipLevelKey] + skipLevelsValue, _ := ns.ObjectMeta.Annotations[SkipLevelKey] + skipReasonsValue, _ := ns.ObjectMeta.Annotations[SkipReasonKey] + + nsSkipConfigs := append(parseSkipConfig(SKIP_BY_LEVEL, &skipLevelsValue), parseSkipConfig(SKIP_BY_REASON, &skipReasonsValue)...) + lookupMap := make(map[string]struct{}) - skipLevels := parseSkipLevels(&value, app.globalSkipEventLevels...) - app.nsSkipEventLevels[ns.Name] = skipLevels + for _, skipConfig := range nsSkipConfigs { + lookupMap[skipConfig] = struct{}{} + } + + if len(lookupMap) == 0 { + delete(app.nsSkipEventCriteria, ns.Name) + } else { + app.nsSkipEventCriteria[ns.Name] = lookupMap + } } @@ -188,7 +204,7 @@ func (app *application) handleNsDelete(goneNS interface{}) { ns, _ := goneNS.(*v1.Namespace) - delete(app.nsSkipEventLevels, ns.Name) + delete(app.nsSkipEventCriteria, ns.Name) } func (app *application) handlePodUpdate(oldObj, newObj interface{}) { @@ -198,6 +214,12 @@ func (app *application) handlePodUpdate(oldObj, newObj interface{}) { return } + ignore, ok := pod.Annotations[SkipPodModificationEvent] + + if ok && strings.ToLower(ignore) == "true" { + return + } + var sentryEvent *sentry.Event // The Pod is still running. Check if one of its containers terminated with a non-zero exit code. @@ -294,7 +316,7 @@ func (app application) handleEventAdd(obj interface{}) { return } - if skipEvent(evt, app.nsSkipEventLevels, app.globalSkipEventLevels) { + if skipEvent(evt, app.nsSkipEventCriteria) { return } diff --git a/application_test.go b/application_test.go index 1621675..b0e1ca2 100644 --- a/application_test.go +++ b/application_test.go @@ -13,23 +13,42 @@ func TestSkipEvent(t *testing.T) { evt := &v1.Event{Type: v1.EventTypeNormal} - skipLevels := []string{"normal"} + skipLevels := map[string]map[string]struct{}{ + "default": { + skipConfigLookupKey(SKIP_BY_REASON, "Pod", "created"): {}, + skipConfigLookupKey(SKIP_BY_REASON, "", "puLLeD"): {}, + skipConfigLookupKey(SKIP_BY_LEVEL, "Service", "warning"): {}, + skipConfigLookupKey(SKIP_BY_LEVEL, "", "info"): {}, + }, + } + evt.Type = v1.EventTypeNormal + evt.ObjectMeta.Namespace = "default" + evt.InvolvedObject.Kind = "Pod" + evt.Reason = "created" if !skipEvent(evt, skipLevels) { - t.Error("Normal events must be skipped") + t.Error("pod:created should be skipped by reason") } + evt.Reason = "pulled" + if !skipEvent(evt, skipLevels) { + t.Error("[any kind]:pulled should be skipped by reason") + } + + evt.InvolvedObject.Kind = "serVICE" evt.Type = v1.EventTypeWarning - if skipEvent(evt, skipLevels) { - t.Error("Warnings events must not be skipped") + if !skipEvent(evt, skipLevels) { + t.Error("service:warning should be skipped by level") } - evt.Type = "Error" - if skipEvent(evt, skipLevels) { - t.Error("Error events must not be skipped") + evt.InvolvedObject.Kind = "ConfigMap" + evt.Type = v1.EventTypeNormal + if !skipEvent(evt, skipLevels) { + t.Error("[any kind]:info should be skipped by level") } evt.Type = "Unknown" + evt.Reason = "killed" if skipEvent(evt, skipLevels) { t.Error("Unknown event types must not be skipped") } diff --git a/main.go b/main.go index a66850c..901377d 100644 --- a/main.go +++ b/main.go @@ -62,13 +62,20 @@ func main() { log.Fatalf("Error creating kubernetes client: %v", err) } - skipEnv, _ := os.LookupEnv("SKIP_EVENT_LEVELS") - var skipLevels = parseSkipLevels(&skipEnv, v1.EventTypeNormal) + skipLevelEnv, _ := os.LookupEnv(SkipEventLevelsEnv) + skipReasonEnv, _ := os.LookupEnv(SkipEventReasonsEnv) + skipLevels := parseSkipConfig(SKIP_BY_LEVEL, &skipLevelEnv, v1.EventTypeNormal) + skipReasons := parseSkipConfig(SKIP_BY_REASON, &skipReasonEnv) + + skipConfig := make(map[string]struct{}) + for _, config := range append(skipReasons, skipLevels...) { + skipConfig[config] = struct{}{} + } app := application{ - clientset: clientset, - defaultEnvironment: os.Getenv("SENTRY_ENVIRONMENT"), - globalSkipEventLevels: skipLevels, + clientset: clientset, + defaultEnvironment: os.Getenv("SENTRY_ENVIRONMENT"), + globalSkipConfig: skipConfig, } namespace := os.Getenv("NAMESPACE") diff --git a/skip.go b/skip.go index 12f7c02..5ad3cd0 100644 --- a/skip.go +++ b/skip.go @@ -2,10 +2,24 @@ package main import ( v1 "k8s.io/api/core/v1" + "log" + "strconv" "strings" ) const SkipLevelKey string = "secunet.sentry/skip-event-levels" +const SkipReasonKey string = "secunet.sentry/skip-event-reasons" +const SkipPodModificationEvent string = "secunet.sentry/ignore-pod-updates" + +const SkipEventReasonsEnv = "SKIP_EVENT_REASONS" +const SkipEventLevelsEnv = "SKIP_EVENT_LEVELS" + +type SkipCriteria int + +const ( + SKIP_BY_REASON SkipCriteria = iota + SKIP_BY_LEVEL SkipCriteria = iota +) func trim(mess []string) []string { @@ -18,38 +32,122 @@ func trim(mess []string) []string { return result } -func parseSkipLevels(raw *string, fallback ...string) []string { +func skipConfigLookupKey(criteriaType SkipCriteria, resourceType string, criteriaValue string) string { + + var result strings.Builder + + result.WriteString(strconv.Itoa(int(criteriaType))) + if len(resourceType) > 0 { + result.WriteString("-") + result.WriteString(resourceType) + } + + result.WriteString("-") + result.WriteString(criteriaValue) + + return strings.ToLower(result.String()) +} + +// Parses SkipConfig declarations from either environment variables or namespace annotation values. The supported +// declaration format is: [involvedObjectType:]skipCriteria[,...]. +// +// Some examples: +// +// Filter Pod, Service and PV events by their reason field: +// Set env SkipEventReasonsEnv or the namespace annotation SkipReasonKey to e.g.: +// Pod:created,Service:AllocationFailed,PersistentVolume:PersistentVolumeDeleted +// +// Filter Pod, Service and PV events by their level: +// Set env SkipEventLevelsEnv or the namespace annotation SkipLevelKey to e.g.: +// Pod:normal,Service:warning,PersistentVolume:normal +// +// Generally filter events with level normal, while selectively also filter warnings related to Pods or Services: +// Set env SkipEventLevelsEnv or the namespace annotation SkipLevelKey to: +// normal,Pod:warning,Service:warning +// +// To find out which combinations of event level, object and reason currently exist, run: +// kubectl get events --all-namespaces -o custom-columns=OTYPE:.involvedObject.kind,LEVEL:.type,REASON:.reason,NAMESPACE:.metadata.namespace | sort | uniq +// +func parseSkipConfig(criteriaType SkipCriteria, raw *string, fallback ...string) []string { + + var criteriaList []string if raw == nil || len(strings.TrimSpace(*raw)) == 0 { - var dflt []string + return fallback + + } else { + criteriaList = trim(strings.Split(strings.ToLower(*raw), ",")) + } + + var result []string - for _, val := range fallback { - dflt = append(dflt, strings.ToLower(val)) + for _, criteria := range criteriaList { + + // expecting format of either "resourceType:criteria" or just "criteria" + // E.g. + // "Pod:created" means, if criteriaType is set to SKIP_BY_REASON: + // filter events whose involved object is a pod and the reason is "created" + // ":created" or "created" means, if criteriaType is set to SKIP_BY_REASON: + // filter ALL events of reason "created" + // ... the same applies for filtering by level, so: + // "ConfigMap:normal" means, if criteriaType is set to SKIP_BY_LEVEL: + // filter events whose involved object is a configmap and the event level is "normal" + typeAndCriteria := trim(strings.Split(strings.ToLower(criteria), ":")) + + if len(typeAndCriteria) > 2 || len(typeAndCriteria) == 0 { + // declaration error, more than one ":" delimiter not supported + log.Printf("Illegal skip event config declaration, ignoring: %s\n", criteria) + continue } - return trim(dflt) - } else { - return trim(strings.Split(strings.ToLower(*raw), ",")) + var resourceType string + var criteriaValue string + + if len(typeAndCriteria) == 2 { + resourceType = strings.TrimSpace(strings.ToLower(typeAndCriteria[0])) + criteriaValue = strings.TrimSpace(strings.ToLower(typeAndCriteria[1])) + } else { + criteriaValue = strings.TrimSpace(strings.ToLower(strings.TrimSpace(typeAndCriteria[0]))) + } + + result = append(result, skipConfigLookupKey(criteriaType, resourceType, criteriaValue)) } + return result } -func skipEvent(evt *v1.Event, nsSkipLevels map[string][]string, defaultSkipLevels []string) bool { +func skipEvent(evt *v1.Event, nsSkipLevels map[string]map[string]struct{}) bool { - evtType := strings.ToLower(evt.Type) - evtNs := evt.Namespace + ns := evt.Namespace + reason := strings.ToLower(evt.Reason) + level := strings.ToLower(evt.Type) + oType := strings.ToLower(evt.InvolvedObject.Kind) - appliedSkipLevels, hasNsMapping := nsSkipLevels[evt.Namespace] + appliedSkipLevels, hasNsMapping := nsSkipLevels[ns] - if len(evtNs) == 0 || !hasNsMapping { - appliedSkipLevels = defaultSkipLevels + if len(ns) == 0 || !hasNsMapping { + appliedSkipLevels = nsSkipLevels[AnyNS] } - for _, level := range appliedSkipLevels { - if level == evtType { - return true - } + _, hasOtypeSpecificReasonFilter := appliedSkipLevels[skipConfigLookupKey(SKIP_BY_REASON, oType, reason)] + if hasOtypeSpecificReasonFilter { + return true + } + + _, hasOtypeAgnosticReasonFilter := appliedSkipLevels[skipConfigLookupKey(SKIP_BY_REASON, "", reason)] + if hasOtypeAgnosticReasonFilter { + return true + } + + _, hasOtypeSpecificLevelFilter := appliedSkipLevels[skipConfigLookupKey(SKIP_BY_LEVEL, oType, level)] + if hasOtypeSpecificLevelFilter { + return true + } + + _, hasOtypeAgnosticLevelFilter := appliedSkipLevels[skipConfigLookupKey(SKIP_BY_LEVEL, "", level)] + if hasOtypeAgnosticLevelFilter { + return true } return false diff --git a/test_pod.yaml b/test_pod.yaml index d1636c2..d7f9841 100644 --- a/test_pod.yaml +++ b/test_pod.yaml @@ -3,6 +3,8 @@ kind: Pod metadata: name: test namespace: default + annotations: + secunet.sentry/ignore-pod-updates: "true" spec: containers: - command: From 402512fb8a7554d52660b8b01ef55aa9019f06fd Mon Sep 17 00:00:00 2001 From: paxbit Date: Tue, 11 May 2021 12:42:10 +0200 Subject: [PATCH 2/2] fix: annotation prefix "mismatch" --- skip.go | 6 +++--- test_pod.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/skip.go b/skip.go index 5ad3cd0..17d9072 100644 --- a/skip.go +++ b/skip.go @@ -7,9 +7,9 @@ import ( "strings" ) -const SkipLevelKey string = "secunet.sentry/skip-event-levels" -const SkipReasonKey string = "secunet.sentry/skip-event-reasons" -const SkipPodModificationEvent string = "secunet.sentry/ignore-pod-updates" +const SkipLevelKey string = "sentry/skip-event-levels" +const SkipReasonKey string = "sentry/skip-event-reasons" +const SkipPodModificationEvent string = "sentry/ignore-pod-updates" const SkipEventReasonsEnv = "SKIP_EVENT_REASONS" const SkipEventLevelsEnv = "SKIP_EVENT_LEVELS" diff --git a/test_pod.yaml b/test_pod.yaml index d7f9841..fdb924f 100644 --- a/test_pod.yaml +++ b/test_pod.yaml @@ -4,7 +4,7 @@ metadata: name: test namespace: default annotations: - secunet.sentry/ignore-pod-updates: "true" + sentry/ignore-pod-updates: "true" spec: containers: - command: