feat: skip by reason and skip pod updates...

application.go

@@ -26,6 +26,7 @@ import (
 	"log"
 	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/getsentry/sentry-go"
@@ -44,14 +45,16 @@ type terminationKey struct {
 }
 
 type application struct {
-	clientset             *kubernetes.Clientset
-	defaultEnvironment    string
-	globalSkipEventLevels []string
-	nsSkipEventLevels     map[string][]string
-	terminationsSeen      *lru.Cache
-	namespaces            []string
+	clientset           *kubernetes.Clientset
+	defaultEnvironment  string
+	globalSkipConfig    map[string]struct{}
+	nsSkipEventCriteria map[string]map[string]struct{}
+	terminationsSeen    *lru.Cache
+	namespaces          []string
 }
 
+const AnyNS = "__GLOBAL__"
+
 func (app *application) Run() (chan struct{}, error) {
 	terminationsSeen, err := lru.New(500)
 	if err != nil {
@@ -63,7 +66,9 @@ func (app *application) Run() (chan struct{}, error) {
 
 	errGroup, ctx := errgroup.WithContext(context.Background())
 
-	app.nsSkipEventLevels = make(map[string][]string)
+	app.nsSkipEventCriteria = make(map[string]map[string]struct{})
+	app.nsSkipEventCriteria[AnyNS] = app.globalSkipConfig
+
 	errGroup.Go(func() error {
 		return app.monitorNamespaces(stop, ctx, func() {
 			for _, namespace := range app.namespaces {
@@ -105,7 +110,7 @@ func (app application) monitorNamespaces(stop chan struct{}, ctx context.Context
 	_, controller := cache.NewInformer(
 		watchList,
 		&v1.Namespace{},
-		time.Second*60,
+		time.Second*120,
 		cache.ResourceEventHandlerFuncs{
 			AddFunc:    app.handleNsAdd,
 			UpdateFunc: app.handleNsUpdate,
@@ -177,18 +182,29 @@ func (app *application) handleNsUpdate(_, newNS interface{}) {
 
 	ns, _ := newNS.(*v1.Namespace)
 
-	value, _ := ns.ObjectMeta.Annotations[SkipLevelKey]
+	skipLevelsValue, _ := ns.ObjectMeta.Annotations[SkipLevelKey]
+	skipReasonsValue, _ := ns.ObjectMeta.Annotations[SkipReasonKey]
+
+	nsSkipConfigs := append(parseSkipConfig(SKIP_BY_LEVEL, &skipLevelsValue), parseSkipConfig(SKIP_BY_REASON, &skipReasonsValue)...)
+	lookupMap := make(map[string]struct{})
 
-	skipLevels := parseSkipLevels(&value, app.globalSkipEventLevels...)
-	app.nsSkipEventLevels[ns.Name] = skipLevels
+	for _, skipConfig := range nsSkipConfigs {
+		lookupMap[skipConfig] = struct{}{}
+	}
+
+	if len(lookupMap) == 0 {
+		delete(app.nsSkipEventCriteria, ns.Name)
+	} else {
+		app.nsSkipEventCriteria[ns.Name] = lookupMap
+	}
 
 }
 
 func (app *application) handleNsDelete(goneNS interface{}) {
 
 	ns, _ := goneNS.(*v1.Namespace)
 
-	delete(app.nsSkipEventLevels, ns.Name)
+	delete(app.nsSkipEventCriteria, ns.Name)
 }
 
 func (app *application) handlePodUpdate(oldObj, newObj interface{}) {
@@ -198,6 +214,12 @@ func (app *application) handlePodUpdate(oldObj, newObj interface{}) {
 		return
 	}
 
+	ignore, ok := pod.Annotations[SkipPodModificationEvent]
+
+	if ok && strings.ToLower(ignore) == "true" {
+		return
+	}
+
 	var sentryEvent *sentry.Event
 
 	// The Pod is still running. Check if one of its containers terminated with a non-zero exit code.
@@ -294,7 +316,7 @@ func (app application) handleEventAdd(obj interface{}) {
 		return
 	}
 
-	if skipEvent(evt, app.nsSkipEventLevels, app.globalSkipEventLevels) {
+	if skipEvent(evt, app.nsSkipEventCriteria) {
 		return
 	}
 

application_test.go

@@ -13,23 +13,42 @@ func TestSkipEvent(t *testing.T) {
 
 	evt := &v1.Event{Type: v1.EventTypeNormal}
 
-	skipLevels := []string{"normal"}
+	skipLevels := map[string]map[string]struct{}{
+		"default": {
+			skipConfigLookupKey(SKIP_BY_REASON, "Pod", "created"):    {},
+			skipConfigLookupKey(SKIP_BY_REASON, "", "puLLeD"):        {},
+			skipConfigLookupKey(SKIP_BY_LEVEL, "Service", "warning"): {},
+			skipConfigLookupKey(SKIP_BY_LEVEL, "", "info"):           {},
+		},
+	}
 
+	evt.Type = v1.EventTypeNormal
+	evt.ObjectMeta.Namespace = "default"
+	evt.InvolvedObject.Kind = "Pod"
+	evt.Reason = "created"
 	if !skipEvent(evt, skipLevels) {
-		t.Error("Normal events must be skipped")
+		t.Error("pod:created should be skipped by reason")
 	}
 
+	evt.Reason = "pulled"
+	if !skipEvent(evt, skipLevels) {
+		t.Error("[any kind]:pulled should be skipped by reason")
+	}
+
+	evt.InvolvedObject.Kind = "serVICE"
 	evt.Type = v1.EventTypeWarning
-	if skipEvent(evt, skipLevels) {
-		t.Error("Warnings events must not be skipped")
+	if !skipEvent(evt, skipLevels) {
+		t.Error("service:warning should be skipped by level")
 	}
 
-	evt.Type = "Error"
-	if skipEvent(evt, skipLevels) {
-		t.Error("Error events must not be skipped")
+	evt.InvolvedObject.Kind = "ConfigMap"
+	evt.Type = v1.EventTypeNormal
+	if !skipEvent(evt, skipLevels) {
+		t.Error("[any kind]:info should be skipped by level")
 	}
 
 	evt.Type = "Unknown"
+	evt.Reason = "killed"
 	if skipEvent(evt, skipLevels) {
 		t.Error("Unknown event types must not be skipped")
 	}

main.go

@@ -62,13 +62,20 @@ func main() {
 		log.Fatalf("Error creating kubernetes client: %v", err)
 	}
 
-	skipEnv, _ := os.LookupEnv("SKIP_EVENT_LEVELS")
-	var skipLevels = parseSkipLevels(&skipEnv, v1.EventTypeNormal)
+	skipLevelEnv, _ := os.LookupEnv(SkipEventLevelsEnv)
+	skipReasonEnv, _ := os.LookupEnv(SkipEventReasonsEnv)
+	skipLevels := parseSkipConfig(SKIP_BY_LEVEL, &skipLevelEnv, v1.EventTypeNormal)
+	skipReasons := parseSkipConfig(SKIP_BY_REASON, &skipReasonEnv)
+
+	skipConfig := make(map[string]struct{})
+	for _, config := range append(skipReasons, skipLevels...) {
+		skipConfig[config] = struct{}{}
+	}
 
 	app := application{
-		clientset:             clientset,
-		defaultEnvironment:    os.Getenv("SENTRY_ENVIRONMENT"),
-		globalSkipEventLevels: skipLevels,
+		clientset:          clientset,
+		defaultEnvironment: os.Getenv("SENTRY_ENVIRONMENT"),
+		globalSkipConfig:   skipConfig,
 	}
 
 	namespace := os.Getenv("NAMESPACE")

skip.go

@@ -2,10 +2,24 @@ package main
 
 import (
 	v1 "k8s.io/api/core/v1"
+	"log"
+	"strconv"
 	"strings"
 )
 
-const SkipLevelKey string = "secunet.sentry/skip-event-levels"
+const SkipLevelKey string = "sentry/skip-event-levels"
+const SkipReasonKey string = "sentry/skip-event-reasons"
+const SkipPodModificationEvent string = "sentry/ignore-pod-updates"
+
+const SkipEventReasonsEnv = "SKIP_EVENT_REASONS"
+const SkipEventLevelsEnv = "SKIP_EVENT_LEVELS"
+
+type SkipCriteria int
+
+const (
+	SKIP_BY_REASON SkipCriteria = iota
+	SKIP_BY_LEVEL  SkipCriteria = iota
+)
 
 func trim(mess []string) []string {
 
@@ -18,38 +32,122 @@ func trim(mess []string) []string {
 	return result
 }
 
-func parseSkipLevels(raw *string, fallback ...string) []string {
+func skipConfigLookupKey(criteriaType SkipCriteria, resourceType string, criteriaValue string) string {
+
+	var result strings.Builder
+
+	result.WriteString(strconv.Itoa(int(criteriaType)))
+	if len(resourceType) > 0 {
+		result.WriteString("-")
+		result.WriteString(resourceType)
+	}
+
+	result.WriteString("-")
+	result.WriteString(criteriaValue)
+
+	return strings.ToLower(result.String())
+}
+
+// Parses SkipConfig declarations from either environment variables or namespace annotation values. The supported
+// declaration format is: [involvedObjectType:]skipCriteria[,...].
+//
+// Some examples:
+//
+// Filter Pod, Service and PV events by their reason field:
+// Set env SkipEventReasonsEnv or the namespace annotation SkipReasonKey to e.g.:
+//   Pod:created,Service:AllocationFailed,PersistentVolume:PersistentVolumeDeleted
+//
+// Filter Pod, Service and PV events by their level:
+// Set env SkipEventLevelsEnv or the namespace annotation SkipLevelKey to e.g.:
+//   Pod:normal,Service:warning,PersistentVolume:normal
+//
+// Generally filter events with level normal, while selectively also filter warnings related to Pods or Services:
+// Set env SkipEventLevelsEnv or the namespace annotation SkipLevelKey to:
+//   normal,Pod:warning,Service:warning
+//
+// To find out which combinations of event level, object and reason currently exist, run:
+//   kubectl get events --all-namespaces -o custom-columns=OTYPE:.involvedObject.kind,LEVEL:.type,REASON:.reason,NAMESPACE:.metadata.namespace | sort | uniq
+//
+func parseSkipConfig(criteriaType SkipCriteria, raw *string, fallback ...string) []string {
+
+	var criteriaList []string
 
 	if raw == nil || len(strings.TrimSpace(*raw)) == 0 {
 
-		var dflt []string
+		return fallback
+
+	} else {
+		criteriaList = trim(strings.Split(strings.ToLower(*raw), ","))
+	}
+
+	var result []string
 
-		for _, val := range fallback {
-			dflt = append(dflt, strings.ToLower(val))
+	for _, criteria := range criteriaList {
+
+		// expecting format of either "resourceType:criteria" or just "criteria"
+		// E.g.
+		// "Pod:created" means, if criteriaType is set to SKIP_BY_REASON:
+		// filter events whose involved object is a pod and the reason is "created"
+		// ":created" or "created" means, if criteriaType is set to SKIP_BY_REASON:
+		// filter ALL events of reason "created"
+		// ... the same applies for filtering by level, so:
+		// "ConfigMap:normal" means, if criteriaType is set to SKIP_BY_LEVEL:
+		// filter events whose involved object is a configmap and the event level is "normal"
+		typeAndCriteria := trim(strings.Split(strings.ToLower(criteria), ":"))
+
+		if len(typeAndCriteria) > 2 || len(typeAndCriteria) == 0 {
+			// declaration error, more than one ":" delimiter not supported
+			log.Printf("Illegal skip event config declaration, ignoring: %s\n", criteria)
+			continue
 		}
-		return trim(dflt)
 
-	} else {
-		return trim(strings.Split(strings.ToLower(*raw), ","))
+		var resourceType string
+		var criteriaValue string
+
+		if len(typeAndCriteria) == 2 {
+			resourceType = strings.TrimSpace(strings.ToLower(typeAndCriteria[0]))
+			criteriaValue = strings.TrimSpace(strings.ToLower(typeAndCriteria[1]))
+		} else {
+			criteriaValue = strings.TrimSpace(strings.ToLower(strings.TrimSpace(typeAndCriteria[0])))
+		}
+
+		result = append(result, skipConfigLookupKey(criteriaType, resourceType, criteriaValue))
 	}
 
+	return result
 }
 
-func skipEvent(evt *v1.Event, nsSkipLevels map[string][]string, defaultSkipLevels []string) bool {
+func skipEvent(evt *v1.Event, nsSkipLevels map[string]map[string]struct{}) bool {
 
-	evtType := strings.ToLower(evt.Type)
-	evtNs := evt.Namespace
+	ns := evt.Namespace
+	reason := strings.ToLower(evt.Reason)
+	level := strings.ToLower(evt.Type)
+	oType := strings.ToLower(evt.InvolvedObject.Kind)
 
-	appliedSkipLevels, hasNsMapping := nsSkipLevels[evt.Namespace]
+	appliedSkipLevels, hasNsMapping := nsSkipLevels[ns]
 
-	if len(evtNs) == 0 || !hasNsMapping {
-		appliedSkipLevels = defaultSkipLevels
+	if len(ns) == 0 || !hasNsMapping {
+		appliedSkipLevels = nsSkipLevels[AnyNS]
 	}
 
-	for _, level := range appliedSkipLevels {
-		if level == evtType {
-			return true
-		}
+	_, hasOtypeSpecificReasonFilter := appliedSkipLevels[skipConfigLookupKey(SKIP_BY_REASON, oType, reason)]
+	if hasOtypeSpecificReasonFilter {
+		return true
+	}
+
+	_, hasOtypeAgnosticReasonFilter := appliedSkipLevels[skipConfigLookupKey(SKIP_BY_REASON, "", reason)]
+	if hasOtypeAgnosticReasonFilter {
+		return true
+	}
+
+	_, hasOtypeSpecificLevelFilter := appliedSkipLevels[skipConfigLookupKey(SKIP_BY_LEVEL, oType, level)]
+	if hasOtypeSpecificLevelFilter {
+		return true
+	}
+
+	_, hasOtypeAgnosticLevelFilter := appliedSkipLevels[skipConfigLookupKey(SKIP_BY_LEVEL, "", level)]
+	if hasOtypeAgnosticLevelFilter {
+		return true
 	}
 
 	return false

test_pod.yaml

@@ -3,6 +3,8 @@ kind: Pod
 metadata:
   name: test
   namespace: default
+  annotations:
+    sentry/ignore-pod-updates: "true"
 spec:
   containers:
     - command: