metrics: Wait for a minute before deleting metrics for deleted pods

Instead of deleting metrics immediately after a pod is deleted, use a workqueue
to delay the deletion a minute. This allows the scraper to scrape last values
of the metrics. It's particularly useful when the cluster has short-lived pods
- with immediate deletion the scraper could completely miss metrics for them.

Signed-off-by: Anna Kapuscinska <anna@isovalent.com>

pkg/metrics/metrics.go

@@ -6,6 +6,7 @@ package metrics
 import (
 	"net/http"
 	"sync"
+	"time"
 
 	"github.com/cilium/tetragon/pkg/grpc/tracing"
 	"github.com/cilium/tetragon/pkg/logger"
@@ -26,10 +27,13 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/workqueue"
 )
 
 var (
 	metricsWithPod []*prometheus.MetricVec
+	podQueue       workqueue.DelayingInterface
+	deleteDelay    = 1 * time.Minute
 	once           sync.Once
 )
 
@@ -59,14 +63,22 @@ func RegisterPodDeleteHandler() {
 						default:
 							return
 						}
-						DeleteMetricsForPod(pod)
+						queue := GetPodQueue()
+						queue.AddAfter(pod, deleteDelay)
 					},
 				},
 			)
 		},
 	})
 }
 
+func GetPodQueue() workqueue.DelayingInterface {
+	if podQueue == nil {
+		podQueue = workqueue.NewDelayingQueue()
+	}
+	return podQueue
+}
+
 // ListMetricsWithPod returns the global list of all metrics that have "pod"
 // and "namespace" labels, initializing it if needed.
 func ListMetricsWithPod() []*prometheus.MetricVec {
@@ -105,6 +117,20 @@ func InitAllMetrics(registry *prometheus.Registry) {
 func EnableMetrics(address string) {
 	reg := prometheus.NewRegistry()
 	InitAllMetrics(reg)
+
+	// Start handling metrics deletion on pod delete events
+	go func() {
+		queue := GetPodQueue()
+		for {
+			pod, quit := queue.Get()
+			if quit {
+				return
+			}
+			DeleteMetricsForPod(pod.(*corev1.Pod))
+		}
+	}()
+
+	// Start the metrics server
 	logger.GetLogger().WithField("addr", address).Info("Starting metrics server")
 	http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg}))
 	http.ListenAndServe(address, nil)