Adjust dropped metrics from cAdvisor

This change drops pod-centric metrics without a non-empty 'container' label. Previously we dropped pod-centric metrics without a (pod, namespace) label set however these can be critical for debugging.
prometheus-operator · Sep 28, 2021 · c8fb959 · c8fb959
1 parent 55df3c1
commit c8fb959
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 4 deletions.
diff --git a/jsonnet/kube-prometheus/kube-prometheus-insecure-kubelet.libsonnet b/jsonnet/kube-prometheus/kube-prometheus-insecure-kubelet.libsonnet
@@ -37,9 +37,7 @@
                   action: 'drop',
                   regex: '(' + std.join('|',
                                         [
-                                          'container_fs_.*',  // add filesystem read/write data (nodes*disks*services*4)
                                           'container_spec_.*',  // everything related to cgroup specification and thus static data (nodes*services*5)
-                                          'container_blkio_device_usage_total',  // useful for containers, but not for system services (nodes*disks*services*operations*2)
                                           'container_file_descriptors',  // file descriptors limits and global numbers are exposed via (nodes*services)
                                           'container_sockets',  // used sockets in cgroup. Usually not important for system services (nodes*services)
                                           'container_threads_max',  // max number of threads in cgroup. Usually for system services it is not limited (nodes*services)
@@ -48,6 +46,14 @@
                                           'container_last_seen',  // not needed as system services are always running (nodes*services)
                                         ]) + ');;',
                 },
+                {
+                  sourceLabels: ['__name__', 'container'],
+                  action: 'drop',
+                  regex: '(' + std.join('|',
+                                        [
+                                          'container_blkio_device_usage_total',
+                                        ]) + ');.+',
+                },
               ],
             },
           ],

diff --git a/jsonnet/kube-prometheus/kube-prometheus-kubespray.libsonnet b/jsonnet/kube-prometheus/kube-prometheus-kubespray.libsonnet
@@ -20,15 +20,15 @@ local service(name, namespace, labels, selector, ports) = {
       'kube-controller-manager-prometheus-discovery',
       'kube-system',
       { 'k8s-app': 'kube-controller-manager' },
-      { 'component': 'kube-controller-manager' },
+      { component: 'kube-controller-manager' },
       [{ name: 'https-metrics', port: 10257, targetPort: 10257 }]
     ),
 
     kubeSchedulerPrometheusDiscoveryService: service(
       'kube-scheduler-prometheus-discovery',
       'kube-system',
       { 'k8s-app': 'kube-scheduler' },
-      { 'component': 'kube-scheduler' },
+      { component: 'kube-scheduler' },
       [{ name: 'https-metrics', port: 10259, targetPort: 10259 }],
     ),