From c8fb959b6ff5470cbbd725bfad0eb84de0186ea6 Mon Sep 17 00:00:00 2001 From: Philip Gough Date: Tue, 28 Sep 2021 10:30:18 +0100 Subject: [PATCH] Adjust dropped metrics from cAdvisor This change drops pod-centric metrics without a non-empty 'container' label. Previously we dropped pod-centric metrics without a (pod, namespace) label set however these can be critical for debugging. --- .../kube-prometheus-insecure-kubelet.libsonnet | 10 ++++++++-- .../kube-prometheus-kubespray.libsonnet | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/jsonnet/kube-prometheus/kube-prometheus-insecure-kubelet.libsonnet b/jsonnet/kube-prometheus/kube-prometheus-insecure-kubelet.libsonnet index 4ac6773f10..9fd74a9634 100644 --- a/jsonnet/kube-prometheus/kube-prometheus-insecure-kubelet.libsonnet +++ b/jsonnet/kube-prometheus/kube-prometheus-insecure-kubelet.libsonnet @@ -37,9 +37,7 @@ action: 'drop', regex: '(' + std.join('|', [ - 'container_fs_.*', // add filesystem read/write data (nodes*disks*services*4) 'container_spec_.*', // everything related to cgroup specification and thus static data (nodes*services*5) - 'container_blkio_device_usage_total', // useful for containers, but not for system services (nodes*disks*services*operations*2) 'container_file_descriptors', // file descriptors limits and global numbers are exposed via (nodes*services) 'container_sockets', // used sockets in cgroup. Usually not important for system services (nodes*services) 'container_threads_max', // max number of threads in cgroup. Usually for system services it is not limited (nodes*services) @@ -48,6 +46,14 @@ 'container_last_seen', // not needed as system services are always running (nodes*services) ]) + ');;', }, + { + sourceLabels: ['__name__', 'container'], + action: 'drop', + regex: '(' + std.join('|', + [ + 'container_blkio_device_usage_total', + ]) + ');.+', + }, ], }, ], diff --git a/jsonnet/kube-prometheus/kube-prometheus-kubespray.libsonnet b/jsonnet/kube-prometheus/kube-prometheus-kubespray.libsonnet index 1f0048de6a..465a756aca 100644 --- a/jsonnet/kube-prometheus/kube-prometheus-kubespray.libsonnet +++ b/jsonnet/kube-prometheus/kube-prometheus-kubespray.libsonnet @@ -20,7 +20,7 @@ local service(name, namespace, labels, selector, ports) = { 'kube-controller-manager-prometheus-discovery', 'kube-system', { 'k8s-app': 'kube-controller-manager' }, - { 'component': 'kube-controller-manager' }, + { component: 'kube-controller-manager' }, [{ name: 'https-metrics', port: 10257, targetPort: 10257 }] ), @@ -28,7 +28,7 @@ local service(name, namespace, labels, selector, ports) = { 'kube-scheduler-prometheus-discovery', 'kube-system', { 'k8s-app': 'kube-scheduler' }, - { 'component': 'kube-scheduler' }, + { component: 'kube-scheduler' }, [{ name: 'https-metrics', port: 10259, targetPort: 10259 }], ),