From 47e55a460eca911ded3ad35d38d75f16c84b5740 Mon Sep 17 00:00:00 2001 From: Philip Gough Date: Mon, 30 Aug 2021 12:24:12 +0100 Subject: [PATCH] jsonnet: Drop cAdvisor metrics with no (pod, namespace) labels while preserving ability to monitor system services resource usage The following provides a description and cardinality estimation based on the tests in a local cluster: container_blkio_device_usage_total - useful for containers, but not for system services (nodes*disks*services*operations*2) container_fs_.* - add filesystem read/write data (nodes*disks*services*4) container_file_descriptors - file descriptors limits and global numbers are exposed via (nodes*services) container_threads_max - max number of threads in cgroup. Usually for system services it is not limited (nodes*services) container_threads - used threads in cgroup. Usually not important for system services (nodes*services) container_sockets - used sockets in cgroup. Usually not important for system services (nodes*services) container_start_time_seconds - container start. Possibly not needed for system services (nodes*services) container_last_seen - Not needed as system services are always running (nodes*services) container_spec_.* - Everything related to cgroup specification and thus static data (nodes*services*5) --- .../kube-prometheus-insecure-kubelet.libsonnet | 17 +++++++++++++++++ .../prometheus/prometheus.libsonnet | 17 +++++++++++++++++ manifests/prometheus-serviceMonitorKubelet.yaml | 6 ++++++ 3 files changed, 40 insertions(+) diff --git a/jsonnet/kube-prometheus/kube-prometheus-insecure-kubelet.libsonnet b/jsonnet/kube-prometheus/kube-prometheus-insecure-kubelet.libsonnet index ab5dceae5c..f0cd85b344 100644 --- a/jsonnet/kube-prometheus/kube-prometheus-insecure-kubelet.libsonnet +++ b/jsonnet/kube-prometheus/kube-prometheus-insecure-kubelet.libsonnet @@ -37,6 +37,23 @@ regex: 'container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s)', action: 'drop', }, + // Drop cAdvisor metrics with no (pod, namespace) labels while preserving ability to monitor system services resource usage (cardinality estimation) + { + sourceLabels: ['__name__', 'pod', 'namespace'], + action: 'drop', + regex: '(' + std.join('|', + [ + 'container_fs_.*', // add filesystem read/write data (nodes*disks*services*4) + 'container_spec_.*', // everything related to cgroup specification and thus static data (nodes*services*5) + 'container_blkio_device_usage_total', // useful for containers, but not for system services (nodes*disks*services*operations*2) + 'container_file_descriptors', // file descriptors limits and global numbers are exposed via (nodes*services) + 'container_sockets', // used sockets in cgroup. Usually not important for system services (nodes*services) + 'container_threads_max', // max number of threads in cgroup. Usually for system services it is not limited (nodes*services) + 'container_threads', // used threads in cgroup. Usually not important for system services (nodes*services) + 'container_start_time_seconds', // container start. Possibly not needed for system services (nodes*services) + 'container_last_seen', // not needed as system services are always running (nodes*services) + ]) + ');;', + }, ], }, ], diff --git a/jsonnet/kube-prometheus/prometheus/prometheus.libsonnet b/jsonnet/kube-prometheus/prometheus/prometheus.libsonnet index 2ab96c53ef..51e09351aa 100644 --- a/jsonnet/kube-prometheus/prometheus/prometheus.libsonnet +++ b/jsonnet/kube-prometheus/prometheus/prometheus.libsonnet @@ -322,6 +322,23 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; regex: 'container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s)', action: 'drop', }, + // Drop cAdvisor metrics with no (pod, namespace) labels while preserving ability to monitor system services resource usage (cardinality estimation) + { + sourceLabels: ['__name__', 'pod', 'namespace'], + action: 'drop', + regex: '(' + std.join('|', + [ + 'container_fs_.*', // add filesystem read/write data (nodes*disks*services*4) + 'container_spec_.*', // everything related to cgroup specification and thus static data (nodes*services*5) + 'container_blkio_device_usage_total', // useful for containers, but not for system services (nodes*disks*services*operations*2) + 'container_file_descriptors', // file descriptors limits and global numbers are exposed via (nodes*services) + 'container_sockets', // used sockets in cgroup. Usually not important for system services (nodes*services) + 'container_threads_max', // max number of threads in cgroup. Usually for system services it is not limited (nodes*services) + 'container_threads', // used threads in cgroup. Usually not important for system services (nodes*services) + 'container_start_time_seconds', // container start. Possibly not needed for system services (nodes*services) + 'container_last_seen', // not needed as system services are always running (nodes*services) + ]) + ');;', + }, ], }, { diff --git a/manifests/prometheus-serviceMonitorKubelet.yaml b/manifests/prometheus-serviceMonitorKubelet.yaml index afa853aa9d..f4529c92f5 100644 --- a/manifests/prometheus-serviceMonitorKubelet.yaml +++ b/manifests/prometheus-serviceMonitorKubelet.yaml @@ -59,6 +59,12 @@ spec: regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) sourceLabels: - __name__ + - action: drop + regex: (container_fs_.*|container_spec_.*|container_blkio_device_usage_total|container_file_descriptors|container_sockets|container_threads_max|container_threads|container_start_time_seconds|container_last_seen);; + sourceLabels: + - __name__ + - pod + - namespace path: /metrics/cadvisor port: https-metrics relabelings: