From 4c12eeb1d29f9e30580c0be22ea1cf484198dc98 Mon Sep 17 00:00:00 2001 From: Rock Baek Date: Thu, 16 Dec 2021 01:09:17 -0800 Subject: [PATCH] merge to main (#107) 1.1.8 Co-authored-by: Mei Wang Co-authored-by: foram-splunk <89519924+foram-splunk@users.noreply.github.com> Co-authored-by: hbrewster Co-authored-by: harshit-splunk <89519921+harshit-splunk@users.noreply.github.com> Co-authored-by: harshit-splunk --- Gemfile.lock | 2 +- VERSION | 2 +- docker/Gemfile.lock | 23 +++++----- lib/fluent/plugin/in_kubernetes_metrics.rb | 51 +++++++++++++++------- test/metrics_cadvisor.txt | 8 ++++ test/plugin/test_in_kubernetes_metrics.rb | 27 ++++++------ 6 files changed, 71 insertions(+), 42 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 5f60be1..aebbcab 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - fluent-plugin-kubernetes-metrics (1.1.7) + fluent-plugin-kubernetes-metrics (1.1.8) fluentd (>= 1.9.1) kubeclient (~> 4.6.0) multi_json (~> 1.14.1) diff --git a/VERSION b/VERSION index a5ba932..db15278 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.7 \ No newline at end of file +1.1.8 \ No newline at end of file diff --git a/docker/Gemfile.lock b/docker/Gemfile.lock index dc78522..05491a5 100644 --- a/docker/Gemfile.lock +++ b/docker/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: gem specs: - fluent-plugin-kubernetes-metrics (1.1.7) + fluent-plugin-kubernetes-metrics (1.1.8) fluentd (>= 1.9.1) kubeclient (~> 4.6.0) multi_json (~> 1.14.1) @@ -10,14 +10,13 @@ PATH GEM remote: https://rubygems.org/ specs: - activemodel (6.1.4.1) - activesupport (= 6.1.4.1) - activesupport (6.1.4.1) + activemodel (7.0.0) + activesupport (= 7.0.0) + activesupport (7.0.0) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) tzinfo (~> 2.0) - zeitwerk (~> 2.3) addressable (2.8.0) public_suffix (>= 2.0.2, < 5.0) aes_key_wrap (1.1.0) @@ -36,7 +35,7 @@ GEM fluent-plugin-jq (0.5.1) fluentd (>= 0.14.10, < 2) multi_json (~> 1.13) - fluent-plugin-kubernetes_metadata_filter (2.9.2) + fluent-plugin-kubernetes_metadata_filter (2.9.3) fluentd (>= 0.14.0, < 1.15) kubeclient (>= 4.0.0, < 5.0.0) lru_redux @@ -45,16 +44,16 @@ GEM prometheus-client (>= 2.1.0) fluent-plugin-record-modifier (2.1.0) fluentd (>= 1.0, < 2) - fluent-plugin-splunk-hec (1.2.7) + fluent-plugin-splunk-hec (1.2.9) fluentd (>= 1.4) multi_json (~> 1.13) net-http-persistent (~> 3.1) openid_connect (~> 1.1.8) prometheus-client (>= 2.1.0) - fluentd (1.14.2) + fluentd (1.14.3) bundler cool.io (>= 1.4.5, < 2.0.0) - http_parser.rb (>= 0.5.1, < 0.8.0) + http_parser.rb (>= 0.5.1, < 0.9.0) msgpack (>= 1.3.1, < 2.0.0) serverengine (>= 2.2.2, < 3.0.0) sigdump (~> 0.2.2) @@ -89,11 +88,11 @@ GEM lru_redux (1.1.0) mail (2.7.1) mini_mime (>= 0.1.1) - mime-types (3.3.1) + mime-types (3.4.1) mime-types-data (~> 3.2015) - mime-types-data (3.2021.0901) + mime-types-data (3.2021.1115) mini_mime (1.1.2) - minitest (5.14.4) + minitest (5.15.0) msgpack (1.4.2) multi_json (1.14.1) net-http-persistent (3.1.0) diff --git a/lib/fluent/plugin/in_kubernetes_metrics.rb b/lib/fluent/plugin/in_kubernetes_metrics.rb index 5bd257b..1face14 100644 --- a/lib/fluent/plugin/in_kubernetes_metrics.rb +++ b/lib/fluent/plugin/in_kubernetes_metrics.rb @@ -90,8 +90,13 @@ def start super timer_execute :metric_scraper, @interval, &method(:scrape_metrics) - timer_execute :stats_metric_scraper, @interval, &method(:scrape_stats_metrics) timer_execute :cadvisor_metric_scraper, @interval, &method(:scrape_cadvisor_metrics) + # It is done to optionally fetch from 'stats' for k8s version <1.21 + if is_stats_endpoint_available? + timer_execute :stats_metric_scraper, @interval, &method(:scrape_stats_metrics) + else + log.info "'/stats' endpoint is not available. It has been deprecated since k8s v1.15, disabled since v1.18, and removed in v1.21 and onwards" + end end def close @@ -597,33 +602,35 @@ def emit_stats_metrics(metrics) emit_stats_breakdown(metrics['stats']) unless metrics['stats'].nil? end + # Make sure regex has only one capturing group + def grep_using_regex(metric, regex) + match = metric.match(regex) + return nil if match.nil? + match[1] + end + def emit_cadvisor_metrics(metrics) metrics = metrics.split("\n") metrics.each do |metric| - next unless metric.include? 'container_name=' - - next unless metric.match(/^((?!container_name="").)*$/) && metric[0] != '#' - + + next if metric[0] == '#' or not container_name = grep_using_regex(metric, /container(?:_name)?="([^"]*)"/) + next if container_name.empty? + metric_str, metric_val = metric.split(' ') metric_val = metric_val.to_f if metric_val.is_a? String first_occur = metric_str.index('{') metric_name = metric_str[0..first_occur - 1] - pod_name = metric.match(/pod_name="\S*"/).to_s - pod_name = pod_name.split('"')[1] - image_name = metric.match(/image="\S*"/).to_s - image_name = image_name.split('"')[1] - namespace = metric.match(/namespace="\S*"/).to_s - namespace = namespace.split('"')[1] + pod_name = grep_using_regex(metric, /pod(?:_name)?="([^"]*)"/).to_s + image_name = grep_using_regex(metric, /image="([^"]*)"/).to_s + namespace = grep_using_regex(metric, /namespace="([^"]*)"/).to_s metric_labels = { 'pod_name' => pod_name, 'image' => image_name, 'namespace' => namespace, 'value' => metric_val, 'node' => @node_name } - if metric =~ /^((?!container_name="POD").)*$/ + if container_name=="POD" tag = 'pod' tag = generate_tag("#{tag}#{metric_name.tr('_', '.')}") tag = tag.gsub('container', '') else - container_name = metric.match(/container_name="\S*"/).to_s - container_name = container_name.split('"')[1] container_label = { 'container_name' => container_name } - metric_labels.merge(container_label) + metric_labels.merge!(container_label) tag = generate_tag(metric_name.tr('_', '.').to_s) end router.emit tag, @scraped_at_cadvisor, metric_labels @@ -642,6 +649,20 @@ def scrape_metrics end end + def is_stats_endpoint_available? + if @use_rest_client + response_stats = RestClient::Request.execute request_options_stats + else + @node_names.each do |node| + @node_name = node + response_stats = stats_proxy_api(node).get(@client.headers) + end + end + true + rescue RestClient::NotFound + false + end + def scrape_stats_metrics if @use_rest_client response_stats = RestClient::Request.execute request_options_stats diff --git a/test/metrics_cadvisor.txt b/test/metrics_cadvisor.txt index c19c85e..bda90aa 100644 --- a/test/metrics_cadvisor.txt +++ b/test/metrics_cadvisor.txt @@ -907,6 +907,7 @@ container_network_receive_bytes_total{container_name="POD",id="/kubepods/burstab container_network_receive_bytes_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 1.786404787e+09 container_network_receive_bytes_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 1.786399729e+09 container_network_receive_bytes_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 1.786393994e+09 +container_network_receive_bytes_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad00container_network_receive_errors_total4a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 1.786399238e+09 # HELP container_network_receive_errors_total Cumulative count of errors encountered while receiving # TYPE container_network_receive_errors_total counter container_network_receive_errors_total{container_name="",id="/",image="",interface="eth0",name="",namespace="",pod_name=""} 0 @@ -918,6 +919,7 @@ container_network_receive_errors_total{container_name="POD",id="/kubepods/bursta container_network_receive_errors_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 0 container_network_receive_errors_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 0 container_network_receive_errors_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 0 +container_network_receive_errors_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad004a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 0 # HELP container_network_receive_packets_dropped_total Cumulative count of packets dropped while receiving # TYPE container_network_receive_packets_dropped_total counter container_network_receive_packets_dropped_total{container_name="",id="/",image="",interface="eth0",name="",namespace="",pod_name=""} 0 @@ -929,6 +931,7 @@ container_network_receive_packets_dropped_total{container_name="POD",id="/kubepo container_network_receive_packets_dropped_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 0 container_network_receive_packets_dropped_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 0 container_network_receive_packets_dropped_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 0 +container_network_receive_packets_dropped_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad004a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 0 # HELP container_network_receive_packets_total Cumulative count of packets received # TYPE container_network_receive_packets_total counter container_network_receive_packets_total{container_name="",id="/",image="",interface="eth0",name="",namespace="",pod_name=""} 7.074054e+06 @@ -940,6 +943,7 @@ container_network_receive_packets_total{container_name="POD",id="/kubepods/burst container_network_receive_packets_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 7.073992e+06 container_network_receive_packets_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 7.073969e+06 container_network_receive_packets_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 7.073917e+06 +container_network_receive_packets_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad004a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 7.073237e+06 # HELP container_network_tcp_usage_total tcp connection usage statistic for container # TYPE container_network_tcp_usage_total gauge container_network_tcp_usage_total{container_name="",id="/",image="",name="",namespace="",pod_name="",tcp_state="close"} 0 @@ -1250,6 +1254,7 @@ container_network_transmit_bytes_total{container_name="POD",id="/kubepods/bursta container_network_transmit_bytes_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 3.253087401e+09 container_network_transmit_bytes_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 3.253065965e+09 container_network_transmit_bytes_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 3.253037394e+09 +container_network_transmit_bytes_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad004a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 3.253037394e+09 # HELP container_network_transmit_errors_total Cumulative count of errors encountered while transmitting # TYPE container_network_transmit_errors_total counter container_network_transmit_errors_total{container_name="",id="/",image="",interface="eth0",name="",namespace="",pod_name=""} 0 @@ -1261,6 +1266,7 @@ container_network_transmit_errors_total{container_name="POD",id="/kubepods/burst container_network_transmit_errors_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 0 container_network_transmit_errors_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 0 container_network_transmit_errors_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 0 +container_network_transmit_errors_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad004a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 0 # HELP container_network_transmit_packets_dropped_total Cumulative count of packets dropped while transmitting # TYPE container_network_transmit_packets_dropped_total counter container_network_transmit_packets_dropped_total{container_name="",id="/",image="",interface="eth0",name="",namespace="",pod_name=""} 0 @@ -1272,6 +1278,7 @@ container_network_transmit_packets_dropped_total{container_name="POD",id="/kubep container_network_transmit_packets_dropped_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 0 container_network_transmit_packets_dropped_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 0 container_network_transmit_packets_dropped_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 0 +container_network_transmit_packets_dropped_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad004a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 0 # HELP container_network_transmit_packets_total Cumulative count of packets transmitted # TYPE container_network_transmit_packets_total counter container_network_transmit_packets_total{container_name="",id="/",image="",interface="eth0",name="",namespace="",pod_name=""} 7.727546e+06 @@ -1283,6 +1290,7 @@ container_network_transmit_packets_total{container_name="POD",id="/kubepods/burs container_network_transmit_packets_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 7.727485e+06 container_network_transmit_packets_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 7.727451e+06 container_network_transmit_packets_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 7.727389e+06 +container_network_transmit_packets_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad004a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 7.727481e+06 # HELP container_network_udp_usage_total udp connection usage statistic for container # TYPE container_network_udp_usage_total gauge container_network_udp_usage_total{container_name="",id="/",image="",name="",namespace="",pod_name="",udp_state="dropped"} 0 diff --git a/test/plugin/test_in_kubernetes_metrics.rb b/test/plugin/test_in_kubernetes_metrics.rb index 3b2431d..7ae81a2 100644 --- a/test/plugin/test_in_kubernetes_metrics.rb +++ b/test/plugin/test_in_kubernetes_metrics.rb @@ -35,6 +35,9 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase ).freeze setup do + stub_k8s_requests + + return unless @@hash_map_test.empty? Fluent::Test.setup @@parsed_unit_string = JSON.parse(get_unit_parsed_string) @@ -45,8 +48,6 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase get_cadvisor_parsed_string = f.read end.close - stub_k8s_requests - @@ca_driver = create_driver @@ca_driver.run timeout: 20, expect_emits: 1, shutdown: true @@ -56,8 +57,12 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase metrics = get_cadvisor_parsed_string.split("\n") metrics.each do |metric| next unless metric.include? 'container_name=' + next unless metric[0] != '#' - next unless metric.match(/^((?!container_name="").)*$/) && metric[0] != '#' + container_name = metric.match(/container_name="\S*"/).to_s + container_name = container_name.split('"')[1] + + next if container_name.empty? metric_str, metric_val = metric.split(' ') metric_val = metric_val.to_f if metric_val.is_a? String @@ -70,13 +75,11 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase namespace = metric.match(/namespace="\S*"/).to_s namespace = namespace.split('"')[1] metric_labels = { 'pod_name' => pod_name, 'image' => image_name, 'namespace' => namespace, 'value' => metric_val, 'node' => @node_name } - if metric =~ /^((?!container_name="POD").)*$/ + if container_name == 'POD' tag = 'pod' tag = generate_tag("#{tag}#{metric_name.tr('_', '.')}", @@driver.instance.tag) tag = tag.gsub('container', '') else - container_name = metric.match(/container_name="\S*"/).to_s - container_name = container_name.split('"')[1] container_label = { 'container_name' => container_name } metric_labels.merge(container_label) tag = generate_tag(metric_name.tr('_', '.').to_s, @@driver.instance.tag) @@ -252,11 +255,10 @@ def create_driver(conf = CONFIG) assert_true @@hash_map_cadvisor.key?('kube.container.fs.read.seconds.total') assert_equal @@hash_map_cadvisor['kube.container.fs.read.seconds.total'], @@hash_map_test['kube.container.fs.read.seconds.total'][2]['value'] end - - # TODO: Current Test does not work - metric present in metrics_cadvisor.txt but not being parsed by connector in test/working in production + test 'Test - metrics cadvisor: container_fs_reads_bytes_total' do - assert_false @@hash_map_cadvisor.key?('kube.container.fs.reads.bytes.total') - # assert_equal @@hash_map_cadvisor['kube.container.fs.reads.bytes.total'], @@hash_map_test["kube.container.fs.reads.bytes.total"][2]["value"] + assert_true @@hash_map_cadvisor.key?('kube.container.fs.reads.bytes.total') + assert_equal @@hash_map_cadvisor['kube.container.fs.reads.bytes.total'], @@hash_map_test["kube.container.fs.reads.bytes.total"][2]["value"] end test 'Test - metrics cadvisor: container_fs_reads_merged_total' do @@ -289,10 +291,9 @@ def create_driver(conf = CONFIG) assert_equal @@hash_map_cadvisor['kube.container.fs.write.seconds.total'], @@hash_map_test['kube.container.fs.write.seconds.total'][2]['value'] end - # TODO: Current Test does not work - metric present in metrics_cadvisor.txt but not being parsed by connector in test/working in production test 'Test - metrics cadvisor: container_fs_writes_bytes_total' do - assert_false @@hash_map_cadvisor.key?('kube.container.fs.writes.bytes.total') - # assert_equal @@hash_map_cadvisor['kube.container.fs.writes.bytes.total'], @@hash_map_test["kube.container.fs.writes.bytes.total"][2]["value"] + assert_true @@hash_map_cadvisor.key?('kube.container.fs.writes.bytes.total') + assert_equal @@hash_map_cadvisor['kube.container.fs.writes.bytes.total'], @@hash_map_test["kube.container.fs.writes.bytes.total"][2]["value"] end test 'Test - metrics cadvisor: container_fs_writes_merged_total' do