merge to main (#107)

1.1.8 Co-authored-by: Mei Wang <meiw@splunk.com> Co-authored-by: foram-splunk <89519924+foram-splunk@users.noreply.github.com> Co-authored-by: hbrewster <hbrewster@splunk.com> Co-authored-by: harshit-splunk <89519921+harshit-splunk@users.noreply.github.com> Co-authored-by: harshit-splunk <hvaghani@splunk.com>
splunk · Dec 16, 2021 · 4c12eeb · 4c12eeb
1 parent 2d88541
commit 4c12eeb
Show file tree

Hide file tree

Showing 6 changed files with 71 additions and 42 deletions.
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-kubernetes-metrics (1.1.7)
+    fluent-plugin-kubernetes-metrics (1.1.8)
       fluentd (>= 1.9.1)
       kubeclient (~> 4.6.0)
       multi_json (~> 1.14.1)

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-1.1.7
+1.1.8
diff --git a/docker/Gemfile.lock b/docker/Gemfile.lock
@@ -1,7 +1,7 @@
 PATH
   remote: gem
   specs:
-    fluent-plugin-kubernetes-metrics (1.1.7)
+    fluent-plugin-kubernetes-metrics (1.1.8)
       fluentd (>= 1.9.1)
       kubeclient (~> 4.6.0)
       multi_json (~> 1.14.1)
@@ -10,14 +10,13 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (6.1.4.1)
-      activesupport (= 6.1.4.1)
-    activesupport (6.1.4.1)
+    activemodel (7.0.0)
+      activesupport (= 7.0.0)
+    activesupport (7.0.0)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-      zeitwerk (~> 2.3)
     addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
     aes_key_wrap (1.1.0)
@@ -36,7 +35,7 @@ GEM
     fluent-plugin-jq (0.5.1)
       fluentd (>= 0.14.10, < 2)
       multi_json (~> 1.13)
-    fluent-plugin-kubernetes_metadata_filter (2.9.2)
+    fluent-plugin-kubernetes_metadata_filter (2.9.3)
       fluentd (>= 0.14.0, < 1.15)
       kubeclient (>= 4.0.0, < 5.0.0)
       lru_redux
@@ -45,16 +44,16 @@ GEM
       prometheus-client (>= 2.1.0)
     fluent-plugin-record-modifier (2.1.0)
       fluentd (>= 1.0, < 2)
-    fluent-plugin-splunk-hec (1.2.7)
+    fluent-plugin-splunk-hec (1.2.9)
       fluentd (>= 1.4)
       multi_json (~> 1.13)
       net-http-persistent (~> 3.1)
       openid_connect (~> 1.1.8)
       prometheus-client (>= 2.1.0)
-    fluentd (1.14.2)
+    fluentd (1.14.3)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
-      http_parser.rb (>= 0.5.1, < 0.8.0)
+      http_parser.rb (>= 0.5.1, < 0.9.0)
       msgpack (>= 1.3.1, < 2.0.0)
       serverengine (>= 2.2.2, < 3.0.0)
       sigdump (~> 0.2.2)
@@ -89,11 +88,11 @@ GEM
     lru_redux (1.1.0)
     mail (2.7.1)
       mini_mime (>= 0.1.1)
-    mime-types (3.3.1)
+    mime-types (3.4.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2021.0901)
+    mime-types-data (3.2021.1115)
     mini_mime (1.1.2)
-    minitest (5.14.4)
+    minitest (5.15.0)
     msgpack (1.4.2)
     multi_json (1.14.1)
     net-http-persistent (3.1.0)

diff --git a/lib/fluent/plugin/in_kubernetes_metrics.rb b/lib/fluent/plugin/in_kubernetes_metrics.rb
@@ -90,8 +90,13 @@ def start
         super
 
         timer_execute :metric_scraper, @interval, &method(:scrape_metrics)
-        timer_execute :stats_metric_scraper, @interval, &method(:scrape_stats_metrics)
         timer_execute :cadvisor_metric_scraper, @interval, &method(:scrape_cadvisor_metrics)
+        # It is done to optionally fetch from 'stats' for k8s version <1.21
+        if is_stats_endpoint_available?
+          timer_execute :stats_metric_scraper, @interval, &method(:scrape_stats_metrics)
+        else
+          log.info "'/stats' endpoint is not available. It has been deprecated since k8s v1.15, disabled since v1.18, and removed in v1.21 and onwards"
+        end
       end
 
       def close
@@ -597,33 +602,35 @@ def emit_stats_metrics(metrics)
         emit_stats_breakdown(metrics['stats']) unless metrics['stats'].nil?
       end
 
+      # Make sure regex has only one capturing group
+      def grep_using_regex(metric, regex)
+        match = metric.match(regex)
+        return nil if match.nil?
+        match[1]
+      end
+
       def emit_cadvisor_metrics(metrics)
         metrics = metrics.split("\n")
         metrics.each do |metric|
-          next unless metric.include? 'container_name='
-
-          next unless metric.match(/^((?!container_name="").)*$/) && metric[0] != '#'
-
+
+          next if metric[0] == '#' or not container_name = grep_using_regex(metric, /container(?:_name)?="([^"]*)"/)
+          next if container_name.empty?
+          
           metric_str, metric_val = metric.split(' ')
           metric_val = metric_val.to_f if metric_val.is_a? String
           first_occur = metric_str.index('{')
           metric_name = metric_str[0..first_occur - 1]
-          pod_name = metric.match(/pod_name="\S*"/).to_s
-          pod_name = pod_name.split('"')[1]
-          image_name = metric.match(/image="\S*"/).to_s
-          image_name = image_name.split('"')[1]
-          namespace = metric.match(/namespace="\S*"/).to_s
-          namespace = namespace.split('"')[1]
+          pod_name = grep_using_regex(metric, /pod(?:_name)?="([^"]*)"/).to_s
+          image_name = grep_using_regex(metric, /image="([^"]*)"/).to_s
+          namespace = grep_using_regex(metric, /namespace="([^"]*)"/).to_s
           metric_labels = { 'pod_name' => pod_name, 'image' => image_name, 'namespace' => namespace, 'value' => metric_val, 'node' => @node_name }
-          if metric =~ /^((?!container_name="POD").)*$/
+          if container_name=="POD"
             tag = 'pod'
             tag = generate_tag("#{tag}#{metric_name.tr('_', '.')}")
             tag = tag.gsub('container', '')
           else
-            container_name = metric.match(/container_name="\S*"/).to_s
-            container_name = container_name.split('"')[1]
             container_label = { 'container_name' => container_name }
-            metric_labels.merge(container_label)
+            metric_labels.merge!(container_label)
             tag = generate_tag(metric_name.tr('_', '.').to_s)
           end
           router.emit tag, @scraped_at_cadvisor, metric_labels
@@ -642,6 +649,20 @@ def scrape_metrics
         end
       end
 
+      def is_stats_endpoint_available?
+        if @use_rest_client
+          response_stats = RestClient::Request.execute request_options_stats
+        else
+          @node_names.each do |node|
+            @node_name = node
+            response_stats = stats_proxy_api(node).get(@client.headers)
+          end
+        end
+        true
+        rescue RestClient::NotFound
+          false
+      end
+
       def scrape_stats_metrics
         if @use_rest_client
           response_stats = RestClient::Request.execute request_options_stats