Add the job annotation to all alloy instances and ksm. (#1087)

Make the job annotation configurable again

Signed-off-by: Pete Wall <pete.wall@grafana.com>

Author: petewall

charts/k8s-monitoring/charts/feature-cluster-metrics/values.schema.json

@@ -313,6 +313,14 @@
                         }
                     }
                 },
+                "podAnnotations": {
+                    "type": "object",
+                    "properties": {
+                        "k8s.grafana.com/logs.job": {
+                            "type": "string"
+                        }
+                    }
+                },
                 "prometheusScrape": {
                     "type": "boolean"
                 },

charts/k8s-monitoring/charts/feature-cluster-metrics/values.yaml

@@ -449,6 +449,10 @@ kube-state-metrics:
   nodeSelector:
     kubernetes.io/os: linux
 
+  # @ignored
+  podAnnotations:
+    k8s.grafana.com/logs.job: integrations/kubernetes/kube-state-metrics
+
   # @ignored - Enable the release label
   releaseLabel: true
 

charts/k8s-monitoring/charts/feature-pod-logs/templates/_common_pod_discovery.alloy.tpl

@@ -42,7 +42,7 @@ discovery.relabel "filtered_pods" {
 
   // set the job label from the k8s.grafana.com/logs.job annotation if it exists
   rule {
-    source_labels = ["{{ include "pod_annotation" "k8s.grafana.com/logs.job" }}"]
+    source_labels = ["{{ include "pod_annotation" .Values.annotations.job }}"]
     regex = "(.+)"
     target_label = "job"
   }