Queue and Poller metrics

wpa_queue_messages
wpa_queue_messages_sent_per_minute
wpa_worker_idle
wpa_worker_current
wpa_worker_desired

README.md

@@ -2,7 +2,7 @@
 
 [![GoDoc Widget]][GoDoc] [![CI Status](https://api.travis-ci.com/practo/k8s-worker-pod-autoscaler.svg?token=yTs54joHywqVVdshXhPm&branch=master)](https://travis-ci.com/practo/k8s-worker-pod-autoscaler)
 
-<img src="/artifacts/images/worker-pod-autoscaler.png" width="120">
+<img src="/artifacts/images/wpa.png" width="120">
 
 ----
 
@@ -152,9 +152,23 @@ wpa_controller_loop_duration_seconds{workerpodautoscaler="example-wpa", namespac
 wpa_log_messages_total{severity="ERROR"} 0
 wpa_log_messages_total{severity="WARNING"} 0
 
+wpa_queue_messages{workerpodautoscaler="example-wpa", namespace="example-namespace", queueName="example-q"} 87
+wpa_queue_messages_sent_per_minute{workerpodautoscaler="example-wpa", namespace="example-namespace", queueName="example-q"} 2007
+
+wpa_queue_messages{workerpodautoscaler="example-wpa", namespace="example-namespace", queueName="example-q"} 87
+wpa_queue_messages_sent_per_minute{workerpodautoscaler="example-wpa", namespace="example-namespace", queueName="example-q"} 2007
+
+wpa_worker_current{workerpodautoscaler="example-wpa", namespace="example-namespace", queueName="example-q"} 27
+wpa_worker_desired{workerpodautoscaler="example-wpa", namespace="example-namespace", queueName="example-q"} 5
+wpa_worker_idle{workerpodautoscaler="example-wpa", namespace="example-namespace", queueName="example-q"} 0
+
 go_goroutines{endpoint="workerpodautoscaler-metrics"} 40
 ```
 
+Using these metrics, scaling trends can be better analysed, comparing the Replicas Vs Queue:
+
+<img src="/artifacts/images/wpa-queue-worker-metrics-dashboard.png" width="700" height="280">
+
 If you have [ServiceMonitor](https://github.com/coreos/prometheus-operator/blob/master/Documentation/user-guides/getting-started.md) installed in your cluster. You can bring these metrics to Prometheus by running the following:
 ```
 kubectl create -f artifacts/service.yaml

pkg/controller/controller.go

@@ -76,11 +76,66 @@ var (
 		},
 		[]string{"workerpodautoscaler", "namespace"},
 	)
+
+	qMsgs = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "wpa",
+			Subsystem: "queue",
+			Name:      "messages",
+			Help:      "Number of unprocessed messages in the queue",
+		},
+		[]string{"workerpodautoscaler", "namespace", "queueName"},
+	)
+
+	qMsgsSPM = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "wpa",
+			Subsystem: "queue",
+			Name:      "messages_sent_per_minute",
+			Help:      "Number of messages sent to the queue per minute",
+		},
+		[]string{"workerpodautoscaler", "namespace", "queueName"},
+	)
+
+	workersIdle = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "wpa",
+			Subsystem: "worker",
+			Name:      "idle",
+			Help:      "Number of idle workers",
+		},
+		[]string{"workerpodautoscaler", "namespace", "queueName"},
+	)
+
+	workersCurrent = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "wpa",
+			Subsystem: "worker",
+			Name:      "current",
+			Help:      "Number of current workers",
+		},
+		[]string{"workerpodautoscaler", "namespace", "queueName"},
+	)
+
+	workersDesired = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "wpa",
+			Subsystem: "worker",
+			Name:      "desired",
+			Help:      "Number of desired workers",
+		},
+		[]string{"workerpodautoscaler", "namespace", "queueName"},
+	)
 )
 
 func init() {
 	prometheus.MustRegister(loopDurationSeconds)
 	prometheus.MustRegister(loopCountSuccess)
+	prometheus.MustRegister(qMsgs)
+	prometheus.MustRegister(qMsgsSPM)
+	prometheus.MustRegister(workersIdle)
+	prometheus.MustRegister(workersCurrent)
+	prometheus.MustRegister(workersDesired)
 }
 
 type WokerPodAutoScalerEvent struct {
@@ -355,6 +410,33 @@ func (c *Controller) syncHandler(event WokerPodAutoScalerEvent) error {
 	)
 	klog.Infof("%s: messages: %d, idle: %d, desired: %d", queueName, queueMessages, idleWorkers, desiredWorkers)
 
+	// set metrics
+	qMsgs.WithLabelValues(
+		name,
+		namespace,
+		queueName,
+	).Set(float64(queueMessages))
+	qMsgsSPM.WithLabelValues(
+		name,
+		namespace,
+		queueName,
+	).Set(messagesSentPerMinute)
+	workersIdle.WithLabelValues(
+		name,
+		namespace,
+		queueName,
+	).Set(float64(idleWorkers))
+	workersCurrent.WithLabelValues(
+		name,
+		namespace,
+		queueName,
+	).Set(float64(deployment.Status.AvailableReplicas))
+	workersDesired.WithLabelValues(
+		name,
+		namespace,
+		queueName,
+	).Set(float64(desiredWorkers))
+
 	if desiredWorkers != *deployment.Spec.Replicas {
 		c.updateDeployment(workerPodAutoScaler.Namespace, deploymentName, &desiredWorkers)
 	}