Add histogram to track number samples/metadata/span sent per request

This commit also removes the metric promscale_ingest_items_received_total which is not used anywhere and the same information can be now obtained using the newly introduced `promscale_ingest_items_received_*` histogram metric. Signed-off-by: Arunprasad Rajkumar <ar.arunprasad@gmail.com>
timescale · Nov 29, 2022 · d028f46 · d028f46
1 parent ac1cf99
commit d028f46
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 8 deletions.
diff --git a/pkg/api/metrics.go b/pkg/api/metrics.go
@@ -26,8 +26,8 @@ type Metrics struct {
 func updateIngestMetrics(code string, duration, receivedSamples, receivedMetadata float64) {
 	pgMetrics.IngestorRequests.With(prometheus.Labels{"type": "metric", "code": code}).Inc()
 	pgMetrics.IngestorDuration.With(prometheus.Labels{"type": "metric", "code": code}).Observe(duration)
-	pgMetrics.IngestorItemsReceived.With(prometheus.Labels{"type": "metric", "kind": "sample"}).Add(receivedSamples)
-	pgMetrics.IngestorItemsReceived.With(prometheus.Labels{"type": "metric", "kind": "metadata"}).Add(receivedMetadata)
+	pgMetrics.IngestorItemsReceived.With(prometheus.Labels{"type": "metric", "kind": "sample"}).Observe(receivedSamples)
+	pgMetrics.IngestorItemsReceived.With(prometheus.Labels{"type": "metric", "kind": "metadata"}).Observe(receivedMetadata)
 }
 
 func updateQueryMetrics(handler, code string, duration float64) {

diff --git a/pkg/pgmodel/ingestor/trace/writer.go b/pkg/pgmodel/ingestor/trace/writer.go
@@ -139,7 +139,7 @@ func (t *traceWriterImpl) InsertTraces(ctx context.Context, traces ptrace.Traces
 	startIngest := time.Now() // Time taken for complete ingestion => Processing + DB insert.
 	code := "500"
 	metrics.IngestorActiveWriteRequests.With(traceSpanLabel).Inc()
-	metrics.IngestorItemsReceived.With(traceSpanLabel).Add(float64(traces.SpanCount()))
+	metrics.IngestorItemsReceived.With(traceSpanLabel).Observe(float64(traces.SpanCount()))
 	defer func() {
 		metrics.IngestorDuration.With(prometheus.Labels{"type": "trace", "code": code}).Observe(time.Since(startIngest).Seconds())
 		metrics.IngestorActiveWriteRequests.With(traceSpanLabel).Dec()

diff --git a/pkg/pgmodel/metrics/ingest.go b/pkg/pgmodel/metrics/ingest.go
@@ -141,13 +141,15 @@ var (
 			Help:      "Total number of items (sample/metadata/span) ingested",
 		}, []string{"type", "kind", "subsystem"},
 	)
-	IngestorItemsReceived = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
+	IngestorItemsReceived = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
 			Namespace: util.PromNamespace,
 			Subsystem: "ingest",
-			Name:      "items_received_total",
-			Help:      "Total items (samples/exemplars/spans) received.",
-		}, []string{"type", "kind"},
+			Name:      "items_received",
+			Help:      "Number of (samples/exemplars/spans) received",
+			Buckets:   []float64{10, 50, 100, 200, 500, 1000, 2000, 4000, 6000, 8000, 10000, 20000, 30000},
+		},
+		[]string{"type", "kind"},
 	)
 	IngestorBytes = prometheus.NewCounterVec(
 		prometheus.CounterOpts{