diff --git a/chart/Chart.yaml b/chart/Chart.yaml index 4f28d9db..c4051db3 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -12,7 +12,7 @@ keywords: - monitoring - tracing - opentelemetry -version: 17.12.2 +version: 17.13.0 # TODO(paulfantom): Enable after kubernetes 1.22 reaches EOL (2022-10-28) # kubeVersion: ">= 1.23.0" dependencies: diff --git a/chart/dashboards/promscale.json b/chart/dashboards/promscale.json index 18aa03c5..ac538460 100644 --- a/chart/dashboards/promscale.json +++ b/chart/dashboards/promscale.json @@ -58,6 +58,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 2, + "id": 31, "links": [], "liveNow": false, "panels": [ @@ -127,7 +128,7 @@ }, "showHeader": true }, - "pluginVersion": "9.2.2", + "pluginVersion": "9.2.3", "targets": [ { "datasource": { @@ -263,10 +264,12 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "sum by (kind) (rate(promscale_ingest_items_total{namespace=~\"$namespace\"}[$__rate_interval]))", + "expr": "sum by (kind) (rate(promscale_ingest_items_total{namespace=~\"$namespace\",type=~\"$datatype\"}[$__rate_interval]))", "interval": "", "legendFormat": "{{ kind }}", + "range": true, "refId": "A" } ], @@ -346,7 +349,7 @@ }, "textMode": "auto" }, - "pluginVersion": "9.2.2", + "pluginVersion": "9.2.3", "targets": [ { "datasource": { @@ -388,6 +391,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -420,7 +425,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -465,7 +471,7 @@ "refId": "A" } ], - "title": "Requests to Ingestor", + "title": "Requests", "type": "timeseries" }, { @@ -479,6 +485,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -511,7 +519,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -549,14 +558,16 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, "expr": "sum by (type) (rate(promscale_ingest_requests_total{code=~\"5..\",namespace=~\"$namespace\"}[$__rate_interval]))", "interval": "", "legendFormat": "{{ type }}", + "range": true, "refId": "A" } ], - "title": "Errors (HTTP)", + "title": "Errors", "type": "timeseries" }, { @@ -570,6 +581,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -603,7 +616,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -644,7 +658,7 @@ "exemplar": true, "expr": "histogram_quantile(0.5, rate(promscale_ingest_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval]))", "interval": "", - "legendFormat": "p50 {{ type }}", + "legendFormat": "p50", "refId": "A" }, { @@ -656,7 +670,7 @@ "expr": "histogram_quantile(0.90, rate(promscale_ingest_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval]))", "hide": false, "interval": "", - "legendFormat": "p90 {{ type }}", + "legendFormat": "p90", "refId": "B" }, { @@ -665,16 +679,30 @@ "uid": "${datasource}" }, "exemplar": true, - "expr": "histogram_quantile(0.90, rate(promscale_ingest_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval]))", + "expr": "histogram_quantile(0.95, rate(promscale_ingest_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval]))", "hide": false, "interval": "", - "legendFormat": "p95 {{ type }}", + "legendFormat": "p95", "refId": "C" } ], - "title": "Duration (HTTP)", + "title": "Duration", "type": "timeseries" - }, + } + ], + "title": "Ingest - metric", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 64, + "panels": [ { "datasource": { "type": "prometheus", @@ -686,6 +714,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -719,7 +749,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -735,7 +766,7 @@ "h": 9, "w": 8, "x": 0, - "y": 19 + "y": 11 }, "id": 9, "interval": "2m", @@ -757,14 +788,16 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, "expr": "rate(grpc_server_msg_received_total{grpc_method=~\"(WriteSpan|WriteSpanStream|Export)\",namespace=~\"$namespace\"}[$__rate_interval])", "interval": "", "legendFormat": "{{ grpc_service }}", + "range": true, "refId": "A" } ], - "title": "Requests (gRPC)", + "title": "Requests", "type": "timeseries" }, { @@ -778,6 +811,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -811,7 +846,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -827,7 +863,7 @@ "h": 9, "w": 8, "x": 8, - "y": 19 + "y": 11 }, "id": 7, "interval": "2m", @@ -856,7 +892,7 @@ "refId": "A" } ], - "title": "Errors (gRPC)", + "title": "Errors", "type": "timeseries" }, { @@ -870,6 +906,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -903,7 +941,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -919,7 +958,7 @@ "h": 9, "w": 8, "x": 16, - "y": 19 + "y": 11 }, "id": 11, "interval": "2m", @@ -953,7 +992,7 @@ "uid": "${datasource}" }, "exemplar": true, - "expr": "histogram_quantile(0.5, rate(grpc_server_handling_seconds_bucket{grpc_service=\"opentelemetry.proto.collector.trace.v1.TraceService\",grpc_method=~\"Export\",namespace=~\"$namespace\"}[$__rate_interval]))", + "expr": "histogram_quantile(0.9, rate(grpc_server_handling_seconds_bucket{grpc_service=\"opentelemetry.proto.collector.trace.v1.TraceService\",grpc_method=~\"Export\",namespace=~\"$namespace\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "p90 {{ type }}", @@ -965,18 +1004,18 @@ "uid": "${datasource}" }, "exemplar": true, - "expr": "histogram_quantile(0.5, rate(grpc_server_handling_seconds_bucket{grpc_service=\"opentelemetry.proto.collector.trace.v1.TraceService\",grpc_method=~\"Export\",namespace=~\"$namespace\"}[$__rate_interval]))", + "expr": "histogram_quantile(0.95, rate(grpc_server_handling_seconds_bucket{grpc_service=\"opentelemetry.proto.collector.trace.v1.TraceService\",grpc_method=~\"Export\",namespace=~\"$namespace\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "p95 {{ type }}", "refId": "C" } ], - "title": "Duration (gRPC)", + "title": "Duration", "type": "timeseries" } ], - "title": "Ingest", + "title": "Ingest - trace", "type": "row" }, { @@ -989,7 +1028,7 @@ "h": 1, "w": 24, "x": 0, - "y": 10 + "y": 11 }, "id": 32, "panels": [ @@ -1004,6 +1043,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -1036,7 +1077,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1052,7 +1094,7 @@ "h": 9, "w": 8, "x": 0, - "y": 11 + "y": 19 }, "id": 43, "interval": "2m", @@ -1081,7 +1123,7 @@ "refId": "A" } ], - "title": "Requests (HTTP)", + "title": "Requests", "type": "timeseries" }, { @@ -1095,6 +1137,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -1127,7 +1171,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1143,7 +1188,7 @@ "h": 9, "w": 8, "x": 8, - "y": 11 + "y": 19 }, "id": 44, "interval": "2m", @@ -1172,7 +1217,7 @@ "refId": "A" } ], - "title": "Errors (HTTP)", + "title": "Errors", "type": "timeseries" }, { @@ -1186,6 +1231,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -1219,7 +1266,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1235,7 +1283,7 @@ "h": 9, "w": 8, "x": 16, - "y": 11 + "y": 19 }, "id": 45, "interval": "2m", @@ -1274,9 +1322,21 @@ "interval": "", "legendFormat": "p90", "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "histogram_quantile(0.95, sum by (le, instance, job) (rate(promscale_query_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", + "hide": false, + "interval": "", + "legendFormat": "p95", + "refId": "C" } ], - "title": "Duration (HTTP)", + "title": "Duration", "type": "timeseries" } ], @@ -1284,7 +1344,7 @@ "type": "row" }, { - "collapsed": false, + "collapsed": true, "datasource": { "type": "prometheus", "uid": "${datasource}" @@ -1293,929 +1353,930 @@ "h": 1, "w": 24, "x": 0, - "y": 11 + "y": 12 }, "id": 19, - "panels": [], - "title": "Database", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "mappings": [ - { - "options": { - "0": { - "color": "green", - "index": 0, - "text": "OK" + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "green", + "index": 0, + "text": "OK" + }, + "1": { + "color": "red", + "index": 2, + "text": "FAILED" + } + }, + "type": "value" }, - "1": { - "color": "red", - "index": 2, - "text": "FAILED" + { + "options": { + "match": "null+nan", + "result": { + "color": "orange", + "index": 1, + "text": "UNKNOWN" + } + }, + "type": "special" } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] }, - "type": "value" + "unit": "none" }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 20 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.3", + "targets": [ { - "options": { - "match": "null+nan", - "result": { - "color": "orange", - "index": 1, - "text": "UNKNOWN" - } + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "type": "special" + "exemplar": true, + "expr": "max(promscale_sql_database_worker_maintenance_job_failed{namespace=~\"$namespace\"})", + "interval": "", + "legendFormat": "", + "refId": "A" } ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 0, - "y": 12 - }, - "id": 24, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "title": "Last maintenance job status", + "type": "stat" }, - "textMode": "auto" - }, - "pluginVersion": "9.2.2", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "exemplar": true, - "expr": "max(promscale_sql_database_worker_maintenance_job_failed{namespace=~\"$namespace\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Last maintenance job status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 10, - "x": 4, - "y": 12 - }, - "id": 21, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] }, - "exemplar": true, - "expr": "min by (job, instance) (promscale_sql_database_chunks_compressed_count{namespace=~\"$namespace\"})\n/\nmax by (job, instance)(promscale_sql_database_chunks_count{namespace=~\"$namespace\"})", - "interval": "", - "legendFormat": "{{ instance }}", - "refId": "A" - } - ], - "title": "Compressed Chunks", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "gridPos": { + "h": 9, + "w": 10, + "x": 4, + "y": 20 }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 10, - "x": 14, - "y": 12 - }, - "id": 49, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max by (job, instance) (promscale_sql_database_chunks_metrics_expired_count{namespace=~\"$namespace\"})", - "interval": "", - "legendFormat": "metrics-expired", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max by (job, instance) (promscale_sql_database_chunks_metrics_uncompressed_count{namespace=~\"$namespace\"})", - "hide": false, - "interval": "", - "legendFormat": "metrics-uncompressed", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max by (job, instance) (promscale_sql_database_chunks_traces_expired_count{namespace=~\"$namespace\"})", - "hide": false, - "interval": "", - "legendFormat": "traces-expired", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max by (job, instance) (promscale_sql_database_chunks_traces_uncompressed_count{namespace=~\"$namespace\"})", - "hide": false, - "interval": "", - "legendFormat": "traces-uncompressed", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max by (job, instance) (promscale_sql_database_chunks_metrics_delayed_compression_count{namespace=~\"$namespace\"})", - "hide": false, - "interval": "", - "legendFormat": "metrics-compression-delayed", - "range": true, - "refId": "E" - } - ], - "title": "The number of chunks to be processed by maintenance jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Time since the last DB maintenance job started", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 0, - "y": 14 - }, - "id": 15, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" + "exemplar": true, + "expr": "min by (job, instance) (promscale_sql_database_chunks_compressed_count{namespace=~\"$namespace\"})\n/\nmax by (job, instance)(promscale_sql_database_chunks_count{namespace=~\"$namespace\"})", + "interval": "", + "legendFormat": "{{ instance }}", + "refId": "A" + } ], - "fields": "", - "values": false + "title": "Compressed Chunks", + "type": "timeseries" }, - "textMode": "auto" - }, - "pluginVersion": "9.2.2", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "exemplar": false, - "expr": "time() - max(promscale_sql_database_worker_maintenance_job_start_timestamp_seconds{namespace=~\"$namespace\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Time since the last job start", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 10, + "x": 14, + "y": 20 + }, + "id": 49, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 0, - "y": 16 - }, - "id": 23, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "exemplar": true, - "expr": "1 - rate(promscale_sql_database_health_check_errors_total{namespace=~\"$namespace\"}[$__rate_interval]) / rate(promscale_sql_database_health_check_total{namespace=~\"$namespace\"}[$__rate_interval])", - "interval": "", - "legendFormat": "{{ instance }}", - "refId": "A" - } - ], - "title": "Database health", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "editorMode": "code", + "exemplar": true, + "expr": "max by (job, instance) (promscale_sql_database_chunks_metrics_expired_count{namespace=~\"$namespace\"})", + "interval": "", + "legendFormat": "metrics-expired", + "range": true, + "refId": "A" }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max by (job, instance) (promscale_sql_database_chunks_metrics_uncompressed_count{namespace=~\"$namespace\"})", + "hide": false, + "interval": "", + "legendFormat": "metrics-uncompressed", + "range": true, + "refId": "B" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max by (job, instance) (promscale_sql_database_chunks_traces_expired_count{namespace=~\"$namespace\"})", + "hide": false, + "interval": "", + "legendFormat": "traces-expired", + "range": true, + "refId": "C" }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 21 - }, - "id": 27, - "interval": "2m", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "editorMode": "code", + "exemplar": true, + "expr": "max by (job, instance) (promscale_sql_database_chunks_traces_uncompressed_count{namespace=~\"$namespace\"})", + "hide": false, + "interval": "", + "legendFormat": "traces-uncompressed", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max by (job, instance) (promscale_sql_database_chunks_metrics_delayed_compression_count{namespace=~\"$namespace\"})", + "hide": false, + "interval": "", + "legendFormat": "metrics-compression-delayed", + "range": true, + "refId": "E" + } + ], + "title": "The number of chunks to be processed by maintenance jobs", + "type": "timeseries" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "exemplar": true, - "expr": "histogram_quantile(0.5, rate(promscale_database_requests_duration_seconds_bucket{method=~\"query.*\",namespace=~\"$namespace\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "p50 - {{ method }}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "description": "Time since the last DB maintenance job started", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] }, - "exemplar": true, - "expr": "histogram_quantile(0.9, rate(promscale_database_requests_duration_seconds_bucket{method=~\"query.*\",namespace=~\"$namespace\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "p90 - {{ method }}", - "refId": "B" + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 22 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": false, + "expr": "time() - max(promscale_sql_database_worker_maintenance_job_start_timestamp_seconds{namespace=~\"$namespace\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Time since the last job start", + "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "exemplar": true, - "expr": "histogram_quantile(0.95, rate(promscale_database_requests_duration_seconds_bucket{method=~\"query.*\",namespace=~\"$namespace\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "p95 - {{ method }}", - "refId": "C" - } - ], - "title": "Duration (query)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" }, - "thresholdsStyle": { - "mode": "off" - } + "overrides": [] }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 24 }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 21 - }, - "id": 28, - "interval": "2m", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "exemplar": true, - "expr": "histogram_quantile(0.5, rate(promscale_database_requests_duration_seconds_bucket{method!~\"query.*\",namespace=~\"$namespace\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "p50 - {{ method }}", - "refId": "A" + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "1 - rate(promscale_sql_database_health_check_errors_total{namespace=~\"$namespace\"}[$__rate_interval]) / rate(promscale_sql_database_health_check_total{namespace=~\"$namespace\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{ instance }}", + "refId": "A" + } + ], + "title": "Database health", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "exemplar": true, - "expr": "histogram_quantile(0.9, rate(promscale_database_requests_duration_seconds_bucket{method!~\"query.*\",namespace=~\"$namespace\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "p90 - {{ method }}", - "refId": "B" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 29 + }, + "id": 27, + "interval": "2m", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "histogram_quantile(0.5, rate(promscale_database_requests_duration_seconds_bucket{method=~\"query.*\",namespace=~\"$namespace\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "p50 - {{ method }}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "histogram_quantile(0.9, rate(promscale_database_requests_duration_seconds_bucket{method=~\"query.*\",namespace=~\"$namespace\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "p90 - {{ method }}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "histogram_quantile(0.95, rate(promscale_database_requests_duration_seconds_bucket{method=~\"query.*\",namespace=~\"$namespace\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "p95 - {{ method }}", + "refId": "C" + } + ], + "title": "Duration (query)", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "exemplar": true, - "expr": "histogram_quantile(0.95, rate(promscale_database_requests_duration_seconds_bucket{method!~\"query.*\",namespace=~\"$namespace\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "p95 - {{ method }}", - "refId": "C" - } - ], - "title": "Duration (non-query)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 29 + }, + "id": 28, + "interval": "2m", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "histogram_quantile(0.5, rate(promscale_database_requests_duration_seconds_bucket{method!~\"query.*\",namespace=~\"$namespace\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "p50 - {{ method }}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 30 - }, - "id": 26, - "interval": "2m", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "exemplar": true, + "expr": "histogram_quantile(0.9, rate(promscale_database_requests_duration_seconds_bucket{method!~\"query.*\",namespace=~\"$namespace\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "p90 - {{ method }}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "histogram_quantile(0.95, rate(promscale_database_requests_duration_seconds_bucket{method!~\"query.*\",namespace=~\"$namespace\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "p95 - {{ method }}", + "refId": "C" + } + ], + "title": "Duration (non-query)", + "type": "timeseries" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "exemplar": true, - "expr": "rate(promscale_database_requests_total{namespace=~\"$namespace\"}[$__rate_interval])", - "interval": "", - "legendFormat": "{{ method }}", - "refId": "A" - } - ], - "title": "Requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 26, + "interval": "2m", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 30 - }, - "id": 48, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "exemplar": true, + "expr": "rate(promscale_database_requests_total{namespace=~\"$namespace\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{ method }}", + "refId": "A" + } + ], + "title": "Requests", + "type": "timeseries" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "code", - "expr": "promscale_sql_database_network_latency_milliseconds{namespace=~\"$namespace\"}", - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 48, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "promscale_sql_database_network_latency_milliseconds{namespace=~\"$namespace\"}", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Network latency", + "type": "timeseries" } ], - "title": "Network latency", - "type": "timeseries" + "title": "Database", + "type": "row" }, { "collapsed": true, @@ -2223,7 +2284,7 @@ "h": 1, "w": 24, "x": 0, - "y": 39 + "y": 13 }, "id": 55, "panels": [ @@ -2273,8 +2334,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -2286,7 +2346,7 @@ "h": 9, "w": 12, "x": 0, - "y": 40 + "y": 48 }, "id": 51, "interval": "2m", @@ -2408,8 +2468,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -2421,7 +2480,7 @@ "h": 9, "w": 12, "x": 12, - "y": 40 + "y": 48 }, "id": 53, "interval": "2m", @@ -2460,14 +2519,13 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -2501,12 +2559,11 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, - "unit": "none" + "unit": "s" }, "overrides": [] }, @@ -2516,7 +2573,7 @@ "x": 0, "y": 49 }, - "id": 50, + "id": 54, "interval": "2m", "options": { "legend": { @@ -2538,9 +2595,9 @@ }, "editorMode": "code", "exemplar": false, - "expr": "max(promscale_sql_database_worker_maintenance_job_long_running_total{namespace=~\"$namespace\"})", + "expr": "histogram_quantile(0.99, max(rate(promscale_sql_database_worker_maintenance_job_metrics_compression_last_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])) by (le, job))", "interval": "", - "legendFormat": "total", + "legendFormat": "metrics-compression", "range": true, "refId": "A" }, @@ -2551,10 +2608,10 @@ }, "editorMode": "code", "exemplar": false, - "expr": "max(promscale_sql_database_worker_maintenance_job_long_running_buffer_pin{namespace=~\"$namespace\"})", + "expr": "histogram_quantile(0.99, max(rate(promscale_sql_database_worker_maintenance_job_metrics_retention_last_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])) by (le, job))", "hide": false, "interval": "", - "legendFormat": "buffer_pin", + "legendFormat": "metrics-retention", "range": true, "refId": "B" }, @@ -2565,10 +2622,10 @@ }, "editorMode": "code", "exemplar": false, - "expr": "max(promscale_sql_database_worker_maintenance_job_long_running_io{namespace=~\"$namespace\"})", + "expr": "histogram_quantile(0.99, max(rate(promscale_sql_database_worker_maintenance_job_traces_retention_last_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])) by (le, job))", "hide": false, "interval": "", - "legendFormat": "io", + "legendFormat": "traces-retention", "range": true, "refId": "C" }, @@ -2579,57 +2636,15 @@ }, "editorMode": "code", "exemplar": false, - "expr": "max(promscale_sql_database_worker_maintenance_job_long_running_ipc{namespace=~\"$namespace\"})", + "expr": "histogram_quantile(0.99, max(rate(promscale_sql_database_worker_maintenance_job_traces_compression_last_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])) by (le, job))", "hide": false, "interval": "", - "legendFormat": "ipc", + "legendFormat": "traces-compression", "range": true, "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(promscale_sql_database_worker_maintenance_job_long_running_lock{namespace=~\"$namespace\"})", - "hide": false, - "interval": "", - "legendFormat": "lock", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(promscale_sql_database_worker_maintenance_job_long_running_lwlock{namespace=~\"$namespace\"})", - "hide": false, - "interval": "", - "legendFormat": "lwlock", - "range": true, - "refId": "F" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(promscale_sql_database_worker_maintenance_job_long_running_timeout{namespace=~\"$namespace\"})", - "hide": false, - "interval": "", - "legendFormat": "timeout", - "range": true, - "refId": "G" } ], - "title": "Long running maintenance queries by wait event", + "title": "Duration of recent jobs", "type": "timeseries" }, { @@ -2637,14 +2652,13 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -2678,12 +2692,11 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, - "unit": "none" + "unit": "opm" }, "overrides": [] }, @@ -2693,7 +2706,7 @@ "x": 12, "y": 49 }, - "id": 52, + "id": 57, "interval": "2m", "options": { "legend": { @@ -2715,9 +2728,9 @@ }, "editorMode": "code", "exemplar": false, - "expr": "max(promscale_sql_database_worker_maintenance_job_locks_total{namespace=~\"$namespace\"})", + "expr": "max by (job, instance)(rate(promscale_sql_database_worker_maintenance_job_metrics_compression_total_runs_count{namespace=~\"$namespace\"}[$__rate_interval])) * 60", "interval": "", - "legendFormat": "total", + "legendFormat": "metrics-compression-total", "range": true, "refId": "A" }, @@ -2728,10 +2741,10 @@ }, "editorMode": "code", "exemplar": false, - "expr": "max(promscale_sql_database_worker_maintenance_job_locks_share_update_exclusive{namespace=~\"$namespace\"})", + "expr": "max by (job, instance)(rate(promscale_sql_database_worker_maintenance_job_metrics_retention_total_runs_count{namespace=~\"$namespace\"}[$__rate_interval])) * 60", "hide": false, "interval": "", - "legendFormat": "share_update_exclusive", + "legendFormat": "metrics-retention-total", "range": true, "refId": "B" }, @@ -2742,10 +2755,10 @@ }, "editorMode": "code", "exemplar": false, - "expr": "max(promscale_sql_database_worker_maintenance_job_locks_share_row_exclusive{namespace=~\"$namespace\"})", + "expr": "max by (job, instance)(rate(promscale_sql_database_worker_maintenance_job_traces_retention_total_runs_count{namespace=~\"$namespace\"}[$__rate_interval])) * 60", "hide": false, "interval": "", - "legendFormat": "share_row_exclusive", + "legendFormat": "traces-retention-total", "range": true, "refId": "C" }, @@ -2756,10 +2769,10 @@ }, "editorMode": "code", "exemplar": false, - "expr": "max(promscale_sql_database_worker_maintenance_job_locks_share{namespace=~\"$namespace\"})", + "expr": "max by (job, instance)(rate(promscale_sql_database_worker_maintenance_job_metrics_compression_failures_count{namespace=~\"$namespace\"}[$__rate_interval])) * 60", "hide": false, "interval": "", - "legendFormat": "share", + "legendFormat": "metrics-compression-failures", "range": true, "refId": "D" }, @@ -2770,10 +2783,10 @@ }, "editorMode": "code", "exemplar": false, - "expr": "max(promscale_sql_database_worker_maintenance_job_locks_row_share{namespace=~\"$namespace\"})", + "expr": "max by (job, instance)(rate(promscale_sql_database_worker_maintenance_job_metrics_retention_failures_count{namespace=~\"$namespace\"}[$__rate_interval])) * 60", "hide": false, "interval": "", - "legendFormat": "row_share", + "legendFormat": "metrics-retention-failures", "range": true, "refId": "E" }, @@ -2784,10 +2797,10 @@ }, "editorMode": "code", "exemplar": false, - "expr": "max(promscale_sql_database_worker_maintenance_job_locks_row_exclusive{namespace=~\"$namespace\"})", + "expr": "max by (job, instance)(rate(promscale_sql_database_worker_maintenance_job_traces_retention_failures_count{namespace=~\"$namespace\"}[$__rate_interval])) * 60", "hide": false, "interval": "", - "legendFormat": "row_exclusive", + "legendFormat": "traces-retention-failures", "range": true, "refId": "F" }, @@ -2798,10 +2811,10 @@ }, "editorMode": "code", "exemplar": false, - "expr": "max(promscale_sql_database_worker_maintenance_job_locks_exclusive{namespace=~\"$namespace\"})", + "expr": "max by (job, instance)(rate(promscale_sql_database_worker_maintenance_job_traces_compression_failures_count{namespace=~\"$namespace\"}[$__rate_interval])) * 60", "hide": false, "interval": "", - "legendFormat": "exclusive", + "legendFormat": "traces-compression-failures", "range": true, "refId": "G" }, @@ -2812,49 +2825,17 @@ }, "editorMode": "code", "exemplar": false, - "expr": "max(promscale_sql_database_worker_maintenance_job_locks_access_share{namespace=~\"$namespace\"})", + "expr": "max by (job, instance)(rate(promscale_sql_database_worker_maintenance_job_traces_retention_total_runs_count{namespace=~\"$namespace\"}[$__rate_interval])) * 60", "hide": false, "interval": "", - "legendFormat": "access_share", + "legendFormat": "traces-compression-total", "range": true, "refId": "H" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(promscale_sql_database_worker_maintenance_job_locks_access_exclusive{namespace=~\"$namespace\"})", - "hide": false, - "interval": "", - "legendFormat": "access_exclusive", - "range": true, - "refId": "I" } ], - "title": "Locks held by maintenance jobs by lock mode", + "title": "Completion and failure rates", "type": "timeseries" - } - ], - "title": "Maintenance Jobs", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "-BqhIPC7z" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 30, - "panels": [ + }, { "datasource": { "type": "prometheus", @@ -2866,6 +2847,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -2884,7 +2867,7 @@ "type": "linear" }, "showPoints": "auto", - "spanNulls": false, + "spanNulls": true, "stacking": { "group": "A", "mode": "none" @@ -2894,30 +2877,27 @@ } }, "mappings": [], - "max": 1, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "percentunit" + "unit": "none" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 8, + "h": 9, + "w": 12, "x": 0, - "y": 13 + "y": 50 }, - "id": 34, + "id": 50, + "interval": "2m", "options": { "legend": { "calcs": [], @@ -2936,14 +2916,100 @@ "type": "prometheus", "uid": "${datasource}" }, - "exemplar": true, - "expr": "rate(promscale_cache_query_hits_total{type=\"metric\",namespace=~\"$namespace\"}[$__rate_interval]) / rate(promscale_cache_queries_total{type=\"metric\",namespace=~\"$namespace\"}[$__rate_interval])", + "editorMode": "code", + "exemplar": false, + "expr": "max(promscale_sql_database_worker_maintenance_job_long_running_total{namespace=~\"$namespace\"})", "interval": "", - "legendFormat": "{{ name }}", + "legendFormat": "total", + "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(promscale_sql_database_worker_maintenance_job_long_running_buffer_pin{namespace=~\"$namespace\"})", + "hide": false, + "interval": "", + "legendFormat": "buffer_pin", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(promscale_sql_database_worker_maintenance_job_long_running_io{namespace=~\"$namespace\"})", + "hide": false, + "interval": "", + "legendFormat": "io", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(promscale_sql_database_worker_maintenance_job_long_running_ipc{namespace=~\"$namespace\"})", + "hide": false, + "interval": "", + "legendFormat": "ipc", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(promscale_sql_database_worker_maintenance_job_long_running_lock{namespace=~\"$namespace\"})", + "hide": false, + "interval": "", + "legendFormat": "lock", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(promscale_sql_database_worker_maintenance_job_long_running_lwlock{namespace=~\"$namespace\"})", + "hide": false, + "interval": "", + "legendFormat": "lwlock", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(promscale_sql_database_worker_maintenance_job_long_running_timeout{namespace=~\"$namespace\"})", + "hide": false, + "interval": "", + "legendFormat": "timeout", + "range": true, + "refId": "G" } ], - "title": "Hit ratio (metrics)", + "title": "Long running maintenance queries by wait event", "type": "timeseries" }, { @@ -2957,6 +3023,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -2975,7 +3043,7 @@ "type": "linear" }, "showPoints": "auto", - "spanNulls": false, + "spanNulls": true, "stacking": { "group": "A", "mode": "none" @@ -2985,30 +3053,27 @@ } }, "mappings": [], - "max": 1, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "percentunit" + "unit": "none" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 13 + "h": 9, + "w": 12, + "x": 12, + "y": 50 }, - "id": 35, + "id": 52, + "interval": "2m", "options": { "legend": { "calcs": [], @@ -3027,16 +3092,148 @@ "type": "prometheus", "uid": "${datasource}" }, - "exemplar": true, - "expr": "rate(promscale_cache_query_hits_total{type=\"trace\",namespace=~\"$namespace\"}[$__rate_interval]) / rate(promscale_cache_queries_total{type=\"trace\",namespace=~\"$namespace\"}[$__rate_interval])", + "editorMode": "code", + "exemplar": false, + "expr": "max(promscale_sql_database_worker_maintenance_job_locks_total{namespace=~\"$namespace\"})", "interval": "", - "legendFormat": "{{ name }}", + "legendFormat": "total", + "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(promscale_sql_database_worker_maintenance_job_locks_share_update_exclusive{namespace=~\"$namespace\"})", + "hide": false, + "interval": "", + "legendFormat": "share_update_exclusive", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(promscale_sql_database_worker_maintenance_job_locks_share_row_exclusive{namespace=~\"$namespace\"})", + "hide": false, + "interval": "", + "legendFormat": "share_row_exclusive", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(promscale_sql_database_worker_maintenance_job_locks_share{namespace=~\"$namespace\"})", + "hide": false, + "interval": "", + "legendFormat": "share", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(promscale_sql_database_worker_maintenance_job_locks_row_share{namespace=~\"$namespace\"})", + "hide": false, + "interval": "", + "legendFormat": "row_share", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(promscale_sql_database_worker_maintenance_job_locks_row_exclusive{namespace=~\"$namespace\"})", + "hide": false, + "interval": "", + "legendFormat": "row_exclusive", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(promscale_sql_database_worker_maintenance_job_locks_exclusive{namespace=~\"$namespace\"})", + "hide": false, + "interval": "", + "legendFormat": "exclusive", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(promscale_sql_database_worker_maintenance_job_locks_access_share{namespace=~\"$namespace\"})", + "hide": false, + "interval": "", + "legendFormat": "access_share", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(promscale_sql_database_worker_maintenance_job_locks_access_exclusive{namespace=~\"$namespace\"})", + "hide": false, + "interval": "", + "legendFormat": "access_exclusive", + "range": true, + "refId": "I" } ], - "title": "Hit ratio (traces)", + "title": "Locks held by maintenance jobs by lock mode", "type": "timeseries" - }, + } + ], + "title": "Maintenance Jobs", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "-BqhIPC7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 30, + "panels": [ { "datasource": { "type": "prometheus", @@ -3048,6 +3245,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -3076,11 +3275,13 @@ } }, "mappings": [], + "max": 1, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3088,17 +3289,17 @@ } ] }, - "unit": "cps" + "unit": "percentunit" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 13 + "h": 9, + "w": 12, + "x": 0, + "y": 40 }, - "id": 37, + "id": 34, "options": { "legend": { "calcs": [], @@ -3117,14 +3318,16 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "sum by (type, name) (rate(promscale_cache_evictions_total{namespace=~\"$namespace\"}[$__rate_interval]))", + "expr": "rate(promscale_cache_query_hits_total{type=~\"$datatype\",namespace=~\"$namespace\"}[$__rate_interval]) / rate(promscale_cache_queries_total{type=~\"$datatype\",namespace=~\"$namespace\"}[$__rate_interval])", "interval": "", - "legendFormat": "{{ type }} - {{ name }}", + "legendFormat": "{{ name }}", + "range": true, "refId": "A" } ], - "title": "Evictions", + "title": "Hit ratio", "type": "timeseries" }, { @@ -3138,6 +3341,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -3171,7 +3376,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3185,9 +3391,9 @@ }, "gridPos": { "h": 9, - "w": 8, - "x": 0, - "y": 21 + "w": 12, + "x": 12, + "y": 40 }, "id": 41, "interval": "2m", @@ -3209,10 +3415,12 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.5, rate(promscale_cache_query_latency_microseconds_bucket{type=\"metric\",namespace=~\"$namespace\"}[$__rate_interval]))", + "expr": "histogram_quantile(0.5, rate(promscale_cache_query_latency_microseconds_bucket{type=~\"$datatype\",namespace=~\"$namespace\"}[$__rate_interval]))", "interval": "", "legendFormat": "p50 - {{ name }}", + "range": true, "refId": "A" }, { @@ -3220,15 +3428,17 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.90, rate(promscale_cache_query_latency_microseconds_bucket{type=\"metric\",namespace=~\"$namespace\"}[$__rate_interval]))", + "expr": "histogram_quantile(0.90, rate(promscale_cache_query_latency_microseconds_bucket{type=~\"$datatype\",namespace=~\"$namespace\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "p90 - {{ name }}", + "range": true, "refId": "B" } ], - "title": "Latency (metrics)", + "title": "Latency", "type": "timeseries" }, { @@ -3242,6 +3452,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -3270,12 +3482,12 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3283,18 +3495,17 @@ } ] }, - "unit": "ms" + "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 9, - "w": 8, - "x": 8, - "y": 21 + "w": 12, + "x": 0, + "y": 49 }, - "id": 42, - "interval": "2m", + "id": 37, "options": { "legend": { "calcs": [], @@ -3313,26 +3524,16 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.5, rate(promscale_cache_query_latency_microseconds_bucket{type=\"trace\",namespace=~\"$namespace\"}[$__rate_interval]))", + "expr": "sum by (type, name) (rate(promscale_cache_evictions_total{namespace=~\"$namespace\",type=~\"$datatype\"}[$__rate_interval]))", "interval": "", - "legendFormat": "p50 {{ name }}", + "legendFormat": "{{ name }}", + "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "exemplar": true, - "expr": "histogram_quantile(0.90, rate(promscale_cache_query_latency_microseconds_bucket{type=\"trace\",namespace=~\"$namespace\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "p90 {{ name }}", - "refId": "B" } ], - "title": "Latency (traces)", + "title": "Evictions", "type": "timeseries" }, { @@ -3346,6 +3547,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -3379,7 +3582,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3393,9 +3597,9 @@ }, "gridPos": { "h": 9, - "w": 8, - "x": 16, - "y": 21 + "w": 12, + "x": 12, + "y": 49 }, "id": 39, "options": { @@ -3416,10 +3620,12 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "promscale_cache_elements{namespace=~\"$namespace\"} / promscale_cache_capacity_elements{namespace=~\"$namespace\"}", + "expr": "promscale_cache_elements{namespace=~\"$namespace\",type=~\"$datatype\"} / promscale_cache_capacity_elements{namespace=~\"$namespace\",type=~\"$datatype\"}", "interval": "", "legendFormat": "{{ name }}", + "range": true, "refId": "A" } ], @@ -3427,7 +3633,9 @@ "type": "timeseries" } ], - "title": "Cache", + "repeat": "datatype", + "repeatDirection": "h", + "title": "Cache - $datatype", "type": "row" } ], @@ -3437,7 +3645,11 @@ "templating": { "list": [ { - "current": {}, + "current": { + "selected": false, + "text": "Promscale-PromQL", + "value": "Promscale-PromQL" + }, "hide": 0, "includeAll": false, "multi": false, @@ -3479,6 +3691,42 @@ "skipUrlSync": false, "sort": 0, "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "hide": 0, + "includeAll": true, + "multi": true, + "name": "datatype", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "metric", + "value": "metric" + }, + { + "selected": false, + "text": "trace", + "value": "trace" + } + ], + "query": "metric,trace", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" } ] },