chore: update cosmos-node-exporter and alerts

QuokkaStake · Jul 1, 2024 · a483902 · a483902
1 parent acf6d12
commit a483902
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 20 deletions.
diff --git a/templates/monitoring/grafana/dashboards/fullnodes/cosmos_node_stats.json b/templates/monitoring/grafana/dashboards/fullnodes/cosmos_node_stats.json
@@ -19,9 +19,9 @@
           "type": "prometheus",
           "uid": "prometheus"
         },
-        "enable": true,
+        "enable": false,
         "expr": "(1 - up{job=~\"tendermint.*\"})",
-        "hide": true,
+        "hide": false,
         "iconColor": "super-light-red",
         "name": "tendermint down",
         "textFormat": "",
@@ -34,7 +34,7 @@
         },
         "enable": true,
         "expr": "1 - up{job=~\"cosmos-node-exporter.*\"}",
-        "hide": true,
+        "hide": false,
         "iconColor": "dark-red",
         "name": "cosmos-node-exporter down",
         "textFormat": "",
@@ -279,7 +279,7 @@
           },
           "editorMode": "code",
           "exemplar": true,
-          "expr": "cosmos_node_exporter_time_since_latest_block < 60 * 60 * 24 * 365",
+          "expr": "time() - cosmos_node_exporter_latest_block_time",
           "interval": "",
           "legendFormat": "{{ host }}",
           "range": true,
@@ -934,7 +934,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": null
               },
               {
                 "color": "red",
@@ -1117,7 +1118,7 @@
           }
         ]
       },
-      "pluginVersion": "10.4.1",
+      "pluginVersion": "11.1.0",
       "targets": [
         {
           "datasource": {
@@ -1328,7 +1329,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": null
               },
               {
                 "color": "red",
@@ -1365,7 +1367,7 @@
           }
         ]
       },
-      "pluginVersion": "10.4.1",
+      "pluginVersion": "11.1.0",
       "targets": [
         {
           "datasource": {
@@ -1449,7 +1451,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": null
               },
               {
                 "color": "red",
@@ -1511,7 +1514,8 @@
                   "mode": "absolute",
                   "steps": [
                     {
-                      "color": "red"
+                      "color": "red",
+                      "value": null
                     },
                     {
                       "color": "green",
@@ -1591,7 +1595,7 @@
         "showHeader": true,
         "sortBy": []
       },
-      "pluginVersion": "10.4.1",
+      "pluginVersion": "11.1.0",
       "targets": [
         {
           "datasource": {
@@ -1750,7 +1754,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": null
               },
               {
                 "color": "red",
@@ -1812,7 +1817,8 @@
                   "mode": "absolute",
                   "steps": [
                     {
-                      "color": "red"
+                      "color": "red",
+                      "value": null
                     },
                     {
                       "color": "green",
@@ -1981,7 +1987,7 @@
           }
         ]
       },
-      "pluginVersion": "10.4.1",
+      "pluginVersion": "11.1.0",
       "targets": [
         {
           "datasource": {
@@ -2155,7 +2161,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": null
               },
               {
                 "color": "red",
@@ -2209,7 +2216,8 @@
                   "mode": "absolute",
                   "steps": [
                     {
-                      "color": "red"
+                      "color": "red",
+                      "value": null
                     },
                     {
                       "color": "green",
@@ -2254,7 +2262,7 @@
           }
         ]
       },
-      "pluginVersion": "10.4.1",
+      "pluginVersion": "11.1.0",
       "targets": [
         {
           "datasource": {
@@ -2362,6 +2370,6 @@
   "timezone": "",
   "title": "Cosmos Node Stats",
   "uid": "cosmos-node-stats",
-  "version": 17,
+  "version": 19,
   "weekStart": ""
 }
diff --git a/templates/monitoring/prometheus/rules.d/cosmos-node-exporter.yml b/templates/monitoring/prometheus/rules.d/cosmos-node-exporter.yml
@@ -39,7 +39,7 @@ groups:
           description: "Tendermint testnet node has 0 voting power"
 
       - alert: CosmosNodeTimeSinceLatestBlock
-        expr: cosmos_node_exporter_time_since_latest_block > 600
+        expr: time() - cosmos_node_exporter_latest_block_time > 600
         for: 1m
         labels:
           severity: critical

diff --git a/templates/monitoring/prometheus/rules.d/cosmos-wallets-exporter.yml b/templates/monitoring/prometheus/rules.d/cosmos-wallets-exporter.yml
@@ -29,7 +29,7 @@ groups:
           description: "The balance of oracle wallet on {{ $labels.chain }} is below threshold: {{ $value }}"
 
       - alert: CosmosWalletExportePigeonBalanceTooLow
-        expr: cosmos_wallets_exporter_balance{group="pigeon"} < 20
+        expr: cosmos_wallets_exporter_balance{group="pigeon"} < 500
         for: 10m
         labels:
           severity: warning