From 3d422c3d0fb57656eeff47085af72c9f4c1d0be3 Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Tue, 11 Jun 2024 18:03:12 +0300 Subject: [PATCH] prometheus.rules: support multiple cluster in split brain This patch make the alerting rules for split brain safe to use with multi cluster installation. Fixes #2304 Signed-off-by: Amnon Heiman --- prometheus/prom_rules/prometheus.rules.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prometheus/prom_rules/prometheus.rules.yml b/prometheus/prom_rules/prometheus.rules.yml index d7c3c005bd..780712b222 100644 --- a/prometheus/prom_rules/prometheus.rules.yml +++ b/prometheus/prom_rules/prometheus.rules.yml @@ -297,11 +297,11 @@ groups: description: 'Node {{ $labels.instance }} in Joining mode for 1 day' summary: Node {{ $labels.instance }} in Joining mode for 1 day - alert: splitBrain - expr: sum(scylla_gossip_live) < (count(scylla_node_operation_mode==3)-1) * count(scylla_gossip_live) + expr: sum(scylla_gossip_live) by (cluster) < (count(scylla_node_operation_mode==3) by (cluster) -1) * count(scylla_gossip_live) by (cluster) for: 10m labels: severity: "warn" - description: 'Cluster in a split-brain mode' + description: 'Cluster {{ $labels.cluster }} in a split-brain mode' summary: Some nodes in the cluster do not see all of the other live nodes - alert: bloomFilterSize expr: scylla_sstables_bloom_filter_memory_size/scylla_memory_total_memory > 0.2