Better handle failed prometheus server

internal/checks/base.go

@@ -130,7 +130,7 @@ type exprProblem struct {
 	severity Severity
 }
 
-func textAndSeverityFromError(err error, reporter, prom string, s Severity) (text string, severity Severity) {
+func getPromDescAndPerr(err error, prom string) (string, *promapi.FailoverGroupError) {
 	promDesc := fmt.Sprintf("%q", prom)
 	var perr *promapi.FailoverGroupError
 	perrOk := errors.As(err, &perr)
@@ -139,6 +139,12 @@ func textAndSeverityFromError(err error, reporter, prom string, s Severity) (tex
 			promDesc = promText(prom, uri)
 		}
 	}
+	return promDesc, perr
+}
+
+func textAndSeverityFromError(err error, reporter, prom string, s Severity) (text string, severity Severity) {
+	promDesc, perr := getPromDescAndPerr(err, prom)
+	perrOk := perr != nil
 
 	switch {
 	case promapi.IsQueryTooExpensive(err):

internal/checks/promql_counter.go

@@ -82,12 +82,13 @@ func (c CounterCheck) checkNode(ctx context.Context, node *parser.PromQLNode, en
 			}
 		} else {
 			metadata, err := c.prom.Metadata(ctx, s.Name)
-			if err != nil {
-				text, severity := textAndSeverityFromError(err, c.Reporter(), c.prom.Name(), Bug)
+			promDesc, _ := getPromDescAndPerr(err, c.prom.Name())
+			text := fmt.Sprintf("couldn't run %q checks due to missing metrics metadata. %s connection error: %s", c.Reporter(), promDesc, err)
+			if metadata == nil {
 				problems = append(problems, exprProblem{
 					expr:     s.Name,
 					text:     text,
-					severity: severity,
+					severity: Warning,
 				})
 				return problems
 			}

internal/checks/promql_counter_test.go

@@ -202,7 +202,34 @@ func TestCounterCheck(t *testing.T) {
 				},
 			},
 		},
+		{
+			description: "500 error from Prometheus API",
+			content:     "- record: foo\n  expr: rate(foo[5m])\n",
+			checker:     newCounterCheck,
+			prometheus:  newSimpleProm,
+			problems: func(uri string) []checks.Problem {
+				return []checks.Problem{
+					{
+						Fragment: "foo",
+						Lines:    []int{2},
+						Reporter: "promql/counter",
+						Text:     checkCounterErrorUnableToRun(checks.CounterCheckName, "prom", uri, "server_error: internal error"),
+						Severity: checks.Warning,
+					},
+				}
+			},
+			mocks: []*prometheusMock{
+				{
+					conds: []requestCondition{requireMetadataPath},
+					resp:  respondWithInternalError(),
+				},
+			},
+		},
 	}
 
 	runTests(t, testCases)
 }
+
+func checkCounterErrorUnableToRun(c, name, uri, err string) string {
+	return fmt.Sprintf(`couldn't run %q checks due to missing metrics metadata. prometheus %q at %s connection error: %s`, c, name, uri, err)
+}