diff --git a/builtin/credential-cache.c b/builtin/credential-cache.c
index 7f733cb756e03c..3b8130d3d64f9c 100644
--- a/builtin/credential-cache.c
+++ b/builtin/credential-cache.c
@@ -23,7 +23,7 @@ static int connection_closed(int error)
 
 static int connection_fatally_broken(int error)
 {
-	return (error != ENOENT) && (error != ENETDOWN);
+	return (error != ENOENT) && (error != ENETDOWN) && (error != ECONNREFUSED);
 }
 
 #else