diff --git a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt
index 0f3746bf03..115be5fa2f 100644
--- a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt
+++ b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt
@@ -97,6 +97,8 @@ object ReportManager {
         val useFlakyTests = args.flakyTestAttempts > 0
         if (useFlakyTests) JUnitDedupe.modify(testSuite)
 
+        val testSuccessful = if (useFlakyTests) testSuite?.successful() ?: false else matrices.allSuccessful()
+
         listOf(
             CostReport,
             MatrixResultsReport
@@ -104,7 +106,7 @@ object ReportManager {
             it.run(matrices, testSuite, printToStdout = true, args = args)
         }
 
-        if (matrices.allSuccessful().not()) {
+        if (!testSuccessful) {
             listOf(
                 HtmlErrorReport
             ).map { it.run(matrices, testSuite, printToStdout = false, args = args) }
@@ -113,7 +115,14 @@ object ReportManager {
         JUnitReport.run(matrices, testSuite, printToStdout = false, args = args)
         processJunitXml(testSuite, args, testShardChunks)
 
-        return matrices.exitCode()
+        // FTL has a bug with matrix roll-up when using flakyTestAttempts
+        // as a work around, we calculate the success based on JUnit XML results.
+        val exitCode = if (useFlakyTests) {
+            if (testSuccessful) 0 else 1
+        } else {
+            matrices.exitCode()
+        }
+        return exitCode
     }
 
     data class ShardEfficiency(