Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

fix: Removed test case time overhead calculation #2364

Merged
merged 1 commit into from
Apr 13, 2023
Merged

fix: Removed test case time overhead calculation #2364

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion test_runner/src/main/kotlin/ftl/adapter/google/GoogleTestMatrixAdapter.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,6 @@ fun TestOutcome.toApiModel() = Outcome(
testSuiteOverview.flakes,
testSuiteOverview.skipped,
testSuiteOverview.elapsedTime,
testSuiteOverview.overheadTime
)
)

Expand Down
1 change: 0 additions & 1 deletion test_runner/src/main/kotlin/ftl/api/TestMatrix.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,6 @@ object TestMatrix {
val flakes: Int = 0,
val skipped: Int = 0,
val elapsedTime: Double = 0.0,
val overheadTime: Double = 0.0
)

data class BillableMinutes(
Expand Down
5 changes: 1 addition & 4 deletions test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestCase.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,27 +10,24 @@ import ftl.reports.api.millis
internal fun createJUnitTestCases(
testCases: List<TestCase>,
toolResultsStep: ToolResultsStep,
overheadTime: Double
): List<JUnitTest.Case> = testCases.map { testCase ->
createJUnitTestCase(
testCase = testCase,
toolResultsStep = toolResultsStep,
overheadTime = overheadTime
)
}

private fun createJUnitTestCase(
testCase: TestCase,
toolResultsStep: ToolResultsStep,
overheadTime: Double
): JUnitTest.Case {
val stackTraces = mapOf(
testCase.status to testCase.stackTraces?.map(StackTrace::getException)
)
return JUnitTest.Case(
name = testCase.testCaseReference.name,
classname = testCase.testCaseReference.className,
time = (testCase.elapsedTime.millis() + overheadTime).format(),
time = testCase.elapsedTime.millis().format(),
failures = stackTraces["failed"],
errors = stackTraces["error"],
// skipped = true is represented by null. skipped = false is "absent"
Expand Down
1 change: 0 additions & 1 deletion test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestSuite.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ private fun createJUnitTestSuite(
testcases = createJUnitTestCases(
testCases = data.testCases,
toolResultsStep = data.testExecution.toolResultsStep,
overheadTime = overview.overheadTime
).toMutableList(),
time = overview.elapsedTime.format()
)
10 changes: 0 additions & 10 deletions test_runner/src/main/kotlin/ftl/client/junit/CreateTestSuiteOverviewData.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,9 @@ internal fun TestExecutionData.createTestSuiteOverviewData(): TestSuiteOverviewD
flakes = testCases.countFlakes(),
skipped = skipped,
elapsedTime = overview.elapsedTime.millis(),
overheadTime = getOverheadTime(overview, testCases)
)
}

private fun List<TestCase>.countErrors() = count { !it.flaky && it.status == "error" }
private fun List<TestCase>.countFailures() = count { !it.flaky && it.status == "failed" }
private fun List<TestCase>.countFlakes() = count { it.flaky }

private fun getOverheadTime(
overview: TestSuiteOverview,
testCases: List<TestCase>
) = if (testCases.isEmpty())
0.0 else
(overview.elapsedTime.millis() - testCases.sumTime()) / testCases.size

private fun List<TestCase>.sumTime(): Double = sumOf { it.elapsedTime.millis() }
2 changes: 0 additions & 2 deletions test_runner/src/main/kotlin/ftl/reports/api/data/TestSuiteOverviewData.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ data class TestSuiteOverviewData(
val flakes: Int = 0,
val skipped: Int = 0,
val elapsedTime: Double = 0.0,
val overheadTime: Double = 0.0
) {
operator fun plus(nextData: TestSuiteOverviewData?) =
if (nextData == null) this else copy(
Expand All @@ -20,7 +19,6 @@ data class TestSuiteOverviewData(
flakes = this.flakes + nextData.flakes,
skipped = this.skipped + nextData.skipped,
elapsedTime = this.elapsedTime + nextData.elapsedTime,
overheadTime = this.overheadTime + nextData.overheadTime
)

operator fun plus(data: TestSuiteOverview) = copy(
Expand Down
6 changes: 3 additions & 3 deletions test_runner/src/test/kotlin/ftl/json/OutcomeDetailsFormatterTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ internal class OutcomeDetailsFormatterTest {
every { summary } returns StepOutcome.success
every { successDetail } returns mockk { every { otherNativeCrash } returns false }
}
val testSuiteOverviewData = TestSuiteOverviewData(12, 0, 0, 3, 2, 0.0, 0.0)
val testSuiteOverviewData = TestSuiteOverviewData(12, 0, 0, 3, 2, 0.0)
val successCount = with(testSuiteOverviewData) { total - errors - failures - flakes - skipped }
val expectedMessage = "$successCount test cases passed, " +
"${testSuiteOverviewData.skipped} skipped, " +
Expand All @@ -43,7 +43,7 @@ internal class OutcomeDetailsFormatterTest {
every { summary } returns StepOutcome.success
every { successDetail } returns mockk { every { otherNativeCrash } returns true }
}
val testSuiteOverviewData = TestSuiteOverviewData(12, 0, 0, 3, 2, 0.0, 0.0)
val testSuiteOverviewData = TestSuiteOverviewData(12, 0, 0, 3, 2, 0.0)
val successCount = with(testSuiteOverviewData) { total - errors - failures - flakes - skipped }
val expectedMessage = "$successCount test cases passed, " +
"${testSuiteOverviewData.skipped} skipped, " +
Expand All @@ -64,7 +64,7 @@ internal class OutcomeDetailsFormatterTest {
every { summary } returns StepOutcome.failure
every { failureDetail } returns mockk(relaxed = true) {}
}
val testSuiteOverviewData = TestSuiteOverviewData(12, 3, 3, 3, 2, 0.0, 0.0)
val testSuiteOverviewData = TestSuiteOverviewData(12, 3, 3, 3, 2, 0.0)
val expectedMessage = "${testSuiteOverviewData.failures} test cases failed, " +
"${testSuiteOverviewData.errors} errors, " +
"1 passed, " +
Expand Down
11 changes: 5 additions & 6 deletions test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestCaseKtTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,15 +50,15 @@ class CreateJUnitTestCaseKtTest {
JUnitTest.Case(
name = "test1",
classname = "TestClassName",
time = "2.200"
time = "1.100"
).apply {
webLink =
"https://console.firebase.google.com/project/projectId/testlab/histories/historyId/matrices/executionId/executions/stepId/testcases/test1"
},
JUnitTest.Case(
name = "test2",
classname = "TestClassName",
time = "2.200",
time = "1.100",
skipped = null
).apply {
webLink =
Expand All @@ -67,7 +67,7 @@ class CreateJUnitTestCaseKtTest {
JUnitTest.Case(
name = "test3",
classname = "TestClassName",
time = "2.200",
time = "1.100",
errors = listOf("exception")
).apply {
webLink =
Expand All @@ -76,7 +76,7 @@ class CreateJUnitTestCaseKtTest {
JUnitTest.Case(
name = "test4",
classname = "TestClassName",
time = "2.200",
time = "1.100",
failures = listOf("exception")
).apply {
webLink =
Expand All @@ -85,7 +85,7 @@ class CreateJUnitTestCaseKtTest {
JUnitTest.Case(
name = "test5",
classname = "TestClassName",
time = "2.200",
time = "1.100",
failures = listOf("exception")
).apply {
webLink =
Expand All @@ -97,7 +97,6 @@ class CreateJUnitTestCaseKtTest {
val actual = createJUnitTestCases(
testCases = testCases,
toolResultsStep = toolResultsStep,
overheadTime = 1.1
)

// then
Expand Down
4 changes: 2 additions & 2 deletions test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestSuiteKtTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,12 +42,12 @@ class CreateJUnitTestSuiteKtTest {
// given
every {
any<TestExecutionData>().createTestSuiteOverviewData()
} returns TestSuiteOverviewData(1, 1, 1, 1, 1, 1.1, 1.1)
} returns TestSuiteOverviewData(1, 1, 1, 1, 1, 1.1)

val jUnitTestCase = JUnitTest.Case(null, null, "1.1")

every {
createJUnitTestCases(any(), any(), any())
createJUnitTestCases(any(), any())
} returns listOf(jUnitTestCase)

val testExecutionDataList = listOf(
Expand Down
5 changes: 2 additions & 3 deletions test_runner/src/test/kotlin/ftl/reports/api/CreateTestSuiteOverviewDataKtTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ class CreateTestSuiteOverviewDataKtTest {
TestSuiteOverview().apply {
skippedCount = 1
elapsedTime = Duration().apply {
seconds = 8
seconds = 4
}
}
)
Expand All @@ -55,8 +55,7 @@ class CreateTestSuiteOverviewDataKtTest {
failures = 1,
skipped = 1,
flakes = 1,
elapsedTime = 8.0,
overheadTime = 1.0
elapsedTime = 4.0,
)
val actual = testExecutionData.createTestSuiteOverviewData()

Expand Down