From c4acfa9353b469e44b3d99253e96473e1ed3b6ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Thu, 22 Apr 2021 18:40:58 +0200 Subject: [PATCH 1/8] Add sharding implementation for corellium --- corellium/shard/README.md | 92 +++++++++++++++ corellium/shard/build.gradle.kts | 21 ++++ .../kotlin/flank/corellium/shard/Internal.kt | 105 +++++++++++++++++ .../kotlin/flank/corellium/shard/Shard.kt | 36 ++++++ .../flank/corellium/shard/ShardKtTest.kt | 107 ++++++++++++++++++ settings.gradle.kts | 4 +- 6 files changed, 363 insertions(+), 2 deletions(-) create mode 100644 corellium/shard/README.md create mode 100644 corellium/shard/build.gradle.kts create mode 100644 corellium/shard/src/main/kotlin/flank/corellium/shard/Internal.kt create mode 100644 corellium/shard/src/main/kotlin/flank/corellium/shard/Shard.kt create mode 100644 corellium/shard/src/test/kotlin/flank/corellium/shard/ShardKtTest.kt diff --git a/corellium/shard/README.md b/corellium/shard/README.md new file mode 100644 index 0000000000..6559c47b70 --- /dev/null +++ b/corellium/shard/README.md @@ -0,0 +1,92 @@ +# Sharding + +* Splitting test cases from one test apk to run on many devices. +* Grouping test cases from many test apks to run on a single device. +* Mix both of the above options. + +### Input + +```yaml +bundle: + - app: app1 + tests: + - test: app1-test1 + cases: + - "class app1.test1.TestClass#test1" // 1s + - app: app2 + tests: + - test: app2-test1 + cases: + - "class app2.test1.TestClass#test2" // 2s + - "class app2.test1.TestClass#test3" // 3s + - test: app2-test2 + cases: + - "class app2.test2.TestClass#test7" // 7s + - "class app2.test2.TestClass#test8" // 8s + - "class app2.test2.TestClass#test9" // 9s +``` + +### Output + +#### Max shards 3 + +```yaml +shards: + - shard1: + - app: app1 + tests: + - test: app1-test1 + cases: + - "class app1.test1.TestClass#test1" // 1s + - app: app2 + tests: + - test: app2-test2 + cases: + - "class app2.test2.TestClass#test9" // 9s + - shard2: + - app: app2 + test: app2-test1 + cases: + - "class app1.test2.TestClass#test2" // 2s + - app: app2 + test: app2-test2 + cases: + - "class app2.test2.TestClass#test8" // 8s + - shard3: + - app: app2 + test: app2-test1 + cases: + - "class app2.test1.TestClass#test3" // 3s + - app: app2 + test: app2-test2 + cases: + - "class app2.test2.TestClass#test7" // 7s +``` + +#### Max shards 2 + +```yaml +shards: + - shard1: + - app: app1 + tests: + - test: app1-test1 + cases: + - "class app1.test1.TestClass#test1" // 1sz + - app: app2 + tests: + - test: app2-test1 + cases: + - "class app2.test1.TestClass#test2" // 2s + - "class app2.test2.TestClass#test3" // 3s + - test: app2-test2 + cases: + - "class app2.test2.TestClass#test9" // 9s + - shard2: + - app: app2 + tests: + - test: app2-test2 + cases: + - "class app2.test2.TestClass#test7" // 7s + - "class app2.test2.TestClass#test8" // 8s +``` diff --git a/corellium/shard/build.gradle.kts b/corellium/shard/build.gradle.kts new file mode 100644 index 0000000000..461177c967 --- /dev/null +++ b/corellium/shard/build.gradle.kts @@ -0,0 +1,21 @@ +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + +plugins { + kotlin(Plugins.Kotlin.PLUGIN_JVM) + kotlin(Plugins.Kotlin.PLUGIN_SERIALIZATION) version Versions.KOTLIN +} + +repositories { + jcenter() + mavenCentral() + maven(url = "https://kotlin.bintray.com/kotlinx") +} + +tasks.withType { kotlinOptions.jvmTarget = "1.8" } + +dependencies { + implementation(Dependencies.KOTLIN_SERIALIZATION) + implementation(Dependencies.KOTLIN_COROUTINES_CORE) + + testImplementation(Dependencies.JUNIT) +} diff --git a/corellium/shard/src/main/kotlin/flank/corellium/shard/Internal.kt b/corellium/shard/src/main/kotlin/flank/corellium/shard/Internal.kt new file mode 100644 index 0000000000..883f01eb82 --- /dev/null +++ b/corellium/shard/src/main/kotlin/flank/corellium/shard/Internal.kt @@ -0,0 +1,105 @@ +package flank.corellium.shard + +import kotlin.math.min + +// Stage 1 + +/** + * Create flat and easy to iterate list of [Chunk] without loosing info about app and test related to test case. + * @receiver The [List] of [Shard.App]. + * @return The [List] of [Chunk] which is just different representation of input data. + */ +internal fun List.mapToInternalChunks(): List = + mutableListOf().also { result -> + forEach { app -> + app.tests.forEach { test -> + test.cases.forEach { case -> + result.add( + Chunk( + app = app.name, + test = test.name, + case = case.name, + duration = case.duration + ) + ) + } + } + } + } + +/** + * Internal intermediate structure which is representing test case with associated app and test module. + */ +internal class Chunk( + val app: String, + val test: String, + val case: String, + val duration: Long, +) + +// Stage 2 + +/** + * Group the chunks into sub-lists where the standard deviation of group duration should by possibly small. + * @receiver The flat [List] of [Chunk]. + * @return The [List] of [Chunk] groups balanced by the summary duration of each. + */ +internal fun List.groupByDuration(maxCount: Int): List> { + class Chunks( + var duration: Long = 0, + val list: MutableList = mutableListOf() + ) + + // The real part of sharding calculations, + // everything else is just a mapping between structures. + // =================================================== + return sortedByDescending(Chunk::duration).fold( + initial = List(min(size, maxCount)) { Chunks() } + ) { acc, chunk -> + acc.first().apply { + duration += chunk.duration + list += chunk + } + acc.sortedBy(Chunks::duration) + }.map(Chunks::list) + // =================================================== +} + +// Stage 3 + +/** + * Build the final structure of shards. + */ +internal fun List>.mapToShards(): List> { + + // Structure the group of chunks mapping them by app and test. + val list: List>>> = map { group -> + group.groupBy { chunk -> + chunk.app + }.mapValues { (_, chunks) -> + chunks.groupBy { chunk -> + chunk.test + } + } + } + + // Convert grouped chunks into the output structures. + return list.map { map -> + map.map { (app, tests) -> + Shard.App( + name = app, + tests = tests.map { (test, chunks) -> + Shard.Test( + name = test, + cases = chunks.map { chunk -> + Shard.Test.Case( + name = chunk.case, + duration = chunk.duration + ) + } + ) + } + ) + } + } +} diff --git a/corellium/shard/src/main/kotlin/flank/corellium/shard/Shard.kt b/corellium/shard/src/main/kotlin/flank/corellium/shard/Shard.kt new file mode 100644 index 0000000000..0870a33d1d --- /dev/null +++ b/corellium/shard/src/main/kotlin/flank/corellium/shard/Shard.kt @@ -0,0 +1,36 @@ +package flank.corellium.shard + +/** + * Distribute the test cases into the [List] of shards where each shard have similar duration. + * @receiver The bunch of test cases grouped by test and app. + * @return [List] of shards where each shard may contains many apps and test cases. + */ +fun List.calculateShards( + maxCount: Int +): List> = this + .mapToInternalChunks() + .groupByDuration(maxCount) + .mapToShards() + +/** + * Namespace for sharding input and output structures + */ +object Shard { + + class App( + val name: String, + val tests: List + ) + + class Test( + val name: String, + val cases: List + ) { + class Case( + val name: String, + val duration: Long = DEFAULT_DURATION + ) + } + + private const val DEFAULT_DURATION = 120L +} diff --git a/corellium/shard/src/test/kotlin/flank/corellium/shard/ShardKtTest.kt b/corellium/shard/src/test/kotlin/flank/corellium/shard/ShardKtTest.kt new file mode 100644 index 0000000000..01dd51904e --- /dev/null +++ b/corellium/shard/src/test/kotlin/flank/corellium/shard/ShardKtTest.kt @@ -0,0 +1,107 @@ +package flank.corellium.shard + +import org.junit.Assert +import org.junit.Test as JTest + +val apps = listOf( + Shard.App( + name = "app1", + tests = listOf( + Shard.Test( + name = "app1-test1", + cases = listOf( + Shard.Test.Case( + name = "class app1.test1.TestClass#test1", + duration = 1 + ), + ) + ), + ) + ), + Shard.App( + name = "app2", + tests = listOf( + Shard.Test( + name = "app2-test1", + cases = listOf( + Shard.Test.Case( + name = "class app2.test1.TestClass#test2", + duration = 2 + ), + Shard.Test.Case( + name = "class app2.test1.TestClass#test3", + duration = 3 + ), + ) + ), + Shard.Test( + name = "app2-test2", + cases = listOf( + Shard.Test.Case( + name = "class app2.test2.TestClass#test7", + duration = 7 + ), + Shard.Test.Case( + name = "class app2.test2.TestClass#test8", + duration = 8 + ), + Shard.Test.Case( + name = "class app2.test2.TestClass#test9", + duration = 9 + ), + ) + ), + ) + ) +) + +class ShardKtTest { + + @JTest + fun test2() { + apps.calculateShards(2).apply { + printShards() + verifyDurationEqual() + } + } + + @JTest + fun test3() { + apps.calculateShards(3).apply { + printShards() + verifyDurationEqual() + } + } +} + +private fun List>.printShards() { + forEach { + it.forEach { + println(it.name) + it.tests.forEach { + println(it.name) + it.cases.forEach { + println(it.name + ": " + it.duration) + } + } + } + println() + } +} + +private fun List>.verifyDurationEqual() { + map { + it.sumByDouble { + it.tests.sumByDouble { + it.cases.sumByDouble { + it.duration.toDouble() + } + } + } + }.map { + it.toLong() + }.reduce { first, next -> + Assert.assertEquals(first, next) + first + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index e37a558d73..4f788c359a 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -11,10 +11,10 @@ include( "samples:gradle-export-api", "test_projects:android", ":common", - ":corellium", ":corellium:sandbox", ":corellium:client", - ":corellium:log" + ":corellium:log", + ":corellium:shard", ) plugins { From ba8a66247cc5632dee0dd5292cab632157090982 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Thu, 22 Apr 2021 18:57:57 +0200 Subject: [PATCH 2/8] Add sharding structures diagram --- corellium/shard/README.md | 6 +++-- docs/corellium/sharding-class.puml | 37 ++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 docs/corellium/sharding-class.puml diff --git a/corellium/shard/README.md b/corellium/shard/README.md index 6559c47b70..e6fdfe558d 100644 --- a/corellium/shard/README.md +++ b/corellium/shard/README.md @@ -1,7 +1,9 @@ # Sharding -* Splitting test cases from one test apk to run on many devices. -* Grouping test cases from many test apks to run on a single device. +Depending on provided test cases duration, sharding algorithm will: + +* Split test cases from one test apk to run on many devices. +* Group test cases from many test apks to run on a single device. * Mix both of the above options. ### Input diff --git a/docs/corellium/sharding-class.puml b/docs/corellium/sharding-class.puml new file mode 100644 index 0000000000..4c69bb62b0 --- /dev/null +++ b/docs/corellium/sharding-class.puml @@ -0,0 +1,37 @@ +@startuml +'https://plantuml.com/component-diagram + +object calculateShards + +abstract Input { +one: App +} + +abstract Output { +many: List +} + +class App { +name: String +tests: List +} + +class Test { +name: String +cases: List +} + +class Case { +name: String +duration: Long +} + +calculateShards ..> Input : receive +calculateShards ..> Output : return + +Input "1" o-- "1" App +Output "1" o-- "*" App +App "1" o-- "*" Test +Test "1" o-- "*" Case + +@enduml From 4c3ef9a2229db97c52bc876d3bf0b0f2d81a7789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Thu, 22 Apr 2021 19:03:33 +0200 Subject: [PATCH 3/8] Add sharding structures diagram --- corellium/shard/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/corellium/shard/README.md b/corellium/shard/README.md index e6fdfe558d..5aee66e597 100644 --- a/corellium/shard/README.md +++ b/corellium/shard/README.md @@ -6,6 +6,12 @@ Depending on provided test cases duration, sharding algorithm will: * Group test cases from many test apks to run on a single device. * Mix both of the above options. +## Diagram + +![sharding_class_diagram](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/1801_Multi-module_sharding_algorithm/docs/corellium/sharding-class.puml) + +## Example + ### Input ```yaml From ac66081acffd46c4fab0b627b10b9b1b462d32a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20G=C3=B3ral?= <60390247+jan-gogo@users.noreply.github.com> Date: Mon, 26 Apr 2021 10:20:39 +0200 Subject: [PATCH 4/8] Update corellium/shard/src/main/kotlin/flank/corellium/shard/Internal.kt Co-authored-by: Michael Wright --- .../shard/src/main/kotlin/flank/corellium/shard/Internal.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corellium/shard/src/main/kotlin/flank/corellium/shard/Internal.kt b/corellium/shard/src/main/kotlin/flank/corellium/shard/Internal.kt index 883f01eb82..8707952b20 100644 --- a/corellium/shard/src/main/kotlin/flank/corellium/shard/Internal.kt +++ b/corellium/shard/src/main/kotlin/flank/corellium/shard/Internal.kt @@ -5,7 +5,7 @@ import kotlin.math.min // Stage 1 /** - * Create flat and easy to iterate list of [Chunk] without loosing info about app and test related to test case. + * Create flat and easy to iterate list of [Chunk] without losing info about app and test related to test case. * @receiver The [List] of [Shard.App]. * @return The [List] of [Chunk] which is just different representation of input data. */ From dbd6176c34ad7511467326547a17b6bd9b7593a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20G=C3=B3ral?= <60390247+jan-gogo@users.noreply.github.com> Date: Mon, 26 Apr 2021 10:20:52 +0200 Subject: [PATCH 5/8] Update corellium/shard/README.md Co-authored-by: Michael Wright --- corellium/shard/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corellium/shard/README.md b/corellium/shard/README.md index 5aee66e597..4133119a33 100644 --- a/corellium/shard/README.md +++ b/corellium/shard/README.md @@ -4,7 +4,7 @@ Depending on provided test cases duration, sharding algorithm will: * Split test cases from one test apk to run on many devices. * Group test cases from many test apks to run on a single device. -* Mix both of the above options. +* A mix both of the above options. ## Diagram From 7c9a255c2a012fd8d321f56beae55968aa42fa7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20G=C3=B3ral?= <60390247+jan-gogo@users.noreply.github.com> Date: Mon, 26 Apr 2021 10:21:00 +0200 Subject: [PATCH 6/8] Update corellium/shard/README.md Co-authored-by: Michael Wright --- corellium/shard/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corellium/shard/README.md b/corellium/shard/README.md index 4133119a33..7bc6c1a5bc 100644 --- a/corellium/shard/README.md +++ b/corellium/shard/README.md @@ -3,7 +3,7 @@ Depending on provided test cases duration, sharding algorithm will: * Split test cases from one test apk to run on many devices. -* Group test cases from many test apks to run on a single device. +* Group the test cases from many test apks to run on a single device. * A mix both of the above options. ## Diagram From b64422becbc95ae9d2ed2e92463fe229ef65a00c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20G=C3=B3ral?= <60390247+jan-gogo@users.noreply.github.com> Date: Mon, 26 Apr 2021 10:21:05 +0200 Subject: [PATCH 7/8] Update corellium/shard/README.md Co-authored-by: Michael Wright --- corellium/shard/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corellium/shard/README.md b/corellium/shard/README.md index 7bc6c1a5bc..94e8c47e8f 100644 --- a/corellium/shard/README.md +++ b/corellium/shard/README.md @@ -2,7 +2,7 @@ Depending on provided test cases duration, sharding algorithm will: -* Split test cases from one test apk to run on many devices. +* Split the test cases from one test apk to run on many devices. * Group the test cases from many test apks to run on a single device. * A mix both of the above options. From 923a017f106485a9bb265586020a7abaa2683906 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20G=C3=B3ral?= <60390247+jan-gogo@users.noreply.github.com> Date: Mon, 26 Apr 2021 10:21:12 +0200 Subject: [PATCH 8/8] Update corellium/shard/README.md Co-authored-by: Michael Wright --- corellium/shard/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corellium/shard/README.md b/corellium/shard/README.md index 94e8c47e8f..9948281ac7 100644 --- a/corellium/shard/README.md +++ b/corellium/shard/README.md @@ -1,6 +1,6 @@ # Sharding -Depending on provided test cases duration, sharding algorithm will: +Depending on the provided test cases duration, the sharding algorithm will: * Split the test cases from one test apk to run on many devices. * Group the test cases from many test apks to run on a single device.