From b367f843df4e8c53892ef7f0a3663e4702f08bbd Mon Sep 17 00:00:00 2001 From: Markus Hintersteiner Date: Mon, 25 Nov 2024 13:37:28 +0100 Subject: [PATCH 1/9] Firebase Testlab setup PoC --- ...integration-tests-ui-firebase-test-lab.yml | 76 +++++++++++++++++++ .../src/androidTest/AndroidManifest.xml | 19 +++++ 2 files changed, 95 insertions(+) create mode 100644 .github/workflows/integration-tests-ui-firebase-test-lab.yml create mode 100644 sentry-android-integration-tests/sentry-uitest-android/src/androidTest/AndroidManifest.xml diff --git a/.github/workflows/integration-tests-ui-firebase-test-lab.yml b/.github/workflows/integration-tests-ui-firebase-test-lab.yml new file mode 100644 index 0000000000..04e22bbb9c --- /dev/null +++ b/.github/workflows/integration-tests-ui-firebase-test-lab.yml @@ -0,0 +1,76 @@ +name: 'Integration Tests' +on: + push: + branches: + - main + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Ui tests + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + device-model: [MediumPhone.arm] + device-api-level: [ 30, 31, 32, 33 ] + + # we copy the secret to the env variable in order to access it in the workflow + env: + SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} + SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} + + steps: + - name: Git checkout + uses: actions/checkout@v4 + with: + submodules: 'recursive' + + - name: 'Set up Java: 17' + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@707359876a764dbcdb9da0b0ed08291818310c3d # pin@v3 + with: + gradle-home-cache-cleanup: true + + - name: Make assembleUiTests + run: make assembleUiTests + + - name: 'Install jq' + run: 'sudo apt install -y --no-install-recommends jq' + + - name: Install gcloud + uses: 'google-github-actions/setup-gcloud@v2' + with: + version: '>= 363.0.0' + + - name: Configure gcloud + run: | + echo "$SERVICE_ACCOUNT" > /opt/service_account.json + project_id=$(cat /opt/service_account.json | jq -r ".project_id") + gcloud auth activate-service-account --key-file=/opt/service_account.json + gcloud config set project $project_id + + - name: Firebase Test Lab (Device ${{ matrix.device-model }} API ${{ matrix.device-api-level }}) + run: | + appApk=./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/release/sentry-uitest-android-release.apk + testAppApk=./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/androidTest/release/sentry-uitest-android-release-androidTest.apk + attempts=4 + device-model=${{ matrix.device-model }} + device-api-level=${{ matrix.device-api-level }} + + if gcloud firebase test android run --app=$appApk --test=$testAppApk --type=$type --device=$device-model,version=$device-api-level --num-flaky-test-attempts=$attempts + then + echo "Test execution successfully finished" + else + status=$? + echo "Test execution exited with non-zero exit code: " $status + fi diff --git a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/AndroidManifest.xml b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/AndroidManifest.xml new file mode 100644 index 0000000000..5bbdb891e9 --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/AndroidManifest.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + From 186f08dd4f15fc61d798043bbbf59d5cd872c137 Mon Sep 17 00:00:00 2001 From: Markus Hintersteiner Date: Mon, 25 Nov 2024 15:37:54 +0100 Subject: [PATCH 2/9] Fix Firebase setup config --- .../integration-tests-ui-firebase-test-lab.yml | 15 ++++++++------- .../sentry-uitest-android/build.gradle.kts | 1 + 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/integration-tests-ui-firebase-test-lab.yml b/.github/workflows/integration-tests-ui-firebase-test-lab.yml index 04e22bbb9c..92e4ee433a 100644 --- a/.github/workflows/integration-tests-ui-firebase-test-lab.yml +++ b/.github/workflows/integration-tests-ui-firebase-test-lab.yml @@ -16,8 +16,8 @@ jobs: strategy: fail-fast: false matrix: - device-model: [MediumPhone.arm] - device-api-level: [ 30, 31, 32, 33 ] + device-model: [MediumPhone.arm, husky, e1q] # Pixel 8 Pro, Galaxy S24 + device-api-level: [ 34 ] # we copy the secret to the env variable in order to access it in the workflow env: @@ -61,13 +61,14 @@ jobs: - name: Firebase Test Lab (Device ${{ matrix.device-model }} API ${{ matrix.device-api-level }}) run: | - appApk=./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/release/sentry-uitest-android-release.apk - testAppApk=./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/androidTest/release/sentry-uitest-android-release-androidTest.apk + app_apk=./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/release/sentry-uitest-android-release.apk + test_app_apk=./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/androidTest/release/sentry-uitest-android-release-androidTest.apk attempts=4 - device-model=${{ matrix.device-model }} - device-api-level=${{ matrix.device-api-level }} + device_model=${{ matrix.device-model }} + device_api_level=${{ matrix.device-api-level }} + type=instrumentation - if gcloud firebase test android run --app=$appApk --test=$testAppApk --type=$type --device=$device-model,version=$device-api-level --num-flaky-test-attempts=$attempts + if gcloud firebase test android run --app=$app_apk --test=$test_app_apk --type=$type --device=^:^model=$device_model:version=$device_api_level --num-flaky-test-attempts=$attempts then echo "Test execution successfully finished" else diff --git a/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts b/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts index 7755166a4e..f6fbc35a7c 100644 --- a/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts +++ b/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts @@ -125,6 +125,7 @@ dependencies { androidTestImplementation(Config.TestLibs.mockWebserver) androidTestImplementation(Config.TestLibs.androidxJunit) androidTestImplementation(Config.TestLibs.leakCanaryInstrumentation) + androidTestImplementation("com.google.firebase:testlab-instr-lib:0.2") androidTestUtil(Config.TestLibs.androidxTestOrchestrator) } From 8b12669bddb95fe672265b47ba36e2512c0a5a33 Mon Sep 17 00:00:00 2001 From: Markus Hintersteiner Date: Mon, 25 Nov 2024 15:37:59 +0100 Subject: [PATCH 3/9] Fix tests --- .../java/io/sentry/uitest/android/AutomaticSpansTest.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/AutomaticSpansTest.kt b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/AutomaticSpansTest.kt index d5dd0fd80e..9bf72e258c 100644 --- a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/AutomaticSpansTest.kt +++ b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/AutomaticSpansTest.kt @@ -90,7 +90,7 @@ class AutomaticSpansTest : BaseUiTest() { // AGP matrix tests have no frames Assume.assumeTrue(totalFrames > 0) assertNotEquals(totalFrames, 0) - assertTrue(totalFrames > slowFrames + frozenFrames, "Expected total frames ($totalFrames) to be higher than the sum of slow ($slowFrames) and frozen ($frozenFrames) frames.") + assertTrue(totalFrames >= slowFrames + frozenFrames, "Expected total frames ($totalFrames) to be higher than the sum of slow ($slowFrames) and frozen ($frozenFrames) frames.") } assertNoOtherEnvelopes() } @@ -126,7 +126,7 @@ class AutomaticSpansTest : BaseUiTest() { // AGP matrix tests have no frames Assume.assumeTrue(totalFrames > 0) assertNotEquals(totalFrames, 0) - assertTrue(totalFrames > slowFrames + frozenFrames, "Expected total frames ($totalFrames) to be higher than the sum of slow ($slowFrames) and frozen ($frozenFrames) frames.") + assertTrue(totalFrames >= slowFrames + frozenFrames, "Expected total frames ($totalFrames) to be higher than the sum of slow ($slowFrames) and frozen ($frozenFrames) frames.") } assertNoOtherEnvelopes() } From b346ccffbbe97ffd556c8bcaf0de8d983a62bb59 Mon Sep 17 00:00:00 2001 From: Markus Hintersteiner Date: Mon, 25 Nov 2024 15:41:39 +0100 Subject: [PATCH 4/9] Improve env variable naming --- .github/workflows/integration-tests-ui-firebase-test-lab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-ui-firebase-test-lab.yml b/.github/workflows/integration-tests-ui-firebase-test-lab.yml index 92e4ee433a..48b86b710f 100644 --- a/.github/workflows/integration-tests-ui-firebase-test-lab.yml +++ b/.github/workflows/integration-tests-ui-firebase-test-lab.yml @@ -54,7 +54,7 @@ jobs: - name: Configure gcloud run: | - echo "$SERVICE_ACCOUNT" > /opt/service_account.json + echo "$FIREBASE_TEST_LAB_SERVICE_ACCOUNT_JSON" > /opt/service_account.json project_id=$(cat /opt/service_account.json | jq -r ".project_id") gcloud auth activate-service-account --key-file=/opt/service_account.json gcloud config set project $project_id From 40a074022b475e2e35a221a0493be69790245ac1 Mon Sep 17 00:00:00 2001 From: Markus Hintersteiner Date: Mon, 25 Nov 2024 16:45:41 +0100 Subject: [PATCH 5/9] Simplify firebase test lab script --- .../integration-tests-ui-firebase-test-lab.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/integration-tests-ui-firebase-test-lab.yml b/.github/workflows/integration-tests-ui-firebase-test-lab.yml index 48b86b710f..46c2916b32 100644 --- a/.github/workflows/integration-tests-ui-firebase-test-lab.yml +++ b/.github/workflows/integration-tests-ui-firebase-test-lab.yml @@ -15,9 +15,6 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - device-model: [MediumPhone.arm, husky, e1q] # Pixel 8 Pro, Galaxy S24 - device-api-level: [ 34 ] # we copy the secret to the env variable in order to access it in the workflow env: @@ -59,16 +56,20 @@ jobs: gcloud auth activate-service-account --key-file=/opt/service_account.json gcloud config set project $project_id - - name: Firebase Test Lab (Device ${{ matrix.device-model }} API ${{ matrix.device-api-level }}) + - name: Firebase Test Lab run: | app_apk=./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/release/sentry-uitest-android-release.apk test_app_apk=./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/androidTest/release/sentry-uitest-android-release-androidTest.apk attempts=4 - device_model=${{ matrix.device-model }} - device_api_level=${{ matrix.device-api-level }} type=instrumentation - if gcloud firebase test android run --app=$app_apk --test=$test_app_apk --type=$type --device=^:^model=$device_model:version=$device_api_level --num-flaky-test-attempts=$attempts + if gcloud firebase test android run \ + --app=$app_apk --test=$test_app_apk \ + --type=$type \ + --num-flaky-test-attempts=$attempts \ + --device=^:^model=MediumPhone.arm:version=34 \ + --device=^:^model=bluejay:version=32 \ + --device=^:^model=a14xm:version=34 \ then echo "Test execution successfully finished" else From 4993fbdbea78a50ea6fb96aa7a20f9677742c06d Mon Sep 17 00:00:00 2001 From: Markus Hintersteiner Date: Tue, 26 Nov 2024 10:18:35 +0100 Subject: [PATCH 6/9] Fix define correct env vars --- .github/workflows/integration-tests-ui-firebase-test-lab.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests-ui-firebase-test-lab.yml b/.github/workflows/integration-tests-ui-firebase-test-lab.yml index 46c2916b32..71c713d663 100644 --- a/.github/workflows/integration-tests-ui-firebase-test-lab.yml +++ b/.github/workflows/integration-tests-ui-firebase-test-lab.yml @@ -18,8 +18,7 @@ jobs: # we copy the secret to the env variable in order to access it in the workflow env: - SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} - SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} + FIREBASE_TEST_LAB_SERVICE_ACCOUNT_JSON: ${{ secrets.FIREBASE_TEST_LAB_SERVICE_ACCOUNT_JSON }} steps: - name: Git checkout From 76e27faf7123841ace52f87e26d530b261bce54e Mon Sep 17 00:00:00 2001 From: Markus Hintersteiner Date: Tue, 26 Nov 2024 11:57:51 +0100 Subject: [PATCH 7/9] Avoid shell escape chars by base64 encoding --- .../workflows/integration-tests-ui-firebase-test-lab.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-tests-ui-firebase-test-lab.yml b/.github/workflows/integration-tests-ui-firebase-test-lab.yml index 71c713d663..582e847421 100644 --- a/.github/workflows/integration-tests-ui-firebase-test-lab.yml +++ b/.github/workflows/integration-tests-ui-firebase-test-lab.yml @@ -18,7 +18,9 @@ jobs: # we copy the secret to the env variable in order to access it in the workflow env: - FIREBASE_TEST_LAB_SERVICE_ACCOUNT_JSON: ${{ secrets.FIREBASE_TEST_LAB_SERVICE_ACCOUNT_JSON }} + # The contents of service_account.json encoded using base64, to avoid any shell escaping issues + # base64 -i .json + FIREBASE_TEST_LAB_SERVICE_ACCOUNT_JSON_BASE64: ${{ secrets.FIREBASE_TEST_LAB_SERVICE_ACCOUNT_JSON_BASE64 }} steps: - name: Git checkout @@ -50,9 +52,8 @@ jobs: - name: Configure gcloud run: | - echo "$FIREBASE_TEST_LAB_SERVICE_ACCOUNT_JSON" > /opt/service_account.json - project_id=$(cat /opt/service_account.json | jq -r ".project_id") - gcloud auth activate-service-account --key-file=/opt/service_account.json + echo "$FIREBASE_TEST_LAB_SERVICE_ACCOUNT_JSON_BASE64" | base64 -d | gcloud auth activate-service-account --key-file=- + project_id=$(echo "$FIREBASE_TEST_LAB_SERVICE_ACCOUNT_JSON_BASE64" | base64 -d | jq -r ".project_id") gcloud config set project $project_id - name: Firebase Test Lab From 1ecf0d7cd9ed14c5512b7446b0838f60863f352b Mon Sep 17 00:00:00 2001 From: Markus Hintersteiner Date: Tue, 26 Nov 2024 12:44:03 +0100 Subject: [PATCH 8/9] Fix properly fail --- .../integration-tests-ui-firebase-test-lab.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration-tests-ui-firebase-test-lab.yml b/.github/workflows/integration-tests-ui-firebase-test-lab.yml index 582e847421..3eaf968070 100644 --- a/.github/workflows/integration-tests-ui-firebase-test-lab.yml +++ b/.github/workflows/integration-tests-ui-firebase-test-lab.yml @@ -1,4 +1,4 @@ -name: 'Integration Tests' +name: 'Integration Tests (Firebase Test Lab)' on: push: branches: @@ -63,16 +63,16 @@ jobs: attempts=4 type=instrumentation - if gcloud firebase test android run \ + gcloud firebase test android run \ --app=$app_apk --test=$test_app_apk \ --type=$type \ --num-flaky-test-attempts=$attempts \ --device=^:^model=MediumPhone.arm:version=34 \ --device=^:^model=bluejay:version=32 \ - --device=^:^model=a14xm:version=34 \ - then - echo "Test execution successfully finished" - else - status=$? - echo "Test execution exited with non-zero exit code: " $status + --device=^:^model=a14xm:version=34 + + status=$? + if [ $status -ne 0 ]; then + echo "Firebase Test Lab failed with status $status" + exit 1 fi From ec54665cfc3d2d9f69a91660065212893a48eee8 Mon Sep 17 00:00:00 2001 From: Markus Hintersteiner Date: Tue, 26 Nov 2024 14:31:19 +0100 Subject: [PATCH 9/9] Cleanup --- .github/workflows/integration-tests-ui-firebase-test-lab.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/integration-tests-ui-firebase-test-lab.yml b/.github/workflows/integration-tests-ui-firebase-test-lab.yml index 3eaf968070..7a8a56b6ba 100644 --- a/.github/workflows/integration-tests-ui-firebase-test-lab.yml +++ b/.github/workflows/integration-tests-ui-firebase-test-lab.yml @@ -42,11 +42,8 @@ jobs: - name: Make assembleUiTests run: make assembleUiTests - - name: 'Install jq' - run: 'sudo apt install -y --no-install-recommends jq' - - name: Install gcloud - uses: 'google-github-actions/setup-gcloud@v2' + uses: 'google-github-actions/setup-gcloud@6189d56e4096ee891640bb02ac264be376592d6a' # pin 2.1.2 with: version: '>= 363.0.0'