diff --git a/.github/workflows/integration-tests-ui-firebase-test-lab.yml b/.github/workflows/integration-tests-ui-firebase-test-lab.yml new file mode 100644 index 0000000000..3eaf968070 --- /dev/null +++ b/.github/workflows/integration-tests-ui-firebase-test-lab.yml @@ -0,0 +1,78 @@ +name: 'Integration Tests (Firebase Test Lab)' +on: + push: + branches: + - main + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Ui tests + runs-on: ubuntu-latest + strategy: + fail-fast: false + + # we copy the secret to the env variable in order to access it in the workflow + env: + # The contents of service_account.json encoded using base64, to avoid any shell escaping issues + # base64 -i .json + FIREBASE_TEST_LAB_SERVICE_ACCOUNT_JSON_BASE64: ${{ secrets.FIREBASE_TEST_LAB_SERVICE_ACCOUNT_JSON_BASE64 }} + + steps: + - name: Git checkout + uses: actions/checkout@v4 + with: + submodules: 'recursive' + + - name: 'Set up Java: 17' + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@707359876a764dbcdb9da0b0ed08291818310c3d # pin@v3 + with: + gradle-home-cache-cleanup: true + + - name: Make assembleUiTests + run: make assembleUiTests + + - name: 'Install jq' + run: 'sudo apt install -y --no-install-recommends jq' + + - name: Install gcloud + uses: 'google-github-actions/setup-gcloud@v2' + with: + version: '>= 363.0.0' + + - name: Configure gcloud + run: | + echo "$FIREBASE_TEST_LAB_SERVICE_ACCOUNT_JSON_BASE64" | base64 -d | gcloud auth activate-service-account --key-file=- + project_id=$(echo "$FIREBASE_TEST_LAB_SERVICE_ACCOUNT_JSON_BASE64" | base64 -d | jq -r ".project_id") + gcloud config set project $project_id + + - name: Firebase Test Lab + run: | + app_apk=./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/release/sentry-uitest-android-release.apk + test_app_apk=./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/androidTest/release/sentry-uitest-android-release-androidTest.apk + attempts=4 + type=instrumentation + + gcloud firebase test android run \ + --app=$app_apk --test=$test_app_apk \ + --type=$type \ + --num-flaky-test-attempts=$attempts \ + --device=^:^model=MediumPhone.arm:version=34 \ + --device=^:^model=bluejay:version=32 \ + --device=^:^model=a14xm:version=34 + + status=$? + if [ $status -ne 0 ]; then + echo "Firebase Test Lab failed with status $status" + exit 1 + fi diff --git a/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts b/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts index 7755166a4e..f6fbc35a7c 100644 --- a/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts +++ b/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts @@ -125,6 +125,7 @@ dependencies { androidTestImplementation(Config.TestLibs.mockWebserver) androidTestImplementation(Config.TestLibs.androidxJunit) androidTestImplementation(Config.TestLibs.leakCanaryInstrumentation) + androidTestImplementation("com.google.firebase:testlab-instr-lib:0.2") androidTestUtil(Config.TestLibs.androidxTestOrchestrator) } diff --git a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/AndroidManifest.xml b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/AndroidManifest.xml new file mode 100644 index 0000000000..5bbdb891e9 --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/AndroidManifest.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + diff --git a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/AutomaticSpansTest.kt b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/AutomaticSpansTest.kt index d5dd0fd80e..9bf72e258c 100644 --- a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/AutomaticSpansTest.kt +++ b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/AutomaticSpansTest.kt @@ -90,7 +90,7 @@ class AutomaticSpansTest : BaseUiTest() { // AGP matrix tests have no frames Assume.assumeTrue(totalFrames > 0) assertNotEquals(totalFrames, 0) - assertTrue(totalFrames > slowFrames + frozenFrames, "Expected total frames ($totalFrames) to be higher than the sum of slow ($slowFrames) and frozen ($frozenFrames) frames.") + assertTrue(totalFrames >= slowFrames + frozenFrames, "Expected total frames ($totalFrames) to be higher than the sum of slow ($slowFrames) and frozen ($frozenFrames) frames.") } assertNoOtherEnvelopes() } @@ -126,7 +126,7 @@ class AutomaticSpansTest : BaseUiTest() { // AGP matrix tests have no frames Assume.assumeTrue(totalFrames > 0) assertNotEquals(totalFrames, 0) - assertTrue(totalFrames > slowFrames + frozenFrames, "Expected total frames ($totalFrames) to be higher than the sum of slow ($slowFrames) and frozen ($frozenFrames) frames.") + assertTrue(totalFrames >= slowFrames + frozenFrames, "Expected total frames ($totalFrames) to be higher than the sum of slow ($slowFrames) and frozen ($frozenFrames) frames.") } assertNoOtherEnvelopes() }