From df2e41a54bad8ef096c1a2e3466b9d5e9c229592 Mon Sep 17 00:00:00 2001 From: Vinay Potluri Date: Wed, 8 Mar 2023 04:06:11 -0800 Subject: [PATCH] Fixes missing HTTP header X-Goog-User-Project Flank does not pass the the HTTP header flag X-Goog-user-Project with project Id when using end user credentials. This results in HTTP error 403 (PERMISSION_DENIED). This fix sets the flag so the tests can be run by the user. Fixes #2347 --- .../ftl/client/google/AndroidCatalog.kt | 3 ++ .../kotlin/ftl/client/google/GcTestMatrix.kt | 4 +++ .../kotlin/ftl/client/google/GcToolResults.kt | 30 +++++++++++++++++-- .../kotlin/ftl/client/google/IosCatalog.kt | 3 ++ .../kotlin/ftl/client/google/OsVersion.kt | 6 +++- .../google/run/android/GcAndroidTestMatrix.kt | 6 +++- .../client/google/run/ios/GcIosTestMatrix.kt | 6 +++- .../main/kotlin/ftl/config/FtlConstants.kt | 1 + .../reports/outcome/BillableMinutesTest.kt | 1 + 9 files changed, 54 insertions(+), 6 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/client/google/AndroidCatalog.kt b/test_runner/src/main/kotlin/ftl/client/google/AndroidCatalog.kt index c3062e86f9..325ab11eaf 100644 --- a/test_runner/src/main/kotlin/ftl/client/google/AndroidCatalog.kt +++ b/test_runner/src/main/kotlin/ftl/client/google/AndroidCatalog.kt @@ -1,9 +1,11 @@ package ftl.client.google +import com.google.api.client.http.HttpHeaders import com.google.testing.model.AndroidDevice import com.google.testing.model.AndroidDeviceCatalog import com.google.testing.model.AndroidModel import com.google.testing.model.Orientation +import ftl.config.FtlConstants.GCS_PROJECT_HEADER import ftl.http.executeWithRetry import ftl.presentation.cli.firebase.test.reportmanager.ReportManagerState import ftl.presentation.publish @@ -21,6 +23,7 @@ object AndroidCatalog { GcTesting.get.testEnvironmentCatalog() .get("android") .setProjectId(projectId) + .setRequestHeaders(HttpHeaders().set(GCS_PROJECT_HEADER, projectId)) .executeWithRetry() .androidDeviceCatalog .filterDevicesWithoutSupportedVersions() diff --git a/test_runner/src/main/kotlin/ftl/client/google/GcTestMatrix.kt b/test_runner/src/main/kotlin/ftl/client/google/GcTestMatrix.kt index dd82f05063..07be100ed3 100644 --- a/test_runner/src/main/kotlin/ftl/client/google/GcTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/client/google/GcTestMatrix.kt @@ -1,7 +1,9 @@ package ftl.client.google +import com.google.api.client.http.HttpHeaders import com.google.testing.model.CancelTestMatrixResponse import com.google.testing.model.TestMatrix +import ftl.config.FtlConstants.GCS_PROJECT_HEADER import ftl.http.executeWithRetry import ftl.run.exception.FlankGeneralError import kotlinx.coroutines.Dispatchers @@ -38,6 +40,7 @@ object GcTestMatrix { suspend fun refresh(testMatrixId: String, projectId: String): TestMatrix { val getMatrix = withContext(Dispatchers.IO) { GcTesting.get.projects().testMatrices().get(projectId, testMatrixId) + .setRequestHeaders(HttpHeaders().set(GCS_PROJECT_HEADER, projectId)) } var failed = 0 val maxWait = ofHours(1).seconds @@ -57,6 +60,7 @@ object GcTestMatrix { suspend fun cancel(testMatrixId: String, projectId: String): CancelTestMatrixResponse { val cancelMatrix = withContext(Dispatchers.IO) { GcTesting.get.projects().testMatrices().cancel(projectId, testMatrixId) + .setRequestHeaders(HttpHeaders().set(GCS_PROJECT_HEADER, projectId)) } var failed = 0 val maxTries = 3 diff --git a/test_runner/src/main/kotlin/ftl/client/google/GcToolResults.kt b/test_runner/src/main/kotlin/ftl/client/google/GcToolResults.kt index 1ccf447b74..bc315d87ba 100644 --- a/test_runner/src/main/kotlin/ftl/client/google/GcToolResults.kt +++ b/test_runner/src/main/kotlin/ftl/client/google/GcToolResults.kt @@ -1,5 +1,6 @@ package ftl.client.google +import com.google.api.client.http.HttpHeaders import com.google.api.services.toolresults.ToolResults import com.google.api.services.toolresults.model.Environment import com.google.api.services.toolresults.model.Execution @@ -16,6 +17,7 @@ import com.google.testing.model.ToolResultsHistory import com.google.testing.model.ToolResultsStep import ftl.args.IArgs import ftl.config.FtlConstants +import ftl.config.FtlConstants.GCS_PROJECT_HEADER import ftl.config.FtlConstants.JSON_FACTORY import ftl.config.FtlConstants.applicationName import ftl.config.FtlConstants.httpTransport @@ -44,6 +46,7 @@ object GcToolResults { .histories() .list(args.project) .setFilterByName(args.resultsHistoryName) + .setRequestHeaders(HttpHeaders().set(GCS_PROJECT_HEADER, args.project)) .executeWithRetry() return result?.histories ?: emptyList() } @@ -56,6 +59,7 @@ object GcToolResults { .projects() .histories() .create(args.project, history) + .setRequestHeaders(HttpHeaders().set(GCS_PROJECT_HEADER, args.project)) .execute() } @@ -90,6 +94,7 @@ object GcToolResults { toolResultsStep.historyId, toolResultsStep.executionId ) + .setRequestHeaders(HttpHeaders().set(GCS_PROJECT_HEADER, testExecution.projectId)) .executeWithRetry() } @@ -105,6 +110,7 @@ object GcToolResults { toolResultsStep.executionId, toolResultsStep.stepId ) + .setRequestHeaders(HttpHeaders().set(GCS_PROJECT_HEADER, toolResultsStep.projectId)) .executeWithRetry() } @@ -121,6 +127,7 @@ object GcToolResults { toolResultsStep.executionId, toolResultsStep.stepId ) + .setRequestHeaders(HttpHeaders().set(GCS_PROJECT_HEADER, toolResultsStep.projectId)) .executeWithRetry() // Lists Test Cases attached to a Step @@ -141,6 +148,7 @@ object GcToolResults { toolResultsStep.stepId ) .setPageToken(pageToken) + .setRequestHeaders(HttpHeaders().set(GCS_PROJECT_HEADER, toolResultsStep.projectId)) .executeWithRetry() } @@ -155,12 +163,26 @@ object GcToolResults { } fun getDefaultBucket(projectId: String, source: String? = null): String? = try { - service.Projects().initializeSettings(projectId).executeWithRetry().defaultBucket + service.Projects().initializeSettings(projectId) + .setRequestHeaders(HttpHeaders().set(GCS_PROJECT_HEADER, projectId)) + .executeWithRetry() + .defaultBucket } catch (ftlProjectError: FTLProjectError) { // flank needs to rewrap the exception with additional info about project when (ftlProjectError) { - is PermissionDenied -> throw FlankGeneralError(permissionDeniedErrorMessage(projectId, source, ftlProjectError.message)) - is ProjectNotFound -> throw FlankGeneralError(projectNotFoundErrorMessage(projectId, ftlProjectError.message)) + is PermissionDenied -> throw FlankGeneralError( + permissionDeniedErrorMessage( + projectId, + source, + ftlProjectError.message + ) + ) + is ProjectNotFound -> throw FlankGeneralError( + projectNotFoundErrorMessage( + projectId, + ftlProjectError.message + ) + ) is FailureToken -> UserAuth.throwAuthenticationError() } } @@ -190,6 +212,7 @@ object GcToolResults { ) .setPageToken(pageToken) .setPageSize(100) + .setRequestHeaders(HttpHeaders().set(GCS_PROJECT_HEADER, results.projectId)) .executeWithRetry() fun listAllSteps(results: ToolResultsExecution): MutableList { @@ -217,6 +240,7 @@ object GcToolResults { ) .setPageToken(pageToken) .setPageSize(100) + .setRequestHeaders(HttpHeaders().set(GCS_PROJECT_HEADER, results.projectId)) .executeWithRetry() } diff --git a/test_runner/src/main/kotlin/ftl/client/google/IosCatalog.kt b/test_runner/src/main/kotlin/ftl/client/google/IosCatalog.kt index bc39ca9735..7691a747b8 100644 --- a/test_runner/src/main/kotlin/ftl/client/google/IosCatalog.kt +++ b/test_runner/src/main/kotlin/ftl/client/google/IosCatalog.kt @@ -1,9 +1,11 @@ package ftl.client.google +import com.google.api.client.http.HttpHeaders import com.google.testing.model.IosDeviceCatalog import com.google.testing.model.IosModel import com.google.testing.model.Orientation import ftl.config.Device +import ftl.config.FtlConstants.GCS_PROJECT_HEADER import ftl.environment.getLocaleDescription import ftl.environment.ios.getDescription import ftl.environment.ios.iosVersionsToCliTable @@ -50,6 +52,7 @@ object IosCatalog { GcTesting.get.testEnvironmentCatalog() .get("ios") .setProjectId(projectId) + .setRequestHeaders(HttpHeaders().set(GCS_PROJECT_HEADER, projectId)) .executeWithRetry() .iosDeviceCatalog .filterDevicesWithoutSupportedVersions() diff --git a/test_runner/src/main/kotlin/ftl/client/google/OsVersion.kt b/test_runner/src/main/kotlin/ftl/client/google/OsVersion.kt index 60c15177bd..039477a1ee 100644 --- a/test_runner/src/main/kotlin/ftl/client/google/OsVersion.kt +++ b/test_runner/src/main/kotlin/ftl/client/google/OsVersion.kt @@ -1,13 +1,16 @@ package ftl.client.google +import com.google.api.client.http.HttpHeaders import com.google.testing.model.AndroidVersion import com.google.testing.model.IosVersion +import ftl.config.FtlConstants.GCS_PROJECT_HEADER import ftl.http.executeWithRetry fun androidOsVersions(projectId: String): List = GcTesting.get.testEnvironmentCatalog() .get("android") .setProjectId(projectId) + .setRequestHeaders(HttpHeaders().set(GCS_PROJECT_HEADER, projectId)) .executeWithRetry() .androidDeviceCatalog .versions @@ -17,7 +20,8 @@ fun iosOsVersions(projectId: String): List = GcTesting.get.testEnvironmentCatalog() .get("ios") .setProjectId(projectId) + .setRequestHeaders(HttpHeaders().set(GCS_PROJECT_HEADER, projectId)) .executeWithRetry() .iosDeviceCatalog .versions - .orEmpty() + .orEmpty() \ No newline at end of file diff --git a/test_runner/src/main/kotlin/ftl/client/google/run/android/GcAndroidTestMatrix.kt b/test_runner/src/main/kotlin/ftl/client/google/run/android/GcAndroidTestMatrix.kt index 6765240e1a..63dd4fde2c 100644 --- a/test_runner/src/main/kotlin/ftl/client/google/run/android/GcAndroidTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/client/google/run/android/GcAndroidTestMatrix.kt @@ -1,5 +1,6 @@ package ftl.client.google.run.android +import com.google.api.client.http.HttpHeaders import com.google.common.annotations.VisibleForTesting import com.google.testing.Testing import com.google.testing.model.Account @@ -22,6 +23,7 @@ import ftl.api.TestMatrixAndroid import ftl.client.google.GcTesting import ftl.client.google.run.toClientInfoDetailList import ftl.client.google.run.toFileReference +import ftl.config.FtlConstants.GCS_PROJECT_HEADER import ftl.http.executeWithRetry import ftl.run.exception.FlankGeneralError import ftl.util.timeoutToSeconds @@ -45,7 +47,9 @@ private suspend fun executeAndroidTestMatrix( ): List> = coroutineScope { (0 until config.repeatTests).map { runIndex -> async(Dispatchers.IO) { - createAndroidTestMatrix(type, config, typeIndex, runIndex).executeWithRetry() + createAndroidTestMatrix(type, config, typeIndex, runIndex) + .setRequestHeaders(HttpHeaders().set(GCS_PROJECT_HEADER, config.project)) + .executeWithRetry() } } } diff --git a/test_runner/src/main/kotlin/ftl/client/google/run/ios/GcIosTestMatrix.kt b/test_runner/src/main/kotlin/ftl/client/google/run/ios/GcIosTestMatrix.kt index 6daf51eca4..12254cc256 100644 --- a/test_runner/src/main/kotlin/ftl/client/google/run/ios/GcIosTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/client/google/run/ios/GcIosTestMatrix.kt @@ -1,5 +1,6 @@ package ftl.client.google.run.ios +import com.google.api.client.http.HttpHeaders import com.google.testing.Testing import com.google.testing.model.ClientInfo import com.google.testing.model.EnvironmentMatrix @@ -15,6 +16,7 @@ import ftl.client.google.run.mapGcsPathsToFileReference import ftl.client.google.run.mapToIosDeviceFiles import ftl.client.google.run.toClientInfoDetailList import ftl.client.google.run.toIosDeviceFile +import ftl.config.FtlConstants.GCS_PROJECT_HEADER import ftl.http.executeWithRetry import ftl.run.exception.FlankGeneralError import ftl.util.timeoutToSeconds @@ -37,7 +39,9 @@ private suspend fun executeIosTestMatrixAsync( config: TestMatrixIos.Config ): Deferred = coroutineScope { async(Dispatchers.IO) { - createIosTestMatrix(type, config).executeWithRetry() + createIosTestMatrix(type, config) + .setRequestHeaders(HttpHeaders().set(GCS_PROJECT_HEADER, config.project)) + .executeWithRetry() } } diff --git a/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt b/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt index 4fc00421bf..59b0790017 100644 --- a/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt +++ b/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt @@ -25,6 +25,7 @@ object FtlConstants { const val indent = " " const val matrixIdsFile = "matrix_ids.json" const val applicationName = "Flank" + const val GCS_PROJECT_HEADER = "x-goog-user-project" const val GCS_PREFIX = "gs://" const val GCS_STORAGE_LINK = "https://console.developers.google.com/storage/browser/" const val runTimeout = "-1" diff --git a/test_runner/src/test/kotlin/ftl/reports/outcome/BillableMinutesTest.kt b/test_runner/src/test/kotlin/ftl/reports/outcome/BillableMinutesTest.kt index a476c0c4c1..d249486195 100644 --- a/test_runner/src/test/kotlin/ftl/reports/outcome/BillableMinutesTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/outcome/BillableMinutesTest.kt @@ -47,6 +47,7 @@ class BillableMinutesTest { .testEnvironmentCatalog() .get(any()) .setProjectId(any()) + .setRequestHeaders(any()) .executeWithRetry() } returns make { androidDeviceCatalog = make { models = androidModels }