Capture BQ Query Cost (#20124)

Author: ulixius9

ingestion/src/metadata/ingestion/source/database/bigquery/queries.py

@@ -32,7 +32,8 @@
    end_time,
    query as query_text,
    null as schema_name,
-   total_slot_ms as duration
+   total_slot_ms as duration,
+   (total_bytes_billed / POWER(2, 40)) * {cost_per_tib} as cost
 FROM `region-{region}`.INFORMATION_SCHEMA.JOBS_BY_PROJECT
 WHERE creation_time BETWEEN "{start_time}" AND "{end_time}"
   {filters}

ingestion/src/metadata/ingestion/source/database/bigquery/query_parser.py

@@ -66,6 +66,7 @@ def get_sql_statement(self, start_time: datetime, end_time: datetime) -> str:
             region=self.service_connection.usageLocation,
             filters=self.get_filters(),
             result_limit=self.source_config.resultLimit,
+            cost_per_tib=self.service_connection.costPerTB,
         )
 
     @staticmethod

openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json

@@ -116,6 +116,12 @@
     "supportsDataDiff": {
         "title": "Supports Data Diff Extraction.",
         "$ref": "../connectionBasicType.json#/definitions/supportsDataDiff"
+    },
+    "costPerTB": {
+      "title": "Cost Per TiB",
+      "description": "Cost per TiB for BigQuery usage",
+      "type": "number",
+      "default": 6.25
     }
   },
   "additionalProperties": false,

openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/bigQueryConnection.ts

@@ -16,6 +16,10 @@
 export interface BigQueryConnection {
     connectionArguments?: { [key: string]: any };
     connectionOptions?:   { [key: string]: string };
+    /**
+     * Cost per TiB for BigQuery usage
+     */
+    costPerTB?: number;
     /**
      * GCP Credentials
      */