Skip to content

Commit

Permalink
Merged default browser dags (#6737)
Browse files Browse the repository at this point in the history
  • Loading branch information
@wwyc
wwyc authored Dec 30, 2024
1 parent c018a7d commit f9a66ce
Show file tree
Hide file tree
Showing 2 changed files with 1 addition and 20 deletions.
19 changes: 0 additions & 19 deletions dags.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2056,22 +2056,3 @@ bqetl_default_browser_aggregates:
schedule_interval: 0 22 * * *
tags:
- impact/tier_3

bqetl_fx_desktop_background_defaultagent_aggregates:
default_args:
depends_on_past: false
email:
- telemetry-alerts@mozilla.com
- wichan@mozilla.com
email_on_failure: true
email_on_retry: false
owner: wichan@mozilla.com
retries: 2
retry_delay: 30m
start_date: '2024-12-20'
description: |
This DAG builds daily aggregate tables for firefox desktop background default agent tables
repo: bigquery-etl
schedule_interval: 0 22 * * *
tags:
- impact/tier_3
2 changes: 1 addition & 1 deletion ...d-prod/firefox_desktop_background_defaultagent_derived/default_agent_agg_v1/metadata.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ labels:
table_type: aggregate
shredder_mitigation: true
scheduling:
dag_name: bqetl_fx_desktop_background_defaultagent_aggregates
dag_name: bqetl_default_browser_aggregates
bigquery:
time_partitioning:
type: day
Expand Down

1 comment on commit f9a66ce

@dataops-ci-bot
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Integration report for "Merged default browser dags (#6737)"

sql.diff

Click to expand! 
Only in /tmp/workspace/main-generated-sql/dags/: bqetl_fx_desktop_background_defaultagent_aggregates.py
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_default_browser_aggregates.py /tmp/workspace/generated-sql/dags/bqetl_default_browser_aggregates.py
--- /tmp/workspace/main-generated-sql/dags/bqetl_default_browser_aggregates.py	2024-12-30 03:42:58.000000000 +0000
+++ /tmp/workspace/generated-sql/dags/bqetl_default_browser_aggregates.py	2024-12-30 03:45:04.000000000 +0000
@@ -86,6 +86,17 @@
         depends_on_past=False,
     )
 
+    firefox_desktop_background_defaultagent_derived__default_agent_agg__v1 = bigquery_etl_query(
+        task_id="firefox_desktop_background_defaultagent_derived__default_agent_agg__v1",
+        destination_table="default_agent_agg_v1",
+        dataset_id="firefox_desktop_background_defaultagent_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="wichan@mozilla.com",
+        email=["telemetry-alerts@mozilla.com", "wichan@mozilla.com"],
+        date_partition_parameter="submission_date",
+        depends_on_past=False,
+    )
+
     default_browser_agent_derived__default_browser_agg__v1.set_upstream(
         wait_for_copy_deduplicate_all
     )
@@ -93,3 +104,7 @@
     default_browser_agent_derived__default_browser_agg_by_os__v1.set_upstream(
         wait_for_copy_deduplicate_all
     )
+
+    firefox_desktop_background_defaultagent_derived__default_agent_agg__v1.set_upstream(
+        wait_for_copy_deduplicate_all
+    )
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_fx_desktop_background_defaultagent_aggregates.py /tmp/workspace/generated-sql/dags/bqetl_fx_desktop_background_defaultagent_aggregates.py
--- /tmp/workspace/main-generated-sql/dags/bqetl_fx_desktop_background_defaultagent_aggregates.py	2024-12-30 03:42:58.000000000 +0000
+++ /tmp/workspace/generated-sql/dags/bqetl_fx_desktop_background_defaultagent_aggregates.py	1970-01-01 00:00:00.000000000 +0000
@@ -1,80 +0,0 @@
-# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py
-
-from airflow import DAG
-from airflow.sensors.external_task import ExternalTaskMarker
-from airflow.sensors.external_task import ExternalTaskSensor
-from airflow.utils.task_group import TaskGroup
-import datetime
-from operators.gcp_container_operator import GKEPodOperator
-from utils.constants import ALLOWED_STATES, FAILED_STATES
-from utils.gcp import bigquery_etl_query, bigquery_dq_check, bigquery_bigeye_check
-
-docs = """
-### bqetl_fx_desktop_background_defaultagent_aggregates
-
-Built from bigquery-etl repo, [`dags/bqetl_fx_desktop_background_defaultagent_aggregates.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_fx_desktop_background_defaultagent_aggregates.py)
-
-#### Description
-
-This DAG builds daily aggregate tables for firefox desktop background default agent tables
-
-#### Owner
-
-wichan@mozilla.com
-
-#### Tags
-
-* impact/tier_3
-* repo/bigquery-etl
-"""
-
-
-default_args = {
-    "owner": "wichan@mozilla.com",
-    "start_date": datetime.datetime(2024, 12, 20, 0, 0),
-    "end_date": None,
-    "email": ["telemetry-alerts@mozilla.com", "wichan@mozilla.com"],
-    "depends_on_past": False,
-    "retry_delay": datetime.timedelta(seconds=1800),
-    "email_on_failure": True,
-    "email_on_retry": False,
-    "retries": 2,
-}
-
-tags = ["impact/tier_3", "repo/bigquery-etl"]
-
-with DAG(
-    "bqetl_fx_desktop_background_defaultagent_aggregates",
-    default_args=default_args,
-    schedule_interval="0 22 * * *",
-    doc_md=docs,
-    tags=tags,
-) as dag:
-
-    wait_for_copy_deduplicate_all = ExternalTaskSensor(
-        task_id="wait_for_copy_deduplicate_all",
-        external_dag_id="copy_deduplicate",
-        external_task_id="copy_deduplicate_all",
-        execution_delta=datetime.timedelta(seconds=75600),
-        check_existence=True,
-        mode="reschedule",
-        poke_interval=datetime.timedelta(minutes=5),
-        allowed_states=ALLOWED_STATES,
-        failed_states=FAILED_STATES,
-        pool="DATA_ENG_EXTERNALTASKSENSOR",
-    )
-
-    firefox_desktop_background_defaultagent_derived__default_agent_agg__v1 = bigquery_etl_query(
-        task_id="firefox_desktop_background_defaultagent_derived__default_agent_agg__v1",
-        destination_table="default_agent_agg_v1",
-        dataset_id="firefox_desktop_background_defaultagent_derived",
-        project_id="moz-fx-data-shared-prod",
-        owner="wichan@mozilla.com",
-        email=["telemetry-alerts@mozilla.com", "wichan@mozilla.com"],
-        date_partition_parameter="submission_date",
-        depends_on_past=False,
-    )
-
-    firefox_desktop_background_defaultagent_derived__default_agent_agg__v1.set_upstream(
-        wait_for_copy_deduplicate_all
-    )
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/apple_ads_external/ios_app_campaign_stats_v1/bigconfig.yml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/apple_ads_external/ios_app_campaign_stats_v1/bigconfig.yml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/apple_ads_external/ios_app_campaign_stats_v1/bigconfig.yml	2024-12-30 03:37:42.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/apple_ads_external/ios_app_campaign_stats_v1/bigconfig.yml	2024-12-30 03:41:19.000000000 +0000
@@ -1,7 +1,6 @@
 type: BIGCONFIG_FILE
-
 tag_deployments:
-  - collection:
+- collection:
       name: Operational Checks
       notification_channels:
         - slack: '#de-bigeye-triage'
@@ -24,10 +23,10 @@
         metrics:
           - saved_metric_id: is_not_null
             lookback:
-              lookback_type: DATA_TIME
               lookback_window:
                 interval_type: DAYS
                 interval_value: 28
+        lookback_type: DATA_TIME
             rct_overrides:
               - date
       - column_selectors:
@@ -35,10 +34,10 @@
         metrics:
           - saved_metric_id: is_2_char_len
             lookback:
-              lookback_type: DATA_TIME
               lookback_window:
                 interval_type: DAYS
                 interval_value: 28
+        lookback_type: DATA_TIME
             rct_overrides:
               - date
       - column_selectors:
@@ -46,17 +45,17 @@
         metrics:
           - saved_metric_id: volume
             lookback:
-              lookback_type: DATA_TIME
               lookback_window:
                 interval_type: DAYS
                 interval_value: 28
+        lookback_type: DATA_TIME
             rct_overrides:
               - date
           - saved_metric_id: freshness
             lookback:
-              lookback_type: DATA_TIME
               lookback_window:
                 interval_type: DAYS
                 interval_value: 28
+        lookback_type: DATA_TIME
             rct_overrides:
               - date
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_defaultagent_derived/default_agent_agg_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_defaultagent_derived/default_agent_agg_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_defaultagent_derived/default_agent_agg_v1/metadata.yaml	2024-12-30 03:39:30.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_defaultagent_derived/default_agent_agg_v1/metadata.yaml	2024-12-30 03:39:51.000000000 +0000
@@ -8,9 +8,9 @@
   owner1: wichan
   table_type: aggregate
   shredder_mitigation: true
-  dag: bqetl_fx_desktop_background_defaultagent_aggregates
+  dag: bqetl_default_browser_aggregates
 scheduling:
-  dag_name: bqetl_fx_desktop_background_defaultagent_aggregates
+  dag_name: bqetl_default_browser_aggregates
 bigquery:
   time_partitioning:
     type: day
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/google_ads_derived/android_app_campaign_stats_v1/bigconfig.yml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/google_ads_derived/android_app_campaign_stats_v1/bigconfig.yml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/google_ads_derived/android_app_campaign_stats_v1/bigconfig.yml	2024-12-30 03:37:42.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/google_ads_derived/android_app_campaign_stats_v1/bigconfig.yml	2024-12-30 03:41:21.000000000 +0000
@@ -1,7 +1,6 @@
 type: BIGCONFIG_FILE
-
 tag_deployments:
-  - collection:
+- collection:
       name: Operational Checks
       notification_channels:
         - slack: '#de-bigeye-triage'
@@ -23,10 +22,10 @@
         metrics:
           - saved_metric_id: is_not_null
             lookback:
-              lookback_type: DATA_TIME
               lookback_window:
                 interval_type: DAYS
                 interval_value: 28
+        lookback_type: DATA_TIME
             rct_overrides:
               - date
       - column_selectors:
@@ -34,10 +33,10 @@
         metrics:
           - saved_metric_id: is_2_char_len
             lookback:
-              lookback_type: DATA_TIME
               lookback_window:
                 interval_type: DAYS
                 interval_value: 28
+        lookback_type: DATA_TIME
             rct_overrides:
               - date
       - column_selectors:
@@ -45,17 +44,17 @@
         metrics:
           - saved_metric_id: volume
             lookback:
-              lookback_type: DATA_TIME
               lookback_window:
                 interval_type: DAYS
                 interval_value: 28
+        lookback_type: DATA_TIME
             rct_overrides:
               - date
           - saved_metric_id: freshness
             lookback:
-              lookback_type: DATA_TIME
               lookback_window:
                 interval_type: DAYS
                 interval_value: 28
+        lookback_type: DATA_TIME
             rct_overrides:
               - date

Link to full diff

Please # to comment.