-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathOff_daily_xsc_xsac_schedule 0 3 ^^^.py
56 lines (44 loc) · 1.85 KB
/
Off_daily_xsc_xsac_schedule 0 3 ^^^.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
Python 3.6.7rc1 (v3.6.7rc1:311101f7b6, Sep 26 2018, 16:33:24) [MSC v.1900 64 bit (AMD64)] on win32
Type "help", "copyright", "credits" or "license()" for more information.
>>> from airflow import DAG
from airflow.operators.python_operator import PythonOperator
from airflow.macros import datetime
from airflow.contrib.operators.bigquery_operator import BigQueryOperator
from google.cloud.bigquery.schema import SchemaField
from airflow.operators import (
SlackHQOperator
)
dag = DAG(
dag_id='daily_xsc_xsac',
schedule_interval='0 3 * * *',
start_date=datetime(2017, 7, 12),
catchup=True,
default_args={
'retries': 0
})
with open('./scripts/daily_xsc_xsac/daily_tsc_tsac.sql') as f:
daily_tsc_tsac = ' '.join(f.readlines())
with open('./scripts/daily_xsc_xsac/daily_isc_isac_per_source.sql') as f:
daily_isc_isac_per_source = ' '.join(f.readlines())
daily_tsc_tsac_operator = BigQueryOperator(
task_id='bigquery_load_daily_tsc_tsac',
bql=daily_tsc_tsac,
destination_dataset_table='analyst.daily_tsc_tsac_{{ yesterday_ds_nodash }}',
on_failure_callback=SlackHQOperator.on_failure_general_callback,
write_disposition='WRITE_TRUNCATE',
create_disposition='CREATE_IF_NEEDED',
bigquery_conn_id='hqdatawarehouse_bigquery',
use_legacy_sql=False,
dag=dag)
daily_isc_isac_operator = BigQueryOperator(
task_id='bigquery_load_daily_isc_isac',
bql=daily_isc_isac_per_source,
destination_dataset_table='analyst.daily_isc_isac_per_source_{{ yesterday_ds_nodash }}',
on_failure_callback=SlackHQOperator.on_failure_general_callback,
write_disposition='WRITE_TRUNCATE',
create_disposition='CREATE_IF_NEEDED',
bigquery_conn_id='hqdatawarehouse_bigquery',
use_legacy_sql=False,
dag=dag)
daily_tsc_tsac_operator
daily_isc_isac_operator