Add mapping for CGSIS (pen) DAGs

README.md

@@ -128,6 +128,7 @@ do DETRU, do DELOG e da CGINF e demais unidades:
 * CGINF - https://git.economia.gov.br/seges-cginf/airflow-dags/
 * DELOG - https://git.economia.gov.br/seges/airflow-dags-delog/
 * DETRU - https://git.economia.gov.br/seges/airflow-dags-detru/
+* CGSIS - https://git.economia.gov.br/seges/airflow-dags-pen/
 
 Para clonar o repositório da **CGINF**, execute:
 
@@ -147,6 +148,12 @@ Para clonar o repositório do **DETRU**, execute:
 git clone https://git.economia.gov.br/seges/airflow-dags-detru.git
 ```
 
+Para clonar o repositório do **CGSIS**, execute:
+
+```bash
+git clone https://git.economia.gov.br/seges/airflow-dags-pen.git
+```
+
 ## Executar o Airflow
 
 A execução é feita de forma isolada por repositório de DAGs. Acesse o
@@ -174,6 +181,12 @@ Para subir o Airflow com as dags do DETRU, execute:
 docker-compose -f docker-compose-detru.yml up -d
 ```
 
+Para subir o Airflow com as dags do CGSIS, execute:
+
+```bash
+docker-compose -f docker-compose-pen.yml up -d
+```
+
 
 Acesse o Airflow em http://localhost:8080/ o/
 
@@ -326,7 +339,7 @@ Ao abrir o Airflow (localhost:8080) parecerá o aviso:
 > While upgrading the metadatabase, Airflow had to move some bad data in
 > order to apply new constraints. The moved data can be found in the
 > following tables:
-> 
+>
 > (...)
 
 Essas tabelas podem ser apagadas do banco de dados sem problemas.

docker-compose-pen.yml

@@ -0,0 +1,104 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Basic Airflow cluster configuration for CeleryExecutor with Redis and PostgreSQL.
+#
+# WARNING: This configuration is for local development. Do not use it in a production deployment.
+#
+# This configuration supports basic configuration using environment variables or an .env file
+# The following variables are supported:
+#
+# AIRFLOW_IMAGE_NAME         - Docker image name used to run Airflow.
+#                              Default: apache/airflow:master-python3.8
+# AIRFLOW_UID                - User ID in Airflow containers
+#                              Default: 50000
+# AIRFLOW_GID                - Group ID in Airflow containers
+#                              Default: 50000
+# _AIRFLOW_WWW_USER_USERNAME - Username for the administrator account.
+#                              Default: airflow
+# _AIRFLOW_WWW_USER_PASSWORD - Password for the administrator account.
+#                              Default: airflow
+#
+# Feel free to modify this file to suit your needs.
+---
+    version: '3'
+    x-airflow-common:
+      &airflow-common
+      image: ghcr.io/gestaogovbr/airflow2-docker:latest
+      environment:
+        &airflow-common-env
+        AIRFLOW__CORE__EXECUTOR: LocalExecutor
+        AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow
+        AIRFLOW__CORE__ENABLE_XCOM_PICKLING: 'true'
+        AIRFLOW__CORE__FERNET_KEY: 'Uqv6pwYWJn7xifR3QAkifMTkupos8mkxaQL2Q7Ydsqw='
+        AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
+        AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
+        AIRFLOW__API__AUTH_BACKEND: 'airflow.api.auth.backend.basic_auth'
+        AIRFLOW__CORE__DEFAULT_TIMEZONE: 'America/Sao_Paulo'
+        AIRFLOW__WEBSERVER__DEFAULT_UI_TIMEZONE: 'America/Sao_Paulo'
+        AIRFLOW__EMAIL__DEFAULT_EMAIL_ON_RETRY: 'false'
+        AIRFLOW__EMAIL__DEFAULT_EMAIL_ON_FAILURE: 'false'
+        AIRFLOW__WEBSERVER__SECRET_KEY: '42'
+        AIRFLOW__WEBSERVER__WORKERS: 1
+      volumes:
+        - ../airflow-dags-pen/dags:/opt/airflow/dags
+        - ../FastETL:/opt/airflow/plugins/FastETL
+        - ../airflow_commons:/opt/airflow/plugins/airflow_commons
+        - ../airflow-dags/great_expectations:/opt/airflow/great_expectations
+        - /var/run/docker.sock:/var/run/docker.sock
+        # Caso precise usar DockerOperator no ambiente, digite "which docker" e
+        # descomente abaixo a linha correspondente.
+        # - /usr/bin/docker:/bin/docker:ro
+        # - /snap/bin/docker:/bin/docker:ro
+
+      user: "${AIRFLOW_UID:-50000}:${AIRFLOW_GID:-50000}"
+      depends_on:
+        postgres:
+          condition: service_healthy
+
+    services:
+      postgres:
+        image: postgres:13
+        environment:
+          POSTGRES_USER: airflow
+          POSTGRES_PASSWORD: airflow
+          POSTGRES_DB: airflow
+        volumes:
+          - ./mnt/pgdata:/var/lib/postgresql/data
+        healthcheck:
+          test: ["CMD", "pg_isready", "-U", "airflow"]
+          interval: 5s
+          retries: 5
+        restart: always
+
+      airflow-webserver:
+        <<: *airflow-common
+        command: webserver
+        ports:
+          - 8080:8080
+        healthcheck:
+          test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
+          interval: 10s
+          timeout: 10s
+          retries: 5
+        restart: always
+
+      airflow-scheduler:
+        <<: *airflow-common
+        command: scheduler
+        restart: always

docker-compose.yml

@@ -57,13 +57,14 @@
         AIRFLOW__EMAIL__DEFAULT_EMAIL_ON_FAILURE: 'false'
         AIRFLOW__WEBSERVER__SECRET_KEY: '42'
         AIRFLOW__WEBSERVER__WORKERS: 1
-        PYTHONPATH: '/opt/airflow/dags/airflow-dags:/opt/airflow/dags/airflow-dags-detru:/opt/airflow/dags/airflow-dags-delog'
+        PYTHONPATH: '/opt/airflow/dags/airflow-dags:/opt/airflow/dags/airflow-dags-detru:/opt/airflow/dags/airflow-dags-delog:/opt/airflow/dags/airflow-dags-pen'
         RO_DOU__DAG_CONF_DIR: /opt/airflow/dags/airflow-dags/ro_dou/dag_confs
         FASTETL_DEV: "true"
       volumes:
         - ../airflow-dags/dags:/opt/airflow/dags/airflow-dags
         - ../airflow-dags-delog/dags:/opt/airflow/dags/airflow-dags-delog
         - ../airflow-dags-detru/dags:/opt/airflow/dags/airflow-dags-detru
+        - ../airflow-dags-pen/dags:/opt/airflow/dags/airflow-dags-pen
         - ../FastETL/fastetl:/opt/airflow/plugins/fastetl
         - ../airflow_commons:/opt/airflow/plugins/airflow_commons
         - ../Ro-dou/src:/opt/airflow/dags/ro_dou_src