cloudbuild.cdc.yaml

# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

steps:
  # Run CDC DAG generation
  - name: 'gcr.io/kittycorn-public/deploy-kittycorn:v2.0'
    id: config_reader
    waitFor: ['-']
    script: |
      #!/bin/bash
      echo "Configuration (${_CONFIG_FILE}):"
      cat ${_CONFIG_FILE}

      _PJID_SRC_=$(jq -r ."projectIdSource" ${_CONFIG_FILE})
      _DEPLOY_CDC_=$(jq -r ."SAP.deployCDC" ${_CONFIG_FILE})
      _DS_RAW_=$(jq -r ."SAP.datasets.raw" ${_CONFIG_FILE})
      _DS_CDC_=$(jq -r ."SAP.datasets.cdc" ${_CONFIG_FILE})
      _TEST_DATA_=$(jq -r ."testData" ${_CONFIG_FILE})
      _SQL_FLAVOUR_=$(jq -r ."SAP.SQLFlavor" ${_CONFIG_FILE})

      if [[ ${_DEPLOY_CDC_} == "true" ]] ; then
        echo "Running config_reader"
        cd ./src/ && python config_reader.py ${_PJID_SRC_} ${_DS_RAW_} ${_DS_CDC_} ${_TEST_DATA_} ${_SQL_FLAVOUR_}
      fi

  - name: 'gcr.io/kittycorn-public/deploy-kittycorn:v2.0'
    id: copy_dag_py_files_to_gcs
    waitFor: ['config_reader']
    script: |
      #!/bin/bash
      if [[ ${_TGT_BUCKET} != "" ]]
      then
        _TGT_BUCKET_=${_TGT_BUCKET}
      else
        _TGT_BUCKET_=$(jq -r ."targetBucket" ${_CONFIG_FILE})
      fi

      echo "Running copy_dag_py_files_to_gcs"
      generated_files=$(shopt -s nullglob dotglob; echo ./generated_dag/*.py)
      if (( ${#generated_files} ))
      then
        gsutil -m cp -r './generated_dag/*.py' gs://${_TGT_BUCKET_}/dags/
      else
        echo "🔪🔪🔪No Python files found under generated_dag folder or the folder does not exist. Skipping copy.🔪🔪🔪"
      fi

  - name: 'gcr.io/kittycorn-public/deploy-kittycorn:v2.0'
    id: copy_dag_sql_files_to_gcs
    waitFor: ['config_reader']
    script: |
      #!/bin/bash
      if [[ ${_TGT_BUCKET} != "" ]]
      then
        _TGT_BUCKET_=${_TGT_BUCKET}
      else
        _TGT_BUCKET_=$(jq -r ."targetBucket" ${_CONFIG_FILE})
      fi

      echo "Copying DAG SQL files to GCS..."
      generated_files=$(shopt -s nullglob dotglob; echo ./generated_sql/*.sql)
      if (( ${#generated_files} ))
      then
        gsutil -m cp -r './generated_sql/*.sql' gs://${_TGT_BUCKET_}/data/bq_data_replication/
      else
        echo "🔪No SQL files found under generated_sql folder or the folder does not exist. Skipping copy.🔪"
      fi

  - name: gcr.io/kittycorn-public/deploy-kittycorn:v2.0
    id: 'datamesh'
    waitFor: ['config_reader']
    script: |
      #!/bin/bash
      set -e
      _DEPLOY_DATA_MESH_=$(jq -r ."deployDataMesh" ${_CONFIG_FILE})

      if [[ ${_DEPLOY_DATA_MESH_} == "true" ]]; then
        _SQL_FLAVOR_=$(jq -r ."SAP.SQLFlavor" ${_CONFIG_FILE})

        python3 common/data_mesh/deploy_data_mesh.py \
            --config-file ${_CONFIG_FILE} \
            --tag-template-directories config/${_SQL_FLAVOR_}/tag_templates \
            --policy-directories config/${_SQL_FLAVOR_}/policy_taxonomies \
            --lake-directories config/${_SQL_FLAVOR_}/lakes \
            --annotation-directories \
                config/${_SQL_FLAVOR_}/annotations/raw \
                config/${_SQL_FLAVOR_}/annotations/cdc
      else
        echo "==Skipping Data Mesh=="
      fi

logsBucket: "gs://${_GCS_BUCKET}"
timeout: 7200s
substitutions:
  _CONFIG_FILE: "config/config.json"
options:
  substitution_option: "ALLOW_LOOSE"
  automapSubstitutions: true