From 520ccf42838bfa06f8ad3a65adcaa011f2ec5a92 Mon Sep 17 00:00:00 2001 From: Alexander Smolyakov Date: Thu, 19 Jan 2023 13:15:37 +0400 Subject: [PATCH 1/7] Add `pip` workflow implementation --- .../workflows/test-dbt-installation-pip.yml | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-dbt-installation-pip.yml b/.github/workflows/test-dbt-installation-pip.yml index 69c35aa9..a80cd029 100644 --- a/.github/workflows/test-dbt-installation-pip.yml +++ b/.github/workflows/test-dbt-installation-pip.yml @@ -28,14 +28,36 @@ on: permissions: contents: read # required for slack-post-notification workflow +env: + PYTHON_TARGET_VERSION: 3.8 + jobs: pip-integration-test: runs-on: ubuntu-latest steps: - - name: "Output Package Name - ${{ inputs.package_name }}" + - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_TARGET_VERSION }} + + - name: "Install Package - ${{ inputs.package_name }}" + run: | + python -m pip install --user --upgrade pip + python -m pip install pip ${{ inputs.package_name }} + python -m pip --version + + - name: "[DEBUG] Show Installed Packages" + run: | + python -m pip list + + - name: "[DEBUG] Show Package Info - ${{ inputs.package_name }}" + run: | + python -m pip show ${{ inputs.package_name }} + + - name: "Verify ${{ inputs.package_name }} Version" run: | - echo ${{ inputs.package_name }} + dbt --version slack-notification: name: "Post Scheduled Run Failures" From d084c74f9914ae973e8c04de7491566565019ef3 Mon Sep 17 00:00:00 2001 From: Alexander Smolyakov Date: Thu, 19 Jan 2023 13:24:23 +0400 Subject: [PATCH 2/7] Split `Install Package` step --- .github/workflows/test-dbt-installation-pip.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-dbt-installation-pip.yml b/.github/workflows/test-dbt-installation-pip.yml index a80cd029..838e7110 100644 --- a/.github/workflows/test-dbt-installation-pip.yml +++ b/.github/workflows/test-dbt-installation-pip.yml @@ -41,12 +41,18 @@ jobs: with: python-version: ${{ env.PYTHON_TARGET_VERSION }} - - name: "Install Package - ${{ inputs.package_name }}" + - name: "Update Python Packages" run: | python -m pip install --user --upgrade pip - python -m pip install pip ${{ inputs.package_name }} + + - name: "[DEBUG] Show Package Installer Python Version" + run: | python -m pip --version + - name: "Install Package - ${{ inputs.package_name }}" + run: | + python -m pip install pip ${{ inputs.package_name }} + - name: "[DEBUG] Show Installed Packages" run: | python -m pip list From c3e9d18ac9e52a97ee03aa75812ede9ce57a6e7e Mon Sep 17 00:00:00 2001 From: Alexander Smolyakov Date: Thu, 19 Jan 2023 13:27:23 +0400 Subject: [PATCH 3/7] Test: Comment other stages to test `pip` --- .../workflows/test-dbt-installation-main.yml | 96 +++++++++---------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/.github/workflows/test-dbt-installation-main.yml b/.github/workflows/test-dbt-installation-main.yml index e23aaa35..f0a0f735 100644 --- a/.github/workflows/test-dbt-installation-main.yml +++ b/.github/workflows/test-dbt-installation-main.yml @@ -19,19 +19,19 @@ on: # - cron: "0 9,13,18 * * *" # 9:00, 13:00, 18:00 UTC jobs: - dbt-installation-homebrew: - if: ${{ always() }} - strategy: - fail-fast: false - matrix: - package: - ["dbt-postgres", "dbt-redshift", "dbt-snowflake", "dbt-bigquery"] + # dbt-installation-homebrew: + # if: ${{ always() }} + # strategy: + # fail-fast: false + # matrix: + # package: + # ["dbt-postgres", "dbt-redshift", "dbt-snowflake", "dbt-bigquery"] - uses: ./.github/workflows/test-dbt-installation-homebrew.yml - with: - package_name: ${{ matrix.package }} - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TEST_CHANNEL }} + # uses: ./.github/workflows/test-dbt-installation-homebrew.yml + # with: + # package_name: ${{ matrix.package }} + # secrets: + # SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TEST_CHANNEL }} dbt-installation-pip: if: ${{ always() }} @@ -53,42 +53,42 @@ jobs: secrets: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TEST_CHANNEL }} - dbt-installation-docker: - if: ${{ always() }} - strategy: - fail-fast: false - matrix: - package: - [ - "dbt-postgres", - "dbt-redshift", - "dbt-snowflake", - "dbt-bigquery", - "dbt-spark", - ] + # dbt-installation-docker: + # if: ${{ always() }} + # strategy: + # fail-fast: false + # matrix: + # package: + # [ + # "dbt-postgres", + # "dbt-redshift", + # "dbt-snowflake", + # "dbt-bigquery", + # "dbt-spark", + # ] - uses: ./.github/workflows/test-dbt-installation-docker.yml - with: - package_name: ${{ matrix.package }} - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TEST_CHANNEL }} + # uses: ./.github/workflows/test-dbt-installation-docker.yml + # with: + # package_name: ${{ matrix.package }} + # secrets: + # SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TEST_CHANNEL }} - dbt-installation-source: - if: ${{ always() }} - strategy: - fail-fast: false - matrix: - package: - [ - "dbt-postgres", - "dbt-redshift", - "dbt-snowflake", - "dbt-bigquery", - "dbt-spark", - ] + # dbt-installation-source: + # if: ${{ always() }} + # strategy: + # fail-fast: false + # matrix: + # package: + # [ + # "dbt-postgres", + # "dbt-redshift", + # "dbt-snowflake", + # "dbt-bigquery", + # "dbt-spark", + # ] - uses: ./.github/workflows/test-dbt-installation-source.yml - with: - package_name: ${{ matrix.package }} - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TEST_CHANNEL }} + # uses: ./.github/workflows/test-dbt-installation-source.yml + # with: + # package_name: ${{ matrix.package }} + # secrets: + # SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TEST_CHANNEL }} From aa4166ac0b8384b09e73ab14689bbe889f827c5c Mon Sep 17 00:00:00 2001 From: Alexander Smolyakov Date: Thu, 19 Jan 2023 13:56:27 +0400 Subject: [PATCH 4/7] Revert "Test: Comment other stages to test `pip`" This reverts commit c3e9d18ac9e52a97ee03aa75812ede9ce57a6e7e. --- .../workflows/test-dbt-installation-main.yml | 96 +++++++++---------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/.github/workflows/test-dbt-installation-main.yml b/.github/workflows/test-dbt-installation-main.yml index f0a0f735..e23aaa35 100644 --- a/.github/workflows/test-dbt-installation-main.yml +++ b/.github/workflows/test-dbt-installation-main.yml @@ -19,19 +19,19 @@ on: # - cron: "0 9,13,18 * * *" # 9:00, 13:00, 18:00 UTC jobs: - # dbt-installation-homebrew: - # if: ${{ always() }} - # strategy: - # fail-fast: false - # matrix: - # package: - # ["dbt-postgres", "dbt-redshift", "dbt-snowflake", "dbt-bigquery"] + dbt-installation-homebrew: + if: ${{ always() }} + strategy: + fail-fast: false + matrix: + package: + ["dbt-postgres", "dbt-redshift", "dbt-snowflake", "dbt-bigquery"] - # uses: ./.github/workflows/test-dbt-installation-homebrew.yml - # with: - # package_name: ${{ matrix.package }} - # secrets: - # SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TEST_CHANNEL }} + uses: ./.github/workflows/test-dbt-installation-homebrew.yml + with: + package_name: ${{ matrix.package }} + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TEST_CHANNEL }} dbt-installation-pip: if: ${{ always() }} @@ -53,42 +53,42 @@ jobs: secrets: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TEST_CHANNEL }} - # dbt-installation-docker: - # if: ${{ always() }} - # strategy: - # fail-fast: false - # matrix: - # package: - # [ - # "dbt-postgres", - # "dbt-redshift", - # "dbt-snowflake", - # "dbt-bigquery", - # "dbt-spark", - # ] + dbt-installation-docker: + if: ${{ always() }} + strategy: + fail-fast: false + matrix: + package: + [ + "dbt-postgres", + "dbt-redshift", + "dbt-snowflake", + "dbt-bigquery", + "dbt-spark", + ] - # uses: ./.github/workflows/test-dbt-installation-docker.yml - # with: - # package_name: ${{ matrix.package }} - # secrets: - # SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TEST_CHANNEL }} + uses: ./.github/workflows/test-dbt-installation-docker.yml + with: + package_name: ${{ matrix.package }} + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TEST_CHANNEL }} - # dbt-installation-source: - # if: ${{ always() }} - # strategy: - # fail-fast: false - # matrix: - # package: - # [ - # "dbt-postgres", - # "dbt-redshift", - # "dbt-snowflake", - # "dbt-bigquery", - # "dbt-spark", - # ] + dbt-installation-source: + if: ${{ always() }} + strategy: + fail-fast: false + matrix: + package: + [ + "dbt-postgres", + "dbt-redshift", + "dbt-snowflake", + "dbt-bigquery", + "dbt-spark", + ] - # uses: ./.github/workflows/test-dbt-installation-source.yml - # with: - # package_name: ${{ matrix.package }} - # secrets: - # SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TEST_CHANNEL }} + uses: ./.github/workflows/test-dbt-installation-source.yml + with: + package_name: ${{ matrix.package }} + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TEST_CHANNEL }} From 26d6b420abc7544599fdf97f5a1105ebc63df10a Mon Sep 17 00:00:00 2001 From: Alexander Smolyakov Date: Thu, 19 Jan 2023 14:02:17 +0400 Subject: [PATCH 5/7] Update workflow docs --- .github/workflows/test-dbt-installation-pip.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test-dbt-installation-pip.yml b/.github/workflows/test-dbt-installation-pip.yml index 838e7110..35074262 100644 --- a/.github/workflows/test-dbt-installation-pip.yml +++ b/.github/workflows/test-dbt-installation-pip.yml @@ -1,6 +1,12 @@ # **what?** +# This workflow installs the latest version of dbt adapter from PyPI. +# It then runs 'dbt --version' to verify the installation was successful. +# If it fails for the scheduled runs, it will post to our team alerts channel. # **why?** +# This is a simple way to test all adapter installations at a single +# time. It allows us to test them on a schedule as well to check for +# any breaking dependency changes that might happen and alert us on it. # **when?** # This reusable workflow can be called or started manually From e36ae1555161c7d131613e6f10441034b5aab14c Mon Sep 17 00:00:00 2001 From: Alexander Smolyakov Date: Fri, 20 Jan 2023 15:32:23 +0400 Subject: [PATCH 6/7] Update test-dbt-installation-pip.yml --- .github/workflows/test-dbt-installation-pip.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-dbt-installation-pip.yml b/.github/workflows/test-dbt-installation-pip.yml index 35074262..4097bfbd 100644 --- a/.github/workflows/test-dbt-installation-pip.yml +++ b/.github/workflows/test-dbt-installation-pip.yml @@ -34,18 +34,20 @@ on: permissions: contents: read # required for slack-post-notification workflow -env: - PYTHON_TARGET_VERSION: 3.8 - jobs: pip-integration-test: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.7", "3.8", "3.9", "3.10"] + steps: - - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" + - name: "Set up Python - ${{ matrix.python-version }}" uses: actions/setup-python@v4 with: - python-version: ${{ env.PYTHON_TARGET_VERSION }} + python-version: ${{ matrix.python-version }} - name: "Update Python Packages" run: | From 6dfbccbba3dc23a9fa8b16b1ace2fb92e41e6a7e Mon Sep 17 00:00:00 2001 From: Alexander Smolyakov Date: Mon, 23 Jan 2023 17:36:50 +0400 Subject: [PATCH 7/7] Rename job --- .github/workflows/test-dbt-installation-pip.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-dbt-installation-pip.yml b/.github/workflows/test-dbt-installation-pip.yml index 4097bfbd..c208562e 100644 --- a/.github/workflows/test-dbt-installation-pip.yml +++ b/.github/workflows/test-dbt-installation-pip.yml @@ -35,7 +35,7 @@ permissions: contents: read # required for slack-post-notification workflow jobs: - pip-integration-test: + pip-installation-test: runs-on: ubuntu-latest strategy: @@ -75,7 +75,7 @@ jobs: slack-notification: name: "Post Scheduled Run Failures" - needs: pip-integration-test + needs: pip-installation-test if: ${{ failure() && github.event_name == 'schedule' }} uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main