Merge branch 'master' into docs-updates

.github/workflows/bump-version.yml

@@ -51,7 +51,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Create release branch
         run: git checkout -b release/v${{ inputs.cli-version }}
       - name: Initial config
@@ -80,7 +80,7 @@ jobs:
     needs: bump-version
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: create pull request
         uses: repo-sync/pull-request@v2
         with:

.github/workflows/release.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout Elementary
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Python
         uses: actions/setup-python@v4
@@ -27,7 +27,7 @@ jobs:
         run: python -m build --sdist --wheel --outdir dist .
 
       - name: Upload build artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: build
           path: dist
@@ -46,7 +46,7 @@ jobs:
 
     steps:
       - name: Checkout Elementary
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up QEMU for multi-platform support
         uses: docker/setup-qemu-action@v3
@@ -79,7 +79,7 @@ jobs:
   merge-to-docs:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: PR master to docs
         uses: repo-sync/pull-request@v2
         with:

.github/workflows/run-precommit.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Elementary
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python
         uses: actions/setup-python@v4.3.0

.github/workflows/test-github-action.yml

@@ -28,13 +28,13 @@ jobs:
         working-directory: elementary
     steps:
       - name: Checkout Elementary
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: elementary
           ref: ${{ inputs.elementary-ref }}
 
       - name: Checkout dbt package
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: elementary-data/dbt-data-reliability
           path: dbt-data-reliability
@@ -90,14 +90,14 @@ jobs:
             --file-path "report.html"
 
       - name: Upload report
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: report.html
           path: edr_target/report.html
 
       - name: Upload log
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: edr.log
           path: edr_target/edr.log

.github/workflows/test-main-warehouse.yml

@@ -4,6 +4,7 @@ on:
     branches: ["master"]
     paths:
       - elementary/**
+      - .github/**
       - pyproject.toml
   workflow_dispatch:
 

.github/workflows/test-warehouse.yml

@@ -86,13 +86,13 @@ jobs:
       cancel-in-progress: true
     steps:
       - name: Checkout Elementary
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: elementary
           ref: ${{ inputs.elementary-ref }}
 
       - name: Checkout dbt package
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: elementary-data/dbt-data-reliability
           path: dbt-data-reliability
@@ -106,7 +106,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.9"
 
       - name: Install Spark requirements
         if: inputs.warehouse-type == 'spark'
@@ -182,9 +182,9 @@ jobs:
           --project-profile-target "${{ inputs.warehouse-type }}"
 
       - name: Upload report artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: report_${{ inputs.warehouse-type }}_${{ env.BRANCH_NAME }}.html
+          name: report_${{ inputs.warehouse-type }}_${{ env.BRANCH_NAME }}_dbt_${{ inputs.dbt-version }}.html
           path: elementary/edr_target/elementary_report.html
 
       - name: Write GCS keyfile
@@ -218,9 +218,9 @@ jobs:
 
       - name: Upload edr log
         if: ${{ always() }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: edr.log
+          name: edr_${{ inputs.warehouse-type }}_${{ env.BRANCH_NAME }}_dbt_${{ inputs.dbt-version }}.log
           path: elementary/edr_target/edr.log
 
       - name: Run Python package e2e tests

.gitignore

@@ -28,7 +28,6 @@ var/
 logs/
 .venv
 poetry.lock
-package-lock.yml
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -97,4 +96,4 @@ venv/
 
 # elementary outputs
 edr_target/
-tests/tests_with_db/dbt_project/dbt_packages/
+tests/tests_with_db/dbt_project/dbt_packages/

.pre-commit-config.yaml

@@ -43,3 +43,20 @@ repos:
         entry: mypy --no-error-summary
         language: system
         files: ^elementary/.*\.py$
+
+  - repo: local
+    hooks:
+      - id: verify-dbt-project-packages-lock
+        name: Verify internal dbt project package lock
+        entry: |
+          bash -c '
+            dbt_version=$(python -c "from dbt.version import __version__; print(__version__)");
+            required_version="1.8";
+            if [[ $(echo -e "$dbt_version\n$required_version" | sort -V | tail -1) == "$dbt_version" ]]; then
+              dbt deps --lock --project-dir elementary/monitor/dbt_project && git diff --quiet elementary/monitor/dbt_project/package-lock.yml || (echo "Changes detected in package lock file!" && exit 1);
+            else
+              echo "Skipping hook, dbt version is $dbt_version (< 1.8).";
+            fi
+          '
+        language: system
+        pass_filenames: false

.vscode/launch.json

@@ -8,6 +8,29 @@
       "module": "elementary.cli.cli",
       "console": "integratedTerminal",
       "args": "${command:pickArgs}"
+    },
+    {
+      "name": "pytest: Current File",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "pytest",
+      "args": ["-vvv", "-s", "${file}"],
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "pytest: Selector",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "pytest",
+      "args": ["-vvv", "-s", "${file}::${input:selector}"],
+      "console": "integratedTerminal"
+    }
+  ],
+  "inputs": [
+    {
+      "id": "selector",
+      "type": "promptString",
+      "description": "Selector"
     }
   ]
 }

dev-requirements.txt

@@ -16,6 +16,5 @@ types-pytz
 types-jsonschema
 types-PyYAML
 types-setuptools
-pandas-stubs
 types-retry
 types-decorator

docs/_snippets/guides/alerts-code-configuration.mdx

@@ -62,8 +62,8 @@ models:
 ```yml test
 tests:
   - not_null:
-    meta:
-      owner: ["@jessica.jones", "@joe.joseph"]
+      meta:
+        owner: ["@jessica.jones", "@joe.joseph"]
 ```
 
 ```yml test/model config block
@@ -111,8 +111,8 @@ models:
 ```yml test
 tests:
   - not_null:
-    meta:
-      subscribers: ["@jessica.jones", "@joe.joseph"]
+      meta:
+        subscribers: ["@jessica.jones", "@joe.joseph"]
 ```
 
 ```yml test/model config block
@@ -149,8 +149,8 @@ It's recommended to add an explanation of what does it mean if this test fails,
 ```yml test
 tests:
   - not_null:
-    meta:
-      description: "This is the test description"
+      meta:
+        description: "This is the test description"
 ```
 
 ```yml test config block
@@ -190,7 +190,7 @@ models:
 ```yml test
 tests:
   - not_null:
-    tags: ["#marketing", "#data_ops"]
+      tags: ["#marketing", "#data_ops"]
 ```
 
 ```yml test/model config block
@@ -238,8 +238,8 @@ models:
 ```yml test
 tests:
   - not_null:
-    meta:
-      channel: data_ops
+      meta:
+        channel: data_ops
 ```
 
 ```yml test/model config block
@@ -288,8 +288,8 @@ models:
 ```yml test
 tests:
   - not_null:
-    meta:
-      alert_suppression_interval: 12
+      meta:
+        alert_suppression_interval: 12
 ```
 
 ```yml test/model config block
@@ -338,8 +338,8 @@ models:
 ```yml test
 tests:
   - not_null:
-    meta:
-      slack_group_alerts_by: table
+      meta:
+        slack_group_alerts_by: table
 ```
 
 ```yml test/model config block
@@ -398,8 +398,8 @@ models:
 ```yml test
 tests:
   - not_null:
-    meta:
-      alert_fields: ["description", "owners", "tags", "subscribers"]
+      meta:
+        alert_fields: ["description", "owners", "tags", "subscribers"]
 ```
 
 ```yml test/model config block

docs/_snippets/quickstart-package-install.mdx

@@ -39,7 +39,7 @@ Some packages we recommend you check out: [dbt_utils](https://github.com/dbt-lab
     ```yml packages.yml
     packages:
       - package: elementary-data/elementary
-        version: 0.16.0
+        version: 0.16.4
         ## Docs: https://docs.elementary-data.com
     ```
   </Step>

docs/_snippets/setup-teams-integration.mdx

@@ -34,11 +34,13 @@ Call it `Elementary` (or whatever you prefer) and connect it to the workspace of
 
 </Accordion>
 
-Now it is time to setup the webhook for this channel.
+Now it's time to set up a webhook. You have two options for creating a webhook:
 
-<Accordion title="Create Teams Webhook">
+<Accordion title="Option 1: Create Teams Webhook using Connectors (Legacy)">
 
-## Create a webhook
+## Create a webhook using Connectors
+
+**Note:** Microsoft 365 Connectors are being deprecated. Consider using Power Automate Workflows (Option 2) for new integrations.
 
 Go to a channel in your Team and choose `Manage channel`
 
@@ -77,7 +79,38 @@ Copy the URL of the webhook.
 
 </Accordion>
 
-Lastly, pass the webhook to the CLI as a param or in the `config.yml` file:
+<Accordion title="Option 2: Create Teams Webhook using Power Automate (Recommended)">
+
+## Create a webhook using Power Automate
+
+You can create a webhook using Power Automate in two ways:
+
+### Method 1: Directly from Teams (Recommended)
+
+1. Go to your Teams channel
+2. Click the three dots (...) next to the channel name
+3. Select `Workflows`
+4. Choose the template "Post to channel when a webhook request is received"
+5. Copy the webhook URL
+
+### Method 2: From Power Automate Website
+
+1. Go to [Power Automate](https://flow.microsoft.com)
+2. Create a new instant cloud flow
+3. Search for "When a HTTP request is received" as your trigger
+4. In the flow, add a "Post adaptive card in a chat or channel" action
+5. Configure the team and channel where you want to post
+6. Save the flow and copy the HTTP POST URL
+
+**Important Notes:**
+
+- When using Power Automate Workflows, Elementary CLI cannot directly verify if messages were successfully delivered. You'll need to monitor your workflow runs in Power Automate to check for any delivery issues.
+- Workflows can't post in private channels as a flow bot, but can post on behalf of a user
+- Workflows can only be created in your default environment
+
+</Accordion>
+
+Lastly, pass the webhook URL (from either method) to the CLI as a param or in the `config.yml` file:
 
 <Accordion title="Teams config as CLI params">