Merge branch 'master' into pass-inline-key-to-snowflake

Author: vvkh

.github/workflows/formatter.yml

@@ -7,17 +7,16 @@ on:
 
 jobs:
   linter_name:
-    name: runner / black
+    name: runner / ruff
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Check files using the black formatter
-        uses: rickstaa/action-black@v1
-        id: black_formatter
+      - uses: actions/checkout@v3
+      - name: Check files using the ruff formatter
+        uses: chartboost/ruff-action@v1
+        id: ruff_formatter
         with:
-          black_args: ". -l 120"
-      - name: Annotate diff changes using reviewdog
-        if: steps.black_formatter.outputs.is_formatted == 'true'
-        uses: reviewdog/action-suggester@v1
+          args: format
+      - name: Auto commit ruff formatting
+        uses: stefanzweifel/git-auto-commit-action@v5
         with:
-          tool_name: blackfmt
+          commit_message: 'style fixes by ruff'

.gitignore

@@ -147,3 +147,6 @@ benchmark_*.png
 
 # VSCode
 .vscode
+
+# History
+.history

.pre-commit-config.yaml

@@ -0,0 +1,11 @@
+default_language_version:
+  python: python3
+
+repos:
+# Run the Ruff linter.
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  # Ruff version.
+  rev: v0.1.2
+  hooks:
+    # Run the Ruff formatter.
+    - id: ruff-format

CONTRIBUTING.md

@@ -54,6 +54,8 @@ Once inside, you can install the dependencies.
 
 - Option 2: Run `pip install -e .` to install them, and data-diff, in the global context.
 
+- Run `pre-commit install` to automatically format your code before committing.
+
 At the bare minimum, you need MySQL to run the tests.
 
 You can create a local MySQL instance using `docker-compose up mysql`. The URI for it will be `mysql://mysql:Password1@localhost/mysql`. If you're using a different server, make sure to update `TEST_MYSQL_CONN_STRING` in `tests/common.py`. For your convenience, we recommend creating `tests/local_settings.py`, and to override the value there.

README.md

@@ -72,7 +72,7 @@ More information about the algorithm and performance considerations can be found
 # Get started
 
 ## Validating dbt model changes between dev and prod
-⚡ Looking to use `data-diff` in dbt development? Head over to [our `data-diff` + `dbt` documentation](https://docs.datafold.com/development_testing/open_source/) to get started!
+⚡ Looking to use `data-diff` in dbt development? Head over to [our `data-diff` + `dbt` documentation](https://docs.datafold.com/development_testing/how_it_works) to get started!
 
 ## Compare data tables between databases
 🔀 To compare data between databases, install `data-diff` with specific database adapters, e.g.:

data_diff/__main__.py

@@ -267,6 +267,20 @@ def write_usage(self, prog: str, args: str = "", prefix: Optional[str] = None) -
     metavar="PATH",
     help="Specify manifest to utilize for 'prod' comparison paths instead of using configuration.",
 )
+@click.option(
+    "-pd",
+    "--prod-database",
+    "prod_database",
+    default=None,
+    help="Override the dbt production database configuration within dbt_project.yml",
+)
+@click.option(
+    "-ps",
+    "--prod-schema",
+    "prod_schema",
+    default=None,
+    help="Override the dbt production schema configuration within dbt_project.yml",
+)
 def main(conf, run, **kw):
     log_handlers = _get_log_handlers(kw["dbt"])
     if kw["table2"] is None and kw["database2"]:
@@ -323,6 +337,8 @@ def main(conf, run, **kw):
                 where_flag=kw["where"],
                 stats_flag=kw["stats"],
                 columns_flag=kw["columns"],
+                production_database_flag=kw["prod_database"],
+                production_schema_flag=kw["prod_schema"],
             )
         else:
             return _data_diff(
@@ -366,6 +382,8 @@ def _data_diff(
     cloud,
     dbt_profiles_dir,
     dbt_project_dir,
+    prod_database,
+    prod_schema,
     select,
     state,
     threads1=None,
@@ -519,7 +537,7 @@ def _data_diff(
 
     else:
         for op, values in diff_iter:
-            color = COLOR_SCHEME[op]
+            color = COLOR_SCHEME.get(op, "grey62")
 
             if json_output:
                 jsonl = json.dumps([op, list(values)])

data_diff/abcs/database_types.py

@@ -1,6 +1,6 @@
 import decimal
 from abc import ABC, abstractmethod
-from typing import Tuple, Union
+from typing import List, Optional, Tuple, Type, TypeVar, Union
 from datetime import datetime
 
 import attrs
@@ -12,9 +12,24 @@
 DbKey = Union[int, str, bytes, ArithUUID, ArithAlphanumeric]
 DbTime = datetime
 
+N = TypeVar("N")
 
-@attrs.define(frozen=True)
+
+@attrs.define(frozen=True, kw_only=True)
 class ColType:
+    # Arbitrary metadata added and fetched at runtime.
+    _notes: List[N] = attrs.field(factory=list, init=False, hash=False, eq=False)
+
+    def add_note(self, note: N) -> None:
+        self._notes.append(note)
+
+    def get_note(self, cls: Type[N]) -> Optional[N]:
+        """Get the latest added note of type ``cls`` or its descendants."""
+        for note in reversed(self._notes):
+            if isinstance(note, cls):
+                return note
+        return None
+
     @property
     def supported(self) -> bool:
         return True

data_diff/abcs/mixins.py

@@ -1,4 +1,5 @@
 from data_diff.databases.base import MD5_HEXDIGITS, CHECKSUM_HEXDIGITS, QueryError, ConnectError, BaseDialect, Database
+from data_diff.databases.base import CHECKSUM_OFFSET
 from data_diff.databases._connect import connect as connect
 from data_diff.databases._connect import Connect as Connect
 from data_diff.databases.postgresql import PostgreSQL as PostgreSQL