Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

added source freshness fix and macro override test #121

Merged
merged 2 commits into from
May 7, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion dbt/adapters/exasol/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,10 @@ class ExasolAdapter(SQLAdapter):
}

_capabilities = CapabilityDict(
{Capability.SchemaMetadataByRelations: CapabilitySupport(support=Support.Full)}
{
Capability.SchemaMetadataByRelations: CapabilitySupport(support=Support.Full),
Capability.TableLastModifiedMetadata: CapabilitySupport(support=Support.Full),
}
)

@classmethod
Expand Down
21 changes: 21 additions & 0 deletions dbt/include/exasol/macros/metadata.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
{% macro exasol__get_relation_last_modified(information_schema, relations) -%}

{%- call statement('last_modified', fetch_result=True) -%}
select root_name as table_schema,
object_name as identifier,
object_type as table_type,
last_commit as last_modified,
{{ current_timestamp() }} as snapshotted_at
from sys.exa_user_objects
where (
{%- for relation in relations -%}
(upper(root_name) = upper('{{ relation.schema }}') and
upper(object_name) = upper('{{ relation.identifier }}') and
upper(object_type) in ('TABLE', 'VIEW')){%- if not loop.last %} or {% endif -%}
{%- endfor -%}
)
{%- endcall -%}

{{ return(load_result('last_modified')) }}

{% endmacro %}
65 changes: 65 additions & 0 deletions tests/functional/adapter/test_get_relation_last_modified.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
import os
import pytest

from dbt.cli.main import dbtRunner

freshness_via_metadata_schema_yml = """version: 2
sources:
- name: test_source
freshness:
warn_after: {count: 10, period: hour}
error_after: {count: 1, period: day}
schema: "{{ env_var('DBT_GET_LAST_RELATION_TEST_SCHEMA') }}"
loaded_at_field: ts
tables:
- name: test_table
"""

class TestGetRelationLastModified:
@pytest.fixture(scope="class", autouse=True)
def set_env_vars(self, project):
os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"] = project.test_schema
yield
del os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"]

@pytest.fixture(scope="class")
def models(self):
return {"schema.yml": freshness_via_metadata_schema_yml}

@pytest.fixture(scope="class")
def custom_schema(self, project, set_env_vars):
with project.adapter.connection_named("__test"):
relation = project.adapter.Relation.create(
database=project.database, schema=os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"]
)
project.adapter.drop_schema(relation)
project.adapter.create_schema(relation)

yield relation.schema

with project.adapter.connection_named("__test"):
project.adapter.drop_schema(relation)

def test_get_relation_last_modified(self, project, set_env_vars, custom_schema):
project.run_sql(
f"create table {custom_schema}.test_table (id integer, name varchar(100) not null, ts timestamp default current_timestamp)"
)

project.run_sql(
f"insert into {custom_schema}.test_table (id, name) values (1, 'exasol')"
)

project.run_sql("COMMIT")

warning_or_error = False

def probe(e):
nonlocal warning_or_error
if e.info.level in ["warning", "error"]:
warning_or_error = True

runner = dbtRunner(callbacks=[probe])
runner.invoke(["source", "freshness"])

# The 'source freshness' command should succeed without warnings or errors.
assert not warning_or_error
35 changes: 35 additions & 0 deletions tests/functional/artifacts/test_override.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
import pytest
from dbt.tests.util import run_dbt
from dbt.exceptions import CompilationError

model_sql = """
select 1 as id
"""

fail_macros__failure_sql = """
{% macro get_catalog_relations(information_schema, relations) %}
{% do exceptions.raise_compiler_error('rejected: no catalogs for you') %}
{% endmacro %}

"""


class TestDocsGenerateOverride:
@pytest.fixture(scope="class")
def models(self):
return {"model.sql": model_sql}

@pytest.fixture(scope="class")
def macros(self):
return {"failure.sql": fail_macros__failure_sql}

def test_override_used(
self,
project,
):
results = run_dbt(["run"])
assert len(results) == 1
# this should pick up our failure macro and raise a compilation exception
with pytest.raises(CompilationError) as excinfo:
run_dbt(["--warn-error", "docs", "generate"])
assert "rejected: no catalogs for you" in str(excinfo.value)