Skip to content

Commit 2115d5b

Browse files
authored
fix(ingest): remove dbt delete_tests_as_datasets option (#5865)
This option was added for backwards compat and can be dropped now.
1 parent 39fa932 commit 2115d5b

File tree

3 files changed

+188
-1570
lines changed

3 files changed

+188
-1570
lines changed

metadata-ingestion/src/datahub/ingestion/source/dbt.py

Lines changed: 0 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -309,10 +309,6 @@ class DBTConfig(StatefulIngestionConfigBase):
309309
default=None,
310310
description="When fetching manifest files from s3, configuration for aws connection details",
311311
)
312-
delete_tests_as_datasets: bool = Field(
313-
False,
314-
description="Prior to version 0.8.38, dbt tests were represented as datasets. If you ingested dbt tests before, set this flag to True (just needed once) to soft-delete tests that were generated as datasets by previous ingestion.",
315-
)
316312
backcompat_skip_source_on_lineage_edge: bool = Field(
317313
False,
318314
description="Prior to version 0.8.41, lineage edges to sources were directed to the target platform node rather than the dbt source node. This contradicted the established pattern for other lineage edges to point to upstream dbt nodes. To revert lineage logic to this legacy approach, set this flag to true.",
@@ -1440,35 +1436,6 @@ def string_map(input_map: Dict[str, Any]) -> Dict[str, str]:
14401436
self.report.report_workunit(wu)
14411437
yield wu
14421438

1443-
if self.config.delete_tests_as_datasets:
1444-
mce_platform = (
1445-
self.config.target_platform
1446-
if self.config.disable_dbt_node_creation
1447-
else DBT_PLATFORM
1448-
)
1449-
node_datahub_urn = get_urn_from_dbtNode(
1450-
node.database,
1451-
node.schema,
1452-
node.alias or node.name, # previous code used the alias
1453-
mce_platform,
1454-
self.config.env,
1455-
self.config.platform_instance
1456-
if mce_platform == DBT_PLATFORM
1457-
else None,
1458-
)
1459-
soft_delete_mcp = MetadataChangeProposalWrapper(
1460-
entityType="dataset",
1461-
changeType=ChangeTypeClass.UPSERT,
1462-
entityUrn=node_datahub_urn,
1463-
aspectName="status",
1464-
aspect=StatusClass(removed=True),
1465-
)
1466-
soft_delete_wu = MetadataWorkUnit(
1467-
id=f"{node_datahub_urn}-status", mcp=soft_delete_mcp
1468-
)
1469-
self.report.report_workunit(soft_delete_wu)
1470-
yield soft_delete_wu
1471-
14721439
# create workunits from dbt nodes
14731440
def get_workunits(self) -> Iterable[MetadataWorkUnit]:
14741441
if self.config.write_semantics == "PATCH" and not self.ctx.graph:

0 commit comments

Comments
 (0)