Skip to content
This repository has been archived by the owner on Nov 6, 2023. It is now read-only.

fix(snowflake): fix stage parsing error #245

Merged
merged 1 commit into from
Oct 25, 2023
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 23 additions & 15 deletions odd_collector/adapters/snowflake/adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

from .client import SnowflakeClient, SnowflakeClientBase
from .domain import Pipe, Table, View
from .logger import logger
from .map import map_database, map_pipe, map_schemas, map_table, map_view


Expand All @@ -33,22 +34,29 @@ def create_generator(self) -> Generator:
)

def get_data_entity_list(self) -> DataEntityList:
raw_pipes = self._client.get_raw_pipes()
raw_stages = self._client.get_raw_stages()
pipes: List[Pipe] = []
for raw_pipe in raw_pipes:
pipes.extend(
Pipe(
name=raw_pipe.pipe_name,
definition=raw_pipe.definition,
stage_url=raw_stage.stage_url,
stage_type=raw_stage.stage_type,
downstream=raw_pipe.downstream,
pipes_entities = []

# TODO: Create more user-friendly error messages and handle them
try:
raw_pipes = self._client.get_raw_pipes()
raw_stages = self._client.get_raw_stages()
pipes: list[Pipe] = []
for raw_pipe in raw_pipes:
pipes.extend(
Pipe(
name=raw_pipe.pipe_name,
definition=raw_pipe.definition,
stage_url=raw_stage.stage_url,
stage_type=raw_stage.stage_type,
downstream=raw_pipe.downstream,
)
for raw_stage in raw_stages
if raw_pipe.stage_full_name == raw_stage.stage_full_name
)
for raw_stage in raw_stages
if raw_pipe.stage_full_name == raw_stage.stage_full_name
)
pipes_entities = [map_pipe(pipe, self.generator) for pipe in pipes]
pipes_entities = [map_pipe(pipe, self.generator) for pipe in pipes]
except Exception as e:
logger.warning(f"Can't get pipes and stages. {e}")

tables = self._client.get_tables()

tables_with_data_entities: List[
Expand Down
Loading