Skip to content
This repository was archived by the owner on Nov 6, 2023. It is now read-only.

Commit 7e779e8

Browse files
authored
fix(snowflake): fix stage parsing error (#245)
1 parent ccad4cc commit 7e779e8

1 file changed

Lines changed: 23 additions & 15 deletions

File tree

odd_collector/adapters/snowflake/adapter.py

Lines changed: 23 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
from .client import SnowflakeClient, SnowflakeClientBase
1111
from .domain import Pipe, Table, View
12+
from .logger import logger
1213
from .map import map_database, map_pipe, map_schemas, map_table, map_view
1314

1415

@@ -33,22 +34,29 @@ def create_generator(self) -> Generator:
3334
)
3435

3536
def get_data_entity_list(self) -> DataEntityList:
36-
raw_pipes = self._client.get_raw_pipes()
37-
raw_stages = self._client.get_raw_stages()
38-
pipes: List[Pipe] = []
39-
for raw_pipe in raw_pipes:
40-
pipes.extend(
41-
Pipe(
42-
name=raw_pipe.pipe_name,
43-
definition=raw_pipe.definition,
44-
stage_url=raw_stage.stage_url,
45-
stage_type=raw_stage.stage_type,
46-
downstream=raw_pipe.downstream,
37+
pipes_entities = []
38+
39+
# TODO: Create more user-friendly error messages and handle them
40+
try:
41+
raw_pipes = self._client.get_raw_pipes()
42+
raw_stages = self._client.get_raw_stages()
43+
pipes: list[Pipe] = []
44+
for raw_pipe in raw_pipes:
45+
pipes.extend(
46+
Pipe(
47+
name=raw_pipe.pipe_name,
48+
definition=raw_pipe.definition,
49+
stage_url=raw_stage.stage_url,
50+
stage_type=raw_stage.stage_type,
51+
downstream=raw_pipe.downstream,
52+
)
53+
for raw_stage in raw_stages
54+
if raw_pipe.stage_full_name == raw_stage.stage_full_name
4755
)
48-
for raw_stage in raw_stages
49-
if raw_pipe.stage_full_name == raw_stage.stage_full_name
50-
)
51-
pipes_entities = [map_pipe(pipe, self.generator) for pipe in pipes]
56+
pipes_entities = [map_pipe(pipe, self.generator) for pipe in pipes]
57+
except Exception as e:
58+
logger.warning(f"Can't get pipes and stages. {e}")
59+
5260
tables = self._client.get_tables()
5361

5462
tables_with_data_entities: List[

0 commit comments

Comments
 (0)