Skip to content

Commit dbf6dc7

Browse files
author
David Shimamoto
authored
Adjust metadata schema to account for target schema config. (#955)
Adjust metadata schema to account for target schema config. Remove invocation log message for deletion.
1 parent 6dbb9b9 commit dbf6dc7

1 file changed

Lines changed: 54 additions & 62 deletions

File tree

‎macros/metadata/tuva_invocations.sql‎

Lines changed: 54 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,63 +1,59 @@
1-
{% macro create_tuva_invocations_table() %}
2-
3-
{%- set schema_name = var('tuva_schema_prefix') ~ '_metadata' if var('tuva_schema_prefix',None) != None else 'metadata' -%}
4-
5-
{% do adapter.create_schema(api.Relation.create(database=target.database, schema=schema_name)) %}
6-
7-
{%- set table_name = 'tuva_invocations' -%}
8-
{%- set target_type = target.type -%}
1+
{% macro create_tuva_invocations_table(schema_name) %}
2+
{# Creates a tuva invocations table. Returns true if the table is created or exists. Returns false if a non-supported platform is used. #}
3+
{% do adapter.create_schema(api.Relation.create(database=target.database, schema=schema_name)) %}
4+
5+
{%- set table_name = 'tuva_invocations' -%}
6+
{%- set target_type = target.type -%}
97

10-
{%- if target_type in ['redshift','postgres','duckdb'] -%}
11-
{%- set sql -%}
12-
CREATE TABLE IF NOT EXISTS {{ schema_name }}.{{ table_name }}
13-
(
14-
invocation_id {{ dbt.type_string() }},
15-
project_name {{ dbt.type_string() }},
16-
tuva_package_version {{ dbt.type_string() }},
17-
run_started_at {{ dbt.type_timestamp() }}
18-
)
19-
{%- endset -%}
20-
{% do run_query(sql) %}
21-
{{ return(true) }}
22-
{%- elif target_type in ['snowflake','databricks','bigquery'] -%}
23-
{%- set sql -%}
24-
CREATE OR REPLACE TABLE {{ schema_name }}.{{ table_name }}
25-
(
26-
invocation_id {{ dbt.type_string() }},
27-
project_name {{ dbt.type_string() }},
28-
tuva_package_version {{ dbt.type_string() }},
29-
run_started_at {{ dbt.type_timestamp() }}
30-
)
31-
{%- endset -%}
32-
{% do run_query(sql) %}
33-
{{ return(true) }}
34-
{%- if target_type in ['fabric'] -%}
35-
{%- set sql -%}
36-
IF NOT EXISTS (SELECT 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = '{{ table_name }}' AND TABLE_SCHEMA = '{{ schema_name }}')
37-
CREATE TABLE {{ schema_name }}.{{ table_name }}
38-
(
39-
invocation_id {{ dbt.type_string() }},
40-
project_name {{ dbt.type_string() }},
41-
tuva_package_version {{ dbt.type_string() }},
42-
run_started_at {{ dbt.type_timestamp() }}
43-
)
44-
{%- endset -%}
45-
{% do run_query(sql) %}
46-
{{ return(true) }}
8+
{%- if target_type in ['redshift','postgres','duckdb'] -%}
9+
{%- set sql -%}
10+
CREATE TABLE IF NOT EXISTS {{ schema_name }}.{{ table_name }}
11+
(
12+
invocation_id {{ dbt.type_string() }},
13+
project_name {{ dbt.type_string() }},
14+
tuva_package_version {{ dbt.type_string() }},
15+
run_started_at {{ dbt.type_timestamp() }}
16+
)
17+
{%- endset -%}
18+
{% do run_query(sql) %}
19+
{{ return(true) }}
20+
{%- elif target_type in ['snowflake','databricks','bigquery'] -%}
21+
{%- set sql -%}
22+
CREATE OR REPLACE TABLE {{ schema_name }}.{{ table_name }}
23+
(
24+
invocation_id {{ dbt.type_string() }},
25+
project_name {{ dbt.type_string() }},
26+
tuva_package_version {{ dbt.type_string() }},
27+
run_started_at {{ dbt.type_timestamp() }}
28+
)
29+
{%- endset -%}
30+
{% do run_query(sql) %}
31+
{{ return(true) }}
32+
{%- elif target_type in ['fabric'] -%}
33+
{%- set sql -%}
34+
IF NOT EXISTS (SELECT 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = '{{ table_name }}' AND TABLE_SCHEMA = '{{ schema_name }}')
35+
CREATE TABLE {{ schema_name }}.{{ table_name }}
36+
(
37+
invocation_id {{ dbt.type_string() }},
38+
project_name {{ dbt.type_string() }},
39+
tuva_package_version {{ dbt.type_string() }},
40+
run_started_at {{ dbt.type_timestamp() }}
41+
)
42+
{%- endset -%}
43+
{% do run_query(sql) %}
44+
{{ return(true) }}
45+
{%- else -%}
46+
{{ return(false) }}
4747
{%- endif -%}
48-
{%- else -%}
49-
{{ return(false) }}
50-
{%- endif -%}
5148

5249
{% endmacro %}
5350

5451

5552
{% macro log_invocation_start() %}
56-
{# Get the schema name using the same logic as in create_tuva_invocations_table #}
57-
{%- set schema_name = var('tuva_schema_prefix') ~ '_metadata' if var('tuva_schema_prefix',None) != None else 'metadata' -%}
58-
53+
{# Records the invocation start with the tuva project package version. #}
54+
{%- set schema_name = generate_schema_name(custom_schema_name='metadata') %}
5955
{# Capture the boolean result from create_tuva_invocations_table #}
60-
{%- set table_created = the_tuva_project.create_tuva_invocations_table() -%}
56+
{%- set table_created = the_tuva_project.create_tuva_invocations_table(schema_name) -%}
6157

6258
{# Only run DML if table_created is true. It will be false if a non-supported database is used. #}
6359
{%- if table_created -%}
@@ -81,22 +77,18 @@
8177

8278
{% do run_query(query) %}
8379

84-
{%- do the_tuva_project.drop_old_tuva_invocations() -%}
80+
{%- do the_tuva_project.drop_old_tuva_invocations(schema_name) -%}
81+
8582
{%- endif -%}
8683

8784
{% endmacro %}
8885

8986

90-
{% macro drop_old_tuva_invocations() %}
91-
{%- set schema_name = var('tuva_schema_prefix') ~ '_metadata' if var('tuva_schema_prefix',None) != None else 'metadata' -%}
92-
{%- set retention_days = var('tuva_metadata_retention_days', 30) -%}
93-
{%- set retention_days = retention_days | int -%}
94-
95-
{# Delete records older than the retention period #}
96-
{% do log("Deleting Tuva invocations older than " ~ retention_days ~ " days", info=True) %}
87+
{% macro drop_old_tuva_invocations(schema_name) %}
88+
{# Deletes tuva invocation records older than the retention period. #}
89+
{%- set retention_days = var('tuva_metadata_retention_days', 30) | int -%}
9790

98-
{# Use the dateadd function to calculate the cutoff date #}
99-
{% set dateadd = adapter.dispatch('dateadd', 'datetime') %}
91+
{% set dateadd = adapter.dispatch('dateadd', 'datetime') %}
10092
{% set query %}
10193
delete from {{ schema_name }}.tuva_invocations
10294
where run_started_at < cast({{ dateadd('day', -1 * retention_days, current_timestamp()) }} as {{ dbt.type_timestamp() }})

0 commit comments

Comments
 (0)