|
1 | | -{% macro create_tuva_invocations_table() %} |
2 | | - |
3 | | - {%- set schema_name = var('tuva_schema_prefix') ~ '_metadata' if var('tuva_schema_prefix',None) != None else 'metadata' -%} |
4 | | - |
5 | | - {% do adapter.create_schema(api.Relation.create(database=target.database, schema=schema_name)) %} |
6 | | - |
7 | | - {%- set table_name = 'tuva_invocations' -%} |
8 | | - {%- set target_type = target.type -%} |
| 1 | +{% macro create_tuva_invocations_table(schema_name) %} |
| 2 | + {# Creates a tuva invocations table. Returns true if the table is created or exists. Returns false if a non-supported platform is used. #} |
| 3 | + {% do adapter.create_schema(api.Relation.create(database=target.database, schema=schema_name)) %} |
| 4 | + |
| 5 | + {%- set table_name = 'tuva_invocations' -%} |
| 6 | + {%- set target_type = target.type -%} |
9 | 7 |
|
10 | | - {%- if target_type in ['redshift','postgres','duckdb'] -%} |
11 | | - {%- set sql -%} |
12 | | - CREATE TABLE IF NOT EXISTS {{ schema_name }}.{{ table_name }} |
13 | | - ( |
14 | | - invocation_id {{ dbt.type_string() }}, |
15 | | - project_name {{ dbt.type_string() }}, |
16 | | - tuva_package_version {{ dbt.type_string() }}, |
17 | | - run_started_at {{ dbt.type_timestamp() }} |
18 | | - ) |
19 | | - {%- endset -%} |
20 | | - {% do run_query(sql) %} |
21 | | - {{ return(true) }} |
22 | | - {%- elif target_type in ['snowflake','databricks','bigquery'] -%} |
23 | | - {%- set sql -%} |
24 | | - CREATE OR REPLACE TABLE {{ schema_name }}.{{ table_name }} |
25 | | - ( |
26 | | - invocation_id {{ dbt.type_string() }}, |
27 | | - project_name {{ dbt.type_string() }}, |
28 | | - tuva_package_version {{ dbt.type_string() }}, |
29 | | - run_started_at {{ dbt.type_timestamp() }} |
30 | | - ) |
31 | | - {%- endset -%} |
32 | | - {% do run_query(sql) %} |
33 | | - {{ return(true) }} |
34 | | - {%- if target_type in ['fabric'] -%} |
35 | | - {%- set sql -%} |
36 | | - IF NOT EXISTS (SELECT 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = '{{ table_name }}' AND TABLE_SCHEMA = '{{ schema_name }}') |
37 | | - CREATE TABLE {{ schema_name }}.{{ table_name }} |
38 | | - ( |
39 | | - invocation_id {{ dbt.type_string() }}, |
40 | | - project_name {{ dbt.type_string() }}, |
41 | | - tuva_package_version {{ dbt.type_string() }}, |
42 | | - run_started_at {{ dbt.type_timestamp() }} |
43 | | - ) |
44 | | - {%- endset -%} |
45 | | - {% do run_query(sql) %} |
46 | | - {{ return(true) }} |
| 8 | + {%- if target_type in ['redshift','postgres','duckdb'] -%} |
| 9 | + {%- set sql -%} |
| 10 | + CREATE TABLE IF NOT EXISTS {{ schema_name }}.{{ table_name }} |
| 11 | + ( |
| 12 | + invocation_id {{ dbt.type_string() }}, |
| 13 | + project_name {{ dbt.type_string() }}, |
| 14 | + tuva_package_version {{ dbt.type_string() }}, |
| 15 | + run_started_at {{ dbt.type_timestamp() }} |
| 16 | + ) |
| 17 | + {%- endset -%} |
| 18 | + {% do run_query(sql) %} |
| 19 | + {{ return(true) }} |
| 20 | + {%- elif target_type in ['snowflake','databricks','bigquery'] -%} |
| 21 | + {%- set sql -%} |
| 22 | + CREATE OR REPLACE TABLE {{ schema_name }}.{{ table_name }} |
| 23 | + ( |
| 24 | + invocation_id {{ dbt.type_string() }}, |
| 25 | + project_name {{ dbt.type_string() }}, |
| 26 | + tuva_package_version {{ dbt.type_string() }}, |
| 27 | + run_started_at {{ dbt.type_timestamp() }} |
| 28 | + ) |
| 29 | + {%- endset -%} |
| 30 | + {% do run_query(sql) %} |
| 31 | + {{ return(true) }} |
| 32 | + {%- elif target_type in ['fabric'] -%} |
| 33 | + {%- set sql -%} |
| 34 | + IF NOT EXISTS (SELECT 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = '{{ table_name }}' AND TABLE_SCHEMA = '{{ schema_name }}') |
| 35 | + CREATE TABLE {{ schema_name }}.{{ table_name }} |
| 36 | + ( |
| 37 | + invocation_id {{ dbt.type_string() }}, |
| 38 | + project_name {{ dbt.type_string() }}, |
| 39 | + tuva_package_version {{ dbt.type_string() }}, |
| 40 | + run_started_at {{ dbt.type_timestamp() }} |
| 41 | + ) |
| 42 | + {%- endset -%} |
| 43 | + {% do run_query(sql) %} |
| 44 | + {{ return(true) }} |
| 45 | + {%- else -%} |
| 46 | + {{ return(false) }} |
47 | 47 | {%- endif -%} |
48 | | - {%- else -%} |
49 | | - {{ return(false) }} |
50 | | - {%- endif -%} |
51 | 48 |
|
52 | 49 | {% endmacro %} |
53 | 50 |
|
54 | 51 |
|
55 | 52 | {% macro log_invocation_start() %} |
56 | | - {# Get the schema name using the same logic as in create_tuva_invocations_table #} |
57 | | - {%- set schema_name = var('tuva_schema_prefix') ~ '_metadata' if var('tuva_schema_prefix',None) != None else 'metadata' -%} |
58 | | - |
| 53 | + {# Records the invocation start with the tuva project package version. #} |
| 54 | + {%- set schema_name = generate_schema_name(custom_schema_name='metadata') %} |
59 | 55 | {# Capture the boolean result from create_tuva_invocations_table #} |
60 | | - {%- set table_created = the_tuva_project.create_tuva_invocations_table() -%} |
| 56 | + {%- set table_created = the_tuva_project.create_tuva_invocations_table(schema_name) -%} |
61 | 57 |
|
62 | 58 | {# Only run DML if table_created is true. It will be false if a non-supported database is used. #} |
63 | 59 | {%- if table_created -%} |
|
81 | 77 |
|
82 | 78 | {% do run_query(query) %} |
83 | 79 |
|
84 | | - {%- do the_tuva_project.drop_old_tuva_invocations() -%} |
| 80 | + {%- do the_tuva_project.drop_old_tuva_invocations(schema_name) -%} |
| 81 | + |
85 | 82 | {%- endif -%} |
86 | 83 |
|
87 | 84 | {% endmacro %} |
88 | 85 |
|
89 | 86 |
|
90 | | -{% macro drop_old_tuva_invocations() %} |
91 | | - {%- set schema_name = var('tuva_schema_prefix') ~ '_metadata' if var('tuva_schema_prefix',None) != None else 'metadata' -%} |
92 | | - {%- set retention_days = var('tuva_metadata_retention_days', 30) -%} |
93 | | - {%- set retention_days = retention_days | int -%} |
94 | | - |
95 | | - {# Delete records older than the retention period #} |
96 | | - {% do log("Deleting Tuva invocations older than " ~ retention_days ~ " days", info=True) %} |
| 87 | +{% macro drop_old_tuva_invocations(schema_name) %} |
| 88 | + {# Deletes tuva invocation records older than the retention period. #} |
| 89 | + {%- set retention_days = var('tuva_metadata_retention_days', 30) | int -%} |
97 | 90 |
|
98 | | - {# Use the dateadd function to calculate the cutoff date #} |
99 | | - {% set dateadd = adapter.dispatch('dateadd', 'datetime') %} |
| 91 | + {% set dateadd = adapter.dispatch('dateadd', 'datetime') %} |
100 | 92 | {% set query %} |
101 | 93 | delete from {{ schema_name }}.tuva_invocations |
102 | 94 | where run_started_at < cast({{ dateadd('day', -1 * retention_days, current_timestamp()) }} as {{ dbt.type_timestamp() }}) |
|
0 commit comments