Refactor Jinja environment variable handling

CHANGELOG.md

@@ -3,6 +3,22 @@ All notable changes to this project will be documented in this file.
 
 *The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).*
 
+## [4.X.X] - 2025-XX-XX
+### Added
+- Refactored `JinjaEnvVar` into `LocalDataInjection` class with enhanced functionality
+- Added `from_csv()` function to load CSV data as tuples or dictionaries in Jinja templates
+- Added `from_json()` function to load JSON data in Jinja templates
+- Added `from_yaml()` function to load YAML data in Jinja templates
+- Added comprehensive demo showcasing LocalDataInjection vs legacy approaches
+
+### Changed
+- `env_var()` function now provided by `LocalDataInjection` class (backward compatible)
+- Updated Jinja template processor to use `LocalDataInjection` instead of `JinjaEnvVar`
+- Enhanced README.md with LocalDataInjection function documentation
+
+### Deprecated
+- `JinjaEnvVar` class is deprecated in favor of `LocalDataInjection`
+
 ## [4.3.2] - 2026-02-11
 ### Fixed
 - **Checksum stability fix** (#417): Restored trailing semicolon stripping that was accidentally removed in v4.3.0. This fixes checksum drift for ALL scripts ending with `;`, not just those with trailing comments.

README.md

@@ -942,6 +942,22 @@ Return the value of the environmental variable if it exists, otherwise raise an
 {{ env_var('<environmental_variable>') }}
 ```
 
+##### from_csv, from_json, from_yaml
+
+These functions provide access to local data files for use in Jinja templates.
+
+**Examples:**
+```jinja
+-- Load CSV data as dictionaries
+{% set table_defs = from_csv('data/tables.csv', as_dict=true) %}
+
+-- Load JSON configuration
+{% set config = from_json('config/settings.json') %}
+
+-- Load YAML parameters
+{% set params = from_yaml('config/parameters.yaml') %}
+```
+
 ### Environment Variables
 
 #### Why Use Environment Variables?

demo/citibike_demo_jinja_data_injection/1_setup/A__setup.sql

@@ -0,0 +1,99 @@
+SET TARGET_SCHEMA_NAME = '{{ schema_name }}';
+SET TARGET_DB_NAME = '{{ database_name }}'; -- Name of database that will have the SCHEMACHANGE Schema for change tracking.
+-- Dependent Variables; Change the naming pattern if you want but not necessary
+SET ADMIN_ROLE = '"' || $TARGET_DB_NAME || '-ADMIN"'; -- This role will be granted privileges to create objects in any schema in the database
+SET WAREHOUSE_NAME = $TARGET_DB_NAME || '_WH';
+SET SCHEMACHANGE_NAMESPACE = $TARGET_DB_NAME || '.' || $TARGET_SCHEMA_NAME;
+SET SC_M = 'SC_M_' || $TARGET_SCHEMA_NAME;
+SET SC_R = 'SC_R_' || $TARGET_SCHEMA_NAME;
+SET SC_W = 'SC_W_' || $TARGET_SCHEMA_NAME;
+SET SC_C = 'SC_C_' || $TARGET_SCHEMA_NAME;
+
+USE ROLE IDENTIFIER($ADMIN_ROLE);
+USE DATABASE IDENTIFIER($TARGET_DB_NAME);
+USE WAREHOUSE IDENTIFIER($WAREHOUSE_NAME);
+
+CREATE DATABASE ROLE IF NOT EXISTS IDENTIFIER($SC_M);
+CREATE DATABASE ROLE IF NOT EXISTS IDENTIFIER($SC_R);
+CREATE DATABASE ROLE IF NOT EXISTS IDENTIFIER($SC_W);
+CREATE DATABASE ROLE IF NOT EXISTS IDENTIFIER($SC_C);
+
+GRANT DATABASE ROLE IDENTIFIER($SC_M) TO DATABASE ROLE DB_M;
+GRANT DATABASE ROLE IDENTIFIER($SC_R) TO DATABASE ROLE DB_R;
+GRANT DATABASE ROLE IDENTIFIER($SC_W) TO DATABASE ROLE DB_W;
+GRANT DATABASE ROLE IDENTIFIER($SC_C) TO DATABASE ROLE DB_C;
+GRANT DATABASE ROLE IDENTIFIER($SC_M) TO DATABASE ROLE IDENTIFIER($SC_R);
+GRANT DATABASE ROLE IDENTIFIER($SC_R) TO DATABASE ROLE IDENTIFIER($SC_W);
+GRANT DATABASE ROLE IDENTIFIER($SC_W) TO DATABASE ROLE IDENTIFIER($SC_C);
+
+CREATE TRANSIENT SCHEMA IF NOT EXISTS IDENTIFIER($TARGET_SCHEMA_NAME) WITH MANAGED ACCESS;
+
+USE SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE);
+-- SCHEMA
+-- SC_M
+GRANT USAGE ON DATABASE IDENTIFIER($TARGET_DB_NAME) TO DATABASE ROLE IDENTIFIER($SC_M);
+GRANT USAGE ON SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_M);
+-- SC_R
+GRANT MONITOR ON DATABASE IDENTIFIER($TARGET_DB_NAME) TO DATABASE ROLE IDENTIFIER($SC_R);
+GRANT MONITOR ON SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_R);
+-- SC_W
+-- None
+-- SC_C
+
+-- TABLES
+-- SC_M
+GRANT REFERENCES ON ALL TABLES IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_M);
+GRANT REFERENCES ON FUTURE TABLES IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_M);
+-- SC_R
+GRANT SELECT ON ALL TABLES IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_R);
+GRANT SELECT ON FUTURE TABLES IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_R);
+-- SC_W
+GRANT INSERT, UPDATE, DELETE, TRUNCATE, EVOLVE SCHEMA ON ALL TABLES IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_W);
+GRANT INSERT, UPDATE, DELETE, TRUNCATE, EVOLVE SCHEMA ON FUTURE TABLES IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_W);
+-- SC_C
+GRANT CREATE TABLE ON SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_C);
+
+-- STAGES
+-- SC_M
+GRANT USAGE ON ALL STAGES IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_M);
+GRANT USAGE ON FUTURE STAGES IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_M);
+-- SC_R
+GRANT READ ON ALL STAGES IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_R);
+GRANT READ ON FUTURE STAGES IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_R);
+-- SC_W
+GRANT READ,WRITE ON ALL STAGES IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_W);
+GRANT READ,WRITE ON FUTURE STAGES IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_W);
+-- SC_C
+GRANT CREATE STAGE ON SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_C);
+
+
+-- FILE FORMATS
+-- SC_M
+GRANT USAGE ON ALL FILE FORMATS IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_M);
+GRANT USAGE ON FUTURE FILE FORMATS IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_M);
+-- SC_R
+-- N/A
+-- SC_W
+-- N/A
+-- SC_C
+GRANT CREATE FILE FORMAT ON SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_C);
+
+-- STORED PROCEDURES
+-- SC_M
+-- N/A
+-- SC_R
+GRANT USAGE ON ALL PROCEDURES IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_R);
+GRANT USAGE ON FUTURE PROCEDURES IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_R);
+-- SC_W
+-- SC_C
+GRANT CREATE PROCEDURE ON SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_C);
+
+-- FUNCTIONS
+-- SC_M
+-- N/A
+-- SC_R
+GRANT USAGE ON ALL FUNCTIONS IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_R);
+GRANT USAGE ON FUTURE FUNCTIONS IN SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_R);
+-- SC_W
+-- SC_C
+GRANT CREATE FUNCTION ON SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_C);

demo/citibike_demo_jinja_data_injection/2_test/A__render.sql

@@ -0,0 +1,2 @@
+-- This file demonstrates LocalDataInjection functionality
+SELECT 'LocalDataInjection Demo' as message;

demo/citibike_demo_jinja_data_injection/2_test/R__render.sql

@@ -0,0 +1,2 @@
+-- This file demonstrates LocalDataInjection functionality
+SELECT 'LocalDataInjection Demo - Rendered' as message;

demo/citibike_demo_jinja_data_injection/2_test/V1.0.0__render.sql

@@ -0,0 +1,2 @@
+-- This file demonstrates LocalDataInjection functionality
+SELECT 'LocalDataInjection Demo - Version 1.0.0' as message;

demo/citibike_demo_jinja_data_injection/2_test/V1.1.0__initial_database_objects.sql

@@ -0,0 +1,37 @@
+{% from 'modules/create_stage.j2' import create_stage %}
+{% from 'modules/table_operations.j2' import create_table, create_view %}
+{% from 'modules/create_file_format.j2' import create_file_format %}
+
+-- Load data from external files using LocalDataInjection
+{% set trips_definitions = from_csv('2_test/trips.csv', as_dict=true) %}
+{% set weather_definitions = from_csv('2_test/weather.csv', as_dict=true) %}
+{% set file_format_config = from_json('2_test/file_formats.json') %}
+{% set stage_config = from_yaml('2_test/stages.yaml') %}
+
+-- Create the database if it doesn't exist
+USE DATABASE {{database_name}};
+
+-- Set the database and schema context
+USE SCHEMA {{database_name}}.{{schema_name}};
+
+-- Create file formats from JSON configuration
+{% for file_format in file_format_config.file_formats %}
+{{ create_file_format(file_format) }}
+{% endfor %}
+
+-- Create stages from YAML configuration
+{% for stage in stage_config.stages %}
+{{ create_stage(stage.name, stage.url) }}
+{% endfor %}
+
+-- Create landing tables (all VARCHAR with metadata) from CSV configuration
+{{ create_table('TRIPS', trips_definitions, as_varchar=true) }}
+{{ create_table('WEATHER', weather_definitions, as_varchar=true) }}
+
+-- Create final tables from CSV configuration
+{{ create_table('TRIPS', trips_definitions) }}
+{{ create_table('WEATHER', weather_definitions) }}
+
+-- Create views that filter to latest data and apply transformations
+{{ create_view('TRIPS_VIEW', 'TRIPS_LANDING', trips_definitions) }}
+{{ create_view('WEATHER_VIEW', 'WEATHER_LANDING', weather_definitions) }}

demo/citibike_demo_jinja_data_injection/2_test/V1.1.0__initial_database_objects_legacy.sql

@@ -0,0 +1,67 @@
+{% from 'modules/create_stage.j2' import create_stage %}
+{% from 'modules/table_operations.j2' import create_table, create_view %}
+{% from 'modules/create_file_format.j2' import create_file_format %}
+{% from '2_test/trips_legacy.j2' import trips_definitions %}
+{% from '2_test/weather_legacy.j2' import weather_definitions %}
+{% from '2_test/file_formats_legacy.j2' import file_format_config %}
+{% from '2_test/stages_legacy.j2' import stage_config %}
+
+-- LEGACY APPROACH: Without LocalDataInjection - manual data definitions in Jinja
+-- This shows the "before" state that LocalDataInjection improves upon
+-- Instead of using from_csv('trips.csv', as_dict=true), we manually define the data structure
+-- in trips_legacy.j2 and import it with: from '2_test/trips_legacy.j2' import trips_definitions
+
+-- Convert tuples to dictionaries (same as from_csv() would do)
+{% set trips_definitions_dict = [] %}
+{% for row in trips_definitions %}
+    {% set _ = trips_definitions_dict.append({
+        'column_name': row[0],
+        'data_type': row[1],
+        'nullable': row[2],
+        'description': row[3],
+        'source_column': row[4],
+        'landing_transformation': row[5],
+        'final_transformation': row[6]
+    }) %}
+{% endfor %}
+
+{% set weather_definitions_dict = [] %}
+{% for row in weather_definitions %}
+    {% set _ = weather_definitions_dict.append({
+        'column_name': row[0],
+        'data_type': row[1],
+        'nullable': row[2],
+        'description': row[3],
+        'source_column': row[4],
+        'landing_transformation': row[5],
+        'final_transformation': row[6]
+    }) %}
+{% endfor %}
+
+-- Create the database if it doesn't exist
+USE DATABASE {{database_name}};
+
+-- Set the database and schema context
+USE SCHEMA {{database_name}}.{{schema_name}};
+
+-- Create file formats from hardcoded configuration
+{% for file_format in file_format_config.file_formats %}
+{{ create_file_format(file_format) }}
+{% endfor %}
+
+-- Create stages from hardcoded configuration
+{% for stage in stage_config.stages %}
+{{ create_stage(stage.name, stage.url) }}
+{% endfor %}
+
+-- Create landing tables using the SAME macros as LocalDataInjection version
+{{ create_table('TRIPS', trips_definitions_dict, as_varchar=true) }}
+{{ create_table('WEATHER', weather_definitions_dict, as_varchar=true) }}
+
+-- Create final tables using the SAME macros as LocalDataInjection version
+{{ create_table('TRIPS', trips_definitions_dict) }}
+{{ create_table('WEATHER', weather_definitions_dict) }}
+
+-- Create views using the SAME macros as LocalDataInjection version
+{{ create_view('TRIPS_VIEW', 'TRIPS_LANDING', trips_definitions_dict) }}
+{{ create_view('WEATHER_VIEW', 'WEATHER_LANDING', weather_definitions_dict) }}

demo/citibike_demo_jinja_data_injection/2_test/V1.2.0__load_tables_from_s3.sql

@@ -0,0 +1,18 @@
+{% from 'modules/table_operations.j2' import copy_into_landing %}
+
+-- Load configuration from YAML file using LocalDataInjection
+{% set stage_config = from_yaml('2_test/stages.yaml') %}
+{% set trips_definitions = from_csv('2_test/trips.csv', as_dict=true) %}
+{% set weather_definitions = from_csv('2_test/weather.csv', as_dict=true) %}
+
+-- Set the database and schema context
+USE SCHEMA {{database_name}}.{{schema_name}};
+
+-- Load data into landing tables using copy operations defined in YAML
+{% for operation_name, operation in stage_config.copy_operations.items() %}
+    {% if operation.table == 'TRIPS_LANDING' %}
+        {{ copy_into_landing(operation, trips_definitions) }}
+    {% elif operation.table == 'WEATHER_LANDING' %}
+        {{ copy_into_landing(operation, weather_definitions) }}
+    {% endif %}
+{% endfor %}

demo/citibike_demo_jinja_data_injection/2_test/V1.2.0__load_tables_from_s3_legacy.sql

@@ -0,0 +1,48 @@
+{% from 'modules/table_operations.j2' import copy_into_landing %}
+{% from '2_test/stages_legacy.j2' import stage_config %}
+{% from '2_test/trips_legacy.j2' import trips_definitions %}
+{% from '2_test/weather_legacy.j2' import weather_definitions %}
+
+-- LEGACY APPROACH: Without LocalDataInjection - manual data definitions in Jinja
+-- This shows the "before" state that LocalDataInjection improves upon
+-- Instead of using from_yaml('stages.yaml'), we manually define the data structure
+-- in stages_legacy.j2 and import it
+
+-- Convert tuples to dictionaries (same as from_csv() would do)
+{% set trips_definitions_dict = [] %}
+{% for row in trips_definitions %}
+    {% set _ = trips_definitions_dict.append({
+        'column_name': row[0],
+        'data_type': row[1],
+        'nullable': row[2],
+        'description': row[3],
+        'source_column': row[4],
+        'landing_transformation': row[5],
+        'final_transformation': row[6]
+    }) %}
+{% endfor %}
+
+{% set weather_definitions_dict = [] %}
+{% for row in weather_definitions %}
+    {% set _ = weather_definitions_dict.append({
+        'column_name': row[0],
+        'data_type': row[1],
+        'nullable': row[2],
+        'description': row[3],
+        'source_column': row[4],
+        'landing_transformation': row[5],
+        'final_transformation': row[6]
+    }) %}
+{% endfor %}
+
+-- Set the database and schema context
+USE SCHEMA {{database_name}}.{{schema_name}};
+
+-- Load data into landing tables using the SAME macros as LocalDataInjection version
+{% for operation_name, operation in stage_config.copy_operations.items() %}
+    {% if operation.table == 'TRIPS_LANDING' %}
+        {{ copy_into_landing(operation, trips_definitions_dict) }}
+    {% elif operation.table == 'WEATHER_LANDING' %}
+        {{ copy_into_landing(operation, weather_definitions_dict) }}
+    {% endif %}
+{% endfor %}

demo/citibike_demo_jinja_data_injection/2_test/V1.3.0__create_tasks.sql

@@ -0,0 +1,23 @@
+{% from 'modules/task_operations.j2' import create_task %}
+
+-- Load configuration from YAML file using LocalDataInjection
+{% set stage_config = from_yaml('2_test/stages.yaml') %}
+{% set trips_definitions = from_csv('2_test/trips.csv', as_dict=true) %}
+{% set weather_definitions = from_csv('2_test/weather.csv', as_dict=true) %}
+
+-- Set the database and schema context
+USE SCHEMA {{database_name}}.{{schema_name}};
+
+-- Create tasks for processing data from landing to final tables
+{% for task in stage_config.tasks %}
+    {% if task.name == 'PROCESS_TRIPS_DATA' %}
+        {{ create_task(task, trips_definitions) }}
+    {% elif task.name == 'PROCESS_WEATHER_DATA' %}
+        {{ create_task(task, weather_definitions) }}
+    {% endif %}
+{% endfor %}
+
+-- Resume the tasks to start processing
+{% for task in stage_config.tasks %}
+ALTER TASK {{ task.name }} RESUME;
+{% endfor %}