Skip to content

Commit 2b8898f

Browse files
aneiderhiseraneiderhiser
authored
Merge pull request #65 from tuva-health/add-new-workflows-and-profiles
Add duckdb and databricks profiles, adjust workflows
2 parents 127f0c7 + 33cfe05 commit 2b8898f

15 files changed

Lines changed: 377 additions & 489 deletions

‎.github/workflows/dbt_v1.10.15_bigquery_build_full_refresh.yml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
name: BigQuery CICD Full Refresh
2+
on:
3+
# pull_request:
4+
# branches:
5+
# - main
6+
workflow_dispatch: # Allows manual trigger from UI
7+
8+
jobs:
9+
dbt_run:
10+
name: dbt full refresh and test on BigQuery
11+
runs-on: ubuntu-latest
12+
13+
env:
14+
# Environment variables
15+
PYTHON_VERSION: "3.10"
16+
DBT_PROJECT_DIR: "."
17+
DBT_CORE_VERSION: "1.10.15"
18+
# BigQuery connection variables
19+
DBT_BIGQUERY_CI_TOKEN: ${{ secrets.DBT_BIGQUERY_CI_TOKEN }}
20+
DBT_BIGQUERY_CI_PROJECT: ${{ secrets.DBT_BIGQUERY_CI_PROJECT }}
21+
22+
steps:
23+
- name: Checkout Code
24+
uses: actions/checkout@v4
25+
26+
- name: Setup Python
27+
uses: actions/setup-python@v4
28+
with:
29+
python-version: ${{ env.PYTHON_VERSION }}
30+
31+
- name: Install dbt-core and BigQuery adapter
32+
run: |
33+
python -m pip install --upgrade pip
34+
pip install dbt-core==${{ env.DBT_CORE_VERISON }} dbt-bigquery
35+
36+
- name: Create BigQuery credentials file
37+
run: |
38+
echo "${{ env.DBT_BIGQUERY_CI_TOKEN }}" | base64 --decode > ./creds.json
39+
working-directory: ${{ env.DBT_PROJECT_DIR }}
40+
41+
- name: Install dbt dependencies
42+
run: dbt deps --profiles-dir ./integration_tests/profiles/bigquery
43+
working-directory: ${{ env.DBT_PROJECT_DIR }}
44+
45+
- name: Test Connection
46+
run: dbt debug --profiles-dir ./integration_tests/profiles/bigquery
47+
working-directory: ${{ env.DBT_PROJECT_DIR }}
48+
49+
- name: Build dbt models (full-refresh)
50+
run: |
51+
dbt build --full-refresh --profiles-dir ./integration_tests/profiles/bigquery
52+
working-directory: ${{ env.DBT_PROJECT_DIR }}

‎.github/workflows/dbt_v1.10.15_databricks_build_full_refresh.yml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
name: Databricks CICD Full Refresh
2+
on:
3+
# pull_request:
4+
# branches:
5+
# - main
6+
workflow_dispatch: # Allows manual trigger from UI
7+
8+
jobs:
9+
dbt_run:
10+
runs-on: ubuntu-latest
11+
12+
env:
13+
# Environment variables
14+
PYTHON_VERSION: "3.10"
15+
DBT_PROJECT_DIR: "."
16+
DBT_CORE_VERSION: "1.10.15"
17+
# Databricks connection variables
18+
DBT_DATABRICKS_CI_HOST: ${{ secrets.DBT_DATABRICKS_CI_HOST }}
19+
DBT_DATABRICKS_CI_HTTP_PATH: ${{ secrets.DBT_DATABRICKS_CI_HTTP_PATH }}
20+
DBT_DATABRICKS_CI_TOKEN: ${{ secrets.DBT_DATABRICKS_CI_TOKEN }}
21+
DBT_DATABRICKS_CI_CATALOG: ${{ secrets.DBT_DATABRICKS_CI_CATALOG }}
22+
23+
steps:
24+
- uses: actions/checkout@v4
25+
26+
- name: Set up Python
27+
uses: actions/setup-python@v4
28+
with:
29+
python-version: ${{ env.PYTHON_VERSION }}
30+
31+
# Certifi addresses a known certificate issue with databricks and dbt
32+
- name: Install dbt-core and Databricks adapter
33+
run: |
34+
python -m pip install --upgrade pip
35+
pip install dbt-core==1.10.15 dbt-databricks
36+
# Install SSL certificates fix
37+
pip install certifi
38+
39+
- name: Load dbt dependencies
40+
run: dbt deps --profiles-dir ./integration_tests/profiles/databricks
41+
working-directory: ${{ env.DBT_PROJECT_DIR }}
42+
43+
- name: Test connection
44+
run: dbt debug --profiles-dir ./integration_tests/profiles/databricks
45+
working-directory: ${{ env.DBT_PROJECT_DIR }}
46+
47+
- name: Build dbt models (full-refresh)
48+
run: |
49+
dbt build --full-refresh --profiles-dir ./integration_tests/profiles/databricks
50+
working-directory: ${{ env.DBT_PROJECT_DIR }}

‎.github/workflows/dbt_v1.10.15_duckdb_build_full_refresh.yml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: DuckDB/MotherDuck CICD Full Refresh
2+
on:
3+
# pull_request:
4+
# branches:
5+
# - main
6+
workflow_dispatch: # Allows manual trigger from UI
7+
8+
jobs:
9+
dbt_run:
10+
name: dbt full refresh and test on DuckDB/MotherDuck
11+
runs-on: ubuntu-latest
12+
13+
env:
14+
# Environment variables
15+
PYTHON_VERSION: "3.10"
16+
DBT_PROJECT_DIR: "."
17+
DBT_CORE_VERSION: "1.10.15"
18+
19+
# Connection variables
20+
DBT_MOTHERDUCK_CI_TOKEN: ${{ secrets.DBT_MOTHERDUCK_CI_TOKEN }}
21+
DBT_MOTHERDUCK_CI_DATABASE: "my_db"
22+
DBT_MOTHERDUCK_CI_PATH: "md:${{ env.DBT_MOTHERDUCK_CI_DATABASE }}?motherduck_token=${{ env.DBT_MOTHERDUCK_CI_TOKEN }}"
23+
24+
steps:
25+
- name: Checkout Code
26+
uses: actions/checkout@v4
27+
28+
- name: Set up Python
29+
uses: actions/setup-python@v4
30+
with:
31+
python-version: ${{ env.PYTHON_VERSION }}
32+
33+
- name: Install dbt-core and DuckDB adapter
34+
run: |
35+
python -m pip install --upgrade pip
36+
pip install dbt-core==${{ env.DBT_CORE_VERSION}} dbt-duckdb
37+
38+
- name: Install dbt dependencies
39+
run: dbt deps --profiles-dir ./integration_tests/profiles/duckdb
40+
working-directory: ${{ env.DBT_PROJECT_DIR }}
41+
42+
- name: Test Connection
43+
run: dbt debug --profiles-dir ./integration_tests/profiles/duckdb
44+
working-directory: ${{ env.DBT_PROJECT_DIR }}
45+
46+
- name: Build dbt models (full-refresh)
47+
run: |
48+
dbt build --full-refresh --profiles-dir ./integration_tests/profiles/duckdb
49+
working-directory: ${{ env.DBT_PROJECT_DIR }}

‎.github/workflows/dbt_v1.10.15_fabric_build_full_refresh.yml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
name: Fabric CICD Full Refresh
2+
on:
3+
# pull_request:
4+
# branches:
5+
# - main
6+
workflow_dispatch: # Allows manual trigger from UI
7+
8+
jobs:
9+
dbt_run:
10+
name: dbt full refresh and test on Fabric
11+
runs-on: ubuntu-latest
12+
13+
env:
14+
# Environment variables
15+
PYTHON_VERSION: "3.10"
16+
DBT_PROJECT_DIR: "."
17+
DBT_CORE_VERSION: "1.10.15"
18+
# Fabric connection variables
19+
DBT_FABRIC_CI_HOST: ${{ secrets.DBT_FABRIC_CI_HOST }}
20+
DBT_FABRIC_CI_DATABASE: ${{ secrets.DBT_FABRIC_CI_DATABASE }}
21+
DBT_FABRIC_CI_SCHEMA: ${{ secrets.DBT_FABRIC_CI_SCHEMA }}
22+
DBT_FABRIC_CI_SERVICE_PRINCIPAL_ID: ${{ secrets.DBT_FABRIC_SERVICE_PRINCIPAL_ID }}
23+
DBT_FABRIC_CI_SERVICE_PRINCIPAL_SECRET: ${{ secrets.DBT_FABRIC_SERVICE_PRINCIPAL_SECRET }}
24+
DBT_FABRIC_CI_TENANT_ID: ${{ secrets.DBT_FABRIC_CI_TENANT_ID }}
25+
26+
27+
steps:
28+
- name: Checkout Code
29+
uses: actions/checkout@v4
30+
31+
- name: Set up Python
32+
uses: actions/setup-python@v4
33+
with:
34+
python-version: ${{ env.PYTHON_VERSION }}
35+
36+
- name: Install ODBC Driver 18 for SQL Server
37+
run: |
38+
curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
39+
curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list
40+
sudo apt-get update
41+
sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18
42+
43+
- name: Install dbt-core and Fabric adapter
44+
run: |
45+
python -m pip install --upgrade pip
46+
pip install dbt-core==${{ env.DBT_CORE_VERISON }} dbt-fabric
47+
48+
- name: Install dbt dependencies
49+
run: dbt deps --profiles-dir ./integration_tests/profiles/fabric
50+
working-directory: ${{ env.DBT_PROJECT_DIR }}
51+
52+
- name: Test connection
53+
run: dbt debug --profiles-dir ./integration_tests/profiles/fabric
54+
working-directory: ${{ env.DBT_PROJECT_DIR }}
55+
56+
- name: Build dbt models (full-refresh)
57+
run: |
58+
dbt build --full-refresh --profiles-dir ./integration_tests/profiles/fabric
59+
working-directory: ${{ env.DBT_PROJECT_DIR }}

‎.github/workflows/dbt_v1.10.15_redshift_build_full_fresh.yml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
name: Redshift CICD Full Refresh
2+
on:
3+
# pull_request:
4+
# branches:
5+
# - main
6+
workflow_dispatch: # Allows manual trigger from UI
7+
8+
jobs:
9+
dbt_run:
10+
name: dbt full refresh and test on Redshift
11+
runs-on: ubuntu-latest
12+
13+
env:
14+
# Environment variables
15+
DBT_PROJECT_DIR: "."
16+
PYTHON_VERSION: "3.10"
17+
DBT_CORE_VERSION: "1.10.15"
18+
19+
# Connection variables
20+
DBT_REDSHIFT_CI_HOST: ${{ secrets.DBT_REDSHIFT_CI_HOST }}
21+
DBT_REDSHIFT_CI_USER: ${{ secrets.DBT_REDSHIFT_CI_USER }}
22+
DBT_REDSHIFT_CI_PASSWORD: ${{ secrets.DBT_REDSHIFT_CI_PASSWORD }}
23+
DBT_REDSHIFT_CI_DATABASE: ${{ secrets.DBT_REDSHIFT_CI_DATABASE }}
24+
DBT_REDSHIFT_CI_SCHEMA: ${{ secrets.DBT_REDSHIFT_SCHEMA }}
25+
26+
steps:
27+
- name: Checkout Code
28+
uses: actions/checkout@v4
29+
30+
- name: Setup Python
31+
uses: actions/setup-python@v5
32+
with:
33+
python-version: $${{ env.PYTHON_VERSION }}
34+
35+
- name: Install dbt and Redshift adapter
36+
run: |
37+
python -m pip install --upgrade pip
38+
pip install dbt-core==${{ env.DBT_CORE_VERSION }} dbt-redshift
39+
40+
- name: Install dbt dependencies
41+
working-directory: ${{ env.DBT_PROJECT_DIR }}
42+
run: dbt deps --profiles-dir ./integration_tests/profiles/redshift
43+
44+
- name: Test Connection
45+
working-directory: ${{ env.DBT_PROJECT_DIR }}
46+
run: dbt debug --profiles-dir ./integration_tests/profiles/redshift
47+
48+
- name: Build dbt models
49+
working-directory: ${{ env.DBT_PROJECT_DIR }}
50+
run: dbt build --full-refresh --profiles-dir ./integration_tests/profiles/redshift

‎.github/workflows/dbt_v1.10.15_snowflake_build_full_refresh.yml‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
name: Snowflake CICD Full Refresh
2+
on:
3+
pull_request:
4+
branches:
5+
- main
6+
workflow_dispatch: # Allows manual trigger from UI
7+
8+
concurrency:
9+
# Ensure only one instance of this workflow runs across all PRs to prevent database conflicts
10+
group: ci-testing
11+
# Don't cancel in-progress runs automatically
12+
cancel-in-progress: false
13+
14+
jobs:
15+
dbt_run:
16+
name: dbt full refresh and test on Snowflake
17+
runs-on: ubuntu-latest
18+
19+
env:
20+
# Environment variables
21+
DBT_PROJECT_DIR: "."
22+
PYTHON_VERSION: "3.10"
23+
DBT_CORE_VERSION: "1.10.15"
24+
25+
# Connection variables
26+
DBT_SNOWFLAKE_CI_ACCOUNT: ${{ secrets.DBT_SNOWFLAKE_DEV_ACCOUNT }}
27+
DBT_SNOWFLAKE_CI_DATABASE: ${{ secrets.DBT_SNOWFLAKE_DEV_CI_DATABASE }}
28+
DBT_SNOWFLAKE_CI_PASSWORD: ${{ secrets.DBT_SNOWFLAKE_DEV_CI_PASSWORD }}
29+
DBT_SNOWFLAKE_CI_ROLE: ${{ secrets.DBT_SNOWFLAKE_DEV_CI_ROLE }}
30+
DBT_SNOWFLAKE_CI_SCHEMA: ${{ secrets.DBT_SNOWFLAKE_DEV_CI_SCHEMA }}
31+
DBT_SNOWFLAKE_CI_USER: ${{ secrets.DBT_SNOWFLAKE_DEV_CI_USER }}
32+
DBT_SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.DBT_SNOWFLAKE_DEV_CI_WAREHOUSE }}
33+
34+
steps:
35+
- name: Checkout Code
36+
uses: actions/checkout@v4
37+
38+
- name: Setup Python
39+
uses: actions/setup-python@v5
40+
with:
41+
python-version: ${{ env.PYTHON_VERSION }}
42+
43+
- name: Install dbt
44+
run: |
45+
python -m pip install --upgrade pip
46+
pip install dbt-core==${{ env.DBT_CORE_VERSION }} dbt-snowflake
47+
48+
- name: Install dbt dependencies
49+
working-directory: ${{ env.DBT_PROJECT_DIR }}
50+
run: dbt deps --profiles-dir ./integration_tests/profiles/snowflake
51+
52+
- name: Test Connection
53+
working-directory: ${{ env.DBT_PROJECT_DIR }}
54+
run: dbt debug --profiles-dir ./integration_tests/profiles/snowflake
55+
56+
- name: Run build models
57+
working-directory: ${{ env.DBT_PROJECT_DIR }}
58+
run: dbt build --full-refresh --profiles-dir ./integration_tests/profiles/snowflake

0 commit comments

Comments
 (0)