Skip to content

Commit 1012a46

Browse files
mkrahu-z2mkrahu-z2
committed
Add duckdb and databricks profiles, adjust workflows, add fabric profile that was missing previously
1 parent 0257ddd commit 1012a46

7 files changed

Lines changed: 153 additions & 39 deletions

File tree

‎.github/workflows/dbt_v1.10.15_bigquery_build_full_refresh.yml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@ jobs:
1515
PYTHON_VERSION: "3.10"
1616
DBT_PROJECT_DIR: "."
1717
DBT_CORE_VERSION: "1.10.15"
18+
# BigQuery connection variables
19+
TUVA_BIGQUERY_TOKEN: ${{ secrets.TUVA_BIGQUERY_TOKEN }}
20+
TUVA_BIGQUERY_PROJECT: ${{ secrets.TUVA_BIGQUERY_PROJECT }}
1821

1922
steps:
2023
- name: Checkout Code
@@ -29,10 +32,10 @@ jobs:
2932
run: |
3033
python -m pip install --upgrade pip
3134
pip install dbt-core==${{ env.DBT_CORE_VERISON }} dbt-bigquery
32-
35+
3336
- name: Create BigQuery credentials file
3437
run: |
35-
echo "${{ secrets.TUVA_BIGQUERY_TOKEN }}" | base64 --decode > ./creds.json
38+
echo "${{ env.TUVA_BIGQUERY_TOKEN }}" | base64 --decode > ./creds.json
3639
working-directory: ${{ env.DBT_PROJECT_DIR }}
3740

3841
- name: Install dbt dependencies

‎.github/workflows/dbt_v1.10.15_databricks_build_full_refresh.yml‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
name: Databricks CICD Full Refresh
2+
on:
3+
# pull_request:
4+
# branches:
5+
# - main
6+
workflow_dispatch: # Allows manual trigger from UI
7+
8+
jobs:
9+
dbt_run:
10+
runs-on: ubuntu-latest
11+
12+
env:
13+
# Environment variables
14+
PYTHON_VERSION: "3.10"
15+
DBT_PROJECT_DIR: "."
16+
DBT_CORE_VERSION: "1.10.15"
17+
# Databricks connection variables
18+
DATABRICKS_HOST: ${{ secrets.DBT_DATABRICKS_HOST }}
19+
DATABRICKS_HTTP_PATH: ${{ secrets.DBT_DATABRICKS_HTTP_PATH }}
20+
DATABRICKS_TOKEN: ${{ secrets.DBT_DATABRICKS_TOKEN }}
21+
DATABRICKS_CATALOG: ${{ secrets.DBT_DATABRICKS_CATALOG }}
22+
SCHEMA_SUFFIX: ${{ github.run_id }}
23+
24+
steps:
25+
- uses: actions/checkout@v4
26+
27+
- name: Set up Python
28+
uses: actions/setup-python@v4
29+
with:
30+
python-version: ${{ env.PYTHON_VERSION }}
31+
32+
# Certifi addresses a known certificate issue with databricks and dbt
33+
- name: Install dbt-core and Databricks adapter
34+
run: |
35+
python -m pip install --upgrade pip
36+
pip install dbt-core==1.10.15 dbt-databricks
37+
# Install SSL certificates fix
38+
pip install certifi
39+
40+
- name: Load dbt dependencies
41+
run: dbt deps --profiles-dir ./integration_tests/profiles/databricks
42+
working-directory: ${{ env.DBT_PROJECT_DIR }}
43+
44+
- name: Test connection
45+
run: dbt debug --profiles-dir ./integration_tests/profiles/databricks
46+
working-directory: ${{ env.DBT_PROJECT_DIR }}
47+
48+
- name: Build dbt models (full-refresh)
49+
run: |
50+
dbt build --full-refresh --profiles-dir ./integration_tests/profiles/databricks
51+
working-directory: ${{ env.DBT_PROJECT_DIR }}

‎.github/workflows/dbt_v1.10.15_duckdb_build_full_refresh.yml‎

Lines changed: 3 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ jobs:
1818

1919
# Connection variables
2020
MOTHERDUCK_TOKEN: ${{ secrets.DBT_MOTHERDUCK_TOKEN }}
21+
MOTHERDUCK_DB_NAME: "my_db"
22+
MOTHERDUCK_PATH: "md:${{ env.MOTHERDUCK_DB_NAME }}?motherduck_token=${{ env.MOTHERDUCK_TOKEN }}"
2123

2224
steps:
2325
- name: Checkout Code
@@ -33,44 +35,8 @@ jobs:
3335
python -m pip install --upgrade pip
3436
pip install dbt-core==${{ env.DBT_CORE_VERSION}} dbt-duckdb
3537
36-
- name: Create dbt profile for DuckDB/MotherDuck
37-
run: |
38-
mkdir -p ./integration_tests/profiles/duckdb
39-
python3 << 'EOF'
40-
import os
41-
import yaml
42-
43-
# Create unique database name for CI run
44-
db_name = "my_db"
45-
46-
# Build the MotherDuck connection string
47-
motherduck_path = f"md:{db_name}?motherduck_token={os.environ['MOTHERDUCK_TOKEN']}"
48-
49-
profile = {
50-
'default': {
51-
'outputs': {
52-
'dev': {
53-
'type': 'duckdb',
54-
'path': motherduck_path,
55-
'threads': 4,
56-
'settings': {
57-
'custom_user_agent': 'dbt-ci'
58-
}
59-
}
60-
},
61-
'target': 'dev'
62-
}
63-
}
64-
65-
with open('./integration_tests/profiles/duckdb/profiles.yml', 'w') as f:
66-
yaml.dump(profile, f, default_flow_style=False)
67-
68-
print(f"Created DuckDB/MotherDuck profile with database: {db_name}")
69-
EOF
70-
working-directory: ${{ env.DBT_PROJECT_DIR }}
71-
7238
- name: Install dbt dependencies
73-
run: dbt deps --profiles-dir ./integration_tests/profiles/b
39+
run: dbt deps --profiles-dir ./integration_tests/profiles/duckdb
7440
working-directory: ${{ env.DBT_PROJECT_DIR }}
7541

7642
- name: Test Connection

‎.github/workflows/dbt_v1.10.15_fabric_build_full_refresh.yml‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
name: Fabric CICD Full Refresh
2+
on:
3+
# pull_request:
4+
# branches:
5+
# - main
6+
workflow_dispatch: # Allows manual trigger from UI
7+
8+
jobs:
9+
dbt_run:
10+
name: dbt full refresh and test on Fabric
11+
runs-on: ubuntu-latest
12+
13+
env:
14+
# Environment variables
15+
PYTHON_VERSION: "3.10"
16+
DBT_PROJECT_DIR: "."
17+
DBT_CORE_VERSION: "1.10.15"
18+
# Fabric connection variables
19+
DBT_FABRIC_CI_HOST: ${{ secrets.DBT_FABRIC_CI_HOST }}
20+
DBT_FABRIC_SERVICE_PRINCIPAL_ID: ${{ secrets.DBT_FABRIC_SERVICE_PRINCIPAL_ID }}
21+
DBT_FABRIC_SERVICE_PRINCIPAL_SECRET: ${{ secrets.DBT_FABRIC_SERVICE_PRINCIPAL_SECRET }}
22+
DBT_FABRIC_TENANT_ID: ${{ secrets.DBT_FABRIC_TENANT_ID }}
23+
DBT_FABRIC_CI_DATABASE: ${{ secrets.DBT_FABRIC_CI_DATABASE }}
24+
DBT_FABRIC_CI_SCHEMA: ${{ secrets.DBT_FABRIC_CI_SCHEMA }}
25+
26+
steps:
27+
- name: Checkout Code
28+
uses: actions/checkout@v4
29+
30+
- name: Set up Python
31+
uses: actions/setup-python@v4
32+
with:
33+
python-version: ${{ env.PYTHON_VERSION }}
34+
35+
- name: Install ODBC Driver 18 for SQL Server
36+
run: |
37+
curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
38+
curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list
39+
sudo apt-get update
40+
sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18
41+
42+
- name: Install dbt-core and Fabric adapter
43+
run: |
44+
python -m pip install --upgrade pip
45+
pip install dbt-core==${{ env.DBT_CORE_VERISON }} dbt-fabric
46+
47+
- name: Install dbt dependencies
48+
run: dbt deps --profiles-dir ./integration_tests/profiles/fabric
49+
working-directory: ${{ env.DBT_PROJECT_DIR }}
50+
51+
- name: Test connection
52+
run: dbt debug --profiles-dir ./integration_tests/profiles/fabric
53+
working-directory: ${{ env.DBT_PROJECT_DIR }}
54+
55+
- name: Build dbt models (full-refresh)
56+
run: |
57+
dbt build --full-refresh --profiles-dir ./integration_tests/profiles/fabric
58+
working-directory: ${{ env.DBT_PROJECT_DIR }}

‎integration_tests/profiles/databricks/profiles.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
default:
2+
target: ci
3+
outputs:
4+
ci:
5+
type: databricks
6+
host: "${{ env_var('DATABRICKS_HOST') }"
7+
http_path: "${{ env_var('DATABRICKS_HTTP_PATH') }"
8+
token: "${{ env_var('DATABRICKS_TOKEN') }"
9+
catalog: 'hive_metastore'
10+
schema: ci_testing
11+
threads: 8
12+
connect_timeout: 60
13+
connect_retries: 3

‎integration_tests/profiles/duckdb/profiles.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
default:
2+
target: ci
3+
outputs:
4+
ci:
5+
type: duckdb
6+
path: "${{ env_var('MOTHERDUCK_PATH') }}"
7+
threads: 4
8+
settings:
9+
custom_user_agent: 'dbt-ci'

‎integration_tests/profiles/fabric/profiles.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
default:
2+
target: dev
3+
outputs:
4+
dev:
5+
type: fabric
6+
host: "{{ env_var('DBT_FABRIC_CI_HOST') }}"
7+
database: "{{ env_var('DBT_FABRIC_CI_DATABASE') }}"
8+
schema: "{{ env_var('DBT_FABRIC_CI_SCHEMA') }}"
9+
authentication: ServicePrincipal
10+
driver: ODBC Driver 18 for SQL Server
11+
client_id: "{{ env_var('DBT_FABRIC_SERVICE_PRINCIPAL_ID') }}"
12+
client_secret: "{{ env_var('DBT_FABRIC_SERVICE_PRINCIPAL_SECRET') }}"
13+
tenant_id: "{{ env_var('DBT_FABRIC_TENANT_ID') }}"
14+
threads: 8

0 commit comments

Comments
 (0)