Skip to content

Commit f156bec

Browse files
author
sarah-tuva
committed
Add new combined workflow
1 parent 8e90bc4 commit f156bec

1 file changed

Lines changed: 202 additions & 0 deletions

File tree

‎.github/workflows/tuva_dbt_v1.8.6.yml‎

Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
name: tuva_dbt_v1.8.6
2+
on:
3+
workflow_dispatch:
4+
pull_request:
5+
branches:
6+
- main
7+
8+
env:
9+
PYTHON_VERSION: '3.9'
10+
11+
####### Secrets #######
12+
####### BigQuery
13+
DBT_BIGQUERY_TOKEN: ${{ secrets.DBT_BIGQUERY_DEMO_TOKEN }}
14+
DBT_BIGQUERY_DEMO_PROJECT: ${{ secrets.DBT_BIGQUERY_DEMO_PROJECT }}
15+
####### Fabric
16+
DBT_FABRIC_CI_HOST: ${{ secrets.DBT_FABRIC_CI_HOST }}
17+
DBT_FABRIC_SERVICE_PRINCIPAL_ID: ${{ secrets.DBT_FABRIC_SERVICE_PRINCIPAL_ID }}
18+
DBT_FABRIC_SERVICE_PRINCIPAL_SECRET: ${{ secrets.DBT_FABRIC_SERVICE_PRINCIPAL_SECRET }}
19+
DBT_FABRIC_TENANT_ID: ${{ secrets.DBT_FABRIC_TENANT_ID }}
20+
DBT_FABRIC_DEMO_DATABASE: ${{ secrets.DBT_FABRIC_DEMO_DATABASE }}
21+
DBT_FABRIC_CI_SCHEMA: ${{ secrets.DBT_FABRIC_CI_SCHEMA }}
22+
####### Redshift
23+
DBT_REDSHIFT_HOST: ${{ secrets.DBT_REDSHIFT_HOST }}
24+
DBT_REDSHIFT_CI_USER: ${{ secrets.DBT_REDSHIFT_CI_USER }}
25+
DBT_REDSHIFT_CI_PASSWORD: ${{ secrets.DBT_REDSHIFT_CI_PASSWORD }}
26+
DBT_REDSHIFT_CI_PORT: ${{ secrets.DBT_REDSHIFT_CI_PORT }}
27+
DBT_REDSHIFT_DEMO_DATABASE: ${{ secrets.DBT_REDSHIFT_DEMO_DATABASE }}
28+
####### Snowflake
29+
DBT_SNOWFLAKE_DEV_ACCOUNT: ${{ secrets.DBT_SNOWFLAKE_DEV_ACCOUNT }}
30+
DBT_SNOWFLAKE_DEV_DATABASE: ${{ secrets.DBT_SNOWFLAKE_DEV_DATABASE }}
31+
DBT_SNOWFLAKE_DEV_CI_PASSWORD: ${{ secrets.DBT_SNOWFLAKE_DEV_CI_PASSWORD }}
32+
DBT_SNOWFLAKE_DEV_CI_ROLE: ${{ secrets.DBT_SNOWFLAKE_DEV_CI_ROLE }}
33+
DBT_SNOWFLAKE_DEV_CI_SCHEMA: ${{ secrets.DBT_SNOWFLAKE_DEV_CI_SCHEMA }}
34+
DBT_SNOWFLAKE_DEV_CI_USER: ${{ secrets.DBT_SNOWFLAKE_DEV_CI_USER }}
35+
DBT_SNOWFLAKE_DEV_CI_WAREHOUSE: ${{ secrets.DBT_SNOWFLAKE_DEV_CI_WAREHOUSE }}
36+
37+
# Permissions needed for the comment action
38+
permissions:
39+
contents: read
40+
pull-requests: write
41+
42+
jobs:
43+
####### BigQuery
44+
bigquery:
45+
runs-on: ubuntu-latest
46+
47+
steps:
48+
- uses: actions/checkout@v2
49+
50+
- name: Set up Python
51+
uses: actions/setup-python@v4
52+
with:
53+
python-version: ${{ env.PYTHON_VERSION }}
54+
55+
- name: Install dbt-core and BigQuery adapter
56+
run: |
57+
python -m pip install --upgrade pip
58+
pip install dbt-core==1.8.6 dbt-bigquery
59+
60+
- name: Create dbt profiles.yml for BigQuery
61+
run: |
62+
mkdir -p ./profiles/bigquery
63+
echo "default:
64+
outputs:
65+
dev:
66+
type: bigquery
67+
method: service-account
68+
project: "{{ env_var('DBT_BIGQUERY_DEMO_PROJECT') }}"
69+
keyfile: ./creds.json
70+
dataset: connector
71+
threads: 8
72+
timeout_seconds: 300
73+
priority: interactive
74+
target: dev" > ./integration_tests/profiles/bigquery/profiles.yml
75+
76+
- name: Create BigQuery credentials file
77+
run: |
78+
echo "${{ secrets.DBT_BIGQUERY_TOKEN }}" | base64 --decode > ./creds.json
79+
80+
- name: dbt-deps
81+
run: dbt deps --profiles-dir ./integration_tests/profiles/bigquery
82+
83+
- name: dbt-debug
84+
run: dbt debug --profiles-dir ./integration_tests/profiles/bigquery
85+
86+
- name: dbt-build
87+
run: |
88+
dbt build --full-refresh --profiles-dir ./integration_tests/profiles/bigquery
89+
90+
- name: Get the result
91+
if: ${{ always() }}
92+
run: echo "${{ steps.dbt-build.outputs.result }}"
93+
shell: bash
94+
95+
96+
####### Fabric
97+
fabric:
98+
runs-on: ubuntu-latest
99+
100+
steps:
101+
- uses: actions/checkout@v2
102+
103+
- name: Set up Python
104+
uses: actions/setup-python@v4
105+
with:
106+
python-version: ${{ env.PYTHON_VERSION }}
107+
108+
- name: Install ODBC Driver 18 for SQL Server
109+
run: |
110+
curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
111+
curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list
112+
sudo apt-get update
113+
sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18
114+
115+
- name: Install dbt-core and Fabric adapter
116+
run: |
117+
python -m pip install --upgrade pip
118+
pip install dbt-core==1.8.6 dbt-fabric
119+
120+
- name: dbt-deps
121+
run: dbt deps --profiles-dir ./integration_tests/profiles/fabric
122+
123+
- name: dbt-debug
124+
run: dbt debug --profiles-dir ./integration_tests/profiles/fabric
125+
126+
- name: dbt-build
127+
run: |
128+
dbt build --full-refresh --profiles-dir ./integration_tests/profiles/fabric
129+
130+
- name: Get the result
131+
if: ${{ always() }}
132+
run: echo "${{ steps.dbt-build.outputs.result }}"
133+
shell: bash
134+
135+
####### Redshift
136+
redshift:
137+
runs-on: ubuntu-latest
138+
139+
steps:
140+
- uses: actions/checkout@v2
141+
142+
- name: Set up Python
143+
uses: actions/setup-python@v4
144+
with:
145+
python-version: ${{ env.PYTHON_VERSION }}
146+
147+
- name: Install dbt-core and Redshift adapter
148+
run: |
149+
python -m pip install --upgrade pip
150+
pip install dbt-core==1.8.6 dbt-redshift
151+
152+
- name: dbt-deps
153+
run: dbt deps --profiles-dir ./integration_tests/profiles/redshift
154+
155+
- name: dbt-debug
156+
run: dbt debug --profiles-dir ./integration_tests/profiles/redshift
157+
158+
# Breaking up dbt build into seed then compile to try and speed up processing
159+
- name: dbt-seed
160+
run: |
161+
dbt seed --full-refresh --profiles-dir ./integration_tests/profiles/redshift
162+
163+
- name: dbt-compile
164+
run: |
165+
dbt compile --profiles-dir ./integration_tests/profiles/redshift
166+
167+
- name: Get the result
168+
if: ${{ always() }}
169+
run: echo "${{ steps.dbt-build.outputs.result }}"
170+
shell: bash
171+
172+
####### Snowflake
173+
snowflake:
174+
runs-on: ubuntu-latest
175+
176+
steps:
177+
- uses: actions/checkout@v2
178+
179+
- name: Set up Python
180+
uses: actions/setup-python@v4
181+
with:
182+
python-version: ${{ env.PYTHON_VERSION }}
183+
184+
- name: Install dbt-core and Snowflake adapter
185+
run: |
186+
python -m pip install --upgrade pip
187+
pip install dbt-core==1.8.6 dbt-snowflake
188+
189+
- name: dbt-deps
190+
run: dbt deps --profiles-dir ./integration_tests/profiles/snowflake
191+
192+
- name: dbt-debug
193+
run: dbt debug --profiles-dir ./integration_tests/profiles/snowflake
194+
195+
- name: dbt-build
196+
run: |
197+
dbt build --full-refresh --profiles-dir ./integration_tests/profiles/snowflake
198+
199+
- name: Get the result
200+
if: ${{ always() }}
201+
run: echo "${{ steps.dbt-build.outputs.result }}"
202+
shell: bash

0 commit comments

Comments
 (0)