Tuva CI / PR #1322 / snowflake: dbt build --full-refresh #169
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tuva CI | |
| run-name: >- | |
| Tuva CI / PR #${{ | |
| github.event_name == 'workflow_dispatch' && | |
| github.event.inputs.pr_number || | |
| github.event.pull_request.number | |
| }} / ${{ | |
| github.event_name == 'workflow_dispatch' && | |
| ( | |
| github.event.inputs.command_label != '' && | |
| github.event.inputs.command_label || | |
| format('{0}-{1}', github.event.inputs.operation, github.event.inputs.target) | |
| ) || | |
| 'run-snowflake' | |
| }} | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| types: | |
| - opened | |
| workflow_dispatch: | |
| inputs: | |
| pr_number: | |
| description: Pull request number to run against | |
| required: true | |
| type: string | |
| dbt_command: | |
| description: Full dbt command or ordered command sequence, for example `dbt seed --select tag:tuva_demo dbt run --select tag:tuva_demo` | |
| required: false | |
| type: string | |
| default: "" | |
| targets_csv: | |
| description: Comma-separated warehouse list, or `all` | |
| required: false | |
| type: string | |
| default: "" | |
| command_label: | |
| description: Short label for the run name | |
| required: false | |
| type: string | |
| default: "" | |
| operation: | |
| description: CI operation to run | |
| required: false | |
| type: choice | |
| default: run | |
| options: | |
| - run | |
| - build | |
| target: | |
| description: Warehouse target | |
| required: false | |
| type: choice | |
| default: snowflake | |
| options: | |
| - all | |
| - snowflake | |
| - bigquery | |
| - databricks | |
| - fabric | |
| - redshift | |
| - duckdb | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| jobs: | |
| resolve_request: | |
| name: Resolve CI Request | |
| runs-on: ubuntu-latest | |
| outputs: | |
| pr_number: ${{ steps.resolve.outputs.pr_number }} | |
| adapter_matrix: ${{ steps.resolve.outputs.adapter_matrix }} | |
| checkout_ref: ${{ steps.resolve.outputs.checkout_ref }} | |
| dbt_commands_json: ${{ steps.validate.outputs.dbt_commands_json }} | |
| dbt_command_display: ${{ steps.validate.outputs.dbt_command }} | |
| subcommand: ${{ steps.validate.outputs.subcommand }} | |
| requires_seed_baseline: ${{ steps.validate.outputs.requires_seed_baseline }} | |
| refreshes_seeds: ${{ steps.validate.outputs.refreshes_seeds }} | |
| steps: | |
| - name: Checkout workflow utilities | |
| uses: actions/checkout@v4 | |
| - name: Validate requested dbt command | |
| id: validate | |
| run: python scripts/parse_ci_command.py resolve-dispatch | |
| env: | |
| CI_DBT_COMMAND: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.dbt_command || '' }} | |
| CI_OPERATION: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.operation || 'run' }} | |
| CI_TARGET: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.target || 'snowflake' }} | |
| CI_TARGETS_CSV: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.targets_csv || '' }} | |
| - name: Resolve target matrix and guardrails | |
| id: resolve | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const { owner, repo } = context.repo; | |
| const isDispatch = context.eventName === "workflow_dispatch"; | |
| const inputs = context.payload.inputs || {}; | |
| const requestedCommand = `${{ steps.validate.outputs.dbt_command }}`; | |
| const requestedTargets = JSON.parse(String.raw`${{ steps.validate.outputs.targets_json }}`); | |
| const requestedTargetsCsv = `${{ steps.validate.outputs.targets_csv }}`; | |
| const refreshesSeeds = `${{ steps.validate.outputs.refreshes_seeds }}` === "true"; | |
| const requiresSeedBaseline = `${{ steps.validate.outputs.requires_seed_baseline }}` === "true"; | |
| let prNumber; | |
| if (isDispatch) { | |
| const rawPrNumber = String(inputs.pr_number || "").trim(); | |
| if (!/^\d+$/.test(rawPrNumber)) { | |
| core.setFailed(`Invalid workflow_dispatch input pr_number: "${rawPrNumber}".`); | |
| return; | |
| } | |
| prNumber = Number(rawPrNumber); | |
| } else { | |
| prNumber = context.payload.pull_request.number; | |
| } | |
| const { data: pr } = await github.rest.pulls.get({ | |
| owner, | |
| repo, | |
| pull_number: prNumber | |
| }); | |
| if (pr.base.ref !== "main") { | |
| core.setFailed(`PR #${prNumber} does not target main (targets ${pr.base.ref}).`); | |
| return; | |
| } | |
| const repoFullName = `${owner}/${repo}`.toLowerCase(); | |
| const headRepoFullName = String(pr.head.repo.full_name || "").toLowerCase(); | |
| if (headRepoFullName !== repoFullName) { | |
| core.setFailed( | |
| `PR #${prNumber} comes from fork \`${pr.head.repo.full_name}\`. This secrets-backed CI workflow only runs on in-repo PRs. Use the outside contributor bridge workflow first.` | |
| ); | |
| return; | |
| } | |
| const files = await github.paginate(github.rest.pulls.listFiles, { | |
| owner, | |
| repo, | |
| pull_number: prNumber, | |
| per_page: 100 | |
| }); | |
| const changed = files.map((f) => f.filename); | |
| const runGuardPrefixes = ["seeds/", "integration_tests/seeds/"]; | |
| const runGuardExact = new Set([ | |
| "dbt_project.yml", | |
| "integration_tests/dbt_project.yml", | |
| "integration_tests/seeds/_seeds.yml", | |
| "integration_tests/models/_sources.yml", | |
| "macros/cross_database_utils/load_seed.sql" | |
| ]); | |
| const requiresBuild = changed.some( | |
| (path) => | |
| runGuardExact.has(path) || | |
| runGuardPrefixes.some((prefix) => path.startsWith(prefix)) | |
| ); | |
| const touchesSeedState = (paths) => | |
| paths.some( | |
| (path) => | |
| runGuardExact.has(path) || | |
| runGuardPrefixes.some((prefix) => path.startsWith(prefix)) | |
| ); | |
| function parseDisplayTitle(run) { | |
| const prefix = `Tuva CI / PR #${prNumber} / `; | |
| const displayTitle = String(run.display_title || ""); | |
| if (!displayTitle.startsWith(prefix)) { | |
| return null; | |
| } | |
| const label = displayTitle.slice(prefix.length); | |
| const separator = label.indexOf(": "); | |
| if (separator === -1) { | |
| return null; | |
| } | |
| return { | |
| targetLabel: label.slice(0, separator).trim().toLowerCase(), | |
| commandLabel: label.slice(separator + 2).trim().toLowerCase(), | |
| }; | |
| } | |
| function labelIncludesTarget(targetLabel, targetName) { | |
| if (targetLabel === "all") { | |
| return true; | |
| } | |
| return targetLabel | |
| .split(",") | |
| .map((part) => part.trim()) | |
| .filter(Boolean) | |
| .includes(targetName); | |
| } | |
| function labelRefreshesSeeds(commandLabel) { | |
| return commandLabel | |
| .split(" -> ") | |
| .map((part) => part.trim()) | |
| .some((part) => part.startsWith("dbt seed") || part.startsWith("dbt build")); | |
| } | |
| async function hasReusableSeedRefresh(targetName) { | |
| const runs = await github.paginate(github.rest.actions.listWorkflowRuns, { | |
| owner, | |
| repo, | |
| workflow_id: "dbt_ci_modes.yml", | |
| event: "workflow_dispatch", | |
| per_page: 100, | |
| }); | |
| for (const run of runs) { | |
| if (run.status !== "completed" || run.conclusion !== "success") { | |
| continue; | |
| } | |
| const parsedTitle = parseDisplayTitle(run); | |
| if (!parsedTitle) { | |
| continue; | |
| } | |
| if (!labelIncludesTarget(parsedTitle.targetLabel, targetName)) { | |
| continue; | |
| } | |
| if (!labelRefreshesSeeds(parsedTitle.commandLabel)) { | |
| continue; | |
| } | |
| if (run.head_sha === pr.head.sha) { | |
| return true; | |
| } | |
| try { | |
| const comparison = await github.rest.repos.compareCommits({ | |
| owner, | |
| repo, | |
| base: run.head_sha, | |
| head: pr.head.sha, | |
| }); | |
| const comparisonPaths = (comparison.data.files || []).map((file) => file.filename); | |
| if (!touchesSeedState(comparisonPaths)) { | |
| return true; | |
| } | |
| } catch (error) { | |
| core.info( | |
| `Unable to compare ${run.head_sha} to ${pr.head.sha} while checking seed baseline reuse: ${error.message}` | |
| ); | |
| } | |
| } | |
| return false; | |
| } | |
| if (isDispatch && requiresBuild && requiresSeedBaseline) { | |
| const reusableTargets = []; | |
| for (const targetName of requestedTargets) { | |
| if (await hasReusableSeedRefresh(targetName)) { | |
| reusableTargets.push(targetName); | |
| } | |
| } | |
| if (reusableTargets.length !== requestedTargets.length) { | |
| const sample = changed | |
| .filter((path) => touchesSeedState([path])) | |
| .slice(0, 8) | |
| .join(", "); | |
| const ciTargetPrefix = requestedTargetsCsv === "all" ? "all" : requestedTargets.join(" "); | |
| core.setFailed( | |
| `This PR changes seed/config files (${sample}). The requested sequence \`${requestedCommand}\` reaches dbt run/test before refreshing seeds. Run a seed-refreshing command such as \`/ci ${ciTargetPrefix} dbt seed\` or \`/ci ${ciTargetPrefix} dbt build --full-refresh\` before running this sequence.` | |
| ); | |
| return; | |
| } | |
| } | |
| core.setOutput("pr_number", String(prNumber)); | |
| core.setOutput("adapter_matrix", JSON.stringify(requestedTargets)); | |
| core.setOutput("checkout_ref", `refs/pull/${prNumber}/merge`); | |
| run_dbt: | |
| name: >- | |
| Commands / DW: ${{ | |
| matrix.adapter == 'snowflake' && 'Snowflake' || | |
| matrix.adapter == 'bigquery' && 'BigQuery' || | |
| matrix.adapter == 'databricks' && 'Databricks' || | |
| matrix.adapter == 'fabric' && 'Fabric' || | |
| matrix.adapter == 'redshift' && 'Redshift' || | |
| matrix.adapter == 'duckdb' && 'DuckDB/MotherDuck' || | |
| matrix.adapter | |
| }} | |
| needs: resolve_request | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| adapter: ${{ fromJson(needs.resolve_request.outputs.adapter_matrix) }} | |
| concurrency: | |
| group: ci-pr-${{ needs.resolve_request.outputs.pr_number }}-${{ matrix.adapter }} | |
| cancel-in-progress: true | |
| env: | |
| PYTHON_VERSION: "3.10" | |
| DBT_PROJECT_DIR: "./integration_tests" | |
| DBT_CORE_VERSION: "1.10.15" | |
| DUCKDB_VERSION: "1.4.1" | |
| DBT_BIGQUERY_CI_TOKEN: ${{ secrets.DBT_BIGQUERY_CI_TOKEN }} | |
| DBT_BIGQUERY_CI_PROJECT: ${{ secrets.DBT_BIGQUERY_CI_PROJECT }} | |
| DBT_DATABRICKS_CI_HOST: ${{ secrets.DBT_DATABRICKS_CI_HOST }} | |
| DBT_DATABRICKS_CI_HTTP_PATH: ${{ secrets.DBT_DATABRICKS_CI_HTTP_PATH }} | |
| DBT_DATABRICKS_CI_TOKEN: ${{ secrets.DBT_DATABRICKS_CI_TOKEN }} | |
| DBT_DATABRICKS_CI_CATALOG: ${{ secrets.DBT_DATABRICKS_CI_CATALOG }} | |
| DBT_MOTHERDUCK_CI_PATH: ${{ secrets.DBT_MOTHERDUCK_CI_PATH }} | |
| DBT_MOTHERDUCK_CI_DATABASE: "my_db" | |
| DBT_FABRIC_CI_SERVER: ${{ secrets.DBT_FABRIC_CI_SERVER }} | |
| DBT_FABRIC_CI_DATABASE: ${{ secrets.DBT_FABRIC_CI_DATABASE }} | |
| DBT_FABRIC_CI_SCHEMA: ${{ secrets.DBT_FABRIC_CI_SCHEMA }} | |
| DBT_FABRIC_CI_CLIENT_ID: ${{ secrets.DBT_FABRIC_CI_CLIENT_ID }} | |
| DBT_FABRIC_CI_CLIENT_SECRET: ${{ secrets.DBT_FABRIC_CI_CLIENT_SECRET }} | |
| DBT_FABRIC_CI_TENANT_ID: ${{ secrets.DBT_FABRIC_CI_TENANT_ID }} | |
| DBT_REDSHIFT_CI_HOST: ${{ secrets.DBT_REDSHIFT_CI_HOST }} | |
| DBT_REDSHIFT_CI_USER: ${{ secrets.DBT_REDSHIFT_CI_USER }} | |
| DBT_REDSHIFT_CI_PASSWORD: ${{ secrets.DBT_REDSHIFT_CI_PASSWORD }} | |
| DBT_REDSHIFT_CI_DATABASE: ${{ secrets.DBT_REDSHIFT_CI_DATABASE }} | |
| DBT_REDSHIFT_CI_THREADS: ${{ matrix.adapter == 'redshift' && '4' || '' }} | |
| DBT_REDSHIFT_CI_SEED_BATCH_SIZE: ${{ matrix.adapter == 'redshift' && '2000' || '' }} | |
| DBT_SNOWFLAKE_CI_ACCOUNT: ${{ secrets.DBT_SNOWFLAKE_CI_ACCOUNT }} | |
| DBT_SNOWFLAKE_CI_DATABASE: ${{ secrets.DBT_SNOWFLAKE_CI_DATABASE }} | |
| DBT_SNOWFLAKE_CI_PASSWORD: ${{ secrets.DBT_SNOWFLAKE_CI_PASSWORD }} | |
| DBT_SNOWFLAKE_CI_ROLE: ${{ secrets.DBT_SNOWFLAKE_CI_ROLE }} | |
| DBT_SNOWFLAKE_CI_SCHEMA: ${{ secrets.DBT_SNOWFLAKE_CI_SCHEMA }} | |
| DBT_SNOWFLAKE_CI_USER: ${{ secrets.DBT_SNOWFLAKE_CI_USER }} | |
| DBT_SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.DBT_SNOWFLAKE_CI_WAREHOUSE }} | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.resolve_request.outputs.checkout_ref }} | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install ODBC Driver 18 for SQL Server | |
| if: ${{ matrix.adapter == 'fabric' }} | |
| run: | | |
| curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - | |
| curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list | |
| sudo apt-get update | |
| sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 | |
| - name: Install dbt adapter | |
| run: | | |
| python -m pip install --upgrade pip | |
| case "${{ matrix.adapter }}" in | |
| snowflake) | |
| pip install dbt-core==${{ env.DBT_CORE_VERSION }} dbt-snowflake | |
| ;; | |
| bigquery) | |
| pip install dbt-core==${{ env.DBT_CORE_VERSION }} dbt-bigquery | |
| ;; | |
| databricks) | |
| pip install dbt-core==${{ env.DBT_CORE_VERSION }} dbt-databricks | |
| pip install certifi | |
| ;; | |
| fabric) | |
| pip install dbt-core==${{ env.DBT_CORE_VERSION }} dbt-fabric | |
| ;; | |
| redshift) | |
| pip install dbt-core==${{ env.DBT_CORE_VERSION }} dbt-redshift | |
| ;; | |
| duckdb) | |
| pip install dbt-core==${{ env.DBT_CORE_VERSION }} dbt-duckdb duckdb==${{ env.DUCKDB_VERSION }} | |
| ;; | |
| *) | |
| echo "Unsupported adapter: ${{ matrix.adapter }}" >&2 | |
| exit 1 | |
| ;; | |
| esac | |
| - name: Create BigQuery credentials file | |
| if: ${{ matrix.adapter == 'bigquery' }} | |
| run: | | |
| echo "${{ env.DBT_BIGQUERY_CI_TOKEN }}" | base64 --decode > ./creds.json | |
| - name: Install dbt dependencies | |
| run: dbt deps --project-dir ./integration_tests --profiles-dir ./integration_tests/profiles/${{ matrix.adapter }} | |
| - name: Test connection | |
| run: dbt debug --project-dir ./integration_tests --profiles-dir ./integration_tests/profiles/${{ matrix.adapter }} | |
| - name: Assert baseline seed schemas exist for run modes | |
| if: ${{ needs.resolve_request.outputs.requires_seed_baseline == 'true' }} | |
| run: | | |
| dbt run-operation assert_ci_seed_baseline_ready \ | |
| --project-dir ./integration_tests \ | |
| --profiles-dir ./integration_tests/profiles/${{ matrix.adapter }} | |
| - name: Execute dbt | |
| env: | |
| DBT_COMMANDS_JSON: ${{ needs.resolve_request.outputs.dbt_commands_json }} | |
| DBT_ADAPTER: ${{ matrix.adapter }} | |
| CI_REQUIRES_SEED_BASELINE: ${{ needs.resolve_request.outputs.requires_seed_baseline }} | |
| run: | | |
| python - <<'PY' | |
| import json | |
| import os | |
| import shlex | |
| import subprocess | |
| commands = json.loads(os.environ["DBT_COMMANDS_JSON"]) | |
| child_env = os.environ.copy() | |
| seed_refreshing_subcommands = {"seed", "build"} | |
| baseline_required = child_env.get("CI_REQUIRES_SEED_BASELINE") == "true" | |
| seed_refreshed_in_sequence = False | |
| for index, raw_command in enumerate(commands, start=1): | |
| command = list(raw_command) | |
| subcommand = command[1].lower() | |
| use_baseline_seeds = ( | |
| subcommand in seed_refreshing_subcommands | |
| or seed_refreshed_in_sequence | |
| or (baseline_required and subcommand in {"run", "test"}) | |
| ) | |
| command.extend( | |
| [ | |
| "--project-dir", | |
| "./integration_tests", | |
| "--profiles-dir", | |
| f"./integration_tests/profiles/{os.environ['DBT_ADAPTER']}", | |
| ] | |
| ) | |
| child_env["DBT_CI_USE_BASELINE_SEEDS"] = "true" if use_baseline_seeds else "false" | |
| if child_env["DBT_CI_USE_BASELINE_SEEDS"] == "true" and "--no-partial-parse" not in command: | |
| command.append("--no-partial-parse") | |
| print(f"Step {index}: DBT_CI_USE_BASELINE_SEEDS={child_env['DBT_CI_USE_BASELINE_SEEDS']}") | |
| print(f"Running: {shlex.join(command)}") | |
| subprocess.run(command, check=True, env=child_env) | |
| if subcommand in seed_refreshing_subcommands: | |
| seed_refreshed_in_sequence = True | |
| PY |