dbt_ci_modes.yml

name: Tuva CI
run-name: >-
  Tuva CI / PR #${{
    github.event_name == 'workflow_dispatch' &&
    github.event.inputs.pr_number ||
    github.event.pull_request.number
  }} / ${{
    github.event_name == 'workflow_dispatch' &&
    (
      github.event.inputs.command_label != '' &&
      github.event.inputs.command_label ||
      format('{0}-{1}', github.event.inputs.operation, github.event.inputs.target)
    ) ||
    'run-snowflake'
  }}

on:
  pull_request:
    branches:
      - main
    types:
      - opened
  workflow_dispatch:
    inputs:
      pr_number:
        description: Pull request number to run against
        required: true
        type: string
      dbt_command:
        description: Full dbt command or ordered command sequence, for example `dbt seed --select tag:tuva_demo dbt run --select tag:tuva_demo`
        required: false
        type: string
        default: ""
      targets_csv:
        description: Comma-separated warehouse list, or `all`
        required: false
        type: string
        default: ""
      command_label:
        description: Short label for the run name
        required: false
        type: string
        default: ""
      operation:
        description: CI operation to run
        required: false
        type: choice
        default: run
        options:
          - run
          - build
      target:
        description: Warehouse target
        required: false
        type: choice
        default: snowflake
        options:
          - all
          - snowflake
          - bigquery
          - databricks
          - fabric
          - redshift
          - duckdb

permissions:
  contents: read
  pull-requests: read

jobs:
  resolve_request:
    name: Resolve CI Request
    runs-on: ubuntu-latest
    outputs:
      pr_number: ${{ steps.resolve.outputs.pr_number }}
      adapter_matrix: ${{ steps.resolve.outputs.adapter_matrix }}
      checkout_ref: ${{ steps.resolve.outputs.checkout_ref }}
      dbt_commands_json: ${{ steps.validate.outputs.dbt_commands_json }}
      dbt_command_display: ${{ steps.validate.outputs.dbt_command }}
      subcommand: ${{ steps.validate.outputs.subcommand }}
      requires_seed_baseline: ${{ steps.validate.outputs.requires_seed_baseline }}
      refreshes_seeds: ${{ steps.validate.outputs.refreshes_seeds }}
    steps:
      - name: Checkout workflow utilities
        uses: actions/checkout@v4

      - name: Validate requested dbt command
        id: validate
        run: python scripts/parse_ci_command.py resolve-dispatch
        env:
          CI_DBT_COMMAND: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.dbt_command || '' }}
          CI_OPERATION: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.operation || 'run' }}
          CI_TARGET: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.target || 'snowflake' }}
          CI_TARGETS_CSV: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.targets_csv || '' }}

      - name: Resolve target matrix and guardrails
        id: resolve
        uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const { owner, repo } = context.repo;
            const isDispatch = context.eventName === "workflow_dispatch";
            const inputs = context.payload.inputs || {};
            const requestedCommand = `${{ steps.validate.outputs.dbt_command }}`;
            const requestedTargets = JSON.parse(String.raw`${{ steps.validate.outputs.targets_json }}`);
            const requestedTargetsCsv = `${{ steps.validate.outputs.targets_csv }}`;
            const refreshesSeeds = `${{ steps.validate.outputs.refreshes_seeds }}` === "true";
            const requiresSeedBaseline = `${{ steps.validate.outputs.requires_seed_baseline }}` === "true";

            let prNumber;

            if (isDispatch) {
              const rawPrNumber = String(inputs.pr_number || "").trim();
              if (!/^\d+$/.test(rawPrNumber)) {
                core.setFailed(`Invalid workflow_dispatch input pr_number: "${rawPrNumber}".`);
                return;
              }
              prNumber = Number(rawPrNumber);
            } else {
              prNumber = context.payload.pull_request.number;
            }

            const { data: pr } = await github.rest.pulls.get({
              owner,
              repo,
              pull_number: prNumber
            });

            if (pr.base.ref !== "main") {
              core.setFailed(`PR #${prNumber} does not target main (targets ${pr.base.ref}).`);
              return;
            }

            const repoFullName = `${owner}/${repo}`.toLowerCase();
            const headRepoFullName = String(pr.head.repo.full_name || "").toLowerCase();
            if (headRepoFullName !== repoFullName) {
              core.setFailed(
                `PR #${prNumber} comes from fork \`${pr.head.repo.full_name}\`. This secrets-backed CI workflow only runs on in-repo PRs. Use the outside contributor bridge workflow first.`
              );
              return;
            }

            const files = await github.paginate(github.rest.pulls.listFiles, {
              owner,
              repo,
              pull_number: prNumber,
              per_page: 100
            });
            const changed = files.map((f) => f.filename);

            const runGuardPrefixes = ["seeds/", "integration_tests/seeds/"];
            const runGuardExact = new Set([
              "dbt_project.yml",
              "integration_tests/dbt_project.yml",
              "integration_tests/seeds/_seeds.yml",
              "integration_tests/models/_sources.yml",
              "macros/cross_database_utils/load_seed.sql"
            ]);

            const requiresBuild = changed.some(
              (path) =>
                runGuardExact.has(path) ||
                runGuardPrefixes.some((prefix) => path.startsWith(prefix))
            );

            const touchesSeedState = (paths) =>
              paths.some(
                (path) =>
                  runGuardExact.has(path) ||
                  runGuardPrefixes.some((prefix) => path.startsWith(prefix))
              );

            function parseDisplayTitle(run) {
              const prefix = `Tuva CI / PR #${prNumber} / `;
              const displayTitle = String(run.display_title || "");
              if (!displayTitle.startsWith(prefix)) {
                return null;
              }

              const label = displayTitle.slice(prefix.length);
              const separator = label.indexOf(": ");
              if (separator === -1) {
                return null;
              }

              return {
                targetLabel: label.slice(0, separator).trim().toLowerCase(),
                commandLabel: label.slice(separator + 2).trim().toLowerCase(),
              };
            }

            function labelIncludesTarget(targetLabel, targetName) {
              if (targetLabel === "all") {
                return true;
              }
              return targetLabel
                .split(",")
                .map((part) => part.trim())
                .filter(Boolean)
                .includes(targetName);
            }

            function labelRefreshesSeeds(commandLabel) {
              return commandLabel
                .split(" -> ")
                .map((part) => part.trim())
                .some((part) => part.startsWith("dbt seed") || part.startsWith("dbt build"));
            }

            async function hasReusableSeedRefresh(targetName) {
              const runs = await github.paginate(github.rest.actions.listWorkflowRuns, {
                owner,
                repo,
                workflow_id: "dbt_ci_modes.yml",
                event: "workflow_dispatch",
                per_page: 100,
              });

              for (const run of runs) {
                if (run.status !== "completed" || run.conclusion !== "success") {
                  continue;
                }

                const parsedTitle = parseDisplayTitle(run);
                if (!parsedTitle) {
                  continue;
                }

                if (!labelIncludesTarget(parsedTitle.targetLabel, targetName)) {
                  continue;
                }

                if (!labelRefreshesSeeds(parsedTitle.commandLabel)) {
                  continue;
                }

                if (run.head_sha === pr.head.sha) {
                  return true;
                }

                try {
                  const comparison = await github.rest.repos.compareCommits({
                    owner,
                    repo,
                    base: run.head_sha,
                    head: pr.head.sha,
                  });

                  const comparisonPaths = (comparison.data.files || []).map((file) => file.filename);
                  if (!touchesSeedState(comparisonPaths)) {
                    return true;
                  }
                } catch (error) {
                  core.info(
                    `Unable to compare ${run.head_sha} to ${pr.head.sha} while checking seed baseline reuse: ${error.message}`
                  );
                }
              }

              return false;
            }

            if (isDispatch && requiresBuild && requiresSeedBaseline) {
              const reusableTargets = [];
              for (const targetName of requestedTargets) {
                if (await hasReusableSeedRefresh(targetName)) {
                  reusableTargets.push(targetName);
                }
              }

              if (reusableTargets.length !== requestedTargets.length) {
                const sample = changed
                  .filter((path) => touchesSeedState([path]))
                  .slice(0, 8)
                  .join(", ");
                const ciTargetPrefix = requestedTargetsCsv === "all" ? "all" : requestedTargets.join(" ");

                core.setFailed(
                  `This PR changes seed/config files (${sample}). The requested sequence \`${requestedCommand}\` reaches dbt run/test before refreshing seeds. Run a seed-refreshing command such as \`/ci ${ciTargetPrefix} dbt seed\` or \`/ci ${ciTargetPrefix} dbt build --full-refresh\` before running this sequence.`
                );
                return;
              }
            }

            core.setOutput("pr_number", String(prNumber));
            core.setOutput("adapter_matrix", JSON.stringify(requestedTargets));
            core.setOutput("checkout_ref", `refs/pull/${prNumber}/merge`);

  run_dbt:
    name: >-
      Commands / DW: ${{
        matrix.adapter == 'snowflake' && 'Snowflake' ||
        matrix.adapter == 'bigquery' && 'BigQuery' ||
        matrix.adapter == 'databricks' && 'Databricks' ||
        matrix.adapter == 'fabric' && 'Fabric' ||
        matrix.adapter == 'redshift' && 'Redshift' ||
        matrix.adapter == 'duckdb' && 'DuckDB/MotherDuck' ||
        matrix.adapter
      }}
    needs: resolve_request
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        adapter: ${{ fromJson(needs.resolve_request.outputs.adapter_matrix) }}
    concurrency:
      group: ci-pr-${{ needs.resolve_request.outputs.pr_number }}-${{ matrix.adapter }}
      cancel-in-progress: true
    env:
      PYTHON_VERSION: "3.10"
      DBT_PROJECT_DIR: "./integration_tests"
      DBT_CORE_VERSION: "1.10.15"
      DUCKDB_VERSION: "1.4.1"

      DBT_BIGQUERY_CI_TOKEN: ${{ secrets.DBT_BIGQUERY_CI_TOKEN }}
      DBT_BIGQUERY_CI_PROJECT: ${{ secrets.DBT_BIGQUERY_CI_PROJECT }}

      DBT_DATABRICKS_CI_HOST: ${{ secrets.DBT_DATABRICKS_CI_HOST }}
      DBT_DATABRICKS_CI_HTTP_PATH: ${{ secrets.DBT_DATABRICKS_CI_HTTP_PATH }}
      DBT_DATABRICKS_CI_TOKEN: ${{ secrets.DBT_DATABRICKS_CI_TOKEN }}
      DBT_DATABRICKS_CI_CATALOG: ${{ secrets.DBT_DATABRICKS_CI_CATALOG }}

      DBT_MOTHERDUCK_CI_PATH: ${{ secrets.DBT_MOTHERDUCK_CI_PATH }}
      DBT_MOTHERDUCK_CI_DATABASE: "my_db"

      DBT_FABRIC_CI_SERVER: ${{ secrets.DBT_FABRIC_CI_SERVER }}
      DBT_FABRIC_CI_DATABASE: ${{ secrets.DBT_FABRIC_CI_DATABASE }}
      DBT_FABRIC_CI_SCHEMA: ${{ secrets.DBT_FABRIC_CI_SCHEMA }}
      DBT_FABRIC_CI_CLIENT_ID: ${{ secrets.DBT_FABRIC_CI_CLIENT_ID }}
      DBT_FABRIC_CI_CLIENT_SECRET: ${{ secrets.DBT_FABRIC_CI_CLIENT_SECRET }}
      DBT_FABRIC_CI_TENANT_ID: ${{ secrets.DBT_FABRIC_CI_TENANT_ID }}

      DBT_REDSHIFT_CI_HOST: ${{ secrets.DBT_REDSHIFT_CI_HOST }}
      DBT_REDSHIFT_CI_USER: ${{ secrets.DBT_REDSHIFT_CI_USER }}
      DBT_REDSHIFT_CI_PASSWORD: ${{ secrets.DBT_REDSHIFT_CI_PASSWORD }}
      DBT_REDSHIFT_CI_DATABASE: ${{ secrets.DBT_REDSHIFT_CI_DATABASE }}
      DBT_REDSHIFT_CI_THREADS: ${{ matrix.adapter == 'redshift' && '4' || '' }}
      DBT_REDSHIFT_CI_SEED_BATCH_SIZE: ${{ matrix.adapter == 'redshift' && '2000' || '' }}

      DBT_SNOWFLAKE_CI_ACCOUNT: ${{ secrets.DBT_SNOWFLAKE_CI_ACCOUNT }}
      DBT_SNOWFLAKE_CI_DATABASE: ${{ secrets.DBT_SNOWFLAKE_CI_DATABASE }}
      DBT_SNOWFLAKE_CI_PASSWORD: ${{ secrets.DBT_SNOWFLAKE_CI_PASSWORD }}
      DBT_SNOWFLAKE_CI_ROLE: ${{ secrets.DBT_SNOWFLAKE_CI_ROLE }}
      DBT_SNOWFLAKE_CI_SCHEMA: ${{ secrets.DBT_SNOWFLAKE_CI_SCHEMA }}
      DBT_SNOWFLAKE_CI_USER: ${{ secrets.DBT_SNOWFLAKE_CI_USER }}
      DBT_SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.DBT_SNOWFLAKE_CI_WAREHOUSE }}

    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.resolve_request.outputs.checkout_ref }}

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install ODBC Driver 18 for SQL Server
        if: ${{ matrix.adapter == 'fabric' }}
        run: |
          curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
          curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list
          sudo apt-get update
          sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18

      - name: Install dbt adapter
        run: |
          python -m pip install --upgrade pip
          case "${{ matrix.adapter }}" in
            snowflake)
              pip install dbt-core==${{ env.DBT_CORE_VERSION }} dbt-snowflake
              ;;
            bigquery)
              pip install dbt-core==${{ env.DBT_CORE_VERSION }} dbt-bigquery
              ;;
            databricks)
              pip install dbt-core==${{ env.DBT_CORE_VERSION }} dbt-databricks
              pip install certifi
              ;;
            fabric)
              pip install dbt-core==${{ env.DBT_CORE_VERSION }} dbt-fabric
              ;;
            redshift)
              pip install dbt-core==${{ env.DBT_CORE_VERSION }} dbt-redshift
              ;;
            duckdb)
              pip install dbt-core==${{ env.DBT_CORE_VERSION }} dbt-duckdb duckdb==${{ env.DUCKDB_VERSION }}
              ;;
            *)
              echo "Unsupported adapter: ${{ matrix.adapter }}" >&2
              exit 1
              ;;
          esac

      - name: Create BigQuery credentials file
        if: ${{ matrix.adapter == 'bigquery' }}
        run: |
          echo "${{ env.DBT_BIGQUERY_CI_TOKEN }}" | base64 --decode > ./creds.json

      - name: Install dbt dependencies
        run: dbt deps --project-dir ./integration_tests --profiles-dir ./integration_tests/profiles/${{ matrix.adapter }}

      - name: Test connection
        run: dbt debug --project-dir ./integration_tests --profiles-dir ./integration_tests/profiles/${{ matrix.adapter }}

      - name: Assert baseline seed schemas exist for run modes
        if: ${{ needs.resolve_request.outputs.requires_seed_baseline == 'true' }}
        run: |
          dbt run-operation assert_ci_seed_baseline_ready \
            --project-dir ./integration_tests \
            --profiles-dir ./integration_tests/profiles/${{ matrix.adapter }}

      - name: Execute dbt
        env:
          DBT_COMMANDS_JSON: ${{ needs.resolve_request.outputs.dbt_commands_json }}
          DBT_ADAPTER: ${{ matrix.adapter }}
          CI_REQUIRES_SEED_BASELINE: ${{ needs.resolve_request.outputs.requires_seed_baseline }}
        run: |
          python - <<'PY'
          import json
          import os
          import shlex
          import subprocess

          commands = json.loads(os.environ["DBT_COMMANDS_JSON"])
          child_env = os.environ.copy()
          seed_refreshing_subcommands = {"seed", "build"}
          baseline_required = child_env.get("CI_REQUIRES_SEED_BASELINE") == "true"
          seed_refreshed_in_sequence = False

          for index, raw_command in enumerate(commands, start=1):
              command = list(raw_command)
              subcommand = command[1].lower()
              use_baseline_seeds = (
                  subcommand in seed_refreshing_subcommands
                  or seed_refreshed_in_sequence
                  or (baseline_required and subcommand in {"run", "test"})
              )

              command.extend(
                  [
                      "--project-dir",
                      "./integration_tests",
                      "--profiles-dir",
                      f"./integration_tests/profiles/{os.environ['DBT_ADAPTER']}",
                  ]
              )

              child_env["DBT_CI_USE_BASELINE_SEEDS"] = "true" if use_baseline_seeds else "false"
              if child_env["DBT_CI_USE_BASELINE_SEEDS"] == "true" and "--no-partial-parse" not in command:
                  command.append("--no-partial-parse")

              print(f"Step {index}: DBT_CI_USE_BASELINE_SEEDS={child_env['DBT_CI_USE_BASELINE_SEEDS']}")
              print(f"Running: {shlex.join(command)}")
              subprocess.run(command, check=True, env=child_env)

              if subcommand in seed_refreshing_subcommands:
                  seed_refreshed_in_sequence = True
          PY