dbt_ci_modes.yml

name: Tuva CI
run-name: >-
  Tuva CI / PR #${{
    github.event_name == 'workflow_dispatch' &&
    github.event.inputs.pr_number ||
    github.event.pull_request.number
  }} / Mode: ${{
    github.event_name == 'workflow_dispatch' &&
    (github.event.inputs.mode == 'large' && 'Large' ||
    github.event.inputs.mode == 'medium' && 'Medium' ||
    'Small') ||
    contains(join(github.event.pull_request.labels.*.name, ','), 'ci:large') && 'Large' ||
    contains(join(github.event.pull_request.labels.*.name, ','), 'ci:medium') && 'Medium' ||
    'Small'
  }} / DW: ${{
    ((github.event_name == 'workflow_dispatch' &&
    (github.event.inputs.mode == 'large' || github.event.inputs.mode == 'medium')) ||
    contains(join(github.event.pull_request.labels.*.name, ','), 'ci:large') ||
    contains(join(github.event.pull_request.labels.*.name, ','), 'ci:medium')) &&
    'All 6 Warehouses' ||
    'Snowflake'
  }}

on:
  pull_request:
    branches:
      - main
    types:
      - opened
      - synchronize
      - reopened
  workflow_dispatch:
    inputs:
      pr_number:
        description: Pull request number to run against
        required: true
        type: string
      mode:
        description: CI mode to run
        required: true
        type: choice
        options:
          - small
          - medium
          - large

permissions:
  contents: read
  pull-requests: read

jobs:
  resolve_mode:
    name: Resolve CI Mode
    runs-on: ubuntu-latest
    outputs:
      pr_number: ${{ steps.resolve.outputs.pr_number }}
      mode: ${{ steps.resolve.outputs.mode }}
      adapter_matrix: ${{ steps.resolve.outputs.adapter_matrix }}
      checkout_ref: ${{ steps.resolve.outputs.checkout_ref }}
    steps:
      - name: Resolve mode, labels, and large-mode guard
        id: resolve
        uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const { owner, repo } = context.repo;
            const eventName = context.eventName;
            const isDispatch = eventName === "workflow_dispatch";
            const inputs = context.payload.inputs || {};

            let prNumber;
            let requestedMode = null;
            let labels = [];

            if (isDispatch) {
              const rawPrNumber = String(inputs.pr_number || "").trim();
              if (!/^\d+$/.test(rawPrNumber)) {
                core.setFailed(`Invalid workflow_dispatch input pr_number: "${rawPrNumber}".`);
                return;
              }

              prNumber = Number(rawPrNumber);
              requestedMode = String(inputs.mode || "").trim().toLowerCase();
              if (!["small", "medium", "large"].includes(requestedMode)) {
                core.setFailed(`Invalid workflow_dispatch input mode: "${requestedMode}". Use one of small, medium, large.`);
                return;
              }

              const { data: pr } = await github.rest.pulls.get({
                owner,
                repo,
                pull_number: prNumber
              });

              if (pr.base.ref !== "main") {
                core.setFailed(`PR #${prNumber} does not target main (targets ${pr.base.ref}).`);
                return;
              }

              labels = (pr.labels || []).map((l) => l.name);
            } else {
              prNumber = context.payload.pull_request.number;
              labels = (context.payload.pull_request.labels || []).map((l) => l.name);
            }

            const ciLabels = labels.filter((name) => /^ci:(small|medium|large)$/.test(name));

            if (ciLabels.length > 1) {
              core.setFailed(`Multiple CI mode labels found: ${ciLabels.join(", ")}. Keep only one of ci:small, ci:medium, ci:large.`);
              return;
            }

            const mode = requestedMode || (ciLabels.length === 1 ? ciLabels[0].replace("ci:", "") : "small");
            const adapters =
              mode === "small"
                ? ["snowflake"]
                : ["snowflake", "bigquery", "databricks", "fabric", "redshift", "duckdb"];

            const seedPrefixes = ["seeds/", "integration_tests/seeds/"];
            const seedExact = new Set([
              "dbt_project.yml",
              "integration_tests/dbt_project.yml",
              "integration_tests/seeds/_seeds.yml",
              "integration_tests/models/_sources.yml",
              "macros/cross_database_utils/load_seed.sql"
            ]);

            const files = await github.paginate(github.rest.pulls.listFiles, {
              owner,
              repo,
              pull_number: prNumber,
              per_page: 100
            });
            const changed = files.map((f) => f.filename);
            const requiresLarge = changed.some((path) =>
              seedExact.has(path) || seedPrefixes.some((prefix) => path.startsWith(prefix))
            );

            if (requiresLarge && mode !== "large") {
              const sample = changed
                .filter((path) => seedExact.has(path) || seedPrefixes.some((prefix) => path.startsWith(prefix)))
                .slice(0, 8)
                .join(", ");
              core.setFailed(
                `This PR modifies seed/config/load-seed files (${sample}). Use /ci large so CI refreshes baseline schemas.`
              );
              return;
            }

            core.info(`PR number: ${prNumber}`);
            core.info(`Resolved CI mode: ${mode}`);
            core.info(`Adapters: ${adapters.join(", ")}`);

            core.setOutput("pr_number", String(prNumber));
            core.setOutput("mode", mode);
            core.setOutput("adapter_matrix", JSON.stringify(adapters));
            core.setOutput("checkout_ref", `refs/pull/${prNumber}/merge`);

  run_dbt:
    name: >-
      Mode: ${{
        needs.resolve_mode.outputs.mode == 'small' && 'Small' ||
        needs.resolve_mode.outputs.mode == 'medium' && 'Medium' ||
        'Large'
      }} / DW: ${{
        matrix.adapter == 'snowflake' && 'Snowflake' ||
        matrix.adapter == 'bigquery' && 'BigQuery' ||
        matrix.adapter == 'databricks' && 'Databricks' ||
        matrix.adapter == 'fabric' && 'Fabric' ||
        matrix.adapter == 'redshift' && 'Redshift' ||
        matrix.adapter == 'duckdb' && 'DuckDB/MotherDuck' ||
        matrix.adapter
      }}
    needs: resolve_mode
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        adapter: ${{ fromJson(needs.resolve_mode.outputs.adapter_matrix) }}
    concurrency:
      group: ci-pr-${{ needs.resolve_mode.outputs.pr_number }}-${{ matrix.adapter }}
      cancel-in-progress: true
    env:
      PYTHON_VERSION: "3.10"
      DBT_PROJECT_DIR: "./integration_tests"
      DBT_CORE_VERSION: "1.10.15"

      DBT_BIGQUERY_CI_TOKEN: ${{ secrets.DBT_BIGQUERY_CI_TOKEN }}
      DBT_BIGQUERY_CI_PROJECT: ${{ secrets.DBT_BIGQUERY_CI_PROJECT }}

      DBT_DATABRICKS_CI_HOST: ${{ secrets.DBT_DATABRICKS_CI_HOST }}
      DBT_DATABRICKS_CI_HTTP_PATH: ${{ secrets.DBT_DATABRICKS_CI_HTTP_PATH }}
      DBT_DATABRICKS_CI_TOKEN: ${{ secrets.DBT_DATABRICKS_CI_TOKEN }}
      DBT_DATABRICKS_CI_CATALOG: ${{ secrets.DBT_DATABRICKS_CI_CATALOG }}

      DBT_MOTHERDUCK_CI_PATH: ${{ secrets.DBT_MOTHERDUCK_CI_PATH }}
      DBT_MOTHERDUCK_CI_DATABASE: "my_db"

      DBT_FABRIC_CI_SERVER: ${{ secrets.DBT_FABRIC_CI_SERVER }}
      DBT_FABRIC_CI_DATABASE: ${{ secrets.DBT_FABRIC_CI_DATABASE }}
      DBT_FABRIC_CI_SCHEMA: ${{ secrets.DBT_FABRIC_CI_SCHEMA }}
      DBT_FABRIC_CI_CLIENT_ID: ${{ secrets.DBT_FABRIC_CI_CLIENT_ID }}
      DBT_FABRIC_CI_CLIENT_SECRET: ${{ secrets.DBT_FABRIC_CI_CLIENT_SECRET }}
      DBT_FABRIC_CI_TENANT_ID: ${{ secrets.DBT_FABRIC_CI_TENANT_ID }}

      DBT_REDSHIFT_CI_HOST: ${{ secrets.DBT_REDSHIFT_CI_HOST }}
      DBT_REDSHIFT_CI_USER: ${{ secrets.DBT_REDSHIFT_CI_USER }}
      DBT_REDSHIFT_CI_PASSWORD: ${{ secrets.DBT_REDSHIFT_CI_PASSWORD }}
      DBT_REDSHIFT_CI_DATABASE: ${{ secrets.DBT_REDSHIFT_CI_DATABASE }}

      DBT_SNOWFLAKE_CI_ACCOUNT: ${{ secrets.DBT_SNOWFLAKE_CI_ACCOUNT }}
      DBT_SNOWFLAKE_CI_DATABASE: ${{ secrets.DBT_SNOWFLAKE_CI_DATABASE }}
      DBT_SNOWFLAKE_CI_PASSWORD: ${{ secrets.DBT_SNOWFLAKE_CI_PASSWORD }}
      DBT_SNOWFLAKE_CI_ROLE: ${{ secrets.DBT_SNOWFLAKE_CI_ROLE }}
      DBT_SNOWFLAKE_CI_SCHEMA: ${{ secrets.DBT_SNOWFLAKE_CI_SCHEMA }}
      DBT_SNOWFLAKE_CI_USER: ${{ secrets.DBT_SNOWFLAKE_CI_USER }}
      DBT_SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.DBT_SNOWFLAKE_CI_WAREHOUSE }}

    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.resolve_mode.outputs.checkout_ref }}

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install ODBC Driver 18 for SQL Server
        if: ${{ matrix.adapter == 'fabric' }}
        run: |
          curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
          curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list
          sudo apt-get update
          sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18

      - name: Install dbt adapter
        run: |
          python -m pip install --upgrade pip
          case "${{ matrix.adapter }}" in
            snowflake)
              pip install dbt-core==${{ env.DBT_CORE_VERSION }} dbt-snowflake
              ;;
            bigquery)
              pip install dbt-core==${{ env.DBT_CORE_VERSION }} dbt-bigquery
              ;;
            databricks)
              pip install dbt-core==${{ env.DBT_CORE_VERSION }} dbt-databricks
              pip install certifi
              ;;
            fabric)
              pip install dbt-core==${{ env.DBT_CORE_VERSION }} dbt-fabric
              ;;
            redshift)
              pip install dbt-core==${{ env.DBT_CORE_VERSION }} dbt-redshift
              ;;
            duckdb)
              # MotherDuck currently supports DuckDB up to 1.4.4 in CI.
              pip install dbt-core==${{ env.DBT_CORE_VERSION }} dbt-duckdb duckdb==1.4.4
              ;;
            *)
              echo "Unsupported adapter: ${{ matrix.adapter }}" >&2
              exit 1
              ;;
          esac

      - name: Create BigQuery credentials file
        if: ${{ matrix.adapter == 'bigquery' }}
        run: |
          echo "${{ env.DBT_BIGQUERY_CI_TOKEN }}" | base64 --decode > ./creds.json

      - name: Install dbt dependencies
        run: dbt deps --project-dir ./integration_tests --profiles-dir ./integration_tests/profiles/${{ matrix.adapter }}

      - name: Test connection
        run: dbt debug --project-dir ./integration_tests --profiles-dir ./integration_tests/profiles/${{ matrix.adapter }}

      - name: Assert baseline seed schemas exist for run-only modes
        if: ${{ needs.resolve_mode.outputs.mode != 'large' }}
        run: |
          dbt run-operation assert_ci_seed_baseline_ready \
            --project-dir ./integration_tests \
            --profiles-dir ./integration_tests/profiles/${{ matrix.adapter }}

      - name: Run dbt (mode-aware)
        run: |
          if [[ "${{ needs.resolve_mode.outputs.mode }}" == "large" ]]; then
            dbt build --full-refresh \
              --project-dir ./integration_tests \
              --profiles-dir ./integration_tests/profiles/${{ matrix.adapter }}
          else
            dbt run \
              --project-dir ./integration_tests \
              --profiles-dir ./integration_tests/profiles/${{ matrix.adapter }}
          fi