ci.yml

name: Python & Rust CI

on:
  push:
    branches:
      - main
  pull_request:
  # Merge queue: required checks must report on the queue's speculative merge
  # commit, which arrives via the merge_group event (not pull_request).
  merge_group:
  workflow_dispatch:

env:
  GRZ_CHECK_WORKSPACE: "packages/grz-check"
  RUST_TOOLCHAIN_VERSION: "1.89.0"

jobs:
  determine-changes:
    name: Determine Changed Packages
    runs-on: ubuntu-latest
    outputs:
      python_matrix: ${{ steps.generate_matrix.outputs.python_matrix }}
      rust_matrix: ${{ steps.generate_matrix.outputs.rust_matrix }}
      has_python_changes: ${{ steps.generate_matrix.outputs.has_python_changes }}
      has_rust_changes: ${{ steps.generate_matrix.outputs.has_rust_changes }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true

      - name: Get changed files within packages
        id: changed_package_files
        uses: tj-actions/changed-files@v46
        with:
          files: packages/**

      - name: List changed packages and generate matrix
        id: generate_matrix
        env:
          ALL_CHANGED_FILES: ${{ steps.changed_package_files.outputs.all_changed_files }}
        run: |
          echo "All changed files tracked by tj-actions: $ALL_CHANGED_FILES"
          directly_changed_package_dirs=()
          for file_path in $ALL_CHANGED_FILES; do
            # Extract the direct package directory (e.g., packages/package-a)
            package_dir=$(echo "$file_path" | cut -d/ -f1-2)

            # Check if it's a valid package dir (Python or Rust)
            if [[ -f "$package_dir/pyproject.toml" || -f "$package_dir/Cargo.toml" ]]; then
              if [[ ! " ${directly_changed_package_dirs[@]} " =~ " ${package_dir} " ]]; then
                directly_changed_package_dirs+=("$package_dir")
              fi
            fi
          done
          echo "Directly changed package directories: ${directly_changed_package_dirs[*]}"

          # Define dependency graph (source package name -> space-separated list of dependent package names)
          declare -A DEPENDENCY_GRAPH
          DEPENDENCY_GRAPH["grz-pydantic-models"]="grz-cli grzctl grz-common grz-db"
          DEPENDENCY_GRAPH["grz-pydantic-models-testing"]="grz-pydantic-models grzctl grz-db"
          DEPENDENCY_GRAPH["grz-common"]="grz-cli grzctl"
          DEPENDENCY_GRAPH["grz-db"]="grzctl"
          DEPENDENCY_GRAPH["grz-cli"]="grzctl"
          DEPENDENCY_GRAPH["grz-check"]="grz-cli grzctl grz-common"
          # Add other dependencies as needed: DEPENDENCY_GRAPH["pkg-x"]="pkg-y pkg-z"

          final_test_dirs=("${directly_changed_package_dirs[@]}")

          for changed_dir_path in "${directly_changed_package_dirs[@]}"; do
              pkg_name=$(basename "$changed_dir_path") # e.g., grz-cli from packages/grz-cli
              if [[ -n "${DEPENDENCY_GRAPH[$pkg_name]}" ]]; then
                  echo "Package $pkg_name changed, considering its dependents: ${DEPENDENCY_GRAPH[$pkg_name]}"
                  for dependent_name in ${DEPENDENCY_GRAPH[$pkg_name]}; do
                      dependent_path="packages/$dependent_name"
                      # Add dependent path if not already in the list
                      if [[ ! " ${final_test_dirs[@]} " =~ " ${dependent_path} " ]]; then
                          final_test_dirs+=("$dependent_path")
                          echo "Added dependent to test: $dependent_path"
                      fi
                  done
              fi
          done

          # Deduplicate final list (though the check above should prevent most duplicates)
          unique_test_dirs_array=($(echo "${final_test_dirs[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' '))
          echo "Final unique package directories to test: ${unique_test_dirs_array[*]}"

          python_json_objects=""
          rust_json_objects=""

          for pkg_path in "${unique_test_dirs_array[@]}"; do
            if [[ -f "$pkg_path/pyproject.toml" && -d "$pkg_path/tests" ]]; then
              # Note: We only need the path in the matrix object now
              python_json_objects+=$(jq -n --arg path "$pkg_path" '{"path": $path}')
            fi

            if [[ -f "$pkg_path/Cargo.toml" ]]; then
              rust_json_objects+=$(jq -n --arg path "$pkg_path" '{"path": $path}')
            fi
          done

          python_matrix=$(echo "$python_json_objects" | jq -sc .)
          rust_matrix=$(echo "$rust_json_objects" | jq -sc .)

          echo "python_matrix=$python_matrix" >> $GITHUB_OUTPUT
          echo "rust_matrix=$rust_matrix" >> $GITHUB_OUTPUT

          if [ "$(echo "$python_matrix" | jq 'length')" -gt 0 ]; then
            echo "has_python_changes=true" >> $GITHUB_OUTPUT
            echo "Generated Python matrix: $python_matrix"
          else
            echo "has_python_changes=false" >> $GITHUB_OUTPUT
            echo "No Python package changes detected."
          fi

          if [ "$(echo "$rust_matrix" | jq 'length')" -gt 0 ]; then
            echo "has_rust_changes=true" >> $GITHUB_OUTPUT
            echo "Generated Rust matrix: $rust_matrix"
          else
            echo "has_rust_changes=false" >> $GITHUB_OUTPUT
            echo "No Rust package changes detected."
          fi

  quality-checks-python:
    name: Global Quality Checks (Python)
    runs-on: ubuntu-latest
    # Run always on PRs to main, or if determine-changes says something relevant changed on push to main.
    # This ensures quality checks run even if no "package" code changed but e.g. a workflow file did.
    if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          lfs: true

      - name: Install uv
        uses: astral-sh/setup-uv@v6
        with:
          version-file: "pyproject.toml"
          enable-cache: true

      - name: Set up Python
        run: uv python install 3.12 # Consistent Python for these checks

      - name: Ensure uv lockfile is up-to-date
        run: uv lock --check

      - name: Install project dev dependencies
        run: uv sync --all-extras --all-groups --all-packages

      - name: Check formatting
        run: uv run tox -e format-check

      - name: Linting
        run: uv run tox -e lints

      - name: Run type checking
        run: uv run tox -e typecheck

  quality-checks-rust:
    name: Global Quality Checks (Rust)
    runs-on: ubuntu-latest
    if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install Rust toolchain
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
          components: rustfmt, clippy
          cache-workspaces: ${{ env.GRZ_CHECK_WORKSPACE }}
          cache-shared-key: grz-check-binary-${{ runner.os }}-${{ hashFiles('packages/grz-check/Cargo.lock', 'packages/grz-check/src/**/*.rs') }}

      - name: Check formatting for all Rust packages
        run: |
          find packages -name Cargo.toml -print0 | xargs -0 -I {} bash -c 'cd "$(dirname {})" && cargo fmt --all -- --check'

      - name: Linting with Clippy for all Rust packages
        run: |
          find packages -name Cargo.toml -print0 | xargs -0 -I {} bash -c 'cd "$(dirname {})" && cargo clippy --all-targets -- -D warnings'

  build-grz-check-binary:
    name: Build grz-check Binary
    needs: [ quality-checks-rust ]
    runs-on: ubuntu-latest
    if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install Rust toolchain
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
          cache-workspaces: ${{ env.GRZ_CHECK_WORKSPACE }}
          cache-shared-key: grz-check-binary-${{ runner.os }}-${{ hashFiles('packages/grz-check/Cargo.lock', 'packages/grz-check/src/**/*.rs') }}

      - name: Build grz-check debug binary
        run: cargo build --manifest-path packages/grz-check/Cargo.toml

      - name: Upload grz-check binary as an artifact
        uses: actions/upload-artifact@v4
        with:
          name: grz-check-binary
          path: packages/grz-check/target/debug/grz-check

  test-changed-packages-rust:
    name: Test Changed Rust Packages
    needs: [ build-grz-check-binary, determine-changes, quality-checks-rust ]
    if: ${{ !cancelled() && needs.determine-changes.outputs.has_rust_changes == 'true' }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        package: ${{ fromJson(needs.determine-changes.outputs.rust_matrix) }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install Rust toolchain
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
          cache-workspaces: ${{ env.GRZ_CHECK_WORKSPACE }}
          cache-shared-key: grz-check-binary-${{ runner.os }}-${{ hashFiles('packages/grz-check/Cargo.lock', 'packages/grz-check/src/**/*.rs') }}

      - name: Extract package name from path
        id: pkg_info
        run: echo "name=$(basename ${{ matrix.package.path }})" >> $GITHUB_OUTPUT

      - name: Run tests for ${{ steps.pkg_info.outputs.name }}
        run: cd ${{ matrix.package.path }} && cargo test --all-features

  run-workspace-tests-python:
    name: Run workspace tests (Python)
    runs-on: ubuntu-latest
    needs: [ build-grz-check-binary, test-changed-packages-rust ]
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - "3.12"
          - "3.13"
    if: ${{ !cancelled() && (github.event_name == 'pull_request' || github.event_name == 'merge_group' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && needs.build-grz-check-binary.result == 'success' && (needs.test-changed-packages-rust.result == 'success' || needs.test-changed-packages-rust.result == 'skipped') }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          lfs: true

      - name: Install PostgreSQL
        run: sudo apt-get --update install postgresql

      - name: Download grz-check binary artifact
        uses: actions/download-artifact@v4
        with:
          name: grz-check-binary
          path: ${{ github.workspace }}/bin

      - name: Make grz-check executable
        run: chmod +x ${{ github.workspace }}/bin/grz-check

      - name: Add grz-check to PATH
        run: echo "${{ github.workspace }}/bin" >> $GITHUB_PATH

      - name: Install uv
        uses: astral-sh/setup-uv@v6
        with:
          version-file: "pyproject.toml"
          enable-cache: true

      - name: Set up Python ${{ matrix.python-version }}
        run: uv python install ${{ matrix.python-version }}

      - name: Install project dev dependencies
        run: uv sync --all-extras --all-groups --all-packages

      - name: Verify grz-check is available
        run: which grz-check && grz-check --version

      - name: Test ${{ matrix.python-version }}
        run: uv run tox -e ${{ matrix.python-version }} -- tests/

  test-changed-packages-python:
    name: Test Changed Python Packages
    needs: [ determine-changes, quality-checks-python, build-grz-check-binary, test-changed-packages-rust ]
    if: ${{ !cancelled() && needs.determine-changes.outputs.has_python_changes == 'true' && needs.build-grz-check-binary.result == 'success' && (needs.test-changed-packages-rust.result == 'success' || needs.test-changed-packages-rust.result == 'skipped') }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        package: ${{ fromJson(needs.determine-changes.outputs.python_matrix) }}
        python-version:
          - "3.12"
          - "3.13"
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          lfs: true

      - name: Download grz-check binary artifact
        uses: actions/download-artifact@v4
        with:
          name: grz-check-binary
          path: ${{ github.workspace }}/bin

      - name: Make grz-check executable
        run: chmod +x ${{ github.workspace }}/bin/grz-check

      - name: Add grz-check to PATH
        run: echo "${{ github.workspace }}/bin" >> $GITHUB_PATH

      - name: Install uv
        uses: astral-sh/setup-uv@v6
        with:
          version-file: "pyproject.toml"
          enable-cache: true

      - name: Set up Python ${{ matrix.python-version }}
        run: uv python install ${{ matrix.python-version }}

      - name: Install project dev dependencies
        run: uv sync --all-extras --all-groups

      - name: Extract package name from path
        id: pkg_info
        run: echo "name=$(basename ${{ matrix.package.path }})" >> $GITHUB_OUTPUT

      - name: Ensure package uv lockfile is up-to-date
        run: cd ${{ matrix.package.path }} && uv lock --check

      - name: Run tests for ${{ steps.pkg_info.outputs.name }}
        env:
          GRZ_TOX_PACKAGE: ${{ steps.pkg_info.outputs.name }}
        run: uv run tox -e py${{ matrix.python-version }} -- ${{ matrix.package.path }}

  # Aggregator job: a single, stable check-run name to use as the required
  # status check in branch protection. Stays valid even when matrix jobs are
  # skipped (nothing changed) or individual job names change. Lists every leaf
  # job below - keep this in sync when adding or removing jobs.
  ci:
    name: CI
    needs:
      - determine-changes
      - quality-checks-python
      - quality-checks-rust
      - build-grz-check-binary
      - test-changed-packages-rust
      - run-workspace-tests-python
      - test-changed-packages-python
    runs-on: ubuntu-latest
    if: always()
    steps:
      - name: Verify no upstream job failed
        shell: bash
        run: |
          # Fail only on failure/cancelled. A skipped job is allowed.
          declare -A results=(
            [determine-changes]="${{ needs.determine-changes.result }}"
            [quality-checks-python]="${{ needs.quality-checks-python.result }}"
            [quality-checks-rust]="${{ needs.quality-checks-rust.result }}"
            [build-grz-check-binary]="${{ needs.build-grz-check-binary.result }}"
            [test-changed-packages-rust]="${{ needs.test-changed-packages-rust.result }}"
            [run-workspace-tests-python]="${{ needs.run-workspace-tests-python.result }}"
            [test-changed-packages-python]="${{ needs.test-changed-packages-python.result }}"
          )
          failed=0
          for job in "${!results[@]}"; do
            result="${results[$job]}"
            echo "$job=$result"
            if [ "$result" = "failure" ] || [ "$result" = "cancelled" ]; then
              echo "::error::Upstream job did not pass: $job=$result"
              failed=1
            fi
          done
          if [ "$failed" -ne 0 ]; then
            exit 1
          fi
          echo "All upstream jobs passed or were skipped."