ci(repo): add refresh-lock job and aggregator check-runs (#630) #2185
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python & Rust CI | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| workflow_dispatch: | |
| env: | |
| GRZ_CHECK_WORKSPACE: "packages/grz-check" | |
| RUST_TOOLCHAIN_VERSION: "1.89.0" | |
| jobs: | |
| determine-changes: | |
| name: Determine Changed Packages | |
| runs-on: ubuntu-latest | |
| outputs: | |
| python_matrix: ${{ steps.generate_matrix.outputs.python_matrix }} | |
| rust_matrix: ${{ steps.generate_matrix.outputs.rust_matrix }} | |
| has_python_changes: ${{ steps.generate_matrix.outputs.has_python_changes }} | |
| has_rust_changes: ${{ steps.generate_matrix.outputs.has_rust_changes }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| lfs: true | |
| - name: Get changed files within packages | |
| id: changed_package_files | |
| uses: tj-actions/changed-files@v46 | |
| with: | |
| files: packages/** | |
| - name: List changed packages and generate matrix | |
| id: generate_matrix | |
| env: | |
| ALL_CHANGED_FILES: ${{ steps.changed_package_files.outputs.all_changed_files }} | |
| run: | | |
| echo "All changed files tracked by tj-actions: $ALL_CHANGED_FILES" | |
| directly_changed_package_dirs=() | |
| for file_path in $ALL_CHANGED_FILES; do | |
| # Extract the direct package directory (e.g., packages/package-a) | |
| package_dir=$(echo "$file_path" | cut -d/ -f1-2) | |
| # Check if it's a valid package dir (Python or Rust) | |
| if [[ -f "$package_dir/pyproject.toml" || -f "$package_dir/Cargo.toml" ]]; then | |
| if [[ ! " ${directly_changed_package_dirs[@]} " =~ " ${package_dir} " ]]; then | |
| directly_changed_package_dirs+=("$package_dir") | |
| fi | |
| fi | |
| done | |
| echo "Directly changed package directories: ${directly_changed_package_dirs[*]}" | |
| # Define dependency graph (source package name -> space-separated list of dependent package names) | |
| declare -A DEPENDENCY_GRAPH | |
| DEPENDENCY_GRAPH["grz-pydantic-models"]="grz-cli grzctl grz-common grz-db" | |
| DEPENDENCY_GRAPH["grz-pydantic-models-testing"]="grz-pydantic-models grzctl grz-db" | |
| DEPENDENCY_GRAPH["grz-common"]="grz-cli grzctl" | |
| DEPENDENCY_GRAPH["grz-db"]="grzctl" | |
| DEPENDENCY_GRAPH["grz-cli"]="grzctl" | |
| DEPENDENCY_GRAPH["grz-check"]="grz-cli grzctl grz-common" | |
| # Add other dependencies as needed: DEPENDENCY_GRAPH["pkg-x"]="pkg-y pkg-z" | |
| final_test_dirs=("${directly_changed_package_dirs[@]}") | |
| for changed_dir_path in "${directly_changed_package_dirs[@]}"; do | |
| pkg_name=$(basename "$changed_dir_path") # e.g., grz-cli from packages/grz-cli | |
| if [[ -n "${DEPENDENCY_GRAPH[$pkg_name]}" ]]; then | |
| echo "Package $pkg_name changed, considering its dependents: ${DEPENDENCY_GRAPH[$pkg_name]}" | |
| for dependent_name in ${DEPENDENCY_GRAPH[$pkg_name]}; do | |
| dependent_path="packages/$dependent_name" | |
| # Add dependent path if not already in the list | |
| if [[ ! " ${final_test_dirs[@]} " =~ " ${dependent_path} " ]]; then | |
| final_test_dirs+=("$dependent_path") | |
| echo "Added dependent to test: $dependent_path" | |
| fi | |
| done | |
| fi | |
| done | |
| # Deduplicate final list (though the check above should prevent most duplicates) | |
| unique_test_dirs_array=($(echo "${final_test_dirs[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')) | |
| echo "Final unique package directories to test: ${unique_test_dirs_array[*]}" | |
| python_json_objects="" | |
| rust_json_objects="" | |
| for pkg_path in "${unique_test_dirs_array[@]}"; do | |
| if [[ -f "$pkg_path/pyproject.toml" && -d "$pkg_path/tests" ]]; then | |
| # Note: We only need the path in the matrix object now | |
| python_json_objects+=$(jq -n --arg path "$pkg_path" '{"path": $path}') | |
| fi | |
| if [[ -f "$pkg_path/Cargo.toml" ]]; then | |
| rust_json_objects+=$(jq -n --arg path "$pkg_path" '{"path": $path}') | |
| fi | |
| done | |
| python_matrix=$(echo "$python_json_objects" | jq -sc .) | |
| rust_matrix=$(echo "$rust_json_objects" | jq -sc .) | |
| echo "python_matrix=$python_matrix" >> $GITHUB_OUTPUT | |
| echo "rust_matrix=$rust_matrix" >> $GITHUB_OUTPUT | |
| if [ "$(echo "$python_matrix" | jq 'length')" -gt 0 ]; then | |
| echo "has_python_changes=true" >> $GITHUB_OUTPUT | |
| echo "Generated Python matrix: $python_matrix" | |
| else | |
| echo "has_python_changes=false" >> $GITHUB_OUTPUT | |
| echo "No Python package changes detected." | |
| fi | |
| if [ "$(echo "$rust_matrix" | jq 'length')" -gt 0 ]; then | |
| echo "has_rust_changes=true" >> $GITHUB_OUTPUT | |
| echo "Generated Rust matrix: $rust_matrix" | |
| else | |
| echo "has_rust_changes=false" >> $GITHUB_OUTPUT | |
| echo "No Rust package changes detected." | |
| fi | |
| quality-checks-python: | |
| name: Global Quality Checks (Python) | |
| runs-on: ubuntu-latest | |
| # Run always on PRs to main, or if determine-changes says something relevant changed on push to main. | |
| # This ensures quality checks run even if no "package" code changed but e.g. a workflow file did. | |
| if: github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/main') | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| lfs: true | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v6 | |
| with: | |
| version-file: "pyproject.toml" | |
| enable-cache: true | |
| - name: Set up Python | |
| run: uv python install 3.12 # Consistent Python for these checks | |
| - name: Ensure uv lockfile is up-to-date | |
| run: uv lock --check | |
| - name: Install project dev dependencies | |
| run: uv sync --all-extras --all-groups --all-packages | |
| - name: Check formatting | |
| run: uv run tox -e format-check | |
| - name: Linting | |
| run: uv run tox -e lints | |
| - name: Run type checking | |
| run: uv run tox -e typecheck | |
| quality-checks-rust: | |
| name: Global Quality Checks (Rust) | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/main') | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install Rust toolchain | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} | |
| components: rustfmt, clippy | |
| cache-workspaces: ${{ env.GRZ_CHECK_WORKSPACE }} | |
| cache-shared-key: grz-check-binary-${{ runner.os }}-${{ hashFiles('packages/grz-check/Cargo.lock', 'packages/grz-check/src/**/*.rs') }} | |
| - name: Check formatting for all Rust packages | |
| run: | | |
| find packages -name Cargo.toml -print0 | xargs -0 -I {} bash -c 'cd "$(dirname {})" && cargo fmt --all -- --check' | |
| - name: Linting with Clippy for all Rust packages | |
| run: | | |
| find packages -name Cargo.toml -print0 | xargs -0 -I {} bash -c 'cd "$(dirname {})" && cargo clippy --all-targets -- -D warnings' | |
| build-grz-check-binary: | |
| name: Build grz-check Binary | |
| needs: [ quality-checks-rust ] | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/main') | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install Rust toolchain | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} | |
| cache-workspaces: ${{ env.GRZ_CHECK_WORKSPACE }} | |
| cache-shared-key: grz-check-binary-${{ runner.os }}-${{ hashFiles('packages/grz-check/Cargo.lock', 'packages/grz-check/src/**/*.rs') }} | |
| - name: Build grz-check debug binary | |
| run: cargo build --manifest-path packages/grz-check/Cargo.toml | |
| - name: Upload grz-check binary as an artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: grz-check-binary | |
| path: packages/grz-check/target/debug/grz-check | |
| test-changed-packages-rust: | |
| name: Test Changed Rust Packages | |
| needs: [ build-grz-check-binary, determine-changes, quality-checks-rust ] | |
| if: ${{ !cancelled() && needs.determine-changes.outputs.has_rust_changes == 'true' }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| package: ${{ fromJson(needs.determine-changes.outputs.rust_matrix) }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install Rust toolchain | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} | |
| cache-workspaces: ${{ env.GRZ_CHECK_WORKSPACE }} | |
| cache-shared-key: grz-check-binary-${{ runner.os }}-${{ hashFiles('packages/grz-check/Cargo.lock', 'packages/grz-check/src/**/*.rs') }} | |
| - name: Extract package name from path | |
| id: pkg_info | |
| run: echo "name=$(basename ${{ matrix.package.path }})" >> $GITHUB_OUTPUT | |
| - name: Run tests for ${{ steps.pkg_info.outputs.name }} | |
| run: cd ${{ matrix.package.path }} && cargo test --all-features | |
| run-workspace-tests-python: | |
| name: Run workspace tests (Python) | |
| runs-on: ubuntu-latest | |
| needs: [ build-grz-check-binary, test-changed-packages-rust ] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - "3.12" | |
| - "3.13" | |
| if: ${{ !cancelled() && (github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && needs.build-grz-check-binary.result == 'success' && (needs.test-changed-packages-rust.result == 'success' || needs.test-changed-packages-rust.result == 'skipped') }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| lfs: true | |
| - name: Install PostgreSQL | |
| run: sudo apt-get --update install postgresql | |
| - name: Download grz-check binary artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: grz-check-binary | |
| path: ${{ github.workspace }}/bin | |
| - name: Make grz-check executable | |
| run: chmod +x ${{ github.workspace }}/bin/grz-check | |
| - name: Add grz-check to PATH | |
| run: echo "${{ github.workspace }}/bin" >> $GITHUB_PATH | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v6 | |
| with: | |
| version-file: "pyproject.toml" | |
| enable-cache: true | |
| - name: Set up Python ${{ matrix.python-version }} | |
| run: uv python install ${{ matrix.python-version }} | |
| - name: Install project dev dependencies | |
| run: uv sync --all-extras --all-groups --all-packages | |
| - name: Verify grz-check is available | |
| run: which grz-check && grz-check --version | |
| - name: Test ${{ matrix.python-version }} | |
| run: uv run tox -e ${{ matrix.python-version }} -- tests/ | |
| test-changed-packages-python: | |
| name: Test Changed Python Packages | |
| needs: [ determine-changes, quality-checks-python, build-grz-check-binary, test-changed-packages-rust ] | |
| if: ${{ !cancelled() && needs.determine-changes.outputs.has_python_changes == 'true' && needs.build-grz-check-binary.result == 'success' && (needs.test-changed-packages-rust.result == 'success' || needs.test-changed-packages-rust.result == 'skipped') }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| package: ${{ fromJson(needs.determine-changes.outputs.python_matrix) }} | |
| python-version: | |
| - "3.12" | |
| - "3.13" | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| lfs: true | |
| - name: Download grz-check binary artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: grz-check-binary | |
| path: ${{ github.workspace }}/bin | |
| - name: Make grz-check executable | |
| run: chmod +x ${{ github.workspace }}/bin/grz-check | |
| - name: Add grz-check to PATH | |
| run: echo "${{ github.workspace }}/bin" >> $GITHUB_PATH | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v6 | |
| with: | |
| version-file: "pyproject.toml" | |
| enable-cache: true | |
| - name: Set up Python ${{ matrix.python-version }} | |
| run: uv python install ${{ matrix.python-version }} | |
| - name: Install project dev dependencies | |
| run: uv sync --all-extras --all-groups | |
| - name: Extract package name from path | |
| id: pkg_info | |
| run: echo "name=$(basename ${{ matrix.package.path }})" >> $GITHUB_OUTPUT | |
| - name: Ensure package uv lockfile is up-to-date | |
| run: cd ${{ matrix.package.path }} && uv lock --check | |
| - name: Run tests for ${{ steps.pkg_info.outputs.name }} | |
| env: | |
| GRZ_TOX_PACKAGE: ${{ steps.pkg_info.outputs.name }} | |
| run: uv run tox -e py${{ matrix.python-version }} -- ${{ matrix.package.path }} | |
| # Aggregator job: a single, stable check-run name to use as the required | |
| # status check in branch protection. Stays valid even when matrix jobs are | |
| # skipped (nothing changed) or individual job names change. Lists every leaf | |
| # job below - keep this in sync when adding or removing jobs. | |
| ci: | |
| name: CI | |
| needs: | |
| - determine-changes | |
| - quality-checks-python | |
| - quality-checks-rust | |
| - build-grz-check-binary | |
| - test-changed-packages-rust | |
| - run-workspace-tests-python | |
| - test-changed-packages-python | |
| runs-on: ubuntu-latest | |
| if: always() | |
| steps: | |
| - name: Verify no upstream job failed | |
| shell: bash | |
| run: | | |
| # Fail only on failure/cancelled. A skipped job is allowed. | |
| declare -A results=( | |
| [determine-changes]="${{ needs.determine-changes.result }}" | |
| [quality-checks-python]="${{ needs.quality-checks-python.result }}" | |
| [quality-checks-rust]="${{ needs.quality-checks-rust.result }}" | |
| [build-grz-check-binary]="${{ needs.build-grz-check-binary.result }}" | |
| [test-changed-packages-rust]="${{ needs.test-changed-packages-rust.result }}" | |
| [run-workspace-tests-python]="${{ needs.run-workspace-tests-python.result }}" | |
| [test-changed-packages-python]="${{ needs.test-changed-packages-python.result }}" | |
| ) | |
| failed=0 | |
| for job in "${!results[@]}"; do | |
| result="${results[$job]}" | |
| echo "$job=$result" | |
| if [ "$result" = "failure" ] || [ "$result" = "cancelled" ]; then | |
| echo "::error::Upstream job did not pass: $job=$result" | |
| failed=1 | |
| fi | |
| done | |
| if [ "$failed" -ne 0 ]; then | |
| exit 1 | |
| fi | |
| echo "All upstream jobs passed or were skipped." |