Cleanup for the flatten: dead park code, contradiction fixes, one owner per rule #992
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Leaf Evals | |
| on: | |
| pull_request: | |
| push: | |
| branches: [dev] | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| concurrency: | |
| group: leaf-evals-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| changes: | |
| name: Check changed files | |
| runs-on: blacksmith-2vcpu-ubuntu-2404 | |
| outputs: | |
| leaf: ${{ steps.filter.outputs.leaf }} | |
| steps: | |
| # paths-filter diffs via the API on pull_request but needs a local repo on push. | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Check changed files | |
| id: filter | |
| uses: dorny/paths-filter@6852f92c20ea7fd3b0c25de3b5112db3a98da050 | |
| with: | |
| filters: | | |
| leaf: | |
| - "apps/leaf/**" | |
| - ".github/workflows/leaf-evals.yml" | |
| # One smoke eval per PR: the canonical lookup -> preview -> approval -> write | |
| # flow. The full suite is local-only (bun <file>.eval.ts) to keep model spend down. | |
| eval: | |
| name: Eval | |
| needs: changes | |
| if: needs.changes.outputs.leaf == 'true' | |
| runs-on: blacksmith-4vcpu-ubuntu-2404 | |
| timeout-minutes: 20 | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} | |
| EVAL_FAIL_ON_SCORE: "1" | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Bun | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version: 1.3.14 | |
| - name: Install dependencies | |
| run: bun install --frozen-lockfile | |
| - name: Run eval | |
| working-directory: apps/leaf | |
| run: bun tests/evals/agent/mcp/attach-approval.eval.ts |