Merge trail: chore: lint the build-tagged test files, and remove the … #1205
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Tests | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| agent: | |
| description: 'Run a single agent (leave empty for all)' | |
| required: false | |
| type: choice | |
| options: | |
| - '' | |
| - claude-code | |
| - opencode | |
| - gemini-cli | |
| - factoryai-droid | |
| - cursor-cli | |
| - copilot-cli | |
| - roger-roger | |
| - codex | |
| push: | |
| branches: | |
| - main | |
| # Concurrency: only one E2E job runs at a time | |
| concurrency: | |
| group: e2e-tests | |
| cancel-in-progress: true | |
| jobs: | |
| # Build the matrix dynamically so workflow_dispatch can select a single agent. | |
| matrix-setup: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| agents: ${{ steps.set.outputs.agents }} | |
| steps: | |
| - id: set | |
| run: | | |
| input="${{ github.event.inputs.agent }}" | |
| if [ -n "$input" ]; then | |
| echo "agents=[\"$input\"]" >> "$GITHUB_OUTPUT" | |
| else | |
| echo 'agents=["claude-code","opencode","gemini-cli","factoryai-droid","cursor-cli","copilot-cli","roger-roger","codex"]' >> "$GITHUB_OUTPUT" | |
| fi | |
| e2e-tests: | |
| needs: matrix-setup | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 40 | |
| permissions: | |
| actions: read | |
| contents: read | |
| # Needed by the copilot-cli leg. GitHub does not allow matrix expressions in | |
| # `permissions:`, so this scope is granted to every leg's GITHUB_TOKEN. The | |
| # token is kept away from non-copilot agents in two ways: it is only placed | |
| # in COPILOT_GITHUB_TOKEN for the copilot-cli leg (see env blocks below), and | |
| # checkout runs with persist-credentials: false so it is not left in | |
| # .git/config for an agent process to read. | |
| copilot-requests: write | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| agent: ${{ fromJson(needs.matrix-setup.outputs.agents) }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| # Don't persist the (copilot-scoped) GITHUB_TOKEN in .git/config, where | |
| # an agent process running in the checkout could read it. | |
| persist-credentials: false | |
| - name: Setup mise | |
| uses: jdx/mise-action@f10502fc09dadecfefb962fff68ce77213930204 # v4 | |
| - name: Install system dependencies | |
| run: sudo apt-get update && sudo apt-get install -y tmux | |
| - name: Install agent CLI | |
| run: | | |
| case "${{ matrix.agent }}" in | |
| claude-code) curl -fsSL https://claude.ai/install.sh | bash ;; | |
| opencode) curl -fsSL https://opencode.ai/install | bash ;; | |
| gemini-cli) npm install -g @google/gemini-cli ;; | |
| codex) npm install -g @openai/codex ;; | |
| cursor-cli) curl https://cursor.com/install -fsS | bash ;; | |
| factoryai-droid) curl -fsSL https://app.factory.ai/cli | sh ;; | |
| copilot-cli) npm install -g @github/copilot ;; | |
| roger-roger) ;; # installed by mise (see mise.toml) | |
| esac | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - name: Verify roger-roger agent | |
| if: matrix.agent == 'roger-roger' | |
| run: | | |
| set -euo pipefail | |
| echo "Verifying roger-roger binaries on PATH..." | |
| command -v roger-roger | |
| command -v entire-agent-roger-roger | |
| - name: Bootstrap agent | |
| if: matrix.agent != 'roger-roger' | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| CURSOR_API_KEY: ${{ secrets.CURSOR_API_KEY }} | |
| FACTORY_API_KEY: ${{ secrets.FACTORY_API_KEY }} | |
| # Only the copilot-cli leg gets the token; other agents must not receive it. | |
| COPILOT_GITHUB_TOKEN: ${{ matrix.agent == 'copilot-cli' && github.token || '' }} | |
| run: go run ./e2e/bootstrap | |
| - name: Run E2E Tests | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| E2E_CODEX_MODEL: ${{ matrix.agent == 'codex' && 'gpt-5.4-mini' || '' }} | |
| E2E_GEMINI_MODEL: ${{ matrix.agent == 'gemini-cli' && 'gemini-3.1-flash-lite' || '' }} | |
| CURSOR_API_KEY: ${{ secrets.CURSOR_API_KEY }} | |
| FACTORY_API_KEY: ${{ secrets.FACTORY_API_KEY }} | |
| # Only the copilot-cli leg gets the token; other agents must not receive it. | |
| COPILOT_GITHUB_TOKEN: ${{ matrix.agent == 'copilot-cli' && github.token || '' }} | |
| E2E_CONCURRENT_TEST_LIMIT: ${{ matrix.agent == 'gemini-cli' && '6' || matrix.agent == 'factoryai-droid' && '1' || matrix.agent == 'cursor-cli' && '2' || '' }} | |
| # roger-roger is deterministic, so it runs through its dedicated task, | |
| # which does NOT enable --rerun-fails. Routing it through the default | |
| # task (`test:e2e`) would retry a real regression and mask it. | |
| run: | | |
| if [ "${{ matrix.agent }}" = "roger-roger" ]; then | |
| mise run test:e2e:roger-roger TestExternalAgent | |
| else | |
| mise run test:e2e --agent ${{ matrix.agent }} | |
| fi | |
| - name: Upload artifacts | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| with: | |
| name: e2e-artifacts-${{ matrix.agent }} | |
| path: e2e/artifacts/ | |
| retention-days: 7 | |
| e2e-windows: | |
| uses: ./.github/workflows/e2e-windows.yml | |
| secrets: inherit | |
| notify-slack: | |
| runs-on: ubuntu-latest | |
| needs: [e2e-tests, e2e-windows] | |
| if: ${{ always() && (needs.e2e-tests.result == 'failure' || needs.e2e-windows.result == 'failure') && github.event_name == 'push' }} | |
| steps: | |
| # Classify the failed jobs into two severities: | |
| # RED — a "reliable" agent (or Windows) failed. These pass on every | |
| # healthy run, so a failure here is a real regression. | |
| # YELLOW — only a known-flaky agent failed. Worth investigating, but not | |
| # a `main` regression. Reported so we don't lose visibility. | |
| - name: Classify failures | |
| id: classify | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| set -euo pipefail | |
| # Reliable tier (RED). The e2e-windows reusable job is also RED but | |
| # is matched separately below since it has no "(agent)" suffix. | |
| reliable="claude-code opencode gemini-cli roger-roger" | |
| failed_names=$(gh api --paginate \ | |
| "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs" \ | |
| --jq '.jobs[] | select(.conclusion == "failure") | .name') | |
| red="" | |
| yellow="" | |
| while IFS= read -r name; do | |
| [ -z "$name" ] && continue | |
| case "$name" in | |
| *e2e-windows*) red="$red windows"; continue ;; | |
| esac | |
| # "e2e-tests (cursor-cli)" -> "cursor-cli" | |
| agent="${name#*(}"; agent="${agent%)*}" | |
| case " $reliable " in | |
| *" $agent "*) red="$red $agent" ;; | |
| *) yellow="$yellow $agent" ;; | |
| esac | |
| done <<EOF | |
| $failed_names | |
| EOF | |
| # Each list was built as " a b c" (leading space, space-separated). | |
| # Strip the leading space, then join on ", ". Empty stays empty. | |
| red="${red# }"; red="${red// /, }" | |
| yellow="${yellow# }"; yellow="${yellow// /, }" | |
| if [ -n "$red" ]; then | |
| color="#d50200" | |
| header=":red_circle: *E2E Tests Failed* on \`main\`" | |
| else | |
| color="#daa038" | |
| header=":large_yellow_circle: *E2E flaky-agent failures* on \`main\`" | |
| fi | |
| body="" | |
| [ -n "$red" ] && body="${body}Reliable agents (regression): *${red}*\\n" | |
| [ -n "$yellow" ] && body="${body}Flaky agents (investigate): *${yellow}*\\n" | |
| { | |
| echo "color=$color" | |
| echo "header=$header" | |
| echo "body=$body" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Notify Slack of E2E failure | |
| uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 | |
| with: | |
| webhook: ${{ secrets.E2E_SLACK_WEBHOOK_URL }} | |
| webhook-type: incoming-webhook | |
| payload: | | |
| { | |
| "attachments": [ | |
| { | |
| "color": "${{ steps.classify.outputs.color }}", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "${{ steps.classify.outputs.header }}\n\n${{ steps.classify.outputs.body }}<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run details>" | |
| } | |
| }, | |
| { | |
| "type": "context", | |
| "elements": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "Commit: <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}> by ${{ github.actor }}" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| ] | |
| } |