e2e.yml

name: E2E Tests

on:
  workflow_dispatch:
    inputs:
      agent:
        description: 'Run a single agent (leave empty for all)'
        required: false
        type: choice
        options:
          - ''
          - claude-code
          - opencode
          - gemini-cli
          - factoryai-droid
          - cursor-cli
          - copilot-cli
          - roger-roger
          - codex
  push:
    branches:
      - main

# Concurrency: only one E2E job runs at a time
concurrency:
  group: e2e-tests
  cancel-in-progress: true

jobs:
  # Build the matrix dynamically so workflow_dispatch can select a single agent.
  matrix-setup:
    runs-on: ubuntu-latest
    outputs:
      agents: ${{ steps.set.outputs.agents }}
    steps:
      - id: set
        run: |
          input="${{ github.event.inputs.agent }}"
          if [ -n "$input" ]; then
            echo "agents=[\"$input\"]" >> "$GITHUB_OUTPUT"
          else
            echo 'agents=["claude-code","opencode","gemini-cli","factoryai-droid","cursor-cli","copilot-cli","roger-roger","codex"]' >> "$GITHUB_OUTPUT"
          fi

  e2e-tests:
    needs: matrix-setup
    runs-on: ubuntu-latest
    timeout-minutes: 40
    permissions:
      actions: read
      contents: read
      # Needed by the copilot-cli leg. GitHub does not allow matrix expressions in
      # `permissions:`, so this scope is granted to every leg's GITHUB_TOKEN. The
      # token is kept away from non-copilot agents in two ways: it is only placed
      # in COPILOT_GITHUB_TOKEN for the copilot-cli leg (see env blocks below), and
      # checkout runs with persist-credentials: false so it is not left in
      # .git/config for an agent process to read.
      copilot-requests: write
    strategy:
      fail-fast: false
      matrix:
        agent: ${{ fromJson(needs.matrix-setup.outputs.agents) }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          # Don't persist the (copilot-scoped) GITHUB_TOKEN in .git/config, where
          # an agent process running in the checkout could read it.
          persist-credentials: false

      - name: Setup mise
        uses: jdx/mise-action@f10502fc09dadecfefb962fff68ce77213930204 # v4

      - name: Install system dependencies
        run: sudo apt-get update && sudo apt-get install -y tmux

      - name: Install agent CLI
        run: |
          case "${{ matrix.agent }}" in
            claude-code) curl -fsSL https://claude.ai/install.sh | bash ;;
            opencode)    curl -fsSL https://opencode.ai/install | bash ;;
            gemini-cli)  npm install -g @google/gemini-cli ;;
            codex)       npm install -g @openai/codex ;;
            cursor-cli)  curl https://cursor.com/install -fsS | bash ;;
            factoryai-droid) curl -fsSL https://app.factory.ai/cli | sh ;;
            copilot-cli)  npm install -g @github/copilot  ;;
            roger-roger) ;; # installed by mise (see mise.toml)
          esac
          echo "$HOME/.local/bin" >> $GITHUB_PATH

      - name: Verify roger-roger agent
        if: matrix.agent == 'roger-roger'
        run: |
          set -euo pipefail
          echo "Verifying roger-roger binaries on PATH..."
          command -v roger-roger
          command -v entire-agent-roger-roger

      - name: Bootstrap agent
        if: matrix.agent != 'roger-roger'
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          CURSOR_API_KEY: ${{ secrets.CURSOR_API_KEY }}
          FACTORY_API_KEY: ${{ secrets.FACTORY_API_KEY }}
          # Only the copilot-cli leg gets the token; other agents must not receive it.
          COPILOT_GITHUB_TOKEN: ${{ matrix.agent == 'copilot-cli' && github.token || '' }}
        run: go run ./e2e/bootstrap

      - name: Run E2E Tests
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          E2E_CODEX_MODEL: ${{ matrix.agent == 'codex' && 'gpt-5.4-mini' || '' }}
          E2E_GEMINI_MODEL: ${{ matrix.agent == 'gemini-cli' && 'gemini-3.1-flash-lite' || '' }}
          CURSOR_API_KEY: ${{ secrets.CURSOR_API_KEY }}
          FACTORY_API_KEY: ${{ secrets.FACTORY_API_KEY }}
          # Only the copilot-cli leg gets the token; other agents must not receive it.
          COPILOT_GITHUB_TOKEN: ${{ matrix.agent == 'copilot-cli' && github.token || '' }}
          E2E_CONCURRENT_TEST_LIMIT: ${{ matrix.agent == 'gemini-cli' && '6' || matrix.agent == 'factoryai-droid' && '1' || matrix.agent == 'cursor-cli' && '2' || '' }}
        # roger-roger is deterministic, so it runs through its dedicated task,
        # which does NOT enable --rerun-fails. Routing it through the default
        # task (`test:e2e`) would retry a real regression and mask it.
        run: |
          if [ "${{ matrix.agent }}" = "roger-roger" ]; then
            mise run test:e2e:roger-roger TestExternalAgent
          else
            mise run test:e2e --agent ${{ matrix.agent }}
          fi

      - name: Upload artifacts
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: e2e-artifacts-${{ matrix.agent }}
          path: e2e/artifacts/
          retention-days: 7

  e2e-windows:
    uses: ./.github/workflows/e2e-windows.yml
    secrets: inherit

  notify-slack:
    runs-on: ubuntu-latest
    needs: [e2e-tests, e2e-windows]
    if: ${{ always() && (needs.e2e-tests.result == 'failure' || needs.e2e-windows.result == 'failure') && github.event_name == 'push' }}
    steps:
      # Classify the failed jobs into two severities:
      #   RED    — a "reliable" agent (or Windows) failed. These pass on every
      #            healthy run, so a failure here is a real regression.
      #   YELLOW — only a known-flaky agent failed. Worth investigating, but not
      #            a `main` regression. Reported so we don't lose visibility.
      - name: Classify failures
        id: classify
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          set -euo pipefail
          # Reliable tier (RED). The e2e-windows reusable job is also RED but
          # is matched separately below since it has no "(agent)" suffix.
          reliable="claude-code opencode gemini-cli roger-roger"

          failed_names=$(gh api --paginate \
            "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs" \
            --jq '.jobs[] | select(.conclusion == "failure") | .name')

          red=""
          yellow=""
          while IFS= read -r name; do
            [ -z "$name" ] && continue
            case "$name" in
              *e2e-windows*) red="$red windows"; continue ;;
            esac
            # "e2e-tests (cursor-cli)" -> "cursor-cli"
            agent="${name#*(}"; agent="${agent%)*}"
            case " $reliable " in
              *" $agent "*) red="$red $agent" ;;
              *)            yellow="$yellow $agent" ;;
            esac
          done <<EOF
          $failed_names
          EOF

          # Each list was built as " a b c" (leading space, space-separated).
          # Strip the leading space, then join on ", ". Empty stays empty.
          red="${red# }";       red="${red// /, }"
          yellow="${yellow# }"; yellow="${yellow// /, }"

          if [ -n "$red" ]; then
            color="#d50200"
            header=":red_circle: *E2E Tests Failed* on \`main\`"
          else
            color="#daa038"
            header=":large_yellow_circle: *E2E flaky-agent failures* on \`main\`"
          fi

          body=""
          [ -n "$red" ]    && body="${body}Reliable agents (regression): *${red}*\\n"
          [ -n "$yellow" ] && body="${body}Flaky agents (investigate): *${yellow}*\\n"

          {
            echo "color=$color"
            echo "header=$header"
            echo "body=$body"
          } >> "$GITHUB_OUTPUT"

      - name: Notify Slack of E2E failure
        uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0
        with:
          webhook: ${{ secrets.E2E_SLACK_WEBHOOK_URL }}
          webhook-type: incoming-webhook
          payload: |
            {
              "attachments": [
                {
                  "color": "${{ steps.classify.outputs.color }}",
                  "blocks": [
                    {
                      "type": "section",
                      "text": {
                        "type": "mrkdwn",
                        "text": "${{ steps.classify.outputs.header }}\n\n${{ steps.classify.outputs.body }}<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run details>"
                      }
                    },
                    {
                      "type": "context",
                      "elements": [
                        {
                          "type": "mrkdwn",
                          "text": "Commit: <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}> by ${{ github.actor }}"
                        }
                      ]
                    }
                  ]
                }
              ]
            }