feat: add durable run analysis reports #724
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| pull_request: | |
| branches: [main, dev] | |
| # Without this, nothing ever ran on main: every recorded run came from a | |
| # pull request, so the branch the releases are cut from had no signal of its | |
| # own and a merge could land red without anything turning red afterwards. | |
| # Pull-request runs test the merge commit against the base at open time, not | |
| # what main actually became, so they are not a substitute. | |
| push: | |
| branches: [main] | |
| merge_group: | |
| # The e2e suites below are gated on the merge queue, and no merge queue is | |
| # configured, so they had never run: 11 of 11 recorded runs skipped all three. | |
| # A manual dispatch is the only way to exercise them without changing how | |
| # merges work, and their secrets live in the `e2e` environment, which is | |
| # reachable from Actions and nowhere else. | |
| workflow_dispatch: | |
| concurrency: | |
| # The event belongs in the key. Without it a manual dispatch and the branch's | |
| # pull-request runs share one group, and since the group cancels in progress, | |
| # any push kills a dispatch already running. That makes the e2e suites, which | |
| # take hours, impossible to finish by hand. Pushes still supersede each other | |
| # within their own event, which is the behaviour this cancellation is for. | |
| # Main pushes add their SHA to the key below, so this remains true without | |
| # allowing one merged commit to cancel the verdict for its predecessor. | |
| group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.head_ref || github.ref_name }}-${{ github.event_name == 'push' && github.sha || 'shared' }} | |
| # Main pushes have unique groups; pull-request and manual runs keep the | |
| # existing superseding behavior. | |
| cancel-in-progress: true | |
| jobs: | |
| ci: | |
| runs-on: ubuntu-latest | |
| # `pnpm run test` alone measured 16:52, 17:52, 17:55 and 18:58 across four | |
| # branches on one day, so a 35 minute job left too little headroom for | |
| # everything else and cancelled mid-step whenever a runner was slow. A | |
| # cancelled check reads as neither red nor green, which is worse than a | |
| # failure: it is easy to merge past. The suite's cost is dominated by | |
| # per-file database setup rather than test count, so this ceiling has to | |
| # cover that until the database fixture is shared. | |
| timeout-minutes: 40 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: pnpm/action-setup@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| # Node 24 to match local dev and the Vercel runtime: the dashboard | |
| # test script relies on node --test glob patterns (Node 21+). | |
| node-version: 24 | |
| cache: pnpm | |
| - run: pnpm install --frozen-lockfile | |
| - run: pnpm run typecheck | |
| - run: pnpm run test | |
| - run: pnpm run test:release-notes | |
| - run: pnpm run test:ci | |
| # Durable-workflow tests. They need their own Workflow SDK builder config, | |
| # so vitest.config.ts cannot pick them up alongside the rest of the suite. | |
| - run: pnpm run test:workflow-sdk | |
| e2e-orchestration: | |
| needs: [ci] | |
| if: github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| environment: e2e | |
| env: | |
| E2E_BASE_URL: ${{ secrets.E2E_BASE_URL }} | |
| JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} | |
| JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} | |
| JIRA_PROJECT_KEY: ${{ secrets.JIRA_PROJECT_KEY }} | |
| JIRA_WEBHOOK_SECRET: ${{ secrets.JIRA_WEBHOOK_SECRET }} | |
| JIRA_E2E_COMMENTER_TOKEN: ${{ secrets.JIRA_E2E_COMMENTER_TOKEN }} | |
| COLUMN_AI: ${{ secrets.COLUMN_AI }} | |
| COLUMN_AI_REVIEW: ${{ secrets.COLUMN_AI_REVIEW }} | |
| COLUMN_BACKLOG: ${{ secrets.COLUMN_BACKLOG }} | |
| E2E_GITHUB_APP_ID: ${{ secrets.E2E_GITHUB_APP_ID }} | |
| E2E_GITHUB_APP_PRIVATE_KEY: ${{ secrets.E2E_GITHUB_APP_PRIVATE_KEY }} | |
| E2E_GITHUB_INSTALLATION_ID: ${{ secrets.E2E_GITHUB_INSTALLATION_ID }} | |
| E2E_GITHUB_OWNER: ${{ secrets.E2E_GITHUB_OWNER }} | |
| E2E_GITHUB_REPO: ${{ secrets.E2E_GITHUB_REPO }} | |
| CRON_SECRET: ${{ secrets.CRON_SECRET }} | |
| DATABASE_URL: ${{ secrets.DATABASE_URL }} | |
| VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} | |
| VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} | |
| VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} | |
| VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} | |
| MAX_CONCURRENT_AGENTS: ${{ secrets.MAX_CONCURRENT_AGENTS }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: pnpm/action-setup@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20.12" | |
| cache: pnpm | |
| - run: pnpm install --frozen-lockfile | |
| - name: Verify required e2e secrets present | |
| run: | | |
| : "${DATABASE_URL:?DATABASE_URL missing in e2e environment — must be the Neon branch the deployment under test (ai-workflow-demo) uses}" | |
| : "${VERCEL_TOKEN:?VERCEL_TOKEN missing in e2e environment}" | |
| : "${VERCEL_TEAM_ID:?VERCEL_TEAM_ID missing in e2e environment}" | |
| : "${VERCEL_PROJECT_ID:?VERCEL_PROJECT_ID missing in e2e environment}" | |
| - name: Preflight — verify e2e DATABASE_URL is the migrated branch | |
| working-directory: apps/worker | |
| run: pnpm exec tsx e2e/scripts/check-db.ts | |
| - run: pnpm run test:e2e:orchestration | |
| - if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e-orchestration-results | |
| path: | | |
| apps/worker/e2e/**/*.log | |
| retention-days: 7 | |
| e2e-capacity: | |
| needs: [e2e-orchestration] | |
| if: github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| environment: e2e | |
| env: | |
| E2E_BASE_URL: ${{ secrets.E2E_BASE_URL }} | |
| JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} | |
| JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} | |
| JIRA_PROJECT_KEY: ${{ secrets.JIRA_PROJECT_KEY }} | |
| JIRA_WEBHOOK_SECRET: ${{ secrets.JIRA_WEBHOOK_SECRET }} | |
| JIRA_E2E_COMMENTER_TOKEN: ${{ secrets.JIRA_E2E_COMMENTER_TOKEN }} | |
| COLUMN_AI: ${{ secrets.COLUMN_AI }} | |
| COLUMN_AI_REVIEW: ${{ secrets.COLUMN_AI_REVIEW }} | |
| COLUMN_BACKLOG: ${{ secrets.COLUMN_BACKLOG }} | |
| E2E_GITHUB_APP_ID: ${{ secrets.E2E_GITHUB_APP_ID }} | |
| E2E_GITHUB_APP_PRIVATE_KEY: ${{ secrets.E2E_GITHUB_APP_PRIVATE_KEY }} | |
| E2E_GITHUB_INSTALLATION_ID: ${{ secrets.E2E_GITHUB_INSTALLATION_ID }} | |
| E2E_GITHUB_OWNER: ${{ secrets.E2E_GITHUB_OWNER }} | |
| E2E_GITHUB_REPO: ${{ secrets.E2E_GITHUB_REPO }} | |
| CRON_SECRET: ${{ secrets.CRON_SECRET }} | |
| DATABASE_URL: ${{ secrets.DATABASE_URL }} | |
| VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} | |
| VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} | |
| VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} | |
| VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} | |
| MAX_CONCURRENT_AGENTS: ${{ secrets.MAX_CONCURRENT_AGENTS }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: pnpm/action-setup@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20.12" | |
| cache: pnpm | |
| - run: pnpm install --frozen-lockfile | |
| - name: Verify required e2e secrets present | |
| run: | | |
| : "${DATABASE_URL:?DATABASE_URL missing in e2e environment — must be the Neon branch the deployment under test (ai-workflow-demo) uses}" | |
| : "${VERCEL_TOKEN:?VERCEL_TOKEN missing in e2e environment}" | |
| : "${VERCEL_TEAM_ID:?VERCEL_TEAM_ID missing in e2e environment}" | |
| : "${VERCEL_PROJECT_ID:?VERCEL_PROJECT_ID missing in e2e environment}" | |
| - name: Preflight — verify e2e DATABASE_URL is the migrated branch | |
| working-directory: apps/worker | |
| run: pnpm exec tsx e2e/scripts/check-db.ts | |
| - run: pnpm run test:e2e:capacity | |
| - if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e-capacity-results | |
| path: | | |
| apps/worker/e2e/**/*.log | |
| retention-days: 7 | |
| e2e-agent: | |
| needs: [e2e-capacity] | |
| if: github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| environment: e2e | |
| env: | |
| E2E_BASE_URL: ${{ secrets.E2E_BASE_URL }} | |
| JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} | |
| JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} | |
| JIRA_PROJECT_KEY: ${{ secrets.JIRA_PROJECT_KEY }} | |
| JIRA_WEBHOOK_SECRET: ${{ secrets.JIRA_WEBHOOK_SECRET }} | |
| JIRA_E2E_COMMENTER_TOKEN: ${{ secrets.JIRA_E2E_COMMENTER_TOKEN }} | |
| COLUMN_AI: ${{ secrets.COLUMN_AI }} | |
| COLUMN_AI_REVIEW: ${{ secrets.COLUMN_AI_REVIEW }} | |
| COLUMN_BACKLOG: ${{ secrets.COLUMN_BACKLOG }} | |
| E2E_GITHUB_APP_ID: ${{ secrets.E2E_GITHUB_APP_ID }} | |
| E2E_GITHUB_APP_PRIVATE_KEY: ${{ secrets.E2E_GITHUB_APP_PRIVATE_KEY }} | |
| E2E_GITHUB_INSTALLATION_ID: ${{ secrets.E2E_GITHUB_INSTALLATION_ID }} | |
| E2E_GITHUB_OWNER: ${{ secrets.E2E_GITHUB_OWNER }} | |
| E2E_GITHUB_REPO: ${{ secrets.E2E_GITHUB_REPO }} | |
| CRON_SECRET: ${{ secrets.CRON_SECRET }} | |
| DATABASE_URL: ${{ secrets.DATABASE_URL }} | |
| VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} | |
| VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} | |
| VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} | |
| VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} | |
| MAX_CONCURRENT_AGENTS: ${{ secrets.MAX_CONCURRENT_AGENTS }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: pnpm/action-setup@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20.12" | |
| cache: pnpm | |
| - run: pnpm install --frozen-lockfile | |
| - name: Verify required e2e secrets present | |
| run: | | |
| : "${DATABASE_URL:?DATABASE_URL missing in e2e environment — must be the Neon branch the deployment under test (ai-workflow-demo) uses}" | |
| : "${VERCEL_TOKEN:?VERCEL_TOKEN missing in e2e environment}" | |
| : "${VERCEL_TEAM_ID:?VERCEL_TEAM_ID missing in e2e environment}" | |
| : "${VERCEL_PROJECT_ID:?VERCEL_PROJECT_ID missing in e2e environment}" | |
| - name: Preflight — verify e2e DATABASE_URL is the migrated branch | |
| working-directory: apps/worker | |
| run: pnpm exec tsx e2e/scripts/check-db.ts | |
| - run: pnpm run test:e2e:agent | |
| - if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e-agent-results | |
| path: | | |
| apps/worker/e2e/**/*.log | |
| retention-days: 7 |