Skip to content

Commit cfc718b

Browse files
FelixTJDietrichclaude
FelixTJDietrich
and
claude
authored
ci: parallelize e2e and move visual/perf suites to a nightly workflow (#770)
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent d56ebf0 commit cfc718b

9 files changed

Lines changed: 238 additions & 115 deletions

File tree

‎.github/actions/setup/action.yml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
name: "Setup pnpm + Node + install"
2+
description: >
3+
Installs pnpm, sets up Node (from .nvmrc by default) with the pnpm store
4+
cached, and runs a frozen install. Shared by the PR-health and nightly jobs
5+
so their toolchain pins live in one place. Use after actions/checkout (a
6+
local `./` composite can't perform the initial checkout itself).
7+
8+
inputs:
9+
node-version:
10+
description: "Explicit Node version. Empty (default) uses .nvmrc."
11+
required: false
12+
default: ""
13+
engine-strict:
14+
description: >
15+
Set to "false" to install with --config.engine-strict=false (needed when
16+
testing an older Node floor than the repo's root engines gate).
17+
required: false
18+
default: "true"
19+
20+
runs:
21+
using: composite
22+
steps:
23+
- name: Install pnpm
24+
uses: pnpm/action-setup@ac6db6d3c1f721f886538a378a2d73e85697340a # v6.0.8
25+
26+
- name: Set up Node
27+
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
28+
with:
29+
# Pass exactly one: an empty node-version makes setup-node fall through
30+
# to node-version-file (.nvmrc); a set node-version blanks the file so
31+
# the two never conflict.
32+
node-version: ${{ inputs.node-version }}
33+
node-version-file: ${{ inputs.node-version == '' && '.nvmrc' || '' }}
34+
cache: "pnpm"
35+
36+
- name: Install dependencies
37+
shell: bash
38+
run: pnpm install --frozen-lockfile ${{ inputs.engine-strict == 'false' && '--config.engine-strict=false' || '' }}

‎.github/workflows/nightly-checks.yml‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
name: Nightly Checks
2+
3+
# Heavier rendering / cross-engine guards that don't need to gate every PR:
4+
# - visual-regression (pixel baselines, pinned Playwright container)
5+
# - the chromium document-growth budget (cross-engine confirmation of the
6+
# Firefox budget that already runs per-PR in pr-health-checks.yml)
7+
#
8+
# They run nightly against main. To run them against a specific branch on
9+
# demand (e.g. a PR that intentionally changes rendering), use Actions →
10+
# Nightly Checks → Run workflow and pick the branch.
11+
on:
12+
schedule:
13+
# 03:00 UTC nightly. Scheduled runs only ever use the default branch.
14+
- cron: "0 3 * * *"
15+
workflow_dispatch:
16+
17+
concurrency:
18+
group: ${{ github.workflow }}-${{ github.ref }}
19+
cancel-in-progress: true
20+
21+
permissions:
22+
contents: read
23+
24+
jobs:
25+
# Pinned Playwright container so screenshot baselines diff against one Linux
26+
# rendering stack regardless of contributor OS (see
27+
# docs/contributor/development/visual-tests.md).
28+
visual-regression-tests:
29+
runs-on: ubuntu-latest
30+
timeout-minutes: 30
31+
container:
32+
image: mcr.microsoft.com/playwright:v1.59.1-noble
33+
steps:
34+
- name: Checkout repository
35+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
36+
37+
# The container ships an older Node; the composite installs Node from
38+
# .nvmrc via setup-node instead of curl-ing NodeSource on every run.
39+
- name: Setup pnpm + Node + install
40+
uses: ./.github/actions/setup
41+
42+
- name: Build
43+
run: pnpm run build:lib
44+
45+
- name: Run visual regression tests
46+
run: pnpm --filter @tumaet/webapp exec playwright test tests/visual/
47+
48+
- name: Upload Playwright report
49+
if: ${{ failure() }}
50+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
51+
with:
52+
name: playwright-report-visual
53+
path: standalone/webapp/playwright-report/
54+
retention-days: 14
55+
56+
- name: Upload Playwright test results
57+
if: ${{ failure() }}
58+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
59+
with:
60+
name: playwright-test-results-visual
61+
path: standalone/webapp/test-results/
62+
retention-days: 14
63+
64+
# Cross-engine confirmation of the per-PR Firefox document-growth budget. The
65+
# metric (encoded Yjs bytes / store writes) is engine-independent, so PRs stay
66+
# guarded by the Firefox run; this is the nightly chromium cross-check.
67+
perf-chromium-tests:
68+
runs-on: ubuntu-latest
69+
timeout-minutes: 20
70+
steps:
71+
- name: Checkout repository
72+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
73+
74+
- name: Setup pnpm + Node + install
75+
uses: ./.github/actions/setup
76+
77+
- name: Build
78+
run: pnpm run build:lib
79+
80+
- name: Cache Playwright browsers
81+
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
82+
id: playwright-cache
83+
with:
84+
path: ~/.cache/ms-playwright
85+
key: playwright-chromium-${{ runner.os }}-v1.59.1-${{ hashFiles('standalone/webapp/package.json') }}
86+
87+
- name: Install Playwright browsers
88+
if: steps.playwright-cache.outputs.cache-hit != 'true'
89+
run: pnpm --filter @tumaet/webapp exec playwright install chromium
90+
91+
- name: Install Playwright system deps (bounded + retried)
92+
uses: ./.github/actions/playwright-install-deps
93+
with:
94+
browser: chromium
95+
96+
- name: Run chromium performance budget tests
97+
run: pnpm --filter @tumaet/webapp run test:perf
98+
99+
- name: Upload Playwright report
100+
if: ${{ failure() }}
101+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
102+
with:
103+
name: playwright-report-perf-chromium
104+
path: standalone/webapp/playwright-report/
105+
retention-days: 14
106+
107+
- name: Upload Playwright test results
108+
if: ${{ failure() }}
109+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
110+
with:
111+
name: playwright-test-results-perf-chromium
112+
path: standalone/webapp/test-results/
113+
retention-days: 14
114+
115+
# A scheduled failure is invisible unless someone watches the Actions tab, so
116+
# surface it as a deduplicated tracking issue. Scheduled-only: a human is
117+
# already watching manual (workflow_dispatch) runs.
118+
notify-on-failure:
119+
needs: [visual-regression-tests, perf-chromium-tests]
120+
# Mirror pr-health-gate: always() so this isn't skipped, and treat cancelled
121+
# like failure so a raced/superseded nightly still surfaces. Scheduled-only:
122+
# a human already watches manual (workflow_dispatch) runs.
123+
if: >
124+
always() && github.event_name == 'schedule' &&
125+
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
126+
runs-on: ubuntu-latest
127+
timeout-minutes: 5
128+
permissions:
129+
issues: write
130+
steps:
131+
- name: Open or update a tracking issue
132+
env:
133+
GH_TOKEN: ${{ github.token }}
134+
REPO: ${{ github.repository }}
135+
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
136+
run: |
137+
set -euo pipefail
138+
title="Nightly Checks failed"
139+
# Exact-title dedup, client-side: gh's --search is fuzzy and could match
140+
# a similarly-named issue and suppress a real alert.
141+
number=$(gh issue list --repo "$REPO" --state open --limit 200 \
142+
--json number,title --jq "map(select(.title == \"$title\"))[0].number // empty")
143+
if [ -n "$number" ]; then
144+
gh issue comment "$number" --repo "$REPO" \
145+
--body "Another nightly run did not pass: $RUN_URL"
146+
else
147+
gh issue create --repo "$REPO" --title "$title" \
148+
--body "A scheduled \`Nightly Checks\` run did not pass on \`main\`: $RUN_URL — investigate visual / performance regressions and close this issue once green."
149+
fi

‎.github/workflows/pr-health-checks.yml‎

Lines changed: 25 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,13 @@ jobs:
4545
- '.nvmrc'
4646
- 'tsconfig*.json'
4747
- '.github/workflows/pr-health-checks.yml'
48+
- '.github/actions/**'
4849
library:
4950
- 'library/**'
5051
- 'pnpm-lock.yaml'
5152
- 'pnpm-workspace.yaml'
5253
- '.github/workflows/pr-health-checks.yml'
54+
- '.github/actions/**'
5355
5456
# No path filter: a stale CDN version can drift in via a doc edit OR a bump,
5557
# so check every PR. Pure Node — no pnpm install needed.
@@ -82,17 +84,8 @@ jobs:
8284
- name: Checkout repository
8385
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
8486

85-
- name: Install pnpm
86-
uses: pnpm/action-setup@ac6db6d3c1f721f886538a378a2d73e85697340a # v6.0.8
87-
88-
- name: Set up Node
89-
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
90-
with:
91-
node-version-file: ".nvmrc"
92-
cache: "pnpm"
93-
94-
- name: Install dependencies
95-
run: pnpm install --frozen-lockfile
87+
- name: Setup pnpm + Node + install
88+
uses: ./.github/actions/setup
9689

9790
- name: Run Lint
9891
run: pnpm run lint
@@ -140,17 +133,8 @@ jobs:
140133
- name: Checkout repository
141134
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
142135

143-
- name: Install pnpm
144-
uses: pnpm/action-setup@ac6db6d3c1f721f886538a378a2d73e85697340a # v6.0.8
145-
146-
- name: Set up Node
147-
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
148-
with:
149-
node-version-file: ".nvmrc"
150-
cache: "pnpm"
151-
152-
- name: Install dependencies
153-
run: pnpm install --frozen-lockfile
136+
- name: Setup pnpm + Node + install
137+
uses: ./.github/actions/setup
154138

155139
- name: Build library and server
156140
run: pnpm run build:lib && pnpm run build:server
@@ -167,17 +151,8 @@ jobs:
167151
- name: Checkout repository
168152
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
169153

170-
- name: Install pnpm
171-
uses: pnpm/action-setup@ac6db6d3c1f721f886538a378a2d73e85697340a # v6.0.8
172-
173-
- name: Set up Node
174-
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
175-
with:
176-
node-version-file: ".nvmrc"
177-
cache: "pnpm"
178-
179-
- name: Install dependencies
180-
run: pnpm install --frozen-lockfile
154+
- name: Setup pnpm + Node + install
155+
uses: ./.github/actions/setup
181156

182157
- name: Build
183158
run: pnpm run build:lib
@@ -206,8 +181,11 @@ jobs:
206181
- name: Run E2E tests
207182
run: pnpm --filter @tumaet/webapp exec playwright test tests/e2e/
208183

209-
- name: Run performance budget tests
210-
run: pnpm --filter @tumaet/webapp run test:perf
184+
# The chromium document-growth budget runs in the scheduled
185+
# nightly-checks workflow, not here — the Firefox budget (perf-firefox-
186+
# tests below) is the per-PR guard for the exam-freeze regression, since
187+
# Firefox is the engine the freeze was reported on and its GC surfaces
188+
# unbounded growth sooner.
211189

212190
- name: Upload Playwright report
213191
if: ${{ failure() }}
@@ -241,17 +219,8 @@ jobs:
241219
- name: Checkout repository
242220
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
243221

244-
- name: Install pnpm
245-
uses: pnpm/action-setup@ac6db6d3c1f721f886538a378a2d73e85697340a # v6.0.8
246-
247-
- name: Set up Node
248-
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
249-
with:
250-
node-version-file: ".nvmrc"
251-
cache: "pnpm"
252-
253-
- name: Install dependencies
254-
run: pnpm install --frozen-lockfile
222+
- name: Setup pnpm + Node + install
223+
uses: ./.github/actions/setup
255224

256225
- name: Build
257226
run: pnpm run build:lib
@@ -307,19 +276,13 @@ jobs:
307276
- name: Checkout repository
308277
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
309278

310-
- name: Install pnpm
311-
uses: pnpm/action-setup@ac6db6d3c1f721f886538a378a2d73e85697340a # v6.0.8
312-
313-
- name: Set up Node 22 LTS
314-
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
279+
- name: Setup pnpm + Node 22 + install
280+
# Node 22 (the library's published floor) instead of the repo's .nvmrc,
281+
# and engine-strict off so the root `>=24.15.0` gate doesn't reject it.
282+
uses: ./.github/actions/setup
315283
with:
316284
node-version: "22"
317-
cache: "pnpm"
318-
319-
- name: Install dependencies
320-
# `engine-strict=true` rejects Node 22 against the root `>=24.15.0`
321-
# gate; library workspace is `>=22`, so install with the flag off.
322-
run: pnpm install --frozen-lockfile --config.engine-strict=false
285+
engine-strict: "false"
323286

324287
- name: Build library
325288
run: pnpm --filter @tumaet/apollon run build
@@ -361,53 +324,11 @@ jobs:
361324
# of >10% (size-limit default) per artifact.
362325
run: pnpm run size
363326

364-
# Baseline regeneration: see docs/development/visual-tests.md.
365-
visual-regression-tests:
366-
needs: [detect-changes]
367-
if: needs.detect-changes.outputs.should-skip != 'true' && needs.detect-changes.outputs.code == 'true'
368-
runs-on: ubuntu-latest
369-
timeout-minutes: 30
370-
container:
371-
image: mcr.microsoft.com/playwright:v1.59.1-noble
372-
steps:
373-
- name: Checkout repository
374-
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
375-
376-
- name: Install Node 24
377-
shell: bash
378-
run: |
379-
set -euo pipefail
380-
curl -fsSL https://deb.nodesource.com/setup_24.x | bash -
381-
apt-get install -y nodejs
382-
node --version
383-
384-
- name: Install pnpm
385-
uses: pnpm/action-setup@ac6db6d3c1f721f886538a378a2d73e85697340a # v6.0.8
386-
387-
- name: Install dependencies
388-
run: pnpm install --frozen-lockfile
389-
390-
- name: Build
391-
run: pnpm run build:lib
392-
393-
- name: Run visual regression tests
394-
run: pnpm --filter @tumaet/webapp exec playwright test tests/visual/
395-
396-
- name: Upload Playwright report
397-
if: ${{ failure() }}
398-
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
399-
with:
400-
name: playwright-report-visual
401-
path: standalone/webapp/playwright-report/
402-
retention-days: 14
403-
404-
- name: Upload Playwright test results
405-
if: ${{ failure() }}
406-
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
407-
with:
408-
name: playwright-test-results-visual
409-
path: standalone/webapp/test-results/
410-
retention-days: 14
327+
# NOTE: visual-regression and the chromium document-growth budget moved to
328+
# the scheduled `nightly-checks.yml` workflow (run on demand via Run
329+
# workflow). They are rendering/cross-engine guards, not per-PR correctness
330+
# gates. The Firefox document-growth budget (the exam-freeze guard) stays
331+
# per-PR above.
411332

412333
# Gate job — set this as the required status check in branch protection.
413334
# Correctly handles skipped jobs (docs-only, duplicate runs).
@@ -424,7 +345,6 @@ jobs:
424345
library-node22-compat,
425346
e2e-tests,
426347
perf-firefox-tests,
427-
visual-regression-tests,
428348
]
429349
if: always()
430350
steps:

‎docs/contributor/deployment/github-actions.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ Deployments are fully automatic on merge to `main`; production promotion is one
2020
`version-monotonicity.yml` guards every PR by failing if a workspace
2121
`package.json` version moves backwards.
2222

23+
`nightly-checks.yml` runs the visual-regression and chromium performance-budget
24+
guards on a nightly schedule (and on demand via **Run workflow** against any
25+
branch), opening a deduplicated tracking issue if a scheduled run fails.
26+
2327
See [Releases](npm-publishing) for the release-cut procedure.
2428

2529
## Compose files

0 commit comments

Comments
 (0)