test(INSTUI-5155): SSR layout shift lab — INVESTIGATION ONLY, not for merge #2768
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Visual regression | |
| on: | |
| pull_request: | |
| # `closed` is included so the cleanup job can remove the PR's published | |
| # report from gh-pages; the capture job skips the `closed` event. | |
| types: [opened, synchronize, reopened, closed] | |
| concurrency: | |
| group: gh-pages | |
| cancel-in-progress: false | |
| jobs: | |
| visual-regression: | |
| name: Capture, diff, and publish report | |
| if: github.event.action != 'closed' | |
| runs-on: ubuntu-latest | |
| container: | |
| image: cypress/browsers:latest | |
| options: --user 1001 | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - uses: pnpm/action-setup@v6 | |
| - uses: actions/setup-node@v6 | |
| with: | |
| node-version: '24' | |
| cache: 'pnpm' | |
| - name: Cache Cypress binary | |
| uses: actions/cache@v5 | |
| with: | |
| path: ~/.cache/Cypress | |
| key: ${{ runner.os }}-cypress-${{ hashFiles('regression-test/package-lock.json') }} | |
| - name: Build InstUI | |
| run: pnpm install --frozen-lockfile && pnpm run bootstrap | |
| - name: Install regression-test dependencies | |
| run: npm ci | |
| working-directory: regression-test | |
| - name: Install Cypress binary | |
| run: npx cypress install | |
| working-directory: regression-test | |
| - name: Fetch baselines | |
| run: | | |
| mkdir -p regression-test/.baselines | |
| if git fetch origin visual-baselines 2>/dev/null; then | |
| git worktree add /tmp/baselines-wt origin/visual-baselines | |
| cp -r /tmp/baselines-wt/screenshots/. regression-test/.baselines/ 2>/dev/null || echo "::warning::baselines branch has no screenshots/ dir" | |
| git worktree remove --force /tmp/baselines-wt | |
| else | |
| echo "::warning::visual-baselines branch does not exist — first run, everything will be 'added'" | |
| fi | |
| - name: Run Cypress | |
| id: cypress | |
| uses: cypress-io/github-action@v7 | |
| continue-on-error: true | |
| env: | |
| ELECTRON_EXTRA_LAUNCH_ARGS: "--remote-debugging-port=9222" | |
| with: | |
| install: false | |
| build: npm run build | |
| start: npm start | |
| working-directory: regression-test | |
| - name: Flatten screenshots | |
| run: | | |
| mkdir -p regression-test/.actual | |
| find regression-test/cypress/screenshots -name '*.png' -exec cp {} regression-test/.actual/ \; | |
| - name: Diff and generate report | |
| id: diff | |
| working-directory: regression-test | |
| run: > | |
| npx ui-scripts visual-diff | |
| --actual-dir .actual | |
| --baseline-dir .baselines | |
| --output-dir .report | |
| --no-fail-on-missing-baseline | |
| --pr-number ${{ github.event.pull_request.number }} | |
| --pr-url ${{ github.event.pull_request.html_url }} | |
| --meta cypress/meta.json | |
| --a11y cypress/a11y.json | |
| --source-base-url https://github.com/${{ github.repository }}/blob/${{ github.head_ref }}/regression-test/src/app | |
| --facets canvas,light,dark | |
| --app-path app | |
| continue-on-error: true | |
| # Rebuild the app with a basePath matching its published location, then | |
| # drop it next to the report so the report's "HTML" view can iframe the | |
| # live rendered page. Best-effort: a build hiccup must not block the report | |
| # (the HTML view would just 404 for that run). | |
| - name: Build and bundle app for the report's HTML view | |
| working-directory: regression-test | |
| env: | |
| REGRESSION_APP_BASE_PATH: /visual-regression/pr-${{ github.event.pull_request.number }}/app | |
| run: | | |
| npm run build | |
| cp -r out .report/app | |
| continue-on-error: true | |
| - name: Publish report to gh-pages | |
| uses: peaceiris/actions-gh-pages@v4 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| publish_dir: regression-test/.report | |
| publish_branch: gh-pages | |
| destination_dir: visual-regression/pr-${{ github.event.pull_request.number }} | |
| keep_files: true | |
| commit_message: "visual-regression: PR #${{ github.event.pull_request.number }} (${{ github.sha }})" | |
| - name: Read summary | |
| id: summary | |
| env: | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| run: | | |
| # Quoted heredoc, not `node -e "..."`: in a double-quoted shell string | |
| # the backticks around the rule name below would be command | |
| # substitution, and the rule name would vanish from the comment. | |
| node <<'NODE' | |
| const fs = require('fs'); | |
| const { summary: s, results, a11y = [] } = JSON.parse(fs.readFileSync('regression-test/.report/summary.json', 'utf8')); | |
| const base = 'https://instructure.design/visual-regression/pr-' + process.env.PR_NUMBER; | |
| const interesting = results.filter(r => r.status === 'changed' || r.status === 'removed'); | |
| let md = ''; | |
| if (interesting.length) { | |
| md = '<details>\n<summary>Diff images (' + interesting.length + ')</summary>\n\n'; | |
| for (const r of interesting) { | |
| if (r.status === 'changed') { | |
| md += '#### ' + r.name + ' — ' + r.numDiff + ' pixels differ\n\n'; | |
| md += '\n\n'; | |
| } else { | |
| md += '#### ' + r.name + ' — baseline no longer produced\n\n'; | |
| } | |
| } | |
| md += '</details>'; | |
| } | |
| // Name the a11y rules in the comment. Numbers are per offending | |
| // element; the same rule across themes is one row, matching the | |
| // report's overview panel. | |
| let a11yMd = '✅ No violations.'; | |
| if (a11y.length) { | |
| a11yMd = '⚠️ **' + s.a11yViolations + ' violation' + (s.a11yViolations === 1 ? '' : 's') + | |
| '** on ' + s.a11yPages + ' screenshot' + (s.a11yPages === 1 ? '' : 's') + '.\n\n' + | |
| '| Impact | Issue | Rule | Elements | Screenshots |\n|---|---|---|---:|---:|\n' + | |
| a11y.map(r => '| ' + r.impact + ' | ' + r.title + ' | [`' + r.rule + '`](' + r.helpUrl + ') | ' + | |
| r.nodes + ' | ' + r.screens + ' |').join('\n'); | |
| } | |
| const lines = [ | |
| 'total=' + s.total, | |
| 'unchanged=' + s.unchanged, | |
| 'changed=' + s.changed, | |
| 'added=' + s.added, | |
| 'removed=' + s.removed, | |
| 'status=' + (s.changed > 0 || s.removed > 0 ? '⚠️ **Changes detected.**' : '✅ **No changes.**') | |
| ]; | |
| let out = lines.join('\n') + '\n'; | |
| out += 'diff_images<<DIFFIMAGES_EOF\n' + md + '\nDIFFIMAGES_EOF\n'; | |
| out += 'a11y<<A11Y_EOF\n' + a11yMd + '\nA11Y_EOF\n'; | |
| fs.appendFileSync(process.env.GITHUB_OUTPUT, out); | |
| NODE | |
| - name: Post PR comment | |
| uses: marocchino/sticky-pull-request-comment@v3 | |
| with: | |
| header: visual-regression | |
| message: | | |
| ## Visual regression report | |
| **Cypress suite:** ${{ steps.cypress.outcome == 'failure' && format('❌ **Failing** — see the [run logs]({0}/{1}/actions/runs/{2}).', github.server_url, github.repository, github.run_id) || '✅ Passing' }} | |
| **Visual diff:** ${{ steps.summary.outputs.status }} | |
| | Status | Count | | |
| |---|---| | |
| | Unchanged | ${{ steps.summary.outputs.unchanged }} | | |
| | Changed | ${{ steps.summary.outputs.changed }} | | |
| | New | ${{ steps.summary.outputs.added }} | | |
| | Removed | ${{ steps.summary.outputs.removed }} | | |
| **Accessibility (axe):** ${{ steps.summary.outputs.a11y }} | |
| 📊 **[View full report](https://instructure.design/visual-regression/pr-${{ github.event.pull_request.number }}/index.html)** — click a screenshot's ⚠ badge to see each violation boxed on the image, with the offending element named and contrast failures shown as color swatches. | |
| ${{ steps.summary.outputs.diff_images }} | |
| <sub>Baselines come from the `visual-baselines` branch. They refresh on every merge to `master`. The <b>Cypress suite</b> line covers the a11y and console-error assertions — a ❌ there means the suite found real issues even if the visual diff is clean.</sub> | |
| - name: Fail if regressions or Cypress failed | |
| if: steps.diff.outcome == 'failure' || steps.cypress.outcome == 'failure' | |
| run: exit 1 | |
| cleanup-report: | |
| name: Remove report when PR closes | |
| if: github.event.action == 'closed' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| ref: gh-pages | |
| fetch-depth: 0 | |
| - name: Remove this PR's report directory | |
| run: | | |
| DIR="visual-regression/pr-${{ github.event.pull_request.number }}" | |
| if [ ! -d "$DIR" ]; then | |
| echo "No report directory ($DIR); nothing to remove." | |
| exit 0 | |
| fi | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git rm -rq "$DIR" | |
| git commit -m "visual-regression: remove report for closed PR #${{ github.event.pull_request.number }}" | |
| # gh-pages is written by several workflows; rebase before pushing in | |
| # case it moved since checkout. | |
| git pull --rebase origin gh-pages | |
| git push |