visual-regression.yml

name: Visual regression

on:
  pull_request:
    # `closed` is included so the cleanup job can remove the PR's published
    # report from gh-pages; the capture job skips the `closed` event.
    types: [opened, synchronize, reopened, closed]

concurrency:
  group: gh-pages
  cancel-in-progress: false

jobs:
  visual-regression:
    name: Capture, diff, and publish report
    if: github.event.action != 'closed'
    runs-on: ubuntu-latest
    container:
      image: cypress/browsers:latest
      options: --user 1001
    permissions:
      contents: write
      pull-requests: write
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - uses: pnpm/action-setup@v6
      - uses: actions/setup-node@v6
        with:
          node-version: '24'
          cache: 'pnpm'

      - name: Cache Cypress binary
        uses: actions/cache@v5
        with:
          path: ~/.cache/Cypress
          key: ${{ runner.os }}-cypress-${{ hashFiles('regression-test/package-lock.json') }}

      - name: Build InstUI
        run: pnpm install --frozen-lockfile && pnpm run bootstrap

      - name: Install regression-test dependencies
        run: npm ci
        working-directory: regression-test

      - name: Install Cypress binary
        run: npx cypress install
        working-directory: regression-test

      - name: Fetch baselines
        run: |
          mkdir -p regression-test/.baselines
          if git fetch origin visual-baselines 2>/dev/null; then
            git worktree add /tmp/baselines-wt origin/visual-baselines
            cp -r /tmp/baselines-wt/screenshots/. regression-test/.baselines/ 2>/dev/null || echo "::warning::baselines branch has no screenshots/ dir"
            git worktree remove --force /tmp/baselines-wt
          else
            echo "::warning::visual-baselines branch does not exist — first run, everything will be 'added'"
          fi

      - name: Run Cypress
        id: cypress
        uses: cypress-io/github-action@v7
        continue-on-error: true
        env:
          ELECTRON_EXTRA_LAUNCH_ARGS: "--remote-debugging-port=9222"
        with:
          install: false
          build: npm run build
          start: npm start
          working-directory: regression-test

      - name: Flatten screenshots
        run: |
          mkdir -p regression-test/.actual
          find regression-test/cypress/screenshots -name '*.png' -exec cp {} regression-test/.actual/ \;

      - name: Diff and generate report
        id: diff
        working-directory: regression-test
        run: >
          npx ui-scripts visual-diff
          --actual-dir .actual
          --baseline-dir .baselines
          --output-dir .report
          --no-fail-on-missing-baseline
          --pr-number ${{ github.event.pull_request.number }}
          --pr-url ${{ github.event.pull_request.html_url }}
          --meta cypress/meta.json
          --a11y cypress/a11y.json
          --source-base-url https://github.com/${{ github.repository }}/blob/${{ github.head_ref }}/regression-test/src/app
          --facets canvas,light,dark
          --app-path app
        continue-on-error: true

      # Rebuild the app with a basePath matching its published location, then
      # drop it next to the report so the report's "HTML" view can iframe the
      # live rendered page. Best-effort: a build hiccup must not block the report
      # (the HTML view would just 404 for that run).
      - name: Build and bundle app for the report's HTML view
        working-directory: regression-test
        env:
          REGRESSION_APP_BASE_PATH: /visual-regression/pr-${{ github.event.pull_request.number }}/app
        run: |
          npm run build
          cp -r out .report/app
        continue-on-error: true

      - name: Publish report to gh-pages
        uses: peaceiris/actions-gh-pages@v4
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: regression-test/.report
          publish_branch: gh-pages
          destination_dir: visual-regression/pr-${{ github.event.pull_request.number }}
          keep_files: true
          commit_message: "visual-regression: PR #${{ github.event.pull_request.number }} (${{ github.sha }})"

      - name: Read summary
        id: summary
        env:
          PR_NUMBER: ${{ github.event.pull_request.number }}
        run: |
          # Quoted heredoc, not `node -e "..."`: in a double-quoted shell string
          # the backticks around the rule name below would be command
          # substitution, and the rule name would vanish from the comment.
          node <<'NODE'
            const fs = require('fs');
            const { summary: s, results, a11y = [] } = JSON.parse(fs.readFileSync('regression-test/.report/summary.json', 'utf8'));
            const base = 'https://instructure.design/visual-regression/pr-' + process.env.PR_NUMBER;
            const interesting = results.filter(r => r.status === 'changed' || r.status === 'removed');
            let md = '';
            if (interesting.length) {
              md = '<details>\n<summary>Diff images (' + interesting.length + ')</summary>\n\n';
              for (const r of interesting) {
                if (r.status === 'changed') {
                  md += '#### ' + r.name + ' — ' + r.numDiff + ' pixels differ\n\n';
                  md += '![](' + base + '/diff/' + r.name + ')\n\n';
                } else {
                  md += '#### ' + r.name + ' — baseline no longer produced\n\n';
                }
              }
              md += '</details>';
            }
            // Name the a11y rules in the comment. Numbers are per offending
            // element; the same rule across themes is one row, matching the
            // report's overview panel.
            let a11yMd = '✅ No violations.';
            if (a11y.length) {
              a11yMd = '⚠️ **' + s.a11yViolations + ' violation' + (s.a11yViolations === 1 ? '' : 's') +
                '** on ' + s.a11yPages + ' screenshot' + (s.a11yPages === 1 ? '' : 's') + '.\n\n' +
                '| Impact | Issue | Rule | Elements | Screenshots |\n|---|---|---|---:|---:|\n' +
                a11y.map(r => '| ' + r.impact + ' | ' + r.title + ' | [`' + r.rule + '`](' + r.helpUrl + ') | ' +
                  r.nodes + ' | ' + r.screens + ' |').join('\n');
            }
            const lines = [
              'total=' + s.total,
              'unchanged=' + s.unchanged,
              'changed=' + s.changed,
              'added=' + s.added,
              'removed=' + s.removed,
              'status=' + (s.changed > 0 || s.removed > 0 ? '⚠️ **Changes detected.**' : '✅ **No changes.**')
            ];
            let out = lines.join('\n') + '\n';
            out += 'diff_images<<DIFFIMAGES_EOF\n' + md + '\nDIFFIMAGES_EOF\n';
            out += 'a11y<<A11Y_EOF\n' + a11yMd + '\nA11Y_EOF\n';
            fs.appendFileSync(process.env.GITHUB_OUTPUT, out);
          NODE

      - name: Post PR comment
        uses: marocchino/sticky-pull-request-comment@v3
        with:
          header: visual-regression
          message: |
            ## Visual regression report

            **Cypress suite:** ${{ steps.cypress.outcome == 'failure' && format('❌ **Failing** — see the [run logs]({0}/{1}/actions/runs/{2}).', github.server_url, github.repository, github.run_id) || '✅ Passing' }}

            **Visual diff:** ${{ steps.summary.outputs.status }}

            | Status | Count |
            |---|---|
            | Unchanged | ${{ steps.summary.outputs.unchanged }} |
            | Changed | ${{ steps.summary.outputs.changed }} |
            | New | ${{ steps.summary.outputs.added }} |
            | Removed | ${{ steps.summary.outputs.removed }} |

            **Accessibility (axe):** ${{ steps.summary.outputs.a11y }}

            📊 **[View full report](https://instructure.design/visual-regression/pr-${{ github.event.pull_request.number }}/index.html)** — click a screenshot's ⚠ badge to see each violation boxed on the image, with the offending element named and contrast failures shown as color swatches.

            ${{ steps.summary.outputs.diff_images }}

            <sub>Baselines come from the `visual-baselines` branch. They refresh on every merge to `master`. The <b>Cypress suite</b> line covers the a11y and console-error assertions — a ❌ there means the suite found real issues even if the visual diff is clean.</sub>

      - name: Fail if regressions or Cypress failed
        if: steps.diff.outcome == 'failure' || steps.cypress.outcome == 'failure'
        run: exit 1

  cleanup-report:
    name: Remove report when PR closes
    if: github.event.action == 'closed'
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - uses: actions/checkout@v6
        with:
          ref: gh-pages
          fetch-depth: 0
      - name: Remove this PR's report directory
        run: |
          DIR="visual-regression/pr-${{ github.event.pull_request.number }}"
          if [ ! -d "$DIR" ]; then
            echo "No report directory ($DIR); nothing to remove."
            exit 0
          fi
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git rm -rq "$DIR"
          git commit -m "visual-regression: remove report for closed PR #${{ github.event.pull_request.number }}"
          # gh-pages is written by several workflows; rebase before pushing in
          # case it moved since checkout.
          git pull --rebase origin gh-pages
          git push