Merge pull request #65 from ChanMeng666/ci/nightly-prod-e2e

ci(nightly): run the read-only Playwright subset against production

Author: ChanMeng666

.github/workflows/nightly.yml

@@ -160,33 +160,104 @@ jobs:
       - run: npm test
 
   # ---------------------------------------------------------------------------
-  # PLACEHOLDER — read-only end-to-end checks against production (Playwright).
+  # Read-only end-to-end checks against PRODUCTION (Playwright).
   #
-  # DISABLED ON PURPOSE. This repo has no Playwright config or browser install step
-  # yet; those land with workstreams C (playground E2E) and D (docs E2E). Enabling
-  # this job before then would just fail on a missing config, so it stays commented
-  # out rather than guessing at spec paths that do not exist.
+  # `prod-smoke` above proves the routes answer; this proves the sites still WORK —
+  # the playground boots and compiles a plan in a real browser, the embed page draws
+  # from a `#z=` link, the docs site's machine artifacts are byte-identical to this
+  # checkout, and `<ArchLive>` hydrates into a live compiler rather than a dead
+  # `<pre>`. ci.yml runs the same specs against a locally BUILT site before merge;
+  # this runs a subset against what visitors actually get.
   #
-  # To enable, AFTER C/D have landed their configs: delete the comment markers, point
-  # `--config`/`--project` at whatever those workstreams actually committed (do not
-  # invent paths here), and keep the run READ-ONLY — this drives *production*, so no
-  # form submits, no writes, no state mutation, only navigation and assertions.
+  # Two rules govern this job, and neither is negotiable:
   #
-  # e2e-prod:
-  #   name: E2E against production (read-only)
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
-  #     - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
-  #       with:
-  #         node-version: 22
-  #         cache: npm
-  #     - run: npm ci
-  #     - run: npx playwright install --with-deps chromium
-  #     - run: <the E2E command workstream C/D committed>
-  #       env:
-  #         PLAYWRIGHT_BASE_URL: https://playground.archlang.uk
+  #   1. READ-ONLY. It drives production, so the run is restricted to `--grep @prod`
+  #      — a tag each spec file carries a comment explaining. Tagged cases navigate,
+  #      read and assert; nothing downloads, writes the clipboard, submits a form or
+  #      depends on persisted state. Never widen this to the full suite.
+  #   2. NO BUILD. `E2E_BASE_URL` makes each config drop its `webServer` and point at
+  #      the live origin, so there is nothing to build and no preview server — which
+  #      is also what makes the docs byte-equality cases meaningful here: they compare
+  #      PRODUCTION's bytes against THIS checkout of `main`, i.e. they fail when a
+  #      deploy has silently gone stale.
+  #
+  # A red night here means production is broken or stale, not that a PR is bad — it
+  # lands in the same pinned issue the `report` job writes.
   # ---------------------------------------------------------------------------
+  e2e-prod:
+    name: E2E against production (read-only)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - name: Set up Node.js
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          # Node 22, matching ci.yml's e2e-docs leg: archlive.spec.ts imports the
+          # playground's `#z=` decoder, which needs `DecompressionStream("deflate-raw")`
+          # (Node ≥ 21.2) at module scope even when the tagged subset never calls it.
+          node-version: 22
+          cache: npm
+
+      - name: Install dependencies
+        run: npm ci
+
+      # NB: no `npm run build`, no site build. With E2E_BASE_URL set, neither config
+      # starts a preview server — the specs drive the live origins directly.
+
+      # Keyed on the RESOLVED @playwright/test version: a browser build is pinned
+      # to its driver, so a version bump must miss the cache rather than reuse an
+      # incompatible download. (Same key as ci.yml's e2e jobs, so a nightly can hit
+      # a cache a PR run populated.)
+      - name: Resolve Playwright version
+        id: pw
+        run: |
+          version=$(node -p "require('@playwright/test/package.json').version")
+          echo "version=$version" >> "$GITHUB_OUTPUT"
+
+      - name: Cache Playwright browsers
+        id: pw-cache
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-${{ runner.os }}-${{ steps.pw.outputs.version }}
+
+      # On a cache hit the browser binary is already there but the OS packages it
+      # links against are not (they live outside the cached path), so the two
+      # cases install different things.
+      - name: Install Chromium (browser + system deps)
+        if: steps.pw-cache.outputs.cache-hit != 'true'
+        run: npx playwright install --with-deps chromium
+
+      - name: Install Chromium system deps only (cached browser)
+        if: steps.pw-cache.outputs.cache-hit == 'true'
+        run: npx playwright install-deps chromium
+
+      - name: Playground — https://playground.archlang.uk
+        run: npx playwright test -c playground --grep @prod
+        env:
+          E2E_BASE_URL: https://playground.archlang.uk
+
+      # `!cancelled()`, not the default: if the playground is down, the docs site's
+      # verdict is still worth having tonight. Both still fail the job.
+      - name: Docs site — https://archlang.uk
+        if: ${{ !cancelled() }}
+        run: npx playwright test -c docs-site --grep @prod
+        env:
+          E2E_BASE_URL: https://archlang.uk
+
+      - name: Upload Playwright report
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: playwright-report-prod
+          path: |
+            playground/playwright-report
+            playground/test-results
+            docs-site/playwright-report
+            docs-site/test-results
+          retention-days: 7
+          if-no-files-found: ignore
 
   # ---------------------------------------------------------------------------
   # One writer for the pinned issue. Folding "advisories found" and "a job failed"
@@ -195,7 +266,7 @@ jobs:
   # ---------------------------------------------------------------------------
   report:
     name: Report
-    needs: [prod-smoke, audit, secrets, full-matrix]
+    needs: [prod-smoke, audit, secrets, full-matrix, e2e-prod]
     if: always()
     runs-on: ubuntu-latest
     permissions:

AGENTS.md

@@ -242,6 +242,9 @@ npm run typecheck:all    # full-repo typecheck: root tsconfig.dev.json (src+test
 npm run test:coverage    # vitest run --coverage — report-only v8 coverage over src/ (CI: Node 22 leg)
 npm run e2e:playground   # Playwright E2E against the built playground (build core + playground:build:only first)
 npm run e2e:docs         # Playwright E2E against the built docs site (build core + docs:build:only first)
+                         # Set E2E_BASE_URL=<origin> on either and the config drops its webServer and
+                         # drives THAT origin — no build, no preview server. Nightly pairs it with
+                         # `--grep @prod` (the READ-ONLY subset) against the live sites.
 
 npm run playground:dev   # build core, then run the Vite playground dev server
 npm run docs:build       # build core, then build the VitePress docs site
@@ -252,8 +255,13 @@ CI runs five PR gates in parallel: the Node 18/20/22 test matrix (+ report-only
 the 22 leg), a **builds** job (all four workspaces compile, `typecheck:all`, MCP dist-resource
 freshness, VS Code bundle tests), a **Windows** leg (tests + drift at runner-default line
 endings), two **Playwright E2E** jobs (playground, docs), and **CodeQL**; `nightly.yml` adds
-production smoke (`scripts/smoke.mjs`), `npm audit` (report-only issue), gitleaks and a full
-OS×Node matrix.
+production smoke (`scripts/smoke.mjs`), `npm audit` (report-only issue), gitleaks, a full
+OS×Node matrix, and **`e2e-prod`** — the `@prod`-tagged READ-ONLY Playwright subset re-run
+against the live `playground.archlang.uk` / `archlang.uk` via `E2E_BASE_URL` (no build, no
+preview server). Only tag a case `@prod` if it purely navigates, reads and asserts: no
+downloads, no clipboard, no persisted-state or typing flows. Its docs half is also a
+**deploy-staleness probe** — the raw `/<page>.md` cases compare production's bytes against
+the checkout's, so a stale deploy fails the night.
 
 Export to other formats from the CLI: `-f svg|dxf|txt|pdf|png` (`txt` is the
 zero-dep ASCII plan; `pdf` needs optional `pdfkit`; `png` needs optional

docs-site/e2e/archlive.spec.ts

@@ -10,6 +10,16 @@
  * and this component's inline copy), and `test/share-codec.test.ts` welds the copies by
  * extracting this one's body and evaluating it. That is a static check. This is the runtime
  * one — the real button, in the real bundle, decoded by the playground's real codec.
+ *
+ * TAG `@prod` — ONE DESCRIBE ONLY: "the widget hydrates into a live compiler".
+ * `.github/workflows/nightly.yml` re-runs the tagged subset against the LIVE
+ * https://archlang.uk (`E2E_BASE_URL` + `--grep @prod`), and hydration is the case that
+ * answers the production question — "is what is deployed a live compiler, or a dead <pre>?"
+ * — by loading two pages and looking. The three describes below it are deliberately
+ * UNTAGGED: they all TYPE into the widget (`setArchLiveSource`), including the XSS probe,
+ * and the nightly subset is meant to stay small, read-only and boringly deterministic.
+ * A new describe here does NOT get the tag by default — add it only if the case is pure
+ * load-and-look.
  */
 import { readFileSync } from "node:fs";
 import { join } from "node:path";
@@ -33,7 +43,7 @@ const EXPLICIT_PAGE = "/guide";
 /** A page whose ArchLives come from plain ```arch fences (fallback slot → swap on mount). */
 const FENCE_PAGE = "/relational";
 
-test.describe("the widget hydrates into a live compiler", () => {
+test.describe("the widget hydrates into a live compiler", { tag: "@prod" }, () => {
   test("a plain ```arch fence ships as a highlighted <pre> and becomes an editor on mount", async ({
     page,
     request,

docs-site/e2e/routes.spec.ts

@@ -13,6 +13,16 @@
  *
  * Every route list here is DERIVED from sync-docs.mjs (see fixtures.ts) — adding a page or
  * an example extends this suite automatically.
+ *
+ * TAG `@prod` — WHOLE FILE (one tag per describe, so a describe added later must opt in
+ * explicitly). `.github/workflows/nightly.yml` re-runs the tagged subset against the LIVE
+ * https://archlang.uk (`E2E_BASE_URL` + `--grep @prod`). Everything here is a plain GET or a
+ * navigation, which is why the whole file qualifies — and against production it buys one
+ * thing more than it does in CI: the byte-equality cases below compare the DEPLOYED bytes
+ * against the LOCAL checkout of `main`, so a docs deploy that silently went stale (a failed
+ * build, a rolled-back deployment, a `public/` copy that never re-synced) fails here. That
+ * staleness probe is the point, not an accident — do not relax those assertions to make a
+ * red night green.
  */
 import { expect, test } from "@playwright/test";
 import {
@@ -24,7 +34,7 @@ import {
   ROOT_COPY_ROUTES,
 } from "./fixtures.js";
 
-test.describe("the homepage renders", () => {
+test.describe("the homepage renders", { tag: "@prod" }, () => {
   test("/ serves HTML carrying the hand-written agents band", async ({ page }) => {
     const res = await page.goto("/");
     expect(res?.status()).toBe(200);
@@ -35,7 +45,7 @@ test.describe("the homepage renders", () => {
   });
 });
 
-test.describe("machine-readable root artifacts", () => {
+test.describe("machine-readable root artifacts", { tag: "@prod" }, () => {
   test("every ROOT_COPIES route is served and non-empty", async ({ request }) => {
     // Derived, so /llms.txt, /llms-full.txt, both schemas and the grammar are all covered
     // — and so is anything added to that table later.
@@ -76,7 +86,7 @@ test.describe("machine-readable root artifacts", () => {
   });
 });
 
-test.describe("the raw /<page>.md copies serve the canonical markdown", () => {
+test.describe("the raw /<page>.md copies serve the canonical markdown", { tag: "@prod" }, () => {
   test("there are as many routes as sync-docs publishes pages", () => {
     expect(PAGE_ROUTES.length).toBe(7);
   });
@@ -101,7 +111,7 @@ test.describe("the raw /<page>.md copies serve the canonical markdown", () => {
   }
 });
 
-test.describe("the example gallery", () => {
+test.describe("the example gallery", { tag: "@prod" }, () => {
   test("the derived gallery is non-trivial and holds the flagships", () => {
     expect(GALLERY_EXAMPLES.length).toBeGreaterThan(5);
     for (const flagship of ["studio", "museum", "aquarium", "gallery-l"]) {

docs-site/playwright.config.ts

@@ -23,7 +23,19 @@ import { defineConfig, devices } from "@playwright/test";
  * Serving the built output rather than the dev server is deliberate: SSR'd HTML,
  * hydration and the `public/` passthrough are all part of what is under test, and
  * only the production build exercises them the way a visitor does.
+ *
+ * ESCAPE HATCH — `E2E_BASE_URL`: set it and the suite drives THAT origin with no
+ * local server and no build at all (the `webServer` key is omitted, not disabled
+ * — Playwright would otherwise still try to start it). The nightly workflow uses
+ * this to run the READ-ONLY `@prod` subset against https://archlang.uk, which is
+ * why every spec navigates relatively (`page.goto("/")`, `request.get("/llms.txt")`)
+ * rather than to a hardcoded localhost URL. Note what that turns the raw-`.md`
+ * byte-equality cases into: they compare PRODUCTION's bytes against the LOCAL
+ * checkout's, i.e. a deployment-staleness probe. Unset, everything below behaves
+ * exactly as before.
  */
+const externalBaseUrl = process.env.E2E_BASE_URL;
+
 export default defineConfig({
   testDir: "./e2e",
   // Every spec asserts on a stateless page; they may run in parallel.
@@ -34,27 +46,33 @@ export default defineConfig({
   workers: process.env.CI ? 2 : undefined,
   reporter: process.env.CI ? [["html", { open: "never" }], ["list"]] : [["list"]],
   use: {
-    baseURL: "http://localhost:4174",
+    baseURL: externalBaseUrl ?? "http://localhost:4174",
     trace: "retain-on-failure",
     screenshot: "only-on-failure",
   },
   // Chromium only: the site's own behaviour is what these specs test, not browser
   // support for `CompressionStream` (the `#z=` codec already has a documented
   // fallback and its own unit gate in test/share-codec.test.ts).
   projects: [{ name: "chromium", use: { ...devices["Desktop Chrome"] } }],
-  webServer: {
-    // 4174, not the Vite default 4173: that is the PLAYGROUND E2E's port
-    // (playground/playwright.config.ts), and running both suites locally must not
-    // have one silently answer the other's requests. `--strictPort` makes a clash
-    // fail loudly instead of drifting to the next free port, which would hang the
-    // readiness probe until it timed out.
-    //
-    // NB the URL says `localhost`, not `127.0.0.1`: the preview server binds the
-    // hostname, not the loopback address, so a 127.0.0.1 probe never connects.
-    command: "npm run preview -- --port 4174 --strictPort",
-    cwd: ".",
-    url: "http://localhost:4174/",
-    reuseExistingServer: !process.env.CI,
-    timeout: 60_000,
-  },
+  // Spread, not a `webServer: undefined` key: with an external base URL there is
+  // nothing to start and nothing to build.
+  ...(externalBaseUrl
+    ? {}
+    : {
+        webServer: {
+          // 4174, not the Vite default 4173: that is the PLAYGROUND E2E's port
+          // (playground/playwright.config.ts), and running both suites locally must not
+          // have one silently answer the other's requests. `--strictPort` makes a clash
+          // fail loudly instead of drifting to the next free port, which would hang the
+          // readiness probe until it timed out.
+          //
+          // NB the URL says `localhost`, not `127.0.0.1`: the preview server binds the
+          // hostname, not the loopback address, so a 127.0.0.1 probe never connects.
+          command: "npm run preview -- --port 4174 --strictPort",
+          cwd: ".",
+          url: "http://localhost:4174/",
+          reuseExistingServer: !process.env.CI,
+          timeout: 60_000,
+        },
+      }),
 });

playground/e2e/boot.spec.ts

@@ -9,8 +9,17 @@ import { watchForProblems } from "./fixtures.js";
  * externalisation mistake, or a module-scope throw — none of which the unit
  * suite or `vite build` can see, because both succeed on a page that then dies
  * in the browser.
+ *
+ * TAG `@prod` — WHOLE FILE. `.github/workflows/nightly.yml` re-runs the tagged
+ * subset against the LIVE https://playground.archlang.uk (`E2E_BASE_URL` +
+ * `--grep @prod`), so a tag here is a promise about production, not just about a
+ * preview server. Every case below keeps it: they navigate, read the DOM and
+ * assert — nothing downloads, writes to the clipboard, or depends on state
+ * surviving a reload. (`selectOption` on the examples picker mutates only this
+ * throwaway browser context.) Do NOT tag a case that downloads a file, reads or
+ * writes the clipboard, or asserts on localStorage persistence.
  */
-test.describe("boot", () => {
+test.describe("boot", { tag: "@prod" }, () => {
   test("renders the default example into the stage", async ({ page }) => {
     await page.goto("/");
     const svg = page.locator(".pz-stage svg");