feat: Add a11y regression scan suite

.github/workflows/a11y-scan.yml

@@ -1,4 +1,4 @@
-name: Run Accessibility Scans (IBM Equal Access)
+name: A11y / Playwright Scans
 
 on:
   # TEMP: remove before merging — allows testing the workflow from this
@@ -17,7 +17,7 @@ on:
         required: false
         type: string
       assert:
-        description: "Fail the run on a11y violations (RUN_A11Y_ASSERT)"
+        description: "Fail the run when scans differ from committed baselines"
         required: false
         type: boolean
         default: false
@@ -32,7 +32,7 @@ env:
 
 jobs:
   a11y-scan:
-    name: IBM Accessibility Scans
+    name: IBM Equal Access
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -122,16 +122,17 @@ jobs:
         shell: bash
         env:
           RUN_A11Y: "true"
-          RUN_A11Y_ASSERT: ${{ inputs.assert == true && 'true' || 'false' }}
-          LANGFLOW_DEACTIVE_TRACING: "true"
+          RUN_A11Y_ASSERT: ${{ github.event_name == 'workflow_dispatch' && inputs.assert && 'true' || 'false' }}
+          LANGFLOW_DEACTIVATE_TRACING: "true"
         run: |
           cd src/frontend
           # Only specs that call runA11yScan produce reports; discover them
           # so new scan hosts are picked up without editing this workflow.
           SCAN_SPECS=$(grep -rl "runA11yScan(" tests --include="*.spec.ts" | sort)
           echo "Specs with a11y scans:"
           echo "$SCAN_SPECS"
-          npx playwright test $SCAN_SPECS --project=chromium --workers 1 --retries=2
+          test -n "$SCAN_SPECS"
+          npx playwright test $SCAN_SPECS --project=chromium --workers=1 --retries=2
 
       - name: Build Aggregated A11y Summary
         if: always()

.github/workflows/a11y-unit-tests.yml

@@ -1,4 +1,4 @@
-name: Run Frontend A11y Unit Tests
+name: A11y / Unit Tests
 
 on:
   pull_request:
@@ -24,7 +24,7 @@ env:
 
 jobs:
   a11y-unit-tests:
-    name: Frontend A11y Unit Tests (jest-axe)
+    name: Frontend jest-axe
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -53,6 +53,6 @@ jobs:
         if: always()
         with:
           report_paths: "src/frontend/test-results/junit.xml"
-          check_name: "Frontend A11y Unit Test Results"
+          check_name: "A11y / Unit Test Results"
           fail_on_failure: true
           require_tests: true

Makefile.frontend

@@ -5,7 +5,7 @@
 FRONTEND_DIR = src/frontend
 NPM = npm
 
-.PHONY: install_frontend install_frontendci install_frontendc frontend_deps_check build_frontend run_frontend frontend frontendc format_frontend tests_frontend test_frontend test_frontend_watch test_frontend_coverage test_frontend_verbose test_frontend_ci test_frontend_clean test_frontend_file test_frontend_pattern test_frontend_snapshots test_frontend_config test_frontend_bail test_frontend_silent test_frontend_coverage_open help_frontend storybook storybook_build storybook_network
+.PHONY: install_frontend install_frontendci install_frontendc frontend_deps_check build_frontend run_frontend frontend frontendc format_frontend tests_frontend test_frontend_a11y_e2e test_frontend_a11y_e2e_blocking test_frontend_a11y_e2e_update test_frontend test_frontend_watch test_frontend_coverage test_frontend_verbose test_frontend_ci test_frontend_clean test_frontend_file test_frontend_pattern test_frontend_snapshots test_frontend_config test_frontend_bail test_frontend_silent test_frontend_coverage_open help_frontend storybook storybook_build storybook_network
 
 ######################
 # FRONTEND DEPENDENCIES
@@ -77,6 +77,17 @@ else
 	@cd $(FRONTEND_DIR) && npx playwright test --project=chromium
 endif
 
+test_frontend_a11y_e2e: frontend_deps_check ## run non-blocking Playwright a11y scans and reports
+	@cd $(FRONTEND_DIR) && SCAN_SPECS="$$(grep -rl "runA11yScan(" tests --include="*.spec.ts" | sort)" && echo "Specs with a11y scans:" && echo "$$SCAN_SPECS" && test -n "$$SCAN_SPECS" && RUN_A11Y=true RUN_A11Y_ASSERT=false npx playwright test $$SCAN_SPECS --project=chromium --workers=1
+
+test_frontend_a11y_e2e_blocking: frontend_deps_check ## run blocking Playwright a11y scans against baselines
+	@cd $(FRONTEND_DIR) && SCAN_SPECS="$$(grep -rl "runA11yScan(" tests --include="*.spec.ts" | sort)" && echo "Specs with a11y scans:" && echo "$$SCAN_SPECS" && test -n "$$SCAN_SPECS" && RUN_A11Y=true RUN_A11Y_ASSERT=true npx playwright test $$SCAN_SPECS --project=chromium --workers=1
+
+test_frontend_a11y_e2e_update: frontend_deps_check ## refresh Playwright a11y reports used as baselines
+	@cd $(FRONTEND_DIR) && rm -rf coverage/accessibility-reports && SCAN_SPECS="$$(grep -rl "runA11yScan(" tests --include="*.spec.ts" | sort)" && echo "Specs with a11y scans:" && echo "$$SCAN_SPECS" && test -n "$$SCAN_SPECS" && RUN_A11Y=true RUN_A11Y_ASSERT=false npx playwright test $$SCAN_SPECS --project=chromium --workers=1
+	@mkdir -p $(FRONTEND_DIR)/tests/baselines
+	@cp $(FRONTEND_DIR)/coverage/accessibility-reports/chromium__*.json $(FRONTEND_DIR)/tests/baselines/
+
 ######################
 # FRONTEND UNIT TESTS (JEST)
 ######################
@@ -275,4 +286,4 @@ help_frontend: ## show frontend help
 	@echo "  $(GREEN)make storybook_network$(NC)    - Run Storybook accessible on network"
 	@echo ''
 	@echo "$(GREEN)═══════════════════════════════════════════════════════════════════$(NC)"
-	@echo ''
+	@echo ''

src/frontend/tests/a11y/core-pages.a11y.spec.ts

@@ -0,0 +1,171 @@
+import { expect, type LangflowPage, test } from "../fixtures";
+import { adjustScreenView } from "../utils/adjust-screen-view";
+import { TID } from "../utils/constants/testIds";
+import { TEXTS } from "../utils/constants/texts";
+import { TIMEOUTS } from "../utils/constants/timeouts";
+import { addComponentFromSidebar } from "../utils/flow/add-component-from-sidebar";
+
+async function disableAnimations(page: LangflowPage) {
+  await page.addStyleTag({
+    content: `
+      *,
+      *::before,
+      *::after {
+        animation-duration: 0s !important;
+        animation-delay: 0s !important;
+        transition-duration: 0s !important;
+        transition-delay: 0s !important;
+        scroll-behavior: auto !important;
+      }
+    `,
+  });
+}
+
+async function hideComponentSidebar(page: LangflowPage) {
+  const style = await page.addStyleTag({
+    content: '[data-testid="shad-sidebar"] { display: none !important; }',
+  });
+  await expect(page.getByTestId("shad-sidebar")).toBeHidden();
+  return style;
+}
+
+async function openBlankFlowForA11y(page: LangflowPage) {
+  await page.goto("/");
+  await expect(page.getByTestId(TID.mainpageTitle)).toBeVisible({
+    timeout: TIMEOUTS.standard,
+  });
+
+  const emptyPageButton = page.getByTestId(TID.newProjectBtnEmptyPage);
+  if (await emptyPageButton.isVisible()) {
+    await emptyPageButton.click();
+  } else {
+    await page.getByTestId(TID.newProjectBtn).click();
+  }
+
+  const welcomePanel = page.getByTestId("flow-builder-welcome-panel");
+  const modalTitle = page.getByTestId(TID.modalTitle);
+  await expect(welcomePanel.or(modalTitle)).toBeVisible({
+    timeout: TIMEOUTS.standard,
+  });
+
+  if (await welcomePanel.isVisible()) {
+    await page.getByTestId("flow-builder-welcome-browse-more").click();
+  }
+
+  await expect(page.getByTestId(TID.blankFlow)).toBeVisible({
+    timeout: TIMEOUTS.standard,
+  });
+  await page.getByTestId(TID.blankFlow).click();
+}
+
+test.describe("core application accessibility", () => {
+  test.describe.configure({ mode: "serial" });
+
+  test(
+    "scans flow creation and playground states",
+    { tag: ["@a11y", "@workspace"] },
+    async ({ page }) => {
+      await openBlankFlowForA11y(page);
+      await disableAnimations(page);
+      await expect(page.getByTestId(TID.sidebarSearchInput)).toBeVisible();
+      const emptyCanvasSidebarStyle = await hideComponentSidebar(page);
+
+      await page.runA11yScan("flow-canvas-empty");
+
+      await emptyCanvasSidebarStyle.evaluate((style) => style.remove());
+      await expect(page.getByTestId(TID.sidebarSearchInput)).toBeVisible();
+      await addComponentFromSidebar(page, {
+        search: TEXTS.searchChatOutput,
+        testId: "input_outputChat Output",
+        hoverAdd: true,
+      });
+      await addComponentFromSidebar(page, {
+        search: TEXTS.searchChatInput,
+        testId: "input_outputChat Input",
+        position: { x: 100, y: 100 },
+      });
+      await adjustScreenView(page);
+      await hideComponentSidebar(page);
+
+      await page
+        .getByTestId("handle-chatinput-noshownode-chat message-source")
+        .click();
+      await page
+        .getByTestId("handle-chatoutput-noshownode-inputs-target")
+        .click();
+
+      await page.runA11yScan("flow-canvas-populated");
+
+      await page.getByTestId("title-Chat Input").click();
+      await expect(page.getByTestId(TID.editFieldsButton)).toBeVisible();
+      await page.getByTestId(TID.editFieldsButton).click();
+      await expect(page.getByTestId("showinput_value")).toBeVisible();
+
+      await page.runA11yScan("component-configuration");
+
+      await page.getByTestId(TID.editFieldsButton).click();
+      await page
+        .getByRole("button", { name: TEXTS.playground, exact: true })
+        .click();
+      await expect(page.getByTestId(TID.inputChatPlayground)).toBeVisible({
+        timeout: TIMEOUTS.componentMount,
+      });
+
+      await page.runA11yScan("playground-empty");
+
+      await page
+        .getByTestId(TID.inputChatPlayground)
+        .fill("Accessibility test");
+      await page.getByTestId(TID.buttonSend).first().click();
+      await expect(page.getByTestId(TID.chatMessage)).toHaveText(
+        "Accessibility test",
+        { timeout: TIMEOUTS.componentMount },
+      );
+
+      await page.runA11yScan("playground-with-message");
+    },
+  );
+
+  test(
+    "scans flows landing page",
+    { tag: ["@a11y", "@workspace"] },
+    async ({ page }) => {
+      await page.goto("/flows");
+      await disableAnimations(page);
+      await expect(page.getByTestId(TID.mainpageTitle)).toBeVisible({
+        timeout: TIMEOUTS.standard,
+      });
+
+      await page.runA11yScan("flows-landing");
+    },
+  );
+
+  test(
+    "scans general settings",
+    { tag: ["@a11y", "@workspace"] },
+    async ({ page }) => {
+      await page.goto("/settings/general");
+      await disableAnimations(page);
+      await expect(page.getByTestId(TID.settingsMenuHeader)).toBeVisible({
+        timeout: TIMEOUTS.standard,
+      });
+
+      await page.runA11yScan("settings-general");
+    },
+  );
+
+  test(
+    "scans files page",
+    { tag: ["@a11y", "@workspace"] },
+    async ({ page }) => {
+      await page.goto("/assets/files");
+      await disableAnimations(page);
+      await expect(page.getByTestId(TID.mainpageTitle)).toContainText("Files", {
+        timeout: TIMEOUTS.standard,
+      });
+      await expect(page.getByTestId("upload-file-btn")).toBeVisible();
+
+      await page.runA11yScan("files-page");
+    },
+  );
+});

src/frontend/tests/fixtures.ts

@@ -7,6 +7,7 @@ import "./playwrightCoverage";
 import {
   buildA11yScanLabel,
   buildA11ySummaryAttachment,
+  countNewA11yViolations,
   formatA11yFailure,
   isCheckerReport,
 } from "./utils/accessibility-checker";
@@ -100,10 +101,10 @@ export const test = base.extend<{ page: LangflowPage }, A11yFixtures>({
       );
 
       if (RUN_A11Y_ASSERT) {
-        const returnCode = aChecker.assertCompliance(result.report);
+        const newViolationCount = countNewA11yViolations(result.report);
         const failureMessage = formatA11yFailure(scanLabel, result.report);
 
-        expect(returnCode, failureMessage).toBe(0);
+        expect(newViolationCount, failureMessage).toBe(0);
       }
 
       return result;

src/frontend/tests/utils/accessibility-checker.ts

@@ -74,7 +74,8 @@ export function formatA11yFailure(
 
   const failingIssues = report.results.filter(
     (issue) =>
-      issue.level === "violation" || issue.level === "potentialviolation",
+      !issue.ignored &&
+      (issue.level === "violation" || issue.level === "potentialviolation"),
   );
 
   const groupedIssues = new Map<
@@ -113,9 +114,10 @@ export function formatA11yFailure(
     .slice(0, 8);
 
   const lines = [
-    `IBM accessibility scan failed: ${scanLabel}`,
+    `IBM accessibility regression detected: ${scanLabel}`,
     `Report: ${reportPath}`,
-    `Counts: violation=${counts.violation}, potential=${counts.potentialviolation}, recommendation=${counts.recommendation}, manual=${counts.manual}`,
+    `New issues: ${failingIssues.length}`,
+    `Report counts: violation=${counts.violation}, potential=${counts.potentialviolation}, recommendation=${counts.recommendation}, manual=${counts.manual}`,
     `Top issues (${topIssues.length}/${groupedIssues.size} groups shown):`,
     ...topIssues.map(
       (issue) =>
@@ -129,3 +131,9 @@ export function formatA11yFailure(
 
   return lines.join("\n");
 }
+
+export function countNewA11yViolations(report: ICheckerReport): number {
+  return report.results.filter(
+    (issue) => issue.level === "violation" && !issue.ignored,
+  ).length;
+}