Skip to content

Commit 0007f20

Browse files
feat: Add a11y regression scan suite (#13663)
* test: add component a11y unit tests (jest-axe) Cover 14 components in two waves. 16 tests fail by design - each encodes a known gap from a11y-action-plan (phases 0-3) and flips to a regression lock once the component fix lands. 23 tests pass as regression locks for already-correct semantics. * ci: split a11y unit tests into separate workflow Main jest CI excludes *.a11y.test.* so known-gap failures don't block PRs; new a11y-unit-tests.yml runs them as informational check. * fix(a11y): resolve component gaps flagged by a11y unit tests Fixes all 16 failing jest-axe gap tests: - TableHead defaults scope=col - alert display area gets aria-live region - app header becomes <header> landmark; bell button labeled - icon wrapper decorative by default (aria-hidden), ariaLabel opt-in - Input drops wrapping <label> (name pollution) and hides placeholder span - password toggle back in tab order with aria-label/aria-pressed - CheckBoxDiv exposes checkbox role + state - accordion trigger renders native Radix button - dialogs focus container on open instead of suppressing autofocus - full-screen modal exposes dialog role/aria-modal/ariaLabel - flow list card focusable with keyboard activation Also wraps test focus calls in act() and mocks the icon loader so the a11y suite runs without React warnings. * fix: skip Puppeteer Chrome download in remaining Dockerfiles frontend and base images still ran bare npm install; puppeteer (via accessibility-checker, test-only) tried to fetch Chrome and broke the Docker image build in CI. * test: extend a11y unit test coverage (jest-axe) Adds axe + semantics tests for untested primitives (button, alert, textarea, radio-group, dropdown-menu, tooltip, popover) and two known-gap tests that fail by design until fixes land: - copyFieldAreaComponent copy action is a bare div onClick (plan 2.4) - DialogContentWithouFixed renders an empty fragment (plan 1.6) * fix: make webhook copy field a real button (a11y) Copy action was a bare div onClick - no role, name, or keyboard support (a11y-action-plan 2.4). Drops the no-longer-needed DialogContentWithouFixed known-gap test. * feat: update a11y unit tests and improve accessibility semantics in components * feat: enable pull request trigger for a11y scan workflow * test: add a11y regression scan suite * test: make a11y scans non-blocking * test: discover a11y scan specs * ci: group a11y workflow checks
1 parent 287ca4c commit 0007f20

6 files changed

Lines changed: 208 additions & 16 deletions

File tree

.github/workflows/a11y-scan.yml

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Run Accessibility Scans (IBM Equal Access)
1+
name: A11y / Playwright Scans
22

33
on:
44
# TEMP: remove before merging — allows testing the workflow from this
@@ -17,7 +17,7 @@ on:
1717
required: false
1818
type: string
1919
assert:
20-
description: "Fail the run on a11y violations (RUN_A11Y_ASSERT)"
20+
description: "Fail the run when scans differ from committed baselines"
2121
required: false
2222
type: boolean
2323
default: false
@@ -32,7 +32,7 @@ env:
3232

3333
jobs:
3434
a11y-scan:
35-
name: IBM Accessibility Scans
35+
name: IBM Equal Access
3636
runs-on: ubuntu-latest
3737
permissions:
3838
contents: read
@@ -122,16 +122,17 @@ jobs:
122122
shell: bash
123123
env:
124124
RUN_A11Y: "true"
125-
RUN_A11Y_ASSERT: ${{ inputs.assert == true && 'true' || 'false' }}
126-
LANGFLOW_DEACTIVE_TRACING: "true"
125+
RUN_A11Y_ASSERT: ${{ github.event_name == 'workflow_dispatch' && inputs.assert && 'true' || 'false' }}
126+
LANGFLOW_DEACTIVATE_TRACING: "true"
127127
run: |
128128
cd src/frontend
129129
# Only specs that call runA11yScan produce reports; discover them
130130
# so new scan hosts are picked up without editing this workflow.
131131
SCAN_SPECS=$(grep -rl "runA11yScan(" tests --include="*.spec.ts" | sort)
132132
echo "Specs with a11y scans:"
133133
echo "$SCAN_SPECS"
134-
npx playwright test $SCAN_SPECS --project=chromium --workers 1 --retries=2
134+
test -n "$SCAN_SPECS"
135+
npx playwright test $SCAN_SPECS --project=chromium --workers=1 --retries=2
135136
136137
- name: Build Aggregated A11y Summary
137138
if: always()

.github/workflows/a11y-unit-tests.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Run Frontend A11y Unit Tests
1+
name: A11y / Unit Tests
22

33
on:
44
pull_request:
@@ -24,7 +24,7 @@ env:
2424

2525
jobs:
2626
a11y-unit-tests:
27-
name: Frontend A11y Unit Tests (jest-axe)
27+
name: Frontend jest-axe
2828
runs-on: ubuntu-latest
2929
permissions:
3030
contents: read
@@ -53,6 +53,6 @@ jobs:
5353
if: always()
5454
with:
5555
report_paths: "src/frontend/test-results/junit.xml"
56-
check_name: "Frontend A11y Unit Test Results"
56+
check_name: "A11y / Unit Test Results"
5757
fail_on_failure: true
5858
require_tests: true

Makefile.frontend

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
FRONTEND_DIR = src/frontend
66
NPM = npm
77

8-
.PHONY: install_frontend install_frontendci install_frontendc frontend_deps_check build_frontend run_frontend frontend frontendc format_frontend tests_frontend test_frontend test_frontend_watch test_frontend_coverage test_frontend_verbose test_frontend_ci test_frontend_clean test_frontend_file test_frontend_pattern test_frontend_snapshots test_frontend_config test_frontend_bail test_frontend_silent test_frontend_coverage_open help_frontend storybook storybook_build storybook_network
8+
.PHONY: install_frontend install_frontendci install_frontendc frontend_deps_check build_frontend run_frontend frontend frontendc format_frontend tests_frontend test_frontend_a11y_e2e test_frontend_a11y_e2e_blocking test_frontend_a11y_e2e_update test_frontend test_frontend_watch test_frontend_coverage test_frontend_verbose test_frontend_ci test_frontend_clean test_frontend_file test_frontend_pattern test_frontend_snapshots test_frontend_config test_frontend_bail test_frontend_silent test_frontend_coverage_open help_frontend storybook storybook_build storybook_network
99

1010
######################
1111
# FRONTEND DEPENDENCIES
@@ -77,6 +77,17 @@ else
7777
@cd $(FRONTEND_DIR) && npx playwright test --project=chromium
7878
endif
7979

80+
test_frontend_a11y_e2e: frontend_deps_check ## run non-blocking Playwright a11y scans and reports
81+
@cd $(FRONTEND_DIR) && SCAN_SPECS="$$(grep -rl "runA11yScan(" tests --include="*.spec.ts" | sort)" && echo "Specs with a11y scans:" && echo "$$SCAN_SPECS" && test -n "$$SCAN_SPECS" && RUN_A11Y=true RUN_A11Y_ASSERT=false npx playwright test $$SCAN_SPECS --project=chromium --workers=1
82+
83+
test_frontend_a11y_e2e_blocking: frontend_deps_check ## run blocking Playwright a11y scans against baselines
84+
@cd $(FRONTEND_DIR) && SCAN_SPECS="$$(grep -rl "runA11yScan(" tests --include="*.spec.ts" | sort)" && echo "Specs with a11y scans:" && echo "$$SCAN_SPECS" && test -n "$$SCAN_SPECS" && RUN_A11Y=true RUN_A11Y_ASSERT=true npx playwright test $$SCAN_SPECS --project=chromium --workers=1
85+
86+
test_frontend_a11y_e2e_update: frontend_deps_check ## refresh Playwright a11y reports used as baselines
87+
@cd $(FRONTEND_DIR) && rm -rf coverage/accessibility-reports && SCAN_SPECS="$$(grep -rl "runA11yScan(" tests --include="*.spec.ts" | sort)" && echo "Specs with a11y scans:" && echo "$$SCAN_SPECS" && test -n "$$SCAN_SPECS" && RUN_A11Y=true RUN_A11Y_ASSERT=false npx playwright test $$SCAN_SPECS --project=chromium --workers=1
88+
@mkdir -p $(FRONTEND_DIR)/tests/baselines
89+
@cp $(FRONTEND_DIR)/coverage/accessibility-reports/chromium__*.json $(FRONTEND_DIR)/tests/baselines/
90+
8091
######################
8192
# FRONTEND UNIT TESTS (JEST)
8293
######################
@@ -275,4 +286,4 @@ help_frontend: ## show frontend help
275286
@echo " $(GREEN)make storybook_network$(NC) - Run Storybook accessible on network"
276287
@echo ''
277288
@echo "$(GREEN)═══════════════════════════════════════════════════════════════════$(NC)"
278-
@echo ''
289+
@echo ''
Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
import { expect, type LangflowPage, test } from "../fixtures";
2+
import { adjustScreenView } from "../utils/adjust-screen-view";
3+
import { TID } from "../utils/constants/testIds";
4+
import { TEXTS } from "../utils/constants/texts";
5+
import { TIMEOUTS } from "../utils/constants/timeouts";
6+
import { addComponentFromSidebar } from "../utils/flow/add-component-from-sidebar";
7+
8+
async function disableAnimations(page: LangflowPage) {
9+
await page.addStyleTag({
10+
content: `
11+
*,
12+
*::before,
13+
*::after {
14+
animation-duration: 0s !important;
15+
animation-delay: 0s !important;
16+
transition-duration: 0s !important;
17+
transition-delay: 0s !important;
18+
scroll-behavior: auto !important;
19+
}
20+
`,
21+
});
22+
}
23+
24+
async function hideComponentSidebar(page: LangflowPage) {
25+
const style = await page.addStyleTag({
26+
content: '[data-testid="shad-sidebar"] { display: none !important; }',
27+
});
28+
await expect(page.getByTestId("shad-sidebar")).toBeHidden();
29+
return style;
30+
}
31+
32+
async function openBlankFlowForA11y(page: LangflowPage) {
33+
await page.goto("/");
34+
await expect(page.getByTestId(TID.mainpageTitle)).toBeVisible({
35+
timeout: TIMEOUTS.standard,
36+
});
37+
38+
const emptyPageButton = page.getByTestId(TID.newProjectBtnEmptyPage);
39+
if (await emptyPageButton.isVisible()) {
40+
await emptyPageButton.click();
41+
} else {
42+
await page.getByTestId(TID.newProjectBtn).click();
43+
}
44+
45+
const welcomePanel = page.getByTestId("flow-builder-welcome-panel");
46+
const modalTitle = page.getByTestId(TID.modalTitle);
47+
await expect(welcomePanel.or(modalTitle)).toBeVisible({
48+
timeout: TIMEOUTS.standard,
49+
});
50+
51+
if (await welcomePanel.isVisible()) {
52+
await page.getByTestId("flow-builder-welcome-browse-more").click();
53+
}
54+
55+
await expect(page.getByTestId(TID.blankFlow)).toBeVisible({
56+
timeout: TIMEOUTS.standard,
57+
});
58+
await page.getByTestId(TID.blankFlow).click();
59+
}
60+
61+
test.describe("core application accessibility", () => {
62+
test.describe.configure({ mode: "serial" });
63+
64+
test(
65+
"scans flow creation and playground states",
66+
{ tag: ["@a11y", "@workspace"] },
67+
async ({ page }) => {
68+
await openBlankFlowForA11y(page);
69+
await disableAnimations(page);
70+
await expect(page.getByTestId(TID.sidebarSearchInput)).toBeVisible();
71+
const emptyCanvasSidebarStyle = await hideComponentSidebar(page);
72+
73+
await page.runA11yScan("flow-canvas-empty");
74+
75+
await emptyCanvasSidebarStyle.evaluate((style) => style.remove());
76+
await expect(page.getByTestId(TID.sidebarSearchInput)).toBeVisible();
77+
await addComponentFromSidebar(page, {
78+
search: TEXTS.searchChatOutput,
79+
testId: "input_outputChat Output",
80+
hoverAdd: true,
81+
});
82+
await addComponentFromSidebar(page, {
83+
search: TEXTS.searchChatInput,
84+
testId: "input_outputChat Input",
85+
position: { x: 100, y: 100 },
86+
});
87+
await adjustScreenView(page);
88+
await hideComponentSidebar(page);
89+
90+
await page
91+
.getByTestId("handle-chatinput-noshownode-chat message-source")
92+
.click();
93+
await page
94+
.getByTestId("handle-chatoutput-noshownode-inputs-target")
95+
.click();
96+
97+
await page.runA11yScan("flow-canvas-populated");
98+
99+
await page.getByTestId("title-Chat Input").click();
100+
await expect(page.getByTestId(TID.editFieldsButton)).toBeVisible();
101+
await page.getByTestId(TID.editFieldsButton).click();
102+
await expect(page.getByTestId("showinput_value")).toBeVisible();
103+
104+
await page.runA11yScan("component-configuration");
105+
106+
await page.getByTestId(TID.editFieldsButton).click();
107+
await page
108+
.getByRole("button", { name: TEXTS.playground, exact: true })
109+
.click();
110+
await expect(page.getByTestId(TID.inputChatPlayground)).toBeVisible({
111+
timeout: TIMEOUTS.componentMount,
112+
});
113+
114+
await page.runA11yScan("playground-empty");
115+
116+
await page
117+
.getByTestId(TID.inputChatPlayground)
118+
.fill("Accessibility test");
119+
await page.getByTestId(TID.buttonSend).first().click();
120+
await expect(page.getByTestId(TID.chatMessage)).toHaveText(
121+
"Accessibility test",
122+
{ timeout: TIMEOUTS.componentMount },
123+
);
124+
125+
await page.runA11yScan("playground-with-message");
126+
},
127+
);
128+
129+
test(
130+
"scans flows landing page",
131+
{ tag: ["@a11y", "@workspace"] },
132+
async ({ page }) => {
133+
await page.goto("/flows");
134+
await disableAnimations(page);
135+
await expect(page.getByTestId(TID.mainpageTitle)).toBeVisible({
136+
timeout: TIMEOUTS.standard,
137+
});
138+
139+
await page.runA11yScan("flows-landing");
140+
},
141+
);
142+
143+
test(
144+
"scans general settings",
145+
{ tag: ["@a11y", "@workspace"] },
146+
async ({ page }) => {
147+
await page.goto("/settings/general");
148+
await disableAnimations(page);
149+
await expect(page.getByTestId(TID.settingsMenuHeader)).toBeVisible({
150+
timeout: TIMEOUTS.standard,
151+
});
152+
153+
await page.runA11yScan("settings-general");
154+
},
155+
);
156+
157+
test(
158+
"scans files page",
159+
{ tag: ["@a11y", "@workspace"] },
160+
async ({ page }) => {
161+
await page.goto("/assets/files");
162+
await disableAnimations(page);
163+
await expect(page.getByTestId(TID.mainpageTitle)).toContainText("Files", {
164+
timeout: TIMEOUTS.standard,
165+
});
166+
await expect(page.getByTestId("upload-file-btn")).toBeVisible();
167+
168+
await page.runA11yScan("files-page");
169+
},
170+
);
171+
});

src/frontend/tests/fixtures.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import "./playwrightCoverage";
77
import {
88
buildA11yScanLabel,
99
buildA11ySummaryAttachment,
10+
countNewA11yViolations,
1011
formatA11yFailure,
1112
isCheckerReport,
1213
} from "./utils/accessibility-checker";
@@ -100,10 +101,10 @@ export const test = base.extend<{ page: LangflowPage }, A11yFixtures>({
100101
);
101102

102103
if (RUN_A11Y_ASSERT) {
103-
const returnCode = aChecker.assertCompliance(result.report);
104+
const newViolationCount = countNewA11yViolations(result.report);
104105
const failureMessage = formatA11yFailure(scanLabel, result.report);
105106

106-
expect(returnCode, failureMessage).toBe(0);
107+
expect(newViolationCount, failureMessage).toBe(0);
107108
}
108109

109110
return result;

src/frontend/tests/utils/accessibility-checker.ts

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,8 @@ export function formatA11yFailure(
7474

7575
const failingIssues = report.results.filter(
7676
(issue) =>
77-
issue.level === "violation" || issue.level === "potentialviolation",
77+
!issue.ignored &&
78+
(issue.level === "violation" || issue.level === "potentialviolation"),
7879
);
7980

8081
const groupedIssues = new Map<
@@ -113,9 +114,10 @@ export function formatA11yFailure(
113114
.slice(0, 8);
114115

115116
const lines = [
116-
`IBM accessibility scan failed: ${scanLabel}`,
117+
`IBM accessibility regression detected: ${scanLabel}`,
117118
`Report: ${reportPath}`,
118-
`Counts: violation=${counts.violation}, potential=${counts.potentialviolation}, recommendation=${counts.recommendation}, manual=${counts.manual}`,
119+
`New issues: ${failingIssues.length}`,
120+
`Report counts: violation=${counts.violation}, potential=${counts.potentialviolation}, recommendation=${counts.recommendation}, manual=${counts.manual}`,
119121
`Top issues (${topIssues.length}/${groupedIssues.size} groups shown):`,
120122
...topIssues.map(
121123
(issue) =>
@@ -129,3 +131,9 @@ export function formatA11yFailure(
129131

130132
return lines.join("\n");
131133
}
134+
135+
export function countNewA11yViolations(report: ICheckerReport): number {
136+
return report.results.filter(
137+
(issue) => issue.level === "violation" && !issue.ignored,
138+
).length;
139+
}

0 commit comments

Comments
 (0)