Run automated conformance suite in CI

.github/workflows/coverage.yml

@@ -44,6 +44,15 @@ jobs:
     - name: Run tests
       run: npm run test
 
+    - name: Install Chromium
+      run: npx playwright install --with-deps chromium
+
+    - name: Build workspaces for conformance testing
+      run: npm run build
+
+    - name: Run automated conformance tests
+      run: npm run test:ci --workspace fdc3-conformance
+
     - name: Codecov
       uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
       with:

.gitignore

@@ -34,6 +34,8 @@ tsconfig.tsbuildinfo
 **/html-report/
 packages/fdc3-security/junit.xml
 **/html-test-results/
+toolbox/fdc3-conformance/playwright-report/
+toolbox/fdc3-conformance/test-results/
 toolbox/fdc3-example-apps/directory/static/generated
 toolbox/fdc3-example-apps/directory/.vite
 toolbox/fdc3-example-apps/front-end-apps/*/.vite

CHANGELOG.md

@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Added
 
+* Added a Playwright contribution check that runs all non-manual conformance tests against the FDC3 for Web reference Desktop Agent, including fixes for reliable destructured-method setup and cleanup. ([#2045](https://github.com/finos/FDC3/issues/2045))
 * Added CI dependency checks for the root package and every npm workspace, with documented baselines of existing unused-dependency findings.
 * Added conformance coverage for `ChannelError.NoChannelFound`, `ChannelError.MalformedContext`, and `ChannelError.InvalidArguments`. ([#1779](https://github.com/finos/FDC3/issues/1779))
 * Added conformance coverage verifying that Desktop Agent methods continue to work when destructured from the `fdc3` object. ([#1778](https://github.com/finos/FDC3/issues/1778))

toolbox/fdc3-conformance/README.md

@@ -87,6 +87,23 @@ Successful runs look something like this:
 
 <img src="static/running.png" alt="Success" width="400px" />
 
+#### Running In CI
+
+The automated (non-manual) conformance pack can be run against the FDC3 for Web
+reference Desktop Agent with Playwright. Build the workspaces, install Chromium,
+and run the CI script from the repository root:
+
+```sh
+npm run build
+npx playwright install chromium
+npm run test:ci --workspace fdc3-conformance
+```
+
+Playwright starts the conformance app and reference Desktop Agent, launches the
+runner inside the Desktop Agent, and fails when any automated conformance test
+fails. Tests that require a human to use the intent resolver or channel selector
+remain excluded from this pack.
+
 ### Joining The Conformance Program
 
 If you've had a clean run of all the tests locally, why not join the conformance program?
@@ -100,4 +117,3 @@ Once you have followed these steps, you will be allowed to display the FDC3 Comp
 ### Which Desktop Agents Are Conformant?
 
 We publish the details of conformant desktop agents on the [FDC3 Home Page](https://fdc3.finos.org#conformance).  Please check there to find out who FINOS has certified! 
-

toolbox/fdc3-conformance/e2e/conformance.spec.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2026 Future Edge Group FZE
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { expect, test } from '@playwright/test';
+import { assertConformancePassed } from './conformanceResults';
+
+test('runs all automated conformance tests against the reference Desktop Agent', async ({ page }) => {
+  await page.goto('/static/da/index.html');
+
+  const conformanceApp = page.locator('.da-app-card').filter({ hasText: 'FDC3 Conformance Framework' });
+  await expect(conformanceApp).toBeVisible();
+  await conformanceApp.getByRole('button', { name: 'Start' }).click();
+
+  const runner = page.frameLocator('#app-frames iframe').first();
+  const runButton = runner.getByRole('button', { name: 'Run', exact: true }).first();
+  await expect(runButton).toBeVisible();
+  await runner.locator('#testSuite').selectOption({ label: 'All' });
+  await runButton.click();
+
+  const results = runner.locator('#mocha');
+  await expect(results).toHaveAttribute('data-conformance-status', /passed|failed/, {
+    timeout: 9 * 60 * 1000,
+  });
+
+  const status = await results.getAttribute('data-conformance-status');
+  const passes = Number(await results.getAttribute('data-conformance-passes'));
+  const failures = Number(await results.getAttribute('data-conformance-failures'));
+  const tests = Number(await results.getAttribute('data-conformance-tests'));
+  const failureMessages = JSON.parse((await results.getAttribute('data-conformance-failure-messages')) ?? '[]');
+
+  assertConformancePassed({ status, passes, failures, tests, failureMessages });
+});

toolbox/fdc3-conformance/e2e/conformanceResults.spec.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2026 Future Edge Group FZE
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { expect, test } from '@playwright/test';
+import { assertConformancePassed } from './conformanceResults';
+
+test.describe('conformance result gate', () => {
+  test('accepts a completed passing run', () => {
+    expect(() =>
+      assertConformancePassed({
+        status: 'passed',
+        passes: 42,
+        failures: 0,
+        tests: 42,
+        failureMessages: [],
+      })
+    ).not.toThrow();
+  });
+
+  test('rejects a failed or timed-out conformance test', () => {
+    expect(() =>
+      assertConformancePassed({
+        status: 'failed',
+        passes: 41,
+        failures: 1,
+        tests: 42,
+        failureMessages: ['Desktop Agent test: Timeout of 10000ms exceeded'],
+      })
+    ).toThrow(/1 conformance test\(s\) failed[\s\S]*Timeout of 10000ms exceeded/);
+  });
+
+  test('rejects a run that ends without completing every test', () => {
+    expect(() =>
+      assertConformancePassed({
+        status: 'passed',
+        passes: 41,
+        failures: 0,
+        tests: 42,
+        failureMessages: [],
+      })
+    ).toThrow(/run was incomplete: 41 of 42 test\(s\) completed/);
+  });
+});

toolbox/fdc3-conformance/e2e/conformanceResults.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2026 Future Edge Group FZE
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export interface ConformanceResults {
+  status: string | null;
+  passes: number;
+  failures: number;
+  tests: number;
+  failureMessages: string[];
+}
+
+export function assertConformancePassed(results: ConformanceResults): void {
+  const errors: string[] = [];
+
+  if (results.status !== 'passed') {
+    errors.push(`status was ${JSON.stringify(results.status)}, expected "passed"`);
+  }
+  if (results.passes <= 0) {
+    errors.push('the automated conformance pack did not execute any passing tests');
+  }
+  if (results.failures !== 0) {
+    errors.push(`${results.failures} conformance test(s) failed`);
+  }
+  if (results.passes + results.failures !== results.tests) {
+    errors.push(`the run was incomplete: ${results.passes + results.failures} of ${results.tests} test(s) completed`);
+  }
+
+  if (errors.length > 0) {
+    throw new Error([...errors, ...results.failureMessages].join('\n'));
+  }
+}

toolbox/fdc3-conformance/package.json

@@ -14,6 +14,7 @@
     "build": "vite build && npm run copy && shx cp ../../node_modules/mocha/mocha.css ./dist/lib/mocha.css",
     "serve": "npx http-server ./dist -p 3001 -c-1 --cors",
     "dev": "npm run clean && npm run build && npm run serve",
+    "test:ci": "playwright test",
     "clean": "rimraf dist"
   },
   "dependencies": {
@@ -27,6 +28,7 @@
     "util": "^0.12.5"
   },
   "devDependencies": {
+    "@playwright/test": "^1.62.1",
     "@rollup/plugin-inject": "^5.0.5",
     "@types/chai": "^5.2.3",
     "@types/mocha": "^10.0.10",

toolbox/fdc3-conformance/playwright.config.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2026 Future Edge Group FZE
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  testDir: './e2e',
+  timeout: 10 * 60 * 1000,
+  expect: {
+    timeout: 30 * 1000,
+  },
+  fullyParallel: false,
+  forbidOnly: Boolean(process.env.CI),
+  retries: process.env.CI ? 1 : 0,
+  workers: 1,
+  reporter: process.env.CI ? [['line'], ['html', { open: 'never' }]] : 'list',
+  use: {
+    baseURL: 'http://127.0.0.1:4000',
+    channel: 'chromium',
+    screenshot: 'only-on-failure',
+    trace: 'retain-on-failure',
+  },
+  webServer: [
+    {
+      command: 'npm run dev',
+      url: 'http://127.0.0.1:3001/apps/app/index.html',
+      timeout: 2 * 60 * 1000,
+      reuseExistingServer: !process.env.CI,
+    },
+    {
+      command: 'npm run dev --workspace @finos/demo',
+      url: 'http://127.0.0.1:4000/static/da/index.html',
+      timeout: 2 * 60 * 1000,
+      reuseExistingServer: !process.env.CI,
+    },
+  ],
+});

toolbox/fdc3-conformance/src/test/progressReporter.ts

@@ -3,7 +3,9 @@
  * starts, then updates the indicator to a green tick or red cross when the test completes.
  */
 export class ProgressReporter extends Mocha.reporters.Base {
+  private root: HTMLElement;
   private testElements = new Map<string, HTMLElement>();
+  private failureMessages: string[] = [];
   private suiteStack: HTMLElement[];
   private canvas: HTMLCanvasElement;
   private passCount: HTMLElement;
@@ -15,16 +17,17 @@ export class ProgressReporter extends Mocha.reporters.Base {
   constructor(runner: Mocha.Runner, options?: Mocha.MochaOptions) {
     super(runner, options);
 
-    const root = document.getElementById('mocha')!;
-    root.replaceChildren();
+    this.root = document.getElementById('mocha')!;
+    this.root.replaceChildren();
+    this.root.dataset.conformanceStatus = 'running';
 
     const statsEl = document.createElement('ul');
     statsEl.id = 'mocha-stats';
-    root.appendChild(statsEl);
+    this.root.appendChild(statsEl);
 
     const report = document.createElement('ul');
     report.id = 'mocha-report';
-    root.appendChild(report);
+    this.root.appendChild(report);
 
     // Progress ring canvas
     const progressLi = document.createElement('li');
@@ -107,6 +110,7 @@ export class ProgressReporter extends Mocha.reporters.Base {
 
   private onFail(test: Mocha.Test, err: Error) {
     console.log('Test FAILED: ', test.title);
+    this.failureMessages.push(`${test.fullTitle()}: ${err.message}`);
     const li = this.testElements.get(test.fullTitle());
     if (li) {
       li.className = 'test fail';
@@ -122,6 +126,11 @@ export class ProgressReporter extends Mocha.reporters.Base {
   private onEnd() {
     clearInterval(this.durationTimer);
     this.updateDuration();
+    this.root.dataset.conformanceStatus = this.stats.failures === 0 ? 'passed' : 'failed';
+    this.root.dataset.conformancePasses = String(this.stats.passes);
+    this.root.dataset.conformanceFailures = String(this.stats.failures);
+    this.root.dataset.conformanceTests = String(this.runner.total);
+    this.root.dataset.conformanceFailureMessages = JSON.stringify(this.failureMessages);
   }
 
   private getSpeedClass(test: Mocha.Test): string {

toolbox/fdc3-conformance/src/test/testSuite.ts

@@ -1,4 +1,4 @@
-import mocha from 'mocha';
+import mocha from 'mocha/mocha.js';
 import constants from '../constants';
 import fdc3FindIntent from './advanced/fdc3.findIntent';
 import fdc3FindIntentsByContext from './advanced/fdc3.findIntentsByContext';