Run E2E tests on PRs and wait till deploys reach steady state. (Fixes #172) (#814)

Author: Sancus

.github/workflows/merge.yml

@@ -348,8 +348,10 @@ jobs:
 
   e2e-tests-browserstack-stage:
     name: e2e-tests-browserstack-stage
-    needs: accounts-deploy
-    if: '!cancelled()'
+    needs:
+      - accounts-deploy
+      - keycloak-deploy
+    if: always() && needs.accounts-deploy.result == 'success' && needs.keycloak-deploy.result != 'failure' && needs.keycloak-deploy.result != 'cancelled'
     runs-on: ubuntu-latest
     environment:
       name: staging

.github/workflows/validate.yml

@@ -2,14 +2,16 @@
 name: validate
 
 concurrency:
-  group: validate
+  group: validate-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 on:
+  pull_request:
+    branches:
+      - main
   push:
     branches:
-      - '**'
-      - '!main'
+      - main
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
@@ -25,6 +27,7 @@ jobs:
       IS_CI_AUTOMATION: "yes"
     outputs:
       src-changed: ${{ steps.check.outputs.src-changed }}
+      e2e-changed: ${{ steps.check.outputs.e2e-changed }}
       iac-changed: ${{ steps.check.outputs.iac-changed }}
     steps:
       - uses: actions/checkout@v4
@@ -51,6 +54,14 @@ jobs:
               - 'src/**'
               - 'uv.lock'
               - 'vite.config.mjs'
+            e2e-changed:
+              - '.github/workflows/validate.yml'
+              - '.env.example'
+              - 'config.toml.example'
+              - 'docker-compose.yml'
+              - 'Dockerfile.keycloak'
+              - 'keycloak/**'
+              - 'test/e2e/**'
             iac-changed:
               - 'pulumi/**'
   
@@ -107,8 +118,15 @@ jobs:
           src: "pulumi"
 
   run-e2e-tests-local:
-    needs: detect-changes
-    if: false
+    needs:
+      - detect-changes
+      - validate-accounts
+    if: >-
+      always() &&
+      (needs.detect-changes.outputs.src-changed == 'true' || needs.detect-changes.outputs.e2e-changed == 'true') &&
+      needs.detect-changes.result == 'success' &&
+      needs.validate-accounts.result != 'failure' &&
+      needs.validate-accounts.result != 'cancelled'
     runs-on: ubuntu-latest
     environment:
       name: staging
@@ -129,31 +147,47 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Set up local environment
+        shell: bash
         run: |
           cp .env.example .env
           mkdir -p mail/etc && cp config.toml.example mail/etc/config.toml
-          echo -e "\nFXA_CLIENT_ID=$FXA_CLIENT_ID" >> .env
-          echo "FXA_SECRET=$FXA_SECRET" >> .env
-          echo "SECRET_KEY=$SECRET_KEY" >> .env
-          echo "LOGIN_CODE_SECRET=$LOGIN_CODE_SECRET" >> .env
-          echo "FXA_ENCRYPT_SECRET=$FXA_ENCRYPT_SECRET" >> .env
-          echo "PADDLE_TOKEN=$PADDLE_TOKEN" >> .env
-          echo "PADDLE_API_KEY=$PADDLE_API_KEY" >> .env
-
-      - name: Start appointment stack via docker
+          append_env_if_set() {
+            local key="$1"
+            local value="${!key}"
+
+            if [ -n "$value" ]; then
+              printf '%s=%s\n' "$key" "$value" >> .env
+            fi
+          }
+
+          echo >> .env
+          append_env_if_set FXA_CLIENT_ID
+          append_env_if_set FXA_SECRET
+          append_env_if_set SECRET_KEY
+          append_env_if_set LOGIN_CODE_SECRET
+          append_env_if_set FXA_ENCRYPT_SECRET
+          append_env_if_set PADDLE_TOKEN
+          append_env_if_set PADDLE_API_KEY
+
+      - name: Start accounts stack via docker
         run: docker compose up -d --build -V
 
+      - name: Wait for local services
+        run: |
+          curl --retry 30 --retry-all-errors --retry-delay 2 --fail http://localhost:8087/health
+          curl --retry 30 --retry-all-errors --retry-delay 2 --fail http://localhost:9000/health/ready
+
       - uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: 'npm'
           cache-dependency-path: 'test/e2e/package-lock.json'
 
       - name: Install E2E test dependencies
+        working-directory: test/e2e
         run: |
-          cd ./test/e2e
-          npm install
-          npx playwright install
+          npm ci
+          npx playwright install --with-deps firefox
 
       - name: Create test plans
         run: |
@@ -163,8 +197,8 @@ jobs:
 
       - name: Run E2E tests against local stack
         id: e2e
+        working-directory: test/e2e
         run: |
-          cd ./test/e2e
           cp .env.dev.example .env
           npm run e2e-test
 

pulumi/__main__.py

@@ -17,6 +17,16 @@
 MSG_LB_MATCHING_CLUSTER = 'In this stack, Fargate clusters must have matching load balancer security groups.'
 MSG_CONTAINER_MATCHING_CLUSTER = 'In this stack, Fargate clusters must have matching container security groups.'
 
+
+def wait_for_ecs_service_steady_state(args):
+    if args.type_ != 'aws:ecs/service:Service':
+        return None
+
+    props = dict(args.props)
+    props['wait_for_steady_state'] = True
+    return pulumi.ResourceTransformationResult(props, args.opts)
+
+
 # Set up the project and convenient config access
 project = tb_pulumi.ThunderbirdPulumiProject()
 resources = project.config.get('resources')
@@ -113,7 +123,10 @@
         container_security_groups=[container_sgs[service].resources['sg'].id],
         load_balancer_security_groups=lb_sg_ids,
         desired_count=None if autoscaler_opts is not None else desired_count,
-        opts=pulumi.ResourceOptions(depends_on=depends_on),
+        opts=pulumi.ResourceOptions(
+            depends_on=depends_on,
+            transformations=[wait_for_ecs_service_steady_state],
+        ),
         **opts,
     )
     if autoscaler_opts is not None:

test/e2e/playwright.config.ts

@@ -8,6 +8,13 @@ import dotenv from 'dotenv';
 import path from 'path';
 dotenv.config({ path: path.resolve(__dirname, '.env') });
 
+const ciFirefoxLaunchOptions = process.env.CI ? {
+  firefoxUserPrefs: {
+    'network.dns.disableIPv6': true,
+    'network.dns.native-is-localhost': true,
+  },
+} : undefined;
+
 /**
  * See https://playwright.dev/docs/test-configuration.
  */
@@ -52,7 +59,8 @@ export default defineConfig({
     { name: 'desktop-setup',
       testMatch: /.*\.desktop.setup\.ts/,
       use: {
-        ...devices['Desktop Chrome'],
+        ...devices['Desktop Firefox'],
+        launchOptions: ciFirefoxLaunchOptions,
         screenshot: 'only-on-failure',
        },
     },
@@ -73,6 +81,7 @@ export default defineConfig({
       name: 'firefox',
       use: {
         ...devices['Desktop Firefox'],
+        launchOptions: ciFirefoxLaunchOptions,
         screenshot: 'only-on-failure',
         // Use prepared auth state
         storageState: 'test-results/.auth/user.json',

test/e2e/tests/auth.desktop.setup.ts

@@ -8,20 +8,17 @@ import { navigateToAccountsHubAndSignIn } from '../utils/utils';
 const fs = require('fs');
 const directoryPath = path.join(__dirname, '../test-results/.auth');
 
-fs.mkdir(directoryPath, (err: any) => {
-  if (err) {
-    console.error('error creating auth storage directory:', err);
-    return
-  }
-
-  console.log('created auth storage directory');
-  // when use storageState in browserstack yml, browserstack requires the file to exist already even on the
-  // first time the auth-setup step is run; so must create an emtpy user.json file here
-  const filepath = path.join(directoryPath, 'user.json');
-  const emptyJsonObject = {};
-  const jsonString = JSON.stringify(emptyJsonObject, null, 2); // The '2' adds indentation for readability
+// when use storageState in browserstack yml, browserstack requires the file to exist already even on the
+// first time the auth-setup step is run; so must create an empty user.json file here
+const filepath = path.join(directoryPath, 'user.json');
+const emptyJsonObject = {};
+const jsonString = JSON.stringify(emptyJsonObject, null, 2); // The '2' adds indentation for readability
+try {
+  fs.mkdirSync(directoryPath, { recursive: true });
   fs.writeFileSync(filepath, jsonString);
-});
+} catch (error) {
+  throw new Error(`Failed to initialize auth storage file at ${filepath}`, { cause: error });
+}
 
 // We write it here so it is blown away and re-created at the start of every test run; and is in .gitignore
 const authFile = path.join(__dirname, '../test-results/.auth/user.json');

test/e2e/tests/contact.spec.ts

@@ -152,7 +152,6 @@ test.describe('contact support form', {
   test('contact form displayed correctly when not signed in', async ({ page }) => {
     // clear authentication state for this test
     await page.context().clearCookies();
-    await page.reload();
     await page.waitForTimeout(TIMEOUT_5_SECONDS);
     await contactPage.navigateToContactPage();
 

test/e2e/utils/utils.ts

@@ -47,7 +47,7 @@ export const navigateToAccountsHubAndSignIn = async (page: Page) => {
     const tbAcctsSignInPage = new TBAcctsOIDCPage(page);
     const tbAcctsHubPage = new TBAcctsHubPage(page);
 
-    await page.goto(`${ACCTS_HUB_URL}`);
+    await page.goto(`${ACCTS_HUB_URL}`, { waitUntil: 'domcontentloaded' });
     //await page.waitForTimeout(TIMEOUT_5_SECONDS);
     await waitForVueApp(page);