Run Frontend Tests #1299
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run Frontend Tests | |
| on: | |
| workflow_call: | |
| secrets: | |
| OPENAI_API_KEY: | |
| required: true | |
| STORE_API_KEY: | |
| required: true | |
| ANTHROPIC_API_KEY: | |
| required: true | |
| TAVILY_API_KEY: | |
| required: true | |
| inputs: | |
| suites: | |
| description: "Test suites to run (JSON array)" | |
| required: false | |
| type: string | |
| default: "[]" | |
| release: | |
| description: "Whether this is a release build" | |
| required: false | |
| type: boolean | |
| default: false | |
| tests_folder: | |
| description: "(Optional) Tests to run" | |
| required: false | |
| type: string | |
| default: "tests" | |
| ref: | |
| description: "(Optional) ref to checkout" | |
| required: false | |
| type: string | |
| runs-on: | |
| description: "Runner to use for the tests" | |
| required: false | |
| type: string | |
| default: "ubuntu-latest" | |
| workflow_dispatch: | |
| inputs: | |
| suites: | |
| description: "Test suites to run (JSON array)" | |
| required: false | |
| type: string | |
| default: "[]" | |
| release: | |
| description: "Whether this is a release build" | |
| required: false | |
| type: boolean | |
| default: false | |
| tests_folder: | |
| description: "(Optional) Tests to run" | |
| required: false | |
| type: string | |
| default: "tests" | |
| runs-on: | |
| description: "Runner to use for the tests" | |
| required: false | |
| type: choice | |
| options: | |
| - ubuntu-latest | |
| - windows-latest | |
| - self-hosted | |
| - '["self-hosted", "linux", "ARM64", "langflow-ai-arm64-40gb-ephemeral"]' | |
| default: ubuntu-latest | |
| env: | |
| NODE_VERSION: "22" | |
| PYTHON_VERSION: "3.13" | |
| # Define the directory where Playwright browsers will be installed. | |
| # This path is used for caching across workflows | |
| PLAYWRIGHT_BROWSERS_PATH: "ms-playwright" | |
| PLAYWRIGHT_VERSION: "1.59.1" | |
| LANGFLOW_FEATURE_WXO_DEPLOYMENTS: "true" | |
| jobs: | |
| determine-test-suite: | |
| name: Determine Test Suites and Shard Distribution | |
| runs-on: ${{ (inputs['runs-on'] && startsWith(format('{0}', inputs['runs-on']), '[') && fromJSON(inputs['runs-on'])) || inputs['runs-on'] || github.event.inputs['runs-on'] || 'ubuntu-latest' }} | |
| outputs: | |
| matrix: ${{ steps.setup-matrix.outputs.matrix }} | |
| test_grep: ${{ steps.set-matrix.outputs.test_grep }} | |
| suites: ${{ steps.set-matrix.outputs.suites }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ inputs.ref || github.ref }} | |
| fetch-depth: 0 | |
| - name: Paths Filter | |
| id: filter | |
| uses: dorny/paths-filter@v3 | |
| with: | |
| filters: .github/changes-filter.yaml | |
| - name: Determine Test Suites from Changes | |
| id: set-matrix | |
| shell: bash | |
| run: | | |
| # Start with input suites if provided, otherwise empty array | |
| echo "Changes filter output: $(echo '${{ toJSON(steps.filter.outputs) }}')" | |
| SUITES='${{ inputs.suites }}' | |
| echo "Initial suites: $SUITES" | |
| TEST_GREP="" | |
| echo "Inputs Release: ${{ inputs.release }}" | |
| RELEASE="${{ inputs.release || 'false' }}" | |
| echo "Release build: $RELEASE" | |
| # Only set to release if it's explicitly a release build | |
| if [[ "$RELEASE" == "true" ]]; then | |
| SUITES='["release"]' | |
| echo "Release build detected - setting suites to: $SUITES" | |
| # grep pattern for release is the @release tag - run all tests | |
| TEST_GREP='--grep="@release"' | |
| else | |
| # If input suites were not provided, determine based on changes | |
| if [[ "$SUITES" == "[]" ]]; then | |
| echo "No input suites provided - determining from changes" | |
| SUITES='[]' # Ensure we start with a valid JSON array | |
| TAGS=() | |
| # Add suites and tags based on changed files | |
| if [[ "${{ steps.filter.outputs.components }}" == "true" ]]; then | |
| SUITES=$(echo $SUITES | jq -c '. += ["components"]') | |
| TAGS+=("@components") | |
| echo "Added components suite" | |
| fi | |
| if [[ "${{ steps.filter.outputs.starter-projects }}" == "true" ]]; then | |
| SUITES=$(echo $SUITES | jq -c '. += ["starter-projects"]') | |
| TAGS+=("@starter-projects") | |
| echo "Added starter-projects suite" | |
| fi | |
| if [[ "${{ steps.filter.outputs.workspace }}" == "true" ]]; then | |
| SUITES=$(echo $SUITES | jq -c '. += ["workspace"]') | |
| TAGS+=("@workspace") | |
| echo "Added workspace suite" | |
| fi | |
| if [[ "${{ steps.filter.outputs.api }}" == "true" ]]; then | |
| SUITES=$(echo $SUITES | jq -c '. += ["api"]') | |
| TAGS+=("@api") | |
| echo "Added api suite" | |
| fi | |
| if [[ "${{ steps.filter.outputs.database }}" == "true" ]]; then | |
| SUITES=$(echo $SUITES | jq -c '. += ["database"]') | |
| TAGS+=("@database") | |
| echo "Added database suite" | |
| fi | |
| # Create grep pattern if we have tags | |
| if [ ${#TAGS[@]} -gt 0 ]; then | |
| # Join tags with | for OR logic | |
| REGEX_PATTERN=$(IFS='|'; echo "${TAGS[*]}") | |
| TEST_GREP='--grep="${REGEX_PATTERN}"' | |
| else | |
| # No path-specific changes detected - default to running release tests | |
| # so that FE tests always run even for backend-only changes | |
| SUITES='["release"]' | |
| TEST_GREP='--grep="@release"' | |
| echo "No path-specific changes detected - defaulting to release suite" | |
| fi | |
| else | |
| # Process input suites to tags | |
| # First ensure SUITES is valid JSON | |
| if ! echo "$SUITES" | jq -e . > /dev/null 2>&1; then | |
| echo "Warning: Input suites is not valid JSON, attempting to fix" | |
| # Try to fix common issues like missing quotes | |
| if [[ "$SUITES" =~ ^\[(.*)\]$ ]]; then | |
| # Extract items and add quotes | |
| ITEMS="${BASH_REMATCH[1]}" | |
| QUOTED_ITEMS=$(echo "$ITEMS" | sed 's/\([^,]*\)/"\1"/g') | |
| SUITES="[$QUOTED_ITEMS]" | |
| fi | |
| echo "Fixed suites: $SUITES" | |
| fi | |
| TAGS=() | |
| if echo "$SUITES" | jq -e 'contains(["components"])' > /dev/null; then | |
| TAGS+=("@components") | |
| fi | |
| if echo "$SUITES" | jq -e 'contains(["starter-projects"])' > /dev/null; then | |
| TAGS+=("@starter-projects") | |
| fi | |
| if echo "$SUITES" | jq -e 'contains(["workspace"])' > /dev/null; then | |
| TAGS+=("@workspace") | |
| fi | |
| if echo "$SUITES" | jq -e 'contains(["api"])' > /dev/null; then | |
| TAGS+=("@api") | |
| fi | |
| if echo "$SUITES" | jq -e 'contains(["database"])' > /dev/null; then | |
| TAGS+=("@database") | |
| fi | |
| if [ ${#TAGS[@]} -gt 0 ]; then | |
| # Join tags with | for OR logic | |
| REGEX_PATTERN=$(IFS='|'; echo "${TAGS[*]}") | |
| TEST_GREP='--grep "${REGEX_PATTERN}"' | |
| fi | |
| fi | |
| fi | |
| # Ensure compact JSON output | |
| SUITES=$(echo "$SUITES" | jq -c '.') | |
| echo "Final test suites to run: $SUITES" | |
| echo "Test grep pattern: $TEST_GREP" | |
| # Ensure proper JSON formatting for matrix output | |
| echo "matrix=$(echo $SUITES | jq -c .)" >> $GITHUB_OUTPUT | |
| echo "test_grep=$TEST_GREP" >> $GITHUB_OUTPUT | |
| echo "suites=$SUITES" >> $GITHUB_OUTPUT | |
| - name: Setup Node ${{ env.NODE_VERSION }} | |
| uses: actions/setup-node@v6 | |
| id: setup-node | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| # npm caching is decoupled from setup-node so a transient cache-service | |
| # hiccup (frequent on Windows runners) never fails the job. See #25 / #1. | |
| - name: Get npm cache directory | |
| id: npm-cache-dir | |
| shell: bash | |
| run: echo "dir=$(npm config get cache)" >> "$GITHUB_OUTPUT" | |
| - name: Cache npm dependencies | |
| uses: actions/cache@v5 | |
| continue-on-error: true | |
| with: | |
| path: ${{ steps.npm-cache-dir.outputs.dir }} | |
| key: ${{ runner.os }}-npm-${{ env.NODE_VERSION }}-${{ hashFiles('src/frontend/package-lock.json') }} | |
| restore-keys: | | |
| ${{ runner.os }}-npm-${{ env.NODE_VERSION }}- | |
| - name: Install Frontend Dependencies | |
| run: npm ci | |
| working-directory: ./src/frontend | |
| - name: Calculate Test Shards Distribution | |
| id: setup-matrix | |
| shell: bash | |
| run: | | |
| cd src/frontend | |
| # Get the test count using playwright's built-in grep | |
| if [ -n "${{ steps.set-matrix.outputs.test_grep }}" ]; then | |
| TEST_COUNT=$(npx playwright test ${{ inputs.tests_folder }} ${{ steps.set-matrix.outputs.test_grep }} --list | wc -l) | |
| else | |
| TEST_COUNT=$(npx playwright test ${{ inputs.tests_folder }} --list | wc -l) | |
| fi | |
| echo "Total tests to run: $TEST_COUNT" | |
| # Calculate optimal shard count - 1 shard per 5 tests, min 1, max 70 | |
| SHARD_COUNT=$(( (TEST_COUNT + 4) / 5 )) | |
| if [ $SHARD_COUNT -lt 1 ]; then | |
| SHARD_COUNT=1 | |
| elif [ $SHARD_COUNT -gt 70 ]; then | |
| SHARD_COUNT=70 | |
| fi | |
| # Create the matrix combinations string | |
| MATRIX_COMBINATIONS="" | |
| for i in $(seq 1 $SHARD_COUNT); do | |
| if [ $i -gt 1 ]; then | |
| MATRIX_COMBINATIONS="$MATRIX_COMBINATIONS," | |
| fi | |
| MATRIX_COMBINATIONS="$MATRIX_COMBINATIONS{\"shardIndex\": $i, \"shardTotal\": $SHARD_COUNT}" | |
| done | |
| echo "matrix={\"include\":[$MATRIX_COMBINATIONS]}" >> "$GITHUB_OUTPUT" | |
| setup-and-test: | |
| name: Playwright Tests - Shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }} | |
| runs-on: ${{ (inputs['runs-on'] && startsWith(format('{0}', inputs['runs-on']), '[') && fromJSON(inputs['runs-on'])) || inputs['runs-on'] || github.event.inputs['runs-on'] || 'ubuntu-latest' }} | |
| if: ${{ needs.determine-test-suite.outputs.test_grep != '' }} | |
| needs: determine-test-suite | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.determine-test-suite.outputs.matrix) }} | |
| env: | |
| OPENAI_API_KEY: ${{ inputs.openai_api_key || secrets.OPENAI_API_KEY }} | |
| STORE_API_KEY: ${{ inputs.store_api_key || secrets.STORE_API_KEY }} | |
| SEARCH_API_KEY: "${{ secrets.SEARCH_API_KEY }}" | |
| ASTRA_DB_APPLICATION_TOKEN: "${{ secrets.ASTRA_DB_APPLICATION_TOKEN }}" | |
| ASTRA_DB_API_ENDPOINT: "${{ secrets.ASTRA_DB_API_ENDPOINT }}" | |
| ANTHROPIC_API_KEY: "${{ secrets.ANTHROPIC_API_KEY }}" | |
| TAVILY_API_KEY: "${{ secrets.TAVILY_API_KEY }}" | |
| LANGFLOW_DEACTIVE_TRACING: "true" | |
| outputs: | |
| failed: ${{ steps.check-failure.outputs.failed }} | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ inputs.ref || github.ref }} | |
| - name: Setup Node.js Environment | |
| uses: actions/setup-node@v6 | |
| id: setup-node | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| # npm caching is decoupled from setup-node so a transient cache-service | |
| # hiccup (frequent on Windows runners) never fails the job. See #25 / #1. | |
| - name: Get npm cache directory | |
| id: npm-cache-dir | |
| shell: bash | |
| run: echo "dir=$(npm config get cache)" >> "$GITHUB_OUTPUT" | |
| - name: Cache npm dependencies | |
| uses: actions/cache@v5 | |
| continue-on-error: true | |
| with: | |
| path: ${{ steps.npm-cache-dir.outputs.dir }} | |
| key: ${{ runner.os }}-npm-${{ env.NODE_VERSION }}-${{ hashFiles('src/frontend/package-lock.json') }} | |
| restore-keys: | | |
| ${{ runner.os }}-npm-${{ env.NODE_VERSION }}- | |
| - name: Install Frontend Dependencies | |
| run: npm ci | |
| working-directory: ./src/frontend | |
| # Cache Playwright browsers using a composite key | |
| - name: Cache Playwright Browsers | |
| id: cache-playwright | |
| uses: actions/cache@v5 | |
| continue-on-error: true | |
| with: | |
| path: ${{ env.PLAYWRIGHT_BROWSERS_PATH }} | |
| key: playwright-${{ env.PLAYWRIGHT_VERSION }}-chromium-${{ runner.os }} | |
| restore-keys: | | |
| playwright-${{ env.PLAYWRIGHT_VERSION }}-chromium-${{ runner.os }} | |
| - name: Install Playwright Browser Dependencies (Linux) | |
| if: steps.cache-playwright.outputs.cache-hit != 'true' && runner.os == 'Linux' && !contains(inputs.runs-on, 'self-hosted') | |
| shell: bash | |
| run: | | |
| cd ./src/frontend | |
| npx playwright install --with-deps chromium | |
| - name: Install Playwright Browsers (Windows/macOS/Self-Hosted) | |
| if: steps.cache-playwright.outputs.cache-hit != 'true' && (runner.os != 'Linux' || contains(inputs.runs-on, 'self-hosted')) | |
| shell: bash | |
| run: | | |
| cd ./src/frontend | |
| npx playwright install chromium | |
| - name: "Setup Environment" | |
| uses: astral-sh/setup-uv@v6 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "uv.lock" | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| prune-cache: false | |
| - name: Install Python Dependencies | |
| run: uv sync --extra audio | |
| - name: Configure Environment Variables | |
| shell: bash | |
| run: | | |
| touch .env | |
| echo "${{ secrets.ENV_VARS }}" > .env | |
| - name: Execute Playwright Tests | |
| uses: nick-fields/retry@v3 | |
| with: | |
| # Bumped from 12 → 18 minutes. The shard carrying the | |
| # starter-projects template sweep iterates every starter template; | |
| # under Linux CI load that work can run past 12 minutes once a | |
| # single iteration triggers a per-test Playwright retry. | |
| timeout_minutes: 18 | |
| max_attempts: 2 | |
| # The command block is bash; without this the action falls back to | |
| # the runner default and PowerShell fails to parse the if/grep. | |
| shell: bash | |
| command: | | |
| cd src/frontend | |
| echo 'Running tests with pattern: ${{ needs.determine-test-suite.outputs.test_grep }}' | |
| TEST_LIST="$(npx playwright test ${{ inputs.tests_folder }} ${{ needs.determine-test-suite.outputs.test_grep }} --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --list --retries=3)" | |
| printf '%s\n' "$TEST_LIST" | |
| WORKERS=2 | |
| # The sweep lives in starter-projects-outdated-components.spec.ts | |
| # (renamed from starter-projects-shardN in #13844 — keep this | |
| # pattern in sync or the one-worker mitigation silently no-ops). | |
| if printf '%s\n' "$TEST_LIST" | grep -q 'starter-projects-outdated-components'; then | |
| echo "Detected starter project template sweep in this shard; running it with one worker to avoid shared backend contention." | |
| WORKERS=1 | |
| fi | |
| # echo command before running | |
| echo "npx playwright test ${{ inputs.tests_folder }} ${{ needs.determine-test-suite.outputs.test_grep }} --trace on --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --workers $WORKERS --retries=3" | |
| npx playwright test ${{ inputs.tests_folder }} ${{ needs.determine-test-suite.outputs.test_grep }} --trace on --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --workers "$WORKERS" --retries=3 | |
| - name: Upload Test Results | |
| if: always() | |
| continue-on-error: true | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: blob-report-${{ runner.os }}-${{ matrix.shardIndex }} | |
| path: src/frontend/blob-report | |
| retention-days: 1 | |
| overwrite: true | |
| - name: Upload Playwright Coverage Artifact | |
| if: always() && hashFiles('src/frontend/coverage/playwright/**') != '' | |
| continue-on-error: true | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: playwright-coverage-${{ matrix.shardIndex }} | |
| path: src/frontend/coverage/playwright | |
| retention-days: 7 | |
| overwrite: true | |
| - name: Cleanup UV Cache | |
| run: uv cache prune --ci | |
| merge-reports: | |
| # We need to repeat the condition at every step | |
| # https://github.com/actions/runner/issues/662 | |
| needs: setup-and-test | |
| runs-on: ${{ (inputs['runs-on'] && startsWith(format('{0}', inputs['runs-on']), '[') && fromJSON(inputs['runs-on'])) || inputs['runs-on'] || github.event.inputs['runs-on'] || 'ubuntu-latest' }} | |
| if: always() | |
| env: | |
| EXIT_CODE: ${{!contains(needs.setup-and-test.result, 'failure') && !contains(needs.setup-and-test.result, 'cancelled') && '0' || '1'}} | |
| steps: | |
| - name: "Should Merge Reports" | |
| # If the CI was successful, we don't need to merge the reports | |
| # so we can skip all the steps below | |
| id: should_merge_reports | |
| shell: bash | |
| run: | | |
| if [ "$EXIT_CODE" == "0" ]; then | |
| echo "should_merge_reports=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "should_merge_reports=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Checkout code | |
| if: ${{ steps.should_merge_reports.outputs.should_merge_reports == 'true' }} | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ inputs.ref || github.ref }} | |
| - name: Setup Node.js | |
| if: ${{ steps.should_merge_reports.outputs.should_merge_reports == 'true' }} | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: Download blob reports from GitHub Actions Artifacts | |
| if: ${{ steps.should_merge_reports.outputs.should_merge_reports == 'true' }} | |
| uses: actions/download-artifact@v7 | |
| with: | |
| path: all-blob-reports | |
| pattern: blob-report-* | |
| merge-multiple: true | |
| - name: Merge into HTML Report | |
| if: ${{ steps.should_merge_reports.outputs.should_merge_reports == 'true' }} | |
| shell: bash | |
| run: | | |
| npx playwright merge-reports --reporter html ./all-blob-reports | |
| - name: Upload HTML report | |
| if: ${{ steps.should_merge_reports.outputs.should_merge_reports == 'true' }} | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: html-report--attempt-${{ github.run_attempt }} | |
| path: playwright-report | |
| retention-days: 14 |