validation.yml

name: Validation

on:
  schedule:
    - cron: '30 23 * * *'
  pull_request:
  merge_group:
  workflow_dispatch:
    inputs:
      components:
        description: 'Space-separated component names (e.g. "grid combo-box"), empty for all'
        required: false
        default: ''
      shards:
        description: 'Number of parallel IT shards'
        required: false
        default: '5'
      local-wc-validation:
        description: 'Run validation against local web-components checkout'
        required: false
        default: false
        type: boolean

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.merge_group.head_ref || github.ref }}
  cancel-in-progress: true

env:
  FORK_COUNT: 4

jobs:
  build:
    name: Build
    runs-on: ubuntu-latest
    timeout-minutes: 30
    permissions:
      contents: read
    outputs:
      skip: ${{ steps.skip-ci.outputs.skip }}
      components: ${{ steps.components.outputs.elements }}
    steps:
      - name: Set Maven args
        run: |
          args="-ntp -B"
          [ "${{ runner.debug }}" != "1" ] && args="$args -q"
          echo "MAVEN_ARGS=$args" >> "$GITHUB_ENV"

      - uses: actions/checkout@v6
        with:
          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha || github.sha }}
          fetch-depth: ${{ github.event_name == 'merge_group' && 50 || 1 }}

      - name: Check for skip-ci
        id: skip-ci
        if: github.event_name == 'pull_request' || github.event_name == 'merge_group'
        env:
          PR_TITLE: ${{ github.event.pull_request.title }}
        run: |
          if [ "${{ github.event_name }}" = "merge_group" ]; then
            echo "skip=false" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          if echo "$PR_TITLE" | grep -q '\[skip ci\]'; then
            echo "skip=true" >> "$GITHUB_OUTPUT"
          else
            echo "skip=false" >> "$GITHUB_OUTPUT"
          fi

      # Cache SVG generator build output across runs unless it's sources change.
      - name: Restore charts SVG generator cache
        id: svg-cache
        if: steps.skip-ci.outputs.skip != 'true'
        uses: actions/cache@v5
        with:
          path: vaadin-charts-flow-parent/vaadin-charts-flow-svg-generator/src/main/resources/META-INF/frontend/generated/
          key: ${{ runner.os }}-charts-svg-generator-${{ hashFiles('vaadin-charts-flow-parent/vaadin-charts-flow-svg-generator/**') }}

      - name: Setup JDK 21
        if: steps.skip-ci.outputs.skip != 'true'
        uses: actions/setup-java@v5
        with:
          java-version: '21'
          distribution: 'temurin'
          cache: 'maven'

      - name: Setup Node
        if: steps.skip-ci.outputs.skip != 'true'
        uses: actions/setup-node@v6
        with:
          node-version: '24'

      - name: Show environment info
        if: steps.skip-ci.outputs.skip != 'true'
        run: |
          java -version
          mvn -version
          node --version
          npm --version
          google-chrome --version
          chromedriver --version

      - name: Install artifacts
        if: steps.skip-ci.outputs.skip != 'true'
        env:
          # On an SVG generator cache hit, skip its npm install/build/test and
          # use the restored output instead.
          SKIP_SVG_BUILD: ${{ steps.svg-cache.outputs.cache-hit == 'true' }}
        run: |
          mvn install -Dmaven.test.skip=true -DskipSvgChartsBuild=$SKIP_SVG_BUILD -Drelease -T $FORK_COUNT $MAVEN_ARGS || {
            echo "::warning::First attempt failed (Maven multi-thread race). Retrying..."
            sleep 15
            mvn install -Dmaven.test.skip=true -DskipSvgChartsBuild=$SKIP_SVG_BUILD -Drelease -T $FORK_COUNT $MAVEN_ARGS
          }

      - name: Archive build output
        if: steps.skip-ci.outputs.skip != 'true'
        run: |
          tar -Pczf build-output.tar.gz \
            ~/.m2/repository/com/vaadin \
            vaadin-charts-flow-parent/vaadin-charts-flow-svg-generator/src/main/resources/META-INF/frontend/generated

      - name: Upload build output
        if: steps.skip-ci.outputs.skip != 'true'
        uses: actions/upload-artifact@v6
        with:
          name: build-output
          path: build-output.tar.gz
          retention-days: 1
          overwrite: true

      - name: Detect modified components
        id: components
        if: steps.skip-ci.outputs.skip != 'true'
        env:
          GH_TOKEN: ${{ github.token }}
          INPUT_COMPONENTS: ${{ github.event.inputs.components || '' }}
        run: |
          if [ -n "$INPUT_COMPONENTS" ]; then
            # Explicitly specified components are used as-is, without adding
            # dependent components
            elements="$INPUT_COMPONENTS"
          else
            changed_files=""
            if [ "${{ github.event_name }}" = "pull_request" ]; then
              pr_number="${{ github.event.pull_request.number }}"
              changed_files=$(gh api --paginate "repos/${{ github.repository }}/pulls/$pr_number/files" --jq '.[].filename')
            elif [ "${{ github.event_name }}" = "merge_group" ]; then
              changed_files=$(git diff --name-only "${{ github.event.merge_group.base_sha }}" "${{ github.event.merge_group.head_sha }}")
            fi
            mapfile -t files <<< "$changed_files"
            elements=$(node scripts/computeModules.js --changed-files -- "${files[@]}")
          fi
          if [ -n "$elements" ]; then
            echo "Running partial validation for: $elements"
          else
            echo "Running full validation for all components"
          fi
          echo "elements=$elements" >> "$GITHUB_OUTPUT"

  fast-tests:
    name: Fast tests (${{ matrix.name }})
    needs: build
    if: needs.build.outputs.skip != 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 30
    permissions:
      contents: read
    outputs:
      matrix: ${{ steps.shards.outputs.matrix }}
      has-tests: ${{ steps.shards.outputs.has-tests }}
      unit-failed: ${{ steps.type-failed.outputs.unit }}
      wtr-failed: ${{ steps.type-failed.outputs.wtr }}
    strategy:
      fail-fast: true
      matrix:
        include:
          - name: Unit tests
            type: unit
          - name: WTR tests
            type: wtr
          - name: Package WAR
            type: war
    steps:
      - name: Set Maven args
        run: |
          args="-ntp -B"
          [ "${{ runner.debug }}" != "1" ] && args="$args -q"
          echo "MAVEN_ARGS=$args" >> "$GITHUB_ENV"

      - uses: actions/checkout@v6
        with:
          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha || github.sha }}
          fetch-depth: 1

      - name: Setup Node
        if: matrix.type != 'unit'
        uses: actions/setup-node@v6
        with:
          node-version: '24'

      - name: Checkout web-components
        if: matrix.type != 'unit' && github.event_name == 'schedule' || inputs.local-wc-validation
        uses: actions/checkout@v6
        with:
          repository: vaadin/web-components
          ref: main
          path: web-components

      - name: Install web-components
        if: matrix.type != 'unit' && github.event_name == 'schedule' || inputs.local-wc-validation
        run: yarn install --ignore-engines --ignore-scripts
        working-directory: web-components

      - name: Configure local web-components
        if: matrix.type != 'unit' && github.event_name == 'schedule' || inputs.local-wc-validation
        run: echo "LOCAL_WEB_COMPONENTS_PATH=web-components" > .env

      - name: npm install
        if: matrix.type != 'unit'
        run: npm ci --ignore-scripts

      - name: Merge ITs
        if: matrix.type == 'war'
        run: node scripts/mergeITs.js ${{ needs.build.outputs.components }}

      - name: Setup JDK 21
        uses: actions/setup-java@v5
        with:
          java-version: '21'
          distribution: 'temurin'
          cache: 'maven'

      - name: Install TestBench license
        if: env.TB_LICENSE != ''
        env:
          TB_LICENSE: ${{ secrets.TB_LICENSE }}
        run: |
          mkdir -p ~/.vaadin
          user="${TB_LICENSE%%/*}"
          key="${TB_LICENSE#*/}"
          echo "{\"username\":\"${user}\",\"proKey\":\"${key}\"}" > ~/.vaadin/proKey

      - name: Download build output
        uses: actions/download-artifact@v8
        with:
          name: build-output

      - name: Restore build output
        run: tar -Pxzf build-output.tar.gz

      - name: Run unit tests
        if: matrix.type == 'unit'
        env:
          COMPONENTS: ${{ needs.build.outputs.components }}
        run: |
          shopt -s nullglob
          pl=""
          for component in $COMPONENTS; do
            for pom in vaadin-$component-flow-parent/*/pom.xml; do
              dir=${pom%/pom.xml}
              case "$dir" in
                *-integration-tests|*-testbench) continue ;;
              esac
              pl+="${pl:+,}$dir"
            done
          done
          pl_arg=""
          if [ -n "$pl" ]; then
            echo "Running unit tests for: $pl"
            pl_arg="-pl $pl"
          fi
          mvn test $pl_arg -Drelease -T $FORK_COUNT -Dsurefire.parallel=classes -Dsurefire.threadCount=2 -DskipSvgChartsBuild $MAVEN_ARGS

      - name: Upload unit test reports
        if: always() && matrix.type == 'unit'
        uses: actions/upload-artifact@v6
        with:
          name: surefire-reports
          path: '**/target/surefire-reports/TEST-*.xml'
          retention-days: 1
          if-no-files-found: ignore

      - name: Run WTR tests
        if: matrix.type == 'wtr'
        run: node scripts/wtr.js ${{ needs.build.outputs.components }}

      - name: Upload WTR reports
        if: always() && matrix.type == 'wtr'
        uses: actions/upload-artifact@v6
        with:
          name: wtr-reports
          path: '**/wtr-results.xml'
          retention-days: 1
          if-no-files-found: ignore

      - name: Package integration-tests WAR
        if: matrix.type == 'war'
        run: |
          mvn package -pl integration-tests \
            -Drun-it -Drelease \
            -Dvaadin.productionMode -Dvaadin.force.production.build=true \
            -Dmaven.test.skip=true -DskipJetty $MAVEN_ARGS

      - name: Compile IT test sources
        if: matrix.type == 'war'
        run: mvn test-compile -pl integration-tests -Drun-it -DskipFrontend -DskipJetty -DskipUnitTests $MAVEN_ARGS

      - name: Archive packaged WAR
        if: matrix.type == 'war'
        run: tar -Pczf integration-tests-war.tar.gz integration-tests/pom.xml integration-tests/target

      - name: Upload packaged WAR
        if: matrix.type == 'war'
        uses: actions/upload-artifact@v6
        with:
          name: integration-tests-war
          path: integration-tests-war.tar.gz
          retention-days: 1
          overwrite: true

      - name: Compute IT shards
        if: matrix.type == 'war'
        id: shards
        env:
          SHARDS: ${{ github.event.inputs.shards || '0' }}
          TARGET_PER_SHARD: 35
          MAX_SHARDS: 10
        run: |
          mapfile -t its < <(find integration-tests/src/test/java -name '*IT.java' -printf '%P\n' | sed -e 's|/|.|g' -e 's|\.java$||' | sort)
          count=${#its[@]}
          echo "Found $count IT classes"
          if [ "$count" -eq 0 ]; then
            echo "has-tests=false" >> "$GITHUB_OUTPUT"
            echo 'matrix={"include":[]}' >> "$GITHUB_OUTPUT"
            exit 0
          fi
          n=$SHARDS
          if [ "$n" -eq 0 ]; then
            n=$(( (count + TARGET_PER_SHARD - 1) / TARGET_PER_SHARD ))
            [ "$n" -lt 1 ] && n=1
            [ "$n" -gt "$MAX_SHARDS" ] && n=$MAX_SHARDS
          fi
          if [ "$n" -gt "$count" ]; then n=$count; fi
          echo "$n shards for $count classes (~$(( count / n )) per shard)"
          declare -a buckets
          for i in $(seq 1 $n); do buckets[$i]=""; done
          i=1
          for t in "${its[@]}"; do
            if [ -n "${buckets[$i]}" ]; then buckets[$i]+=","; fi
            buckets[$i]+="$t"
            i=$((i+1))
            [ $i -gt $n ] && i=1
          done
          json='{"include":['
          for k in $(seq 1 $n); do
            [ $k -gt 1 ] && json+=','
            json+='{"shard":"'"$k"'","tests":"'"${buckets[$k]}"'"}'
          done
          json+=']}'
          echo "$json"
          echo "matrix=$json" >> "$GITHUB_OUTPUT"
          echo "has-tests=true" >> "$GITHUB_OUTPUT"

      - name: Record job type on failure
        id: type-failed
        if: always() && failure()
        run: echo "${{ matrix.type }}=true" >> "$GITHUB_OUTPUT"

  its:
    name: ITs (${{ matrix.shard }})
    needs: [build, fast-tests]
    if: needs.fast-tests.outputs.has-tests == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 60
    permissions:
      contents: read
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.fast-tests.outputs.matrix) }}
    steps:
      - name: Set Maven args
        run: |
          args="-ntp -B"
          [ "${{ runner.debug }}" != "1" ] && args="$args -q"
          echo "MAVEN_ARGS=$args" >> "$GITHUB_ENV"

      - uses: actions/checkout@v6
        with:
          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha || github.sha }}
          fetch-depth: 1

      - name: Setup JDK 21
        uses: actions/setup-java@v5
        with:
          java-version: '21'
          distribution: 'temurin'
          cache: 'maven'

      - name: Install TestBench license
        if: env.TB_LICENSE != ''
        env:
          TB_LICENSE: ${{ secrets.TB_LICENSE }}
        run: |
          mkdir -p ~/.vaadin
          user="${TB_LICENSE%%/*}"
          key="${TB_LICENSE#*/}"
          echo "{\"username\":\"${user}\",\"proKey\":\"${key}\"}" > ~/.vaadin/proKey

      - name: Download build output
        uses: actions/download-artifact@v8
        with:
          name: build-output

      - name: Restore build output
        run: tar -Pxzf build-output.tar.gz

      - name: Download packaged WAR
        uses: actions/download-artifact@v8
        with:
          name: integration-tests-war

      - name: Restore packaged WAR
        run: tar -Pxzf integration-tests-war.tar.gz

      - name: Run integration tests (shard ${{ matrix.shard }})
        env:
          SHARD_TESTS: ${{ matrix.tests }}
        run: |
          mvn -pl integration-tests -Drun-it \
            jetty:start-war failsafe:integration-test jetty:stop failsafe:verify \
            -Dvaadin.productionMode \
            -DskipFrontend \
            -Dfailsafe.forkCount=$FORK_COUNT \
            -Dcom.vaadin.testbench.Parameters.testsInParallel=2 \
            -Dfailsafe.rerunFailingTestsCount=2 \
            -Dmaven.test.redirectTestOutputToFile=true \
            -Dtest.reuseDriver=true \
            -Dit.test="$SHARD_TESTS" \
            -B

      - name: Upload error screenshots
        if: failure()
        uses: actions/upload-artifact@v6
        with:
          name: error-screenshots-${{ matrix.shard }}
          path: integration-tests/error-screenshots/
          retention-days: 5
          if-no-files-found: ignore

      - name: Upload failsafe reports
        if: always()
        uses: actions/upload-artifact@v6
        with:
          name: failsafe-reports-${{ matrix.shard }}
          path: integration-tests/target/failsafe-reports/
          retention-days: 1
          if-no-files-found: ignore

  results:
    name: Validation result
    needs: [build, fast-tests, its]
    if: always()
    runs-on: ubuntu-latest
    permissions:
      contents: read
      checks: write
      actions: write
    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha || github.sha }}
          fetch-depth: 1

      - uses: actions/download-artifact@v8
        continue-on-error: true
        with:
          name: surefire-reports
          path: surefire-reports

      - uses: actions/download-artifact@v8
        continue-on-error: true
        with:
          pattern: failsafe-reports-*
          merge-multiple: true
          path: failsafe-reports

      - uses: actions/download-artifact@v8
        continue-on-error: true
        with:
          name: wtr-reports
          path: wtr-reports

      - name: Remove passing test reports
        run: |
          # Detect real failures via testsuite attributes (flaky retries leave
          # <flakyError> tags but keep failures="0" errors="0" on the suite).
          has_failures=false
          for dir in surefire-reports failsafe-reports wtr-reports; do
            [ -d "$dir" ] || continue
            if find "$dir" -name '*.xml' | xargs grep -lE 'failures="[1-9]|errors="[1-9]' 2>/dev/null | grep -q .; then
              has_failures=true
              break
            fi
          done
          echo "has_failures=$has_failures" >> $GITHUB_ENV
          # Red build: keep only failing reports so dorny focuses on failures.
          # Green build: keep all reports so every suite shows as passed.
          if [ "$has_failures" = "true" ]; then
            for dir in surefire-reports failsafe-reports wtr-reports; do
              [ -d "$dir" ] || continue
              find "$dir" -name '*.xml' | while read -r f; do
                grep -qE 'failures="[1-9]|errors="[1-9]' "$f" || rm -f "$f"
              done
            done
          fi

      - name: Check for unit test reports
        id: unit-check
        run: |
          count=$(find surefire-reports -name 'TEST-*.xml' 2>/dev/null | wc -l | tr -d ' ')
          echo "count=$count" >> $GITHUB_OUTPUT

      - name: Unit Tests header
        if: steps.unit-check.outputs.count != '0' && (needs.fast-tests.outputs.unit-failed != 'true' || env.has_failures == 'true')
        run: echo "## Unit Tests" >> $GITHUB_STEP_SUMMARY

      - name: Publish unit test results
        id: unit-dorny
        if: steps.unit-check.outputs.count != '0' && (needs.fast-tests.outputs.unit-failed != 'true' || env.has_failures == 'true')
        uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2
        with:
          name: Unit Tests
          path: surefire-reports/**/TEST-*.xml
          reporter: java-junit
          list-tests: ${{ env.has_failures == 'true' && 'failed' || 'all' }}
          list-suites: ${{ env.has_failures == 'true' && 'failed' || 'all' }}
          fail-on-error: 'false'

      - name: Unit test compilation error
        if: needs.fast-tests.outputs.unit-failed == 'true' && env.has_failures != 'true'
        run: |
          echo ":x: Unit tests compilation failed. Check the **Unit tests** job for details." >> $GITHUB_STEP_SUMMARY

      - name: Check for IT reports
        id: it-check
        run: |
          count=$(find failsafe-reports -name 'TEST-*.xml' 2>/dev/null | wc -l | tr -d ' ')
          echo "count=$count" >> $GITHUB_OUTPUT

      - name: Integration Tests header
        if: steps.it-check.outputs.count != '0' && (needs.fast-tests.result == 'success' || env.has_failures == 'true')
        run: echo "## Integration Tests" >> $GITHUB_STEP_SUMMARY

      - name: Publish integration test results
        id: it-dorny
        if: steps.it-check.outputs.count != '0' && (needs.fast-tests.result == 'success' || env.has_failures == 'true')
        uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2
        with:
          name: Integration Tests
          path: failsafe-reports/TEST-*.xml
          reporter: java-junit
          list-tests: ${{ env.has_failures == 'true' && 'failed' || 'all' }}
          list-suites: ${{ env.has_failures == 'true' && 'failed' || 'all' }}
          fail-on-error: 'false'


      - name: Check for WTR reports
        id: wtr-check
        run: |
          count=$(find wtr-reports -name 'wtr-results.xml' 2>/dev/null | wc -l | tr -d ' ')
          echo "count=$count" >> $GITHUB_OUTPUT

      - name: WTR Tests header
        if: steps.wtr-check.outputs.count != '0' && (needs.fast-tests.result == 'success' || needs.fast-tests.outputs.wtr-failed == 'true')
        run: echo "## WTR Tests" >> $GITHUB_STEP_SUMMARY

      - name: Publish WTR results
        id: wtr-dorny
        if: steps.wtr-check.outputs.count != '0' && (needs.fast-tests.result == 'success' || needs.fast-tests.outputs.wtr-failed == 'true')
        uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2
        with:
          name: WTR Tests
          path: wtr-reports/**/wtr-results.xml
          reporter: java-junit
          list-tests: ${{ env.has_failures == 'true' && 'failed' || 'all' }}
          list-suites: ${{ env.has_failures == 'true' && 'failed' || 'all' }}
          fail-on-error: 'false'

      - name: WTR setup error
        if: steps.wtr-check.outputs.count == '0' && needs.fast-tests.outputs.wtr-failed == 'true'
        run: |
          echo ":x: WTR tests did not produce results. Check the **WTR tests** job for details." >> $GITHUB_STEP_SUMMARY

      - uses: actions/download-artifact@v8
        if: needs.its.result != 'success'
        continue-on-error: true
        with:
          pattern: error-screenshots-*
          merge-multiple: true
          path: error-screenshots

      - name: Upload merged error screenshots
        if: needs.its.result != 'success'
        uses: actions/upload-artifact@v6
        with:
          name: error-screenshots
          path: error-screenshots/
          retention-days: 5
          if-no-files-found: ignore

      - name: Delete intermediate artifacts
        if: always()
        env:
          GH_TOKEN: ${{ github.token }}
          GH_REPO: ${{ github.repository }}
          RUN_ID: ${{ github.run_id }}
        run: |
          gh api --paginate "repos/$GH_REPO/actions/runs/$RUN_ID/artifacts" \
            --jq '.artifacts[] | select(
              (.name | startswith("error-screenshots-")) or
              (.name | startswith("failsafe-reports-")) or
              .name == "wtr-reports" or
              .name == "surefire-reports" or
              .name == "build-output" or
              .name == "integration-tests-war"
            ) | "\(.id) \(.name)"' \
          | while read -r id name; do
              echo "Deleting artifact $name (id=$id)"
              gh api -X DELETE "repos/$GH_REPO/actions/artifacts/$id" || true
            done

      - name: Fail if any job or test failed
        if: always()
        run: |
          failed=false
          # Job-level failures (build/setup/compile/packaging/infra).
          # Skipped is OK: skip-ci skips fast-tests/its, empty builds skip its.
          for r in "${{ needs.build.result }}" "${{ needs.fast-tests.result }}" "${{ needs.its.result }}"; do
            [[ "$r" == "failure" || "$r" == "cancelled" ]] && failed=true
          done
          # Actual test failures captured in the published reports.
          [[ "${{ steps.unit-dorny.outputs.conclusion }}" == "failure" ]] && failed=true
          [[ "${{ steps.wtr-dorny.outputs.conclusion }}"  == "failure" ]] && failed=true
          [[ "${{ steps.it-dorny.outputs.conclusion }}"   == "failure" ]] && failed=true
          [[ "$failed" == "true" ]] && exit 1 || exit 0