integration-tests-mysql-elasticsearch.yml

#  Copyright 2021 Collate
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#  http://www.apache.org/licenses/LICENSE-2.0
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

name: Integration Tests - MySQL + Elasticsearch

on:
  merge_group:
  workflow_dispatch:
  push:
    branches:
      - main
    paths:
      - "openmetadata-service/**"
      - "openmetadata-integration-tests/**"
      - "openmetadata-spec/pom.xml"
      - "openmetadata-spec/src/**"
      - "openmetadata-sdk/**"
      - "openmetadata-shaded-deps/**"
      - "openmetadata-k8s-operator/**"
      - "openmetadata-mcp/**"
      - "common/**"
      - "pom.xml"
      - "Makefile"
      - "bootstrap/**"
      - ".github/scripts/classify_test_outcome.py"
      - ".github/workflows/integration-tests-mysql-elasticsearch.yml"
  pull_request_target:
    types: [labeled, opened, synchronize, reopened, ready_for_review]

permissions:
  contents: read
  pull-requests: read
  checks: write

concurrency:
  group: integration-tests-mysql-es-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || (github.event.label.name == 'safe to test' && github.event.pull_request.head.repo.full_name != github.repository) }}
jobs:
  # Detect whether relevant paths changed. When no matching files are modified
  # the downstream job is skipped via its `if` condition.
  # A job skipped by `if` reports as "Success", so required checks still pass.
  changes:
    name: Detect Changes
    runs-on: ubuntu-latest
    timeout-minutes: 10
    if: ${{ !github.event.pull_request.draft && (github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test') }}
    # pull_request_target uses the GitHub API and does not require a checkout. Merge
    # queue runs check out the merge-queue commit so dorny can diff against
    # merge_group.base_sha and skip the heavy work when nothing backend-relevant
    # changed. Push and workflow_dispatch remain force-true — push is already
    # gated by the trigger paths above, and workflow_dispatch is an explicit
    # "run the full suite" request. Keep the trigger paths and filter list in sync.
    outputs:
      backend: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'true' || steps.filter.outputs.backend }}
    steps:
      - name: Checkout
        uses: actions/checkout@v7
        if: ${{ github.event_name == 'merge_group' }}
        with:
          fetch-depth: 0
          filter: blob:none
          persist-credentials: false
      - uses: dorny/paths-filter@v4
        id: filter
        if: ${{ github.event_name == 'pull_request_target' || github.event_name == 'merge_group' }}
        with:
          base: ${{ github.event_name == 'merge_group' && github.event.merge_group.base_sha || '' }}
          filters: |
            backend:
              - 'openmetadata-service/**'
              - 'openmetadata-integration-tests/**'
              - 'openmetadata-spec/pom.xml'
              - 'openmetadata-spec/src/**'
              - 'openmetadata-sdk/**'
              - 'openmetadata-shaded-deps/**'
              - 'openmetadata-k8s-operator/**'
              - 'openmetadata-mcp/**'
              - 'common/**'
              - 'pom.xml'
              - 'Makefile'
              - 'bootstrap/**'
              - '.github/scripts/classify_test_outcome.py'
              - '.github/workflows/integration-tests-mysql-elasticsearch.yml'

  integration-test-build:
    name: Build Integration Test Runtime
    needs: changes
    runs-on: ubuntu-latest
    timeout-minutes: 60
    if: ${{ needs.changes.outputs.backend == 'true' }}
    steps:
      - name: Wait for the labeler
        uses: lewagon/wait-on-check-action@v1.9.1
        if: ${{ github.event_name == 'pull_request_target' }}
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          check-name: Team Label
          repo-token: ${{ secrets.GITHUB_TOKEN }}
          wait-interval: 90

      - name: Verify PR labels
        uses: jesusvasquez333/verify-pr-label-action@v1.4.0
        if: ${{ github.event_name == 'pull_request_target' }}
        with:
          github-token: '${{ secrets.GITHUB_TOKEN }}'
          valid-labels: 'safe to test'
          pull-request-number: '${{ github.event.pull_request.number }}'
          disable-reviews: true

      - name: Checkout
        uses: actions/checkout@v7
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
          allow-unsafe-pr-checkout: true
          persist-credentials: false
      - name: Cache Maven dependencies
        uses: actions/cache@v6
        with:
          path: ~/.m2
          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-maven-

      - name: Set up JDK 21
        uses: actions/setup-java@v5
        with:
          java-version: '21'
          distribution: 'temurin'

      - name: Build and Bundle Integration Test Runtime
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
        run: |
          project_version="$(mvn --quiet --non-recursive help:evaluate \
            -Dexpression=project.version -DforceStdout -Dstyle.color=never)"
          if [[ ! "$project_version" =~ ^[A-Za-z0-9][A-Za-z0-9._+-]*$ ]]; then
            echo "Invalid Maven project version: $project_version"
            exit 1
          fi

          project_repository="$HOME/.m2/repository/org/open-metadata"
          if [[ -d "$project_repository" ]]; then
            find "$project_repository" -mindepth 2 -maxdepth 2 -type d \
              -name "$project_version" -prune -exec rm -rf -- {} +
          fi

          mvn -DskipTests -DskipITs -Dspring-boot.repackage.skip=true \
            clean install -pl :openmetadata-integration-tests -am

          staging="$RUNNER_TEMP/java-it-reactor-repository"
          bundle="$RUNNER_TEMP/java-it-build"
          mkdir -p "$staging/org/open-metadata" "$bundle"
          version_directory_count=0
          while IFS= read -r -d '' version_directory; do
            artifact_id="$(basename "$(dirname "$version_directory")")"
            mkdir -p "$staging/org/open-metadata/$artifact_id"
            cp -a "$version_directory" "$staging/org/open-metadata/$artifact_id/"
            version_directory_count=$((version_directory_count + 1))
          done < <(find "$project_repository" -mindepth 2 -maxdepth 2 -type d \
            -name "$project_version" -print0)

          if (( version_directory_count == 0 )); then
            echo "The Maven build did not install any reactor artifacts."
            exit 1
          fi

          find "$staging" -type f -name '*-boot.jar' -delete
          tar -C "$staging" -czf "$bundle/reactor-repository.tar.gz" org
          (
            cd "$bundle"
            sha256sum reactor-repository.tar.gz > reactor-repository.sha256
          )
          printf '%s\n' "$project_version" > "$bundle/project-version.txt"
          printf '%s\n' "$SOURCE_SHA" > "$bundle/source-sha.txt"
          du -h "$bundle/reactor-repository.tar.gz" | tee -a "$GITHUB_STEP_SUMMARY"

      - name: Upload Integration Test Runtime
        id: upload-runtime
        continue-on-error: true
        uses: actions/upload-artifact@v7
        with:
          name: java-it-build-mysql-elasticsearch-${{ github.run_id }}
          overwrite: true
          path: ${{ runner.temp }}/java-it-build
          if-no-files-found: error
          retention-days: 1
          compression-level: 0

      # upload-artifact has no built-in retry, and a FinalizeArtifact 403 from the
      # blob-storage intermediary is transient — without a second attempt it fails the
      # build job and every lane that depends on it. overwrite: true keeps the re-upload
      # safe even when the first attempt already staged blobs.
      - name: Upload Integration Test Runtime (retry)
        if: ${{ steps.upload-runtime.outcome == 'failure' }}
        uses: actions/upload-artifact@v7
        with:
          name: java-it-build-mysql-elasticsearch-${{ github.run_id }}
          overwrite: true
          path: ${{ runner.temp }}/java-it-build
          if-no-files-found: error
          retention-days: 1
          compression-level: 0

  integration-test-lanes:
    name: Integration Test Lane (${{ matrix.lane.name }})
    needs: [changes, integration-test-build]
    runs-on: ubuntu-latest
    timeout-minutes: 90
    if: ${{ needs.changes.outputs.backend == 'true' }}
    strategy:
      fail-fast: false
      matrix:
        lane:
          - name: parallel
          - name: global-state
          - name: multi-node
          - name: retry-queue
    steps:
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: true
          android: true
          dotnet: true
          haskell: true
          large-packages: true
          docker-images: false
          swap-storage: true

      # The required build job verifies `safe to test` before this fan-out can run.
      - name: Checkout
        uses: actions/checkout@v7
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
          allow-unsafe-pr-checkout: true
          persist-credentials: false
      - name: Cache Maven dependencies
        uses: actions/cache@v6
        with:
          path: ~/.m2
          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-maven-

      - name: Set up JDK 21
        uses: actions/setup-java@v5
        with:
          java-version: '21'
          distribution: 'temurin'

      - name: Install Ubuntu dependencies
        timeout-minutes: 15
        run: |
          # The hosted-runner Azure apt mirror intermittently blackholes connections;
          # apt then falls back to archive.ubuntu.com and can hang forever with no
          # output, burning the whole job timeout. Bound every fetch and retry.
          sudo tee /etc/apt/apt.conf.d/99-ci-timeouts >/dev/null <<'APTCONF'
          Acquire::Retries "3";
          Acquire::http::Timeout "30";
          Acquire::https::Timeout "30";
          Acquire::http::No-Cache "true";
          APTCONF
          # Keep the whole retry budget strictly inside this step's timeout-minutes.
          # Otherwise the outer deadline truncates a retry and kills the step before
          # the helper can say why it gave up, which is the failure we're fixing.
          APT_BUDGET=600
          APT_DEADLINE=$((SECONDS + APT_BUDGET))
          apt_retry() {
            local limit=$1; shift
            local i remaining
            for i in 1 2 3; do
              remaining=$((APT_DEADLINE - SECONDS))
              if [ "$remaining" -le 30 ]; then
                echo "::error::apt-get $1 gave up: exhausted the ${APT_BUDGET}s apt budget"
                return 1
              fi
              if [ "$limit" -gt "$remaining" ]; then
                limit=$remaining
              fi
              if sudo timeout "$limit" apt-get "$@"; then
                return 0
              fi
              if [ "$i" -lt 3 ]; then
                echo "::warning::apt-get $1 failed or hung (attempt $i/3); retrying in 10s"
                sleep 10
              fi
            done
            echo "::error::apt-get $1 failed after 3 attempts"
            return 1
          }
          apt_retry 240 update
          apt_retry 300 install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
          libevent-dev jq
          sudo make install_antlr_cli

      - name: Download Integration Test Runtime
        uses: actions/download-artifact@v7
        with:
          name: java-it-build-mysql-elasticsearch-${{ github.run_id }}
          path: ${{ runner.temp }}/java-it-build

      - name: Restore Integration Test Runtime
        run: |
          bundle="$RUNNER_TEMP/java-it-build"
          (
            cd "$bundle"
            sha256sum --check reactor-repository.sha256
          )

          project_version="$(<"$bundle/project-version.txt")"
          source_sha="$(<"$bundle/source-sha.txt")"
          if [[ ! "$project_version" =~ ^[A-Za-z0-9][A-Za-z0-9._+-]*$ ]]; then
            echo "Invalid Maven project version: $project_version"
            exit 1
          fi
          if [[ "$(git rev-parse HEAD)" != "$source_sha" ]]; then
            echo "The build artifact does not match the checked-out commit."
            exit 1
          fi

          project_repository="$HOME/.m2/repository/org/open-metadata"
          if [[ -d "$project_repository" ]]; then
            find "$project_repository" -mindepth 2 -maxdepth 2 -type d \
              -name "$project_version" -prune -exec rm -rf -- {} +
          fi
          mkdir -p "$HOME/.m2/repository"
          tar -C "$HOME/.m2/repository" -xzf "$bundle/reactor-repository.tar.gz"

          test -s "$project_repository/platform/$project_version/platform-$project_version.pom"
          test -s "$project_repository/openmetadata-service/$project_version/openmetadata-service-$project_version.jar"

      - name: Run Integration Tests (MySQL + Elasticsearch)
        id: integration-tests
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          IT_LANE: ${{ matrix.lane.name }}
          IT_PARALLELISM: '4'
          IT_MAX_POOL_SIZE: '16'
        run: |
          {
            echo "### Java integration configuration"
            echo "- Profile: mysql-elasticsearch"
            echo "- Lane: ${IT_LANE}"
            echo "- Parallel workers: ${IT_PARALLELISM}"
            echo "- Compensation-thread ceiling: ${IT_MAX_POOL_SIZE}"
            echo "- Runner processors: $(nproc)"
          } | tee -a "$GITHUB_STEP_SUMMARY"

          diagnostics="openmetadata-integration-tests/target/ci-diagnostics"
          mkdir -p "$diagnostics"
          maven_log="$diagnostics/maven.log"
          set +e
          timeout --signal=TERM --kill-after=60s 65m \
            mvn --no-snapshot-updates verify -pl :openmetadata-integration-tests -Pmysql-elasticsearch \
              -DintegrationTests.lane="${IT_LANE}" \
              -DintegrationTests.parallelStrategy=fixed \
              -DintegrationTests.parallelism="${IT_PARALLELISM}" \
              -DintegrationTests.maxPoolSize="${IT_MAX_POOL_SIZE}" \
            2>&1 | tee "$maven_log"
          pipeline_status=("${PIPESTATUS[@]}")
          exit_code="${pipeline_status[0]}"
          if (( exit_code == 0 && pipeline_status[1] != 0 )); then
            exit_code="${pipeline_status[1]}"
          fi
          set -e

          max_log_bytes=20971520
          log_bytes="$(stat --format=%s "$maven_log")"
          if (( log_bytes > max_log_bytes )); then
            tail -c "$max_log_bytes" "$maven_log" > "${maven_log}.tail"
            mv "${maven_log}.tail" "$maven_log"
          fi
          echo "exit_code=${exit_code}" >> "$GITHUB_OUTPUT"
          exit "$exit_code"

      - name: Capture Integration Test Diagnostics
        if: ${{ always() }}
        continue-on-error: true
        env:
          TEST_STEP_OUTCOME: ${{ steps.integration-tests.outcome }}
        run: |
          diagnostics="openmetadata-integration-tests/target/ci-diagnostics"
          mkdir -p "$diagnostics"
          {
            date -u
            uname -a
            java -version
            echo "processors=$(nproc)"
            df -h
            free -h
            docker system df
            docker ps -a --no-trunc
          } > "$diagnostics/environment.txt" 2>&1

          capture_container_logs=false
          if [ "$TEST_STEP_OUTCOME" != "success" ]; then
            capture_container_logs=true
          elif ! find openmetadata-integration-tests/target/failsafe-reports \
            -name 'TEST-*.xml' -print -quit 2>/dev/null | grep -q .; then
            capture_container_logs=true
          fi

          if [ "$capture_container_logs" = "true" ]; then
            while IFS= read -r container; do
              [[ -n "$container" ]] || continue
              name=$(docker inspect --format '{{.Name}}' "$container" | tr -cd '[:alnum:]_.-')
              docker logs --timestamps --since 30m --tail 10000 "$container" 2>&1 \
                | tail -c 10485760 > "$diagnostics/docker-${name}.log" || true
            done < <(docker ps -aq)
          fi

      - name: Classify Integration Test Outcome
        if: ${{ always() }}
        env:
          TEST_STEP_OUTCOME: ${{ steps.integration-tests.outcome }}
          TEST_EXIT_CODE: ${{ steps.integration-tests.outputs.exit_code }}
          TEST_PROFILE: mysql-elasticsearch
          TEST_LANE: ${{ matrix.lane.name }}
        run: |
          python3 .github/scripts/classify_test_outcome.py \
            --step-outcome "$TEST_STEP_OUTCOME" \
            --exit-code "$TEST_EXIT_CODE" \
            --report-glob 'openmetadata-integration-tests/target/failsafe-reports/TEST-*.xml' \
            --diagnostic-glob 'openmetadata-integration-tests/target/ci-diagnostics/**/*' \
            --diagnostic-glob 'openmetadata-integration-tests/target/failsafe-reports/*.dump*' \
            --diagnostic-glob 'openmetadata-integration-tests/target/failsafe-reports/*.txt' \
            --output openmetadata-integration-tests/target/ci-diagnostics/outcome.json

      - name: Upload Integration Test Diagnostics
        if: ${{ always() }}
        continue-on-error: true
        uses: actions/upload-artifact@v7
        with:
          name: java-it-mysql-elasticsearch-${{ matrix.lane.name }}-${{ github.run_id }}-${{ github.run_attempt }}
          path: |
            openmetadata-integration-tests/target/ci-diagnostics
            openmetadata-integration-tests/target/failsafe-reports/*.dump*
            openmetadata-integration-tests/target/failsafe-reports/*.txt
          if-no-files-found: warn
          retention-days: 7

      - name: Upload Integration Test Reports
        if: ${{ always() }}
        continue-on-error: true
        uses: actions/upload-artifact@v7
        with:
          name: java-it-reports-mysql-elasticsearch-${{ matrix.lane.name }}-${{ github.run_id }}-${{ github.run_attempt }}
          path: openmetadata-integration-tests/target/failsafe-reports/TEST-*.xml
          if-no-files-found: ignore
          retention-days: 7

      - name: Clean Up
        if: ${{ always() }}
        continue-on-error: true
        run: |
          cd ./docker/development
          docker compose down --remove-orphans
          sudo rm -rf "${PWD}/docker-volume"

      - name: Publish Test Report
        if: ${{ always() }}
        continue-on-error: true
        uses: ScalableCapital/action-surefire-report@v2
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          check_name: Integration Test Report (mysql-elasticsearch, ${{ matrix.lane.name }})
          fail_on_test_failures: false
          fail_if_no_tests: false
          report_paths: 'openmetadata-integration-tests/target/failsafe-reports/TEST-*.xml'

  integration-tests-mysql-elasticsearch:
    name: integration-tests-mysql-elasticsearch
    needs: [changes, integration-test-build, integration-test-lanes]
    runs-on: ubuntu-latest
    timeout-minutes: 5
    if: ${{ always() && !github.event.pull_request.draft && (github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test') }}
    steps:
      - name: Verify Maven Integration Test Lanes
        env:
          CHANGES_RESULT: ${{ needs.changes.result }}
          BACKEND_CHANGED: ${{ needs.changes.outputs.backend }}
          BUILD_RESULT: ${{ needs.integration-test-build.result }}
          LANES_RESULT: ${{ needs.integration-test-lanes.result }}
        run: |
          if [[ "$CHANGES_RESULT" == "success" && "$BACKEND_CHANGED" != "true" ]]; then
            echo "No backend integration-test paths changed."
            exit 0
          fi

          if [[ "$CHANGES_RESULT" != "success" || "$BUILD_RESULT" != "success" || "$LANES_RESULT" != "success" ]]; then
            echo "Java integration jobs did not complete successfully: changes=${CHANGES_RESULT}, build=${BUILD_RESULT}, lanes=${LANES_RESULT}"
            exit 1
          fi