zxc-e2e-test.yaml

##
# Copyright (C) 2023-2024 Hedera Hashgraph, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##

name: "ZXC: E2E Test"
# The purpose of this reusable workflow is to run the e2e tests on every PR and commit.
# This reusable component is called by the following workflows:
# - .github/workflows/flow-pull-request-checks.yaml
# - .github/workflows/flow-build-application.yaml

on:
  workflow_dispatch:
    inputs:
      consensus-node-version:
        description: "Consensus Node Version:"
        type: string
        required: false
        default: ""
      custom-job-label:
        description: "Custom Job Label:"
        type: string
        required: false
        default: "E2E Test"
      test-script:
        description: "NPM Test Run Script:"
        type: string
        required: false
        default: "test-e2e-standard"
      coverage-subdirectory:
        description: "Coverage Report Subdirectory:"
        type: string
        required: false
        default: "e2e"
      coverage-report-name:
        description: "Coverage Report Name:"
        type: string
        required: false
        default: "E2E Tests Coverage Report"
      local-java-build:
        description: "Build Local Java Code:"
        type: boolean
        required: false
        default: false
      cluster-name:
        description: "Cluster Name:"
        type: string
        required: false
        default: "solo-e2e"
      install-dependencies:
        description: "Install Solo Dependencies:"
        type: boolean
        required: false
        default: true
      runner:
        description: "Runner Label:"
        type: string
        required: false
        default: "hiero-solo-linux-large"
      kind-install:
        description: "Install Kind:"
        type: boolean
        required: false
        default: true
      kind-version:
        description: "Kind Version:"
        type: string
        required: false
        default: "0.29.0"
      kind-node-image:
        description: "Kind Node Image:"
        type: string
        required: false
        default: "kindest/node:v1.31.9@sha256:b94a3a6c06198d17f59cca8c6f486236fa05e2fb359cbd75dabbfc348a10b211"
      kubectl-install:
        description: "Install Kubectl:"
        type: boolean
        required: false
        default: true
      kubectl-version:
        description: "Kubectl Version:"
        type: string
        required: false
        default: "1.32.2"
      helm-install:
        description: "Install Helm:"
        type: boolean
        required: false
        default: true
      helm-version:
        description: "Helm Version:"
        type: string
        required: false
        default: "3.14.2"
      node-version:
        description: "NodeJS Version:"
        type: string
        required: false
        default: "22"
      dual-cluster-node-count:
        description: "Dual Cluster Node Count (SOLO_DUAL_CLUSTER_NODE_COUNT):"
        type: number
        required: false
        default: 3
      test-timeout-minutes:
        description: "Test Timeout (minutes):"
        type: number
        required: false
        default: 60
  workflow_call:
    inputs:
      consensus-node-version:
        description: "Consensus Node Version:"
        type: string
        required: false
        default: ""
      custom-job-label:
        description: "Custom Job Label:"
        type: string
        required: false
        default: "E2E Test"
      test-script:
        description: "NPM Test Run Script:"
        type: string
        required: false
        default: "test-e2e-standard"
      coverage-subdirectory:
        description: "Coverage Report Subdirectory:"
        type: string
        required: false
        default: "e2e"
      coverage-report-name:
        description: "Coverage Report Name:"
        type: string
        required: false
        default: "E2E Tests Coverage Report"
      local-java-build:
        description: "Build Local Java Code:"
        type: boolean
        required: false
        default: false
      cluster-name:
        description: "Cluster Name:"
        type: string
        required: false
        default: "solo-e2e"
      install-dependencies:
        description: "Install Solo Dependencies:"
        type: boolean
        required: false
        default: true
      runner:
        description: "Runner Label:"
        type: string
        required: false
        default: "hiero-solo-linux-large"
      kind-install:
        description: "Install Kind:"
        type: boolean
        required: false
        default: true
      kind-version:
        description: "Kind Version:"
        type: string
        required: false
        default: "0.29.0"
      kind-node-image:
        description: "Kind Node Image:"
        type: string
        required: false
        default: "kindest/node:v1.31.9@sha256:b94a3a6c06198d17f59cca8c6f486236fa05e2fb359cbd75dabbfc348a10b211"
      kubectl-install:
        description: "Install Kubectl:"
        type: boolean
        required: false
        default: true
      kubectl-version:
        description: "Kubectl Version:"
        type: string
        required: false
        default: "1.32.2"
      helm-install:
        description: "Install Helm:"
        type: boolean
        required: false
        default: true
      helm-version:
        description: "Helm Version:"
        type: string
        required: false
        default: "3.14.2"
      node-version:
        description: "NodeJS Version:"
        type: string
        required: false
        default: "22"
      dual-cluster-node-count:
        description: "Dual Cluster Node Count (SOLO_DUAL_CLUSTER_NODE_COUNT):"
        type: number
        required: false
        default: 3
      test-timeout-minutes:
        description: "Test Timeout (minutes):"
        type: number
        required: false
        default: 60

defaults:
  run:
    shell: bash

permissions:
  id-token: write
  contents: read
  actions: read
  pull-requests: write
  checks: write
  statuses: write

env:
  #CG_EXEC: export R_UID=$(id -u); CGROUP_LOGLEVEL=DEBUG cgexec -g cpu,memory:user.slice/user-${R_UID}.slice/user@${R_UID}.service/e2e-${{ github.run_id }} --sticky ionice -c 2 -n 2 nice -n 19
  CG_EXEC: "ionice -c 2 -n 2 nice -n 19"
  KIND_DOCKER_REGISTRY_MIRRORS: ${{ vars.KIND_DOCKER_REGISTRY_MIRRORS }}
  SOLO_CLUSTER_DUALITY: ${{ (inputs.test-script == 'test-e2e-dual-cluster-full' || inputs.test-script == 'test-e2e-external-database') && 2 || 1 }}
  STEP_TIMEOUT_MINUTES: '5'

jobs:
  e2e-test:
    name: ${{ inputs.custom-job-label || 'E2E Test' }}
    runs-on: ${{ inputs.runner || 'hiero-solo-linux-large' }}
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
        with:
          egress-policy: audit

      - name: Checkout Code
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}

      - name: Setup Posix Dependencies
        id: set-posix-dependencies
        if: ${{ inputs.runner != 'windows-latest' && !cancelled() && !failure() }}
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
        run: |
          # Configure APT to use main Ubuntu archive (disable mirror rotation)
          sudo sed -i 's|http://[a-z.]*.ubuntu.com|http://archive.ubuntu.com|g' /etc/apt/sources.list
          # Install jq (a lightweight and flexible command-line JSON processor)
          sudo apt-get update && sudo apt-get install -y jq

      - name: Setup Node
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
        with:
          node-version: ${{ inputs.node-version }}

      - name: Configure npm registry
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
        run: echo "registry=https://registry.npmjs.org/" > .npmrc

      - name: Install wget
        if: ${{ inputs.runner != 'windows-latest' && !cancelled() && !failure() }}
        uses: step-security/retry@38808b4332b98bb36ca031abff0c100231608288 # v4.0.0
        with:
          max_attempts: 3
          timeout_minutes: ${{ env.STEP_TIMEOUT_MINUTES }}
          command: |
            if ! command -v wget >/dev/null 2>&1; then
              sudo apt-get update
              sudo apt-get install -y wget
            fi

      - name: Install Kubectl
        if: ${{ inputs.install-dependencies == true && inputs.kubectl-install == true && !cancelled() && !failure() }}
        uses: step-security/setup-kubectl@b256bc71248a59f1e3a278fc75a65646d6a8a1c4 # pinned to v5.1.0
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
        with:
          version: 'v${{ inputs.kubectl-version }}'
        id: install

      - name: Setup Kind
        if: ${{ inputs.install-dependencies == true && inputs.kind-install == true && !cancelled() && !failure() }}
        uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
        with:
          install_only: true
          node_image: '${{ inputs.kind-node-image }}'
          version: 'v${{ inputs.kind-version }}'
          kubectl_version: 'v${{ inputs.kubectl-version }}'
          verbosity: 3
          wait: 120s

      - name: Setup Helm
        if: ${{ inputs.install-dependencies == true && inputs.helm-install == true && !cancelled() && !failure() }}
        uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
        with:
          version: 'v${{ inputs.helm-version }}'

      - name: Install Dependencies
        id: npm-deps
        uses: step-security/retry@38808b4332b98bb36ca031abff0c100231608288 # v4.0.0
        with:
          max_attempts: 10
          timeout_minutes: ${{ env.STEP_TIMEOUT_MINUTES }}
          command: |
            npm ci

      - name: Install Task
        uses: arduino/setup-task@c0bc642852239c2689f73f4ea6459c29405f3c52 # v3.0.0
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
        with:
          version: 3.49.1
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Setup Go
        uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
        with:
          go-version: '1.22.3'

      - name: Install grpcurl
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
        run: |
          go install github.com/fullstorydev/grpcurl/cmd/grpcurl@latest

      - name: Compile Project
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
        run: task build:compile

      - name: Resolve consensus node version and Java version
        id: resolve-consensus-java
        if: ${{ runner.os == 'linux' && inputs.local-java-build && !cancelled() && !failure() }}
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
        shell: bash
        run: |
          set -euo pipefail
          source .github/workflows/script/helper.sh

          if [ -z "${{ inputs.consensus-node-version }}" ]; then
            consensus_node_version="$(extract_version HEDERA_PLATFORM_VERSION version.ts)"
          else
            consensus_node_version="${{ inputs.consensus-node-version }}"
          fi

          normalized_version="$(echo "${consensus_node_version}" | sed -E 's/^v//; s/-.*$//')"

          java_version="21"
          if printf '%s\n' '0.73.0' "${normalized_version}" | sort -C -V; then
            java_version="25"
          fi

          echo "consensus-node-version=${consensus_node_version}" >> "${GITHUB_OUTPUT}"
          echo "java-version=${java_version}" >> "${GITHUB_OUTPUT}"

      - name: Setup Java
        if: ${{ runner.os == 'linux' && inputs.local-java-build && !cancelled() && !failure() }}
        uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
        with:
          distribution: temurin
          java-version: ${{ steps.resolve-consensus-java.outputs.java-version }}

      - name: Setup Gradle
        if: ${{ runner.os == 'linux' && inputs.local-java-build && !cancelled() && !failure() }}
        uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
        with:
          cache-read-only: false

      - name: Create Diagnostic Script
        if: ${{ !cancelled() && !failure() }}
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
        env:
          WRITE_DIAG_SCRIPT: ${{ github.workspace }}/.github/workflows/script/write_diag_script.sh
        run: |
          sudo chmod +x "${WRITE_DIAG_SCRIPT}"
          ${WRITE_DIAG_SCRIPT}

      - name: Build Hedera code locally
        if: ${{ runner.os == 'linux' && inputs.local-java-build && !cancelled() && !failure() }}
        uses: step-security/retry@38808b4332b98bb36ca031abff0c100231608288 # v4.0.0
        with:
          max_attempts: 3
          timeout_minutes: 30
          command: |
            source .github/workflows/script/helper.sh
            if [ -z "${{ inputs.consensus-node-version }}" ]; then
                export CONSENSUS_NODE_VERSION=$(extract_version HEDERA_PLATFORM_VERSION version.ts)
            else
                export CONSENSUS_NODE_VERSION=${{ inputs.consensus-node-version }}
            fi
            cd ..
            git clone https://github.com/hiero-ledger/hiero-consensus-node.git --depth 1 --branch ${CONSENSUS_NODE_VERSION}
            cd hiero-consensus-node
            ls -ltr
            ${{ env.CG_EXEC }} ./gradlew assemble --stacktrace --info || exit 1
            cd ../solo

      - name: Setup E2E Tests
        if: ${{ inputs.custom-job-label != 'One Shot Single - using Podman' && !cancelled() && !failure() }}
        uses: step-security/retry@38808b4332b98bb36ca031abff0c100231608288 # v4.0.0
        with:
          max_attempts: 3
          timeout_minutes: 10
          shell: bash
          command: |
            npm link
            echo "SOLO_TEST_CLUSTER=${{ inputs.cluster-name }}"
            if [[ "$RUNNER_OS" != "Windows" ]]; then
              SOLO_TEST_CLUSTER=${{ inputs.cluster-name }} ${{ env.CG_EXEC }} ./test/e2e/dual-cluster/setup-dual-e2e.sh
            else
              SOLO_TEST_CLUSTER=${{ inputs.cluster-name }} ./test/e2e/dual-cluster/setup-dual-e2e.sh
            fi

      - name: Run E2E Tests
        id: run-e2e-tests
        if: ${{ inputs.runner != 'windows-latest' && !cancelled() && !failure() }}
        timeout-minutes: ${{ fromJSON(inputs.test-timeout-minutes) }}
        run: |
          export PATH=~/.solo/bin:${PATH}
          echo "PATH: $PATH"
          source .github/workflows/script/helper.sh
          if [[ -z "${{ inputs.consensus-node-version }}" ]]; then
              if [[ "${{ inputs.local-java-build }}" != "true" ]]; then
                  echo "No consensus node version provided, using default from version-test.ts"
              else
                  if [[ "${{ inputs.test-script }}" == "test-e2e-block-node" ]]; then
                    echo "Use default from version.ts for block node test"
                  else
                    export CONSENSUS_NODE_VERSION=$(extract_version HEDERA_PLATFORM_VERSION version.ts)
                  fi
              fi
          else
              export CONSENSUS_NODE_VERSION=${{ inputs.consensus-node-version }}
          fi
          echo SOLO_TEST_CLUSTER=${{ inputs.cluster-name }}-c1 > .env
          echo SOLO_DUAL_CLUSTER_NODE_COUNT=${{ inputs.dual-cluster-node-count || 3 }} >> .env

          if [[ "${{ inputs.custom-job-label }}" != "One Shot Single - using Podman" ]]; then
            # Verify cluster connectivity before running tests
            echo "Verifying cluster connectivity..."
            kubectl cluster-info || {
              echo "Error: Cannot connect to cluster"
              echo "KUBECONFIG: ${KUBECONFIG:-default}"
              kubectl config get-contexts || true
              exit 1
            }
          else 
            echo "SOLO_ONE_SHOT_MINIMAL_SETUP=true" >> .env
            echo "FORCE_PODMAN=true" >> .env
          fi
          
          if [[ "${{ inputs.custom-job-label }}" == "One Shot Single" ]]; then
            echo "ENABLE_IMAGE_CACHE=true" >> .env
          fi
          
          cat .env
          
          # Start resource monitoring in background
          ./.github/workflows/script/monitor-resources.sh start

          # Run the task in the background and trap SIGTERM/SIGINT so that a workflow
          # cancellation (e.g. triggered by a new commit on this PR) terminates the
          # test immediately rather than waiting 30-60 minutes for it to finish on
          # its own.
          TASK_PID=""
          TASK_PGID=""
          TEST_EXIT_CODE=0

          cleanup() {
            if [[ -n "${TASK_PID}" ]]; then
              echo "Received cancellation signal, stopping task (PID ${TASK_PID}, PGID ${TASK_PGID})..."
              if [[ -n "${TASK_PGID}" ]]; then
                kill -TERM -- "-${TASK_PGID}" 2>/dev/null || true
                sleep 5
                kill -KILL -- "-${TASK_PGID}" 2>/dev/null || true
              else
                kill -TERM "${TASK_PID}" 2>/dev/null || true
                sleep 5
                kill -KILL "${TASK_PID}" 2>/dev/null || true
              fi
              wait "${TASK_PID}" 2>/dev/null || true
            fi
            ./.github/workflows/script/monitor-resources.sh stop || true
            exit 130
          }
          trap cleanup SIGTERM SIGINT

          TASK_COMMAND="${{ env.CG_EXEC }} task ${{ inputs.test-script }}"
          if command -v setsid >/dev/null 2>&1; then
            setsid bash -lc "${TASK_COMMAND}" &
          else
            bash -lc "${TASK_COMMAND}" &
          fi
          TASK_PID=$!
          TASK_PGID="$(ps -o pgid= -p "${TASK_PID}" | tr -d '[:space:]' || true)"
          wait "${TASK_PID}" || TEST_EXIT_CODE=$?
          TASK_PID=""
          TASK_PGID=""
          trap - SIGTERM SIGINT

          # Stop resource monitoring and display summary
          ./.github/workflows/script/monitor-resources.sh stop

          exit $TEST_EXIT_CODE

      - name: Log Metrics
        id: log-metrics
        if: ${{ !cancelled() }}
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
        shell: bash
        run: |
          export PATH=~/.solo/bin:${PATH}
          echo "PATH: $PATH"
          echo "json_log={}" >> $GITHUB_OUTPUT
           
          # Ensure logs directory exists
          mkdir -p ~/.solo/logs
          
          # Check if any JSON files exist
          shopt -s nullglob
          json_files=(~/.solo/logs/*.json)
          if [[ ${#json_files[@]} -eq 0 ]]; then
            echo "::notice::No JSON log files found"
          else
            for logFile in "${json_files[@]}"; do
              if [[ -z "$logFile" ]]; then
                continue
              fi
              echo "::group::JSON Format"
              cat $logFile | jq
              echo "::endgroup::" 
              echo "json_log=$(cat $logFile)" >> $GITHUB_OUTPUT
              echo "::group::Spreadsheet Format"
              echo "logFile=$logFile"
              JSON_FILENAME=$logFile ./.github/workflows/script/metrics.sh
              echo "::endgroup::"
            done
          fi
          
          echo "solo_log_dir=$(cd ~/.solo/logs && pwd)" >> $GITHUB_OUTPUT

      - name: Generate ASCII Resource Chart
        id: generate-chart
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
        run: |
          echo "json_log=${{ steps.log-metrics.outputs.json_log }}"
          METRICS_FILE="$HOME/.solo/logs/runner-metrics.jsonl"
          CHART_FILE="$HOME/.solo/logs/runner-metrics-ascii.txt"
          
          DATA_POINTS=$(wc -l < "$METRICS_FILE" 2>/dev/null | tr -d ' ' || echo "0")
          if [[ -f "$METRICS_FILE" && "$DATA_POINTS" -gt 0 ]]; then
            echo "Generating ASCII chart from ${DATA_POINTS} data points..."
            ./.github/workflows/script/plot-runner-metrics-ascii.sh "$METRICS_FILE" "$CHART_FILE"

            if [[ -f "$CHART_FILE" && -s "$CHART_FILE" ]]; then
              echo "chart_generated=true" >> $GITHUB_OUTPUT
              echo "chart_file=$CHART_FILE" >> $GITHUB_OUTPUT
              echo "✅ ASCII chart generated successfully"
            else
              echo "chart_generated=false" >> $GITHUB_OUTPUT
              echo "⚠️  ASCII chart generation produced no output"
            fi
          else
            echo "chart_generated=false" >> $GITHUB_OUTPUT
            echo "ℹ️  No metrics data collected (${DATA_POINTS} data points), skipping chart generation"
          fi

      - name: Write Log Metrics Message File
        id: log-metrics-file
        if: ${{ !cancelled() && steps.run-e2e-tests.conclusion == 'success' }}
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
        env:
          LOG_FILE: "${{ steps.log-metrics.outputs.solo_log_dir }}/log_file.md"
          JSON_INPUT: "${{ toJSON(fromJSON(steps.log-metrics.outputs.json_log)) }}"
        run: |
          echo "## Log Metrics - ${{ inputs.custom-job-label }}" >> $LOG_FILE
          echo "" >> $LOG_FILE
          
          # Add ASCII resource chart if available
          if [[ "${{ steps.generate-chart.outputs.chart_generated }}" == "true" ]]; then
            CHART_FILE="${{ steps.generate-chart.outputs.chart_file }}"
            if [[ -f "$CHART_FILE" ]]; then
              echo "<details>" >> $LOG_FILE
              echo "<summary>📊 Runner Resource Usage</summary>" >> $LOG_FILE
              echo "" >> $LOG_FILE
              echo '```' >> $LOG_FILE
              cat "$CHART_FILE" >> $LOG_FILE
              echo '```' >> $LOG_FILE
              echo "" >> $LOG_FILE
              echo "</details>" >> $LOG_FILE
              echo "" >> $LOG_FILE
            fi
          fi
          
          # Add JSON log details if there's actual data
          if [[ -n "$JSON_INPUT" && "$JSON_INPUT" != "{}" ]]; then
            echo "<details>" >> $LOG_FILE
            echo "<summary>" >> $LOG_FILE
            echo "JSON Log Metrics" >> $LOG_FILE
            echo "</summary>" >> $LOG_FILE
            echo '<pre><code class="JSON">' >> $LOG_FILE
            echo "$JSON_INPUT" >> $LOG_FILE
            echo "</code></pre>" >> $LOG_FILE
            echo "</details>" >> $LOG_FILE
          fi
          
          echo "log_file=$LOG_FILE" >> $GITHUB_OUTPUT

      - name: Write Log Metrics to Workflow Summary
        if: ${{ !cancelled() && steps.log-metrics-file.conclusion == 'success' }}
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
        run: |
          cat "${{ steps.log-metrics-file.outputs.log_file }}" >> $GITHUB_STEP_SUMMARY

      - name: Upload E2E Logs to GitHub
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
        with:
          name: solo-${{ inputs.test-script }}.log
          path: ~/.solo/logs/*
          overwrite: true
          if-no-files-found: warn

      - name: Upload E2E Test Report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: >-
          ${{ github.event_name != 'workflow_dispatch' && inputs.test-script != 'test-e2e-external-database' &&
          steps.npm-deps.conclusion == 'success' && !cancelled() && !failure() }}
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
        with:
          name: e2e_test_report_${{ inputs.test-script }}
          path: "junit-${{ inputs.coverage-subdirectory }}.xml"
          overwrite: true
          if-no-files-found: error

      - name: Publish E2E Coverage Report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: >-
          ${{ github.event_name != 'workflow_dispatch' && inputs.test-script != 'test-e2e-external-database' &&
          !cancelled() && !failure() }}
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
        with:
          name: ${{ inputs.coverage-report-name }}
          path: 'coverage/${{ inputs.coverage-subdirectory }}'

      - name: Cleanup Kind Clusters
        if: ${{ always() }}
        timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
        run: |
          export PATH=~/.solo/bin:${PATH}

          # Helper: collect docker hang diagnostics using non-Docker tools (writes to stderr)
          collect_docker_hang_diagnostics() {
            local hung_command="${1:-unknown command}"
            {
              echo "=== DOCKER HANG DIAGNOSTICS (timed out: '${hung_command}') ==="
              systemctl status docker --no-pager -l 2>/dev/null || service docker status 2>/dev/null || echo "Unable to get docker service status"
              ls -la /var/run/docker.sock 2>/dev/null || echo "Docker socket not found"
              lsof /var/run/docker.sock 2>/dev/null || echo "lsof unavailable or no open handles on docker socket"
              ps aux | grep -E '[d]ocker|[c]ontainerd|[k]ind' 2>/dev/null || true
              free -h 2>/dev/null || true
              journalctl -u docker --no-pager -n 30 2>/dev/null || echo "journalctl unavailable"
              echo "=== END DOCKER HANG DIAGNOSTICS ==="
            } >&2
          }

          for i in $(seq 1 ${{ env.SOLO_CLUSTER_DUALITY }}); do
            timeout 60 kind delete cluster -n "${{ inputs.cluster-name }}-c${i}" || true
          done
          exit_code=0
          timeout 30 docker network rm -f kind || exit_code=$?
          if [[ $exit_code -eq 124 ]]; then
            echo "WARNING: 'docker network rm kind' timed out"
            collect_docker_hang_diagnostics "docker network rm kind"
          elif [[ $exit_code -ne 0 ]]; then
            echo "WARNING: 'docker network rm kind' failed with exit code ${exit_code}"
            collect_docker_hang_diagnostics "docker network rm kind"
          fi
          true