zxc-e2e-test.yaml

##
# Copyright (C) 2023-2024 Hedera Hashgraph, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##

name: "ZXC: E2E Test"
# The purpose of this reusable workflow is to run the e2e tests on every PR and commit.
# This reusable component is called by the following workflows:
# - .github/workflows/flow-pull-request-checks.yaml
# - .github/workflows/flow-build-application.yaml

on:
  workflow_dispatch:
    inputs:
      consensus-node-version:
        description: "Consensus Node Version:"
        type: string
        required: false
        default: ""
      custom-job-label:
        description: "Custom Job Label:"
        type: string
        required: false
        default: "E2E Test"
      test-script:
        description: "NPM Test Run Script:"
        type: string
        required: false
        default: "test-e2e-standard"
      coverage-subdirectory:
        description: "Coverage Report Subdirectory:"
        type: string
        required: false
        default: "e2e"
      coverage-report-name:
        description: "Coverage Report Name:"
        type: string
        required: false
        default: "E2E Tests Coverage Report"
      local-java-build:
        description: "Build Local Java Code:"
        type: boolean
        required: false
        default: false
      cluster-name:
        description: "Cluster Name:"
        type: string
        required: false
        default: "solo-e2e"
      install-dependencies:
        description: "Install Solo Dependencies:"
        type: boolean
        required: false
        default: true
      runner:
        description: "Runner Label:"
        type: string
        required: false
        default: "hiero-solo-linux-large"
      kind-install:
        description: "Install Kind:"
        type: boolean
        required: false
        default: true
      kind-version:
        description: "Kind Version:"
        type: string
        required: false
        default: "0.29.0"
      kind-node-image:
        description: "Kind Node Image:"
        type: string
        required: false
        default: "kindest/node:v1.31.9@sha256:b94a3a6c06198d17f59cca8c6f486236fa05e2fb359cbd75dabbfc348a10b211"
      kubectl-install:
        description: "Install Kubectl:"
        type: boolean
        required: false
        default: true
      kubectl-version:
        description: "Kubectl Version:"
        type: string
        required: false
        default: "1.32.2"
      helm-install:
        description: "Install Helm:"
        type: boolean
        required: false
        default: true
      helm-version:
        description: "Helm Version:"
        type: string
        required: false
        default: "3.14.2"
      enable_s6_image:
        description: "Enable S6 image"
        type: boolean
        required: false
        default: false
      node-version:
        description: "NodeJS Version:"
        type: string
        required: false
        default: "22"
  workflow_call:
    inputs:
      consensus-node-version:
        description: "Consensus Node Version:"
        type: string
        required: false
        default: ""
      custom-job-label:
        description: "Custom Job Label:"
        type: string
        required: false
        default: "E2E Test"
      test-script:
        description: "NPM Test Run Script:"
        type: string
        required: false
        default: "test-e2e-standard"
      coverage-subdirectory:
        description: "Coverage Report Subdirectory:"
        type: string
        required: false
        default: "e2e"
      coverage-report-name:
        description: "Coverage Report Name:"
        type: string
        required: false
        default: "E2E Tests Coverage Report"
      local-java-build:
        description: "Build Local Java Code:"
        type: boolean
        required: false
        default: false
      cluster-name:
        description: "Cluster Name:"
        type: string
        required: false
        default: "solo-e2e"
      install-dependencies:
        description: "Install Solo Dependencies:"
        type: boolean
        required: false
        default: true
      runner:
        description: "Runner Label:"
        type: string
        required: false
        default: "hiero-solo-linux-large"
      kind-install:
        description: "Install Kind:"
        type: boolean
        required: false
        default: true
      kind-version:
        description: "Kind Version:"
        type: string
        required: false
        default: "0.29.0"
      kind-node-image:
        description: "Kind Node Image:"
        type: string
        required: false
        default: "kindest/node:v1.31.9@sha256:b94a3a6c06198d17f59cca8c6f486236fa05e2fb359cbd75dabbfc348a10b211"
      kubectl-install:
        description: "Install Kubectl:"
        type: boolean
        required: false
        default: true
      kubectl-version:
        description: "Kubectl Version:"
        type: string
        required: false
        default: "1.32.2"
      helm-install:
        description: "Install Helm:"
        type: boolean
        required: false
        default: true
      helm-version:
        description: "Helm Version:"
        type: string
        required: false
        default: "3.14.2"
      node-version:
        description: "NodeJS Version:"
        type: string
        required: false
        default: "22"
      enable_s6_image:
        description: "Enable S6 image"
        type: boolean
        required: false
        default: false

defaults:
  run:
    shell: bash

permissions:
  id-token: write
  contents: read
  actions: read
  pull-requests: write
  checks: write
  statuses: write

env:
  #CG_EXEC: export R_UID=$(id -u); CGROUP_LOGLEVEL=DEBUG cgexec -g cpu,memory:user.slice/user-${R_UID}.slice/user@${R_UID}.service/e2e-${{ github.run_id }} --sticky ionice -c 2 -n 2 nice -n 19
  CG_EXEC: "ionice -c 2 -n 2 nice -n 19"
  SOLO_CLUSTER_DUALITY: ${{ (inputs.test-script == 'test-e2e-dual-cluster-full' || inputs.test-script == 'test-e2e-external-database') && 2 || 1 }}
  ENABLE_S6_IMAGE: ${{ inputs.enable_s6_image }}

jobs:
  e2e-test:
    name: ${{ inputs.custom-job-label || 'E2E Test' }}
    runs-on: ${{ inputs.runner || 'hiero-solo-linux-large' }}
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@58077d3c7e43986b6b15fba718e8ea69e387dfcc # v2.15.1
        with:
          egress-policy: audit

      - name: Checkout Code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup Posix Dependencies
        id: set-posix-dependencies
        if: ${{ inputs.runner != 'windows-latest' && !cancelled() && !failure() }}
        run: |
          # Configure APT to use main Ubuntu archive (disable mirror rotation)
          sudo sed -i 's|http://[a-z.]*.ubuntu.com|http://archive.ubuntu.com|g' /etc/apt/sources.list
          # Install jq (a lightweight and flexible command-line JSON processor)
          sudo apt-get update && sudo apt-get install -y jq

      - name: Setup Node
        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        with:
          node-version: ${{ inputs.node-version }}

      - name: Configure npm registry
        run: echo "registry=https://registry.npmjs.org/" > .npmrc

      - name: Install wget
        if: ${{ inputs.runner != 'windows-latest' && !cancelled() && !failure() }}
        run: |
          if ! command -v wget >/dev/null 2>&1; then
            sudo apt-get update
            sudo apt-get install -y wget
          fi

      - name: Install Kubectl
        if: ${{ inputs.install-dependencies == true && inputs.kubectl-install == true }}
        uses: step-security/setup-kubectl@fa3574e17195eaed56b1e9d9095c58d0aee24f03 # pinned to v4.0.1
        with:
          version: 'v${{ inputs.kubectl-version }}'
        id: install

      - name: Setup Kind
        if: ${{ inputs.install-dependencies == true && inputs.kind-install == true }}
        uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0
        with:
          install_only: true
          node_image: '${{ inputs.kind-node-image }}'
          version: 'v${{ inputs.kind-version }}'
          kubectl_version: 'v${{ inputs.kubectl-version }}'
          verbosity: 3
          wait: 120s

      - name: Setup Helm
        if: ${{ inputs.install-dependencies == true && inputs.helm-install == true }}
        uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1
        with:
          version: 'v${{ inputs.helm-version }}'

      - name: Install Dependencies
        id: npm-deps
        uses: step-security/retry@e1d59ce1f574b32f0915e3a8df055cfe9f99be5d # v3.0.4
        with:
          max_attempts: 10
          timeout_minutes: 10
          command: |
            npm ci

      - name: Install Task
        uses: arduino/setup-task@b91d5d2c96a56797b48ac1e0e89220bf64044611 # v2.0.0
        with:
          version: 3.39.2
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Setup Go
        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
        with:
          go-version: '1.22.3'

      - name: Install grpcurl
        run: |
          go install github.com/fullstorydev/grpcurl/cmd/grpcurl@latest

      - name: Compile Project
        run: task build:compile

      - name: Pull Kind Docker Image
        if: ${{ inputs.custom-job-label != 'One Shot Single - using Podman' && inputs.kind-install == true && !cancelled() && !failure() }}
        run: docker image pull kindest/node:v1.31.4@sha256:2cb39f7295fe7eafee0842b1052a599a4fb0f8bcf3f83d96c7f4864c357c6c30

      - name: Setup Java
        if: ${{ runner.os == 'linux' && inputs.local-java-build && !cancelled() && !failure() }}
        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
        with:
          distribution: temurin
          java-version: 21.0.1

      - name: Setup Gradle
        if: ${{ runner.os == 'linux' && inputs.local-java-build && !cancelled() && !failure() }}
        uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2
        with:
          cache-read-only: false

      - name: Create Diagnostic Script
        if: ${{ inputs.runner != 'windows-latest' && !cancelled() && !failure() }}
        env:
          WRITE_DIAG_SCRIPT: ${{ github.workspace }}/.github/workflows/script/write_diag_script.sh
        run: |
          sudo chmod +x "${WRITE_DIAG_SCRIPT}"
          ${WRITE_DIAG_SCRIPT}

      - name: Build Hedera code locally
        if: ${{ runner.os == 'linux' && inputs.local-java-build && !cancelled() && !failure() }}
        uses: step-security/retry@e1d59ce1f574b32f0915e3a8df055cfe9f99be5d # v3.0.4
        with:
          max_attempts: 3
          timeout_minutes: 30
          command: |
            if [ -z "${{ inputs.consensus-node-version }}" ]; then
                export CONSENSUS_NODE_VERSION=$(grep 'TEST_LOCAL_HEDERA_PLATFORM_VERSION' version-test.ts | sed -E "s/.*'([^']+)';/\1/")
            else
                export CONSENSUS_NODE_VERSION=${{ inputs.consensus-node-version }}
            fi
            cd ..
            git clone https://github.com/hiero-ledger/hiero-consensus-node.git --depth 1 --branch ${CONSENSUS_NODE_VERSION}
            cd hiero-consensus-node
            ls -ltr
            ${{ env.CG_EXEC }} ./gradlew assemble --stacktrace --info
            cd ../solo

      - name: Setup E2E Tests
        if: ${{ !cancelled() && inputs.custom-job-label != 'One Shot Single - using Podman' }}
        uses: step-security/retry@e1d59ce1f574b32f0915e3a8df055cfe9f99be5d # v3.0.4
        with:
          max_attempts: 3
          timeout_minutes: 10
          shell: bash
          command: |
            npm link
            echo "SOLO_TEST_CLUSTER=${{ inputs.cluster-name }}"
            if [[ "$RUNNER_OS" != "Windows" ]]; then
              EXEC_CG=${{ env.CG_EXEC }}
            fi
            SOLO_TEST_CLUSTER=${{ inputs.cluster-name }} ${EXEC_CG} ./test/e2e/dual-cluster/setup-dual-e2e.sh

      - name: Build Block Code Locally
        if: ${{ runner.os == 'linux' && (inputs.test-script == 'test-e2e-block-node') && !cancelled() && !failure() }}
        uses: step-security/retry@e1d59ce1f574b32f0915e3a8df055cfe9f99be5d # v3.0.4
        with:
          max_attempts: 3
          timeout_minutes: 20
          command: |
            export TEST_LOCAL_BLOCK_NODE_VERSION=$(grep 'TEST_LOCAL_BLOCK_NODE_VERSION' version-test.ts | sed -E "s/.*'([^']+)';/\1/")
            cd ..
            git clone https://github.com/hiero-ledger/hiero-block-node.git --depth 1 --branch v${TEST_LOCAL_BLOCK_NODE_VERSION}
            cd hiero-block-node
            
            # Configure apt to use multiple mirror sources with fallback
            # This adds fallback mirrors while keeping the original sources
            find . -name "Dockerfile*" -type f | while read dockerfile; do
              # Create a backup of sources.list configuration in the Dockerfile
              sed -i '/RUN apt-get update/i \
            # Add fallback Ubuntu mirrors for reliability\n\
            RUN echo "deb http://archive.ubuntu.com/ubuntu/ mantic main restricted universe multiverse" >> /etc/apt/sources.list && \\\n\
                echo "deb http://archive.ubuntu.com/ubuntu/ mantic-updates main restricted universe multiverse" >> /etc/apt/sources.list && \\\n\
                echo "deb http://archive.ubuntu.com/ubuntu/ mantic-security main restricted universe multiverse" >> /etc/apt/sources.list && \\\n\
                echo "deb http://security.ubuntu.com/ubuntu/ mantic-security main restricted universe multiverse" >> /etc/apt/sources.list && \\\n\
                echo "Acquire::Retries \"3\";" > /etc/apt/apt.conf.d/80-retries && \\\n\
                echo "Acquire::http::Timeout \"30\";" >> /etc/apt/apt.conf.d/80-retries && \\\n\
                echo "Acquire::https::Timeout \"30\";" >> /etc/apt/apt.conf.d/80-retries\n' "$dockerfile"
            done
            
            ./gradlew :block-node-app:createDockerImage
            kind get clusters
            kind load docker-image block-node-server:${TEST_LOCAL_BLOCK_NODE_VERSION} --name ${{ inputs.cluster-name }}-c1
            ${{ env.CG_EXEC }} ./gradlew :block-node-app:createDockerImage
            cd ../solo

      - name: Run E2E Tests
        id: run-e2e-tests
        if: ${{ inputs.runner != 'windows-latest' && !cancelled() && !failure() }}
        shell: bash
        run: |
          export PATH="~/.solo/bin:${PATH}"
          echo "PATH: $PATH"
          if [[ -z "${{ inputs.consensus-node-version }}" ]]; then
              if [[ "${{ inputs.local-java-build }}" != "true" ]]; then
                  echo "No consensus node version provided, using default from version-test.ts"
              else
                  if [[ "${{ inputs.test-script }}" == "test-e2e-block-node" ]]; then
                    echo "Use default from version.ts for block node test"
                  else
                    export CONSENSUS_NODE_VERSION=$(grep 'TEST_LOCAL_HEDERA_PLATFORM_VERSION' version-test.ts | sed -E "s/.*'([^']+)';/\1/")
                  fi
              fi
          else
              export CONSENSUS_NODE_VERSION=${{ inputs.consensus-node-version }}
          fi
          echo SOLO_TEST_CLUSTER=${{ inputs.cluster-name }}-c1 > .env
          
          if [[ "${{ inputs.custom-job-label }}" != "One Shot Single - using Podman" ]]; then
            # Verify cluster connectivity before running tests
            echo "Verifying cluster connectivity..."
            kubectl cluster-info || {
              echo "Error: Cannot connect to cluster"
              echo "KUBECONFIG: ${KUBECONFIG:-default}"
              kubectl config get-contexts || true
              exit 1
            }
          else 
            echo "SOLO_ONE_SHOT_MINIMAL_SETUP=true" >> .env
            echo "FORCE_PODMAN=true" >> .env
          fi
          
          cat .env
          
          # Start resource monitoring in background
          ./.github/workflows/script/monitor-resources.sh start
         
          # Run the test
          ${{ env.CG_EXEC }} task ${{ inputs.test-script }} || TEST_EXIT_CODE=$?
          
          # Stop resource monitoring and display summary
          ./.github/workflows/script/monitor-resources.sh stop
          
          exit $TEST_EXIT_CODE

      - name: Run E2E Tests on Windows
        id: run-e2e-tests-windows
        if: ${{ inputs.runner == 'windows-latest' && !cancelled() && !failure() }}
        shell: bash
        run: |
          export PATH="~/.solo/bin:${PATH}"
          echo "PATH: $PATH"
          ${{ env.CG_EXEC }} task ${{ inputs.test-script }}

      - name: Log Metrics
        id: log-metrics
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          export PATH="~/.solo/bin:${PATH}"
          echo "PATH: $PATH"
          echo "json_log={}" >> $GITHUB_OUTPUT
           
          # Ensure logs directory exists
          mkdir -p ~/.solo/logs
          
          # Check if any JSON files exist
          shopt -s nullglob
          json_files=(~/.solo/logs/*.json)
          if [[ ${#json_files[@]} -eq 0 ]]; then
            echo "::notice::No JSON log files found"
          else
            for logFile in "${json_files[@]}"; do
              if [[ -z "$logFile" ]]; then
                continue
              fi
              echo "::group::JSON Format"
              cat $logFile | jq
              echo "::endgroup::" 
              echo "json_log=$(cat $logFile)" >> $GITHUB_OUTPUT
              echo "::group::Spreadsheet Format"
              echo "logFile=$logFile"
              JSON_FILENAME=$logFile ./.github/workflows/script/metrics.sh
              echo "::endgroup::"
            done
          fi
          
          echo "solo_log_dir=$(cd ~/.solo/logs && pwd)" >> $GITHUB_OUTPUT

      - name: Generate ASCII Resource Chart
        id: generate-chart
        if: ${{ inputs.runner != 'windows-latest' }}
        run: |
          echo "json_log=${{ steps.log-metrics.outputs.json_log }}"
          METRICS_FILE="$HOME/.solo/logs/runner-metrics.jsonl"
          CHART_FILE="$HOME/.solo/logs/runner-metrics-ascii.txt"
          
          if [[ -f "$METRICS_FILE" ]]; then
            echo "Generating ASCII chart from metrics..."
            ./.github/workflows/script/plot-runner-metrics-ascii.sh "$METRICS_FILE" "$CHART_FILE"
          
            if [[ -f "$CHART_FILE" ]]; then
              echo "chart_generated=true" >> $GITHUB_OUTPUT
              echo "chart_file=$CHART_FILE" >> $GITHUB_OUTPUT
              echo "✅ ASCII chart generated successfully"
            else
              echo "chart_generated=false" >> $GITHUB_OUTPUT
              echo "⚠️  ASCII chart generation failed"
            fi
          else
            echo "chart_generated=false" >> $GITHUB_OUTPUT
            echo "ℹ️  No metrics file found, skipping chart generation"
          fi

      - name: Write Log Metrics Message File
        id: log-metrics-file
        if: ${{ inputs.runner != 'windows-latest' && !cancelled() && steps.run-e2e-tests.conclusion == 'success' }}
        env:
          LOG_FILE: "${{ steps.log-metrics.outputs.solo_log_dir }}/log_file.md"
          JSON_INPUT: "${{ toJSON(fromJSON(steps.log-metrics.outputs.json_log)) }}"
        run: |
          echo "## Log Metrics - ${{ inputs.custom-job-label }}" >> $LOG_FILE
          echo "" >> $LOG_FILE
          
          # Add ASCII resource chart if available
          if [[ "${{ steps.generate-chart.outputs.chart_generated }}" == "true" ]]; then
            CHART_FILE="${{ steps.generate-chart.outputs.chart_file }}"
            if [[ -f "$CHART_FILE" ]]; then
              echo "<details>" >> $LOG_FILE
              echo "<summary>📊 Runner Resource Usage</summary>" >> $LOG_FILE
              echo "" >> $LOG_FILE
              echo '```' >> $LOG_FILE
              cat "$CHART_FILE" >> $LOG_FILE
              echo '```' >> $LOG_FILE
              echo "" >> $LOG_FILE
              echo "</details>" >> $LOG_FILE
              echo "" >> $LOG_FILE
            fi
          fi
          
          # Add JSON log details if there's actual data
          if [[ -n "$JSON_INPUT" && "$JSON_INPUT" != "{}" ]]; then
            echo "<details>" >> $LOG_FILE
            echo "<summary>" >> $LOG_FILE
            echo "JSON Log Metrics" >> $LOG_FILE
            echo "</summary>" >> $LOG_FILE
            echo '<pre><code class="JSON">' >> $LOG_FILE
            echo "$JSON_INPUT" >> $LOG_FILE
            echo "</code></pre>" >> $LOG_FILE
            echo "</details>" >> $LOG_FILE
          fi
          
          echo "log_file=$LOG_FILE" >> $GITHUB_OUTPUT

      - name: Comment PR with Log Metrics
        if: >-
          ${{ inputs.runner != 'windows-latest' && !cancelled() && github.event_name == 'pull_request' && 
          steps.log-metrics-file.conclusion == 'success' && github.event.pull_request.head.repo.fork == false }}
        uses: step-security/actions-comment-pull-request@7f6021194300cf361f7ba99a346c4daa6cab809b # v3.0.1
        with:
          mode: recreate
          file-path: ${{ steps.log-metrics-file.outputs.log_file }}
          comment-tag: log-metrics-${{ inputs.test-script }}

      - name: Upload E2E Logs to GitHub
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: solo-${{ inputs.test-script }}.log
          path: ~/.solo/logs/*
          overwrite: true
          if-no-files-found: warn

      - name: Upload E2E Test Report
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        if: ${{ github.event_name != 'workflow_dispatch' && inputs.test-script != 'test-e2e-external-database' && steps.npm-deps.conclusion == 'success' && !cancelled() }}
        with:
          name: e2e_test_report_${{ inputs.test-script }}
          path: "junit-${{ inputs.coverage-subdirectory }}.xml"
          overwrite: true
          if-no-files-found: error

      - name: Publish E2E Coverage Report
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        if: ${{ github.event_name != 'workflow_dispatch' && inputs.test-script != 'test-e2e-external-database' && !cancelled() }}
        with:
          name: ${{ inputs.coverage-report-name }}
          path: 'coverage/${{ inputs.coverage-subdirectory }}'