This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ## | ||
| # Copyright (C) 2023-2024 Hedera Hashgraph, LLC | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| ## | ||
| name: "ZXC: E2E Test" | ||
| # The purpose of this reusable workflow is to run the e2e tests on every PR and commit. | ||
| # This reusable component is called by the following workflows: | ||
| # - .github/workflows/flow-pull-request-checks.yaml | ||
| # - .github/workflows/flow-build-application.yaml | ||
| on: | ||
| workflow_dispatch: | ||
| inputs: | ||
| consensus-node-version: | ||
| description: "Consensus Node Version:" | ||
| type: string | ||
| required: false | ||
| default: "" | ||
| custom-job-label: | ||
| description: "Custom Job Label:" | ||
| type: string | ||
| required: false | ||
| default: "E2E Test" | ||
| test-script: | ||
| description: "NPM Test Run Script:" | ||
| type: string | ||
| required: false | ||
| default: "test-e2e-standard" | ||
| coverage-subdirectory: | ||
| description: "Coverage Report Subdirectory:" | ||
| type: string | ||
| required: false | ||
| default: "e2e" | ||
| coverage-report-name: | ||
| description: "Coverage Report Name:" | ||
| type: string | ||
| required: false | ||
| default: "E2E Tests Coverage Report" | ||
| local-java-build: | ||
| description: "Build Local Java Code:" | ||
| type: boolean | ||
| required: false | ||
| default: false | ||
| cluster-name: | ||
| description: "Cluster Name:" | ||
| type: string | ||
| required: false | ||
| default: "solo-e2e" | ||
| install-dependencies: | ||
| description: "Install Solo Dependencies:" | ||
| type: boolean | ||
| required: false | ||
| default: true | ||
| runner: | ||
| description: "Runner Label:" | ||
| type: string | ||
| required: false | ||
| default: "hiero-solo-linux-large" | ||
| kind-install: | ||
| description: "Install Kind:" | ||
| type: boolean | ||
| required: false | ||
| default: true | ||
| kind-version: | ||
| description: "Kind Version:" | ||
| type: string | ||
| required: false | ||
| default: "0.29.0" | ||
| kind-node-image: | ||
| description: "Kind Node Image:" | ||
| type: string | ||
| required: false | ||
| default: "kindest/node:v1.31.9@sha256:b94a3a6c06198d17f59cca8c6f486236fa05e2fb359cbd75dabbfc348a10b211" | ||
| kubectl-install: | ||
| description: "Install Kubectl:" | ||
| type: boolean | ||
| required: false | ||
| default: true | ||
| kubectl-version: | ||
| description: "Kubectl Version:" | ||
| type: string | ||
| required: false | ||
| default: "1.32.2" | ||
| helm-install: | ||
| description: "Install Helm:" | ||
| type: boolean | ||
| required: false | ||
| default: true | ||
| helm-version: | ||
| description: "Helm Version:" | ||
| type: string | ||
| required: false | ||
| default: "3.14.2" | ||
| node-version: | ||
| description: "NodeJS Version:" | ||
| type: string | ||
| required: false | ||
| default: "22" | ||
| dual-cluster-node-count: | ||
| description: "Dual Cluster Node Count (SOLO_DUAL_CLUSTER_NODE_COUNT):" | ||
| type: number | ||
| required: false | ||
| default: 3 | ||
| test-timeout-minutes: | ||
| description: "Test Timeout (minutes):" | ||
| type: number | ||
| required: false | ||
| default: 60 | ||
| workflow_call: | ||
| inputs: | ||
| consensus-node-version: | ||
| description: "Consensus Node Version:" | ||
| type: string | ||
| required: false | ||
| default: "" | ||
| custom-job-label: | ||
| description: "Custom Job Label:" | ||
| type: string | ||
| required: false | ||
| default: "E2E Test" | ||
| test-script: | ||
| description: "NPM Test Run Script:" | ||
| type: string | ||
| required: false | ||
| default: "test-e2e-standard" | ||
| coverage-subdirectory: | ||
| description: "Coverage Report Subdirectory:" | ||
| type: string | ||
| required: false | ||
| default: "e2e" | ||
| coverage-report-name: | ||
| description: "Coverage Report Name:" | ||
| type: string | ||
| required: false | ||
| default: "E2E Tests Coverage Report" | ||
| local-java-build: | ||
| description: "Build Local Java Code:" | ||
| type: boolean | ||
| required: false | ||
| default: false | ||
| cluster-name: | ||
| description: "Cluster Name:" | ||
| type: string | ||
| required: false | ||
| default: "solo-e2e" | ||
| install-dependencies: | ||
| description: "Install Solo Dependencies:" | ||
| type: boolean | ||
| required: false | ||
| default: true | ||
| runner: | ||
| description: "Runner Label:" | ||
| type: string | ||
| required: false | ||
| default: "hiero-solo-linux-large" | ||
| kind-install: | ||
| description: "Install Kind:" | ||
| type: boolean | ||
| required: false | ||
| default: true | ||
| kind-version: | ||
| description: "Kind Version:" | ||
| type: string | ||
| required: false | ||
| default: "0.29.0" | ||
| kind-node-image: | ||
| description: "Kind Node Image:" | ||
| type: string | ||
| required: false | ||
| default: "kindest/node:v1.31.9@sha256:b94a3a6c06198d17f59cca8c6f486236fa05e2fb359cbd75dabbfc348a10b211" | ||
| kubectl-install: | ||
| description: "Install Kubectl:" | ||
| type: boolean | ||
| required: false | ||
| default: true | ||
| kubectl-version: | ||
| description: "Kubectl Version:" | ||
| type: string | ||
| required: false | ||
| default: "1.32.2" | ||
| helm-install: | ||
| description: "Install Helm:" | ||
| type: boolean | ||
| required: false | ||
| default: true | ||
| helm-version: | ||
| description: "Helm Version:" | ||
| type: string | ||
| required: false | ||
| default: "3.14.2" | ||
| node-version: | ||
| description: "NodeJS Version:" | ||
| type: string | ||
| required: false | ||
| default: "22" | ||
| dual-cluster-node-count: | ||
| description: "Dual Cluster Node Count (SOLO_DUAL_CLUSTER_NODE_COUNT):" | ||
| type: number | ||
| required: false | ||
| default: false | ||
| dual-cluster-node-count: | ||
| description: "Dual Cluster Node Count (SOLO_DUAL_CLUSTER_NODE_COUNT):" | ||
| type: number | ||
| required: false | ||
| default: 3 | ||
| test-timeout-minutes: | ||
| description: "Test Timeout (minutes):" | ||
| type: number | ||
| required: false | ||
| default: 60 | ||
| defaults: | ||
| run: | ||
| shell: bash | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| actions: read | ||
| pull-requests: write | ||
| checks: write | ||
| statuses: write | ||
| env: | ||
| #CG_EXEC: export R_UID=$(id -u); CGROUP_LOGLEVEL=DEBUG cgexec -g cpu,memory:user.slice/user-${R_UID}.slice/user@${R_UID}.service/e2e-${{ github.run_id }} --sticky ionice -c 2 -n 2 nice -n 19 | ||
| CG_EXEC: "ionice -c 2 -n 2 nice -n 19" | ||
| KIND_DOCKER_REGISTRY_MIRRORS: ${{ vars.KIND_DOCKER_REGISTRY_MIRRORS }} | ||
| SOLO_CLUSTER_DUALITY: ${{ (inputs.test-script == 'test-e2e-dual-cluster-full' || inputs.test-script == 'test-e2e-external-database') && 2 || 1 }} | ||
| STEP_TIMEOUT_MINUTES: '5' | ||
| jobs: | ||
| e2e-test: | ||
| name: ${{ inputs.custom-job-label || 'E2E Test' }} | ||
| runs-on: ${{ inputs.runner || 'hiero-solo-linux-large' }} | ||
| steps: | ||
| - name: Harden Runner | ||
| uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| with: | ||
| egress-policy: audit | ||
| - name: Checkout Code | ||
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| - name: Setup Posix Dependencies | ||
| id: set-posix-dependencies | ||
| if: ${{ inputs.runner != 'windows-latest' && !cancelled() && !failure() }} | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| run: | | ||
| # Configure APT to use main Ubuntu archive (disable mirror rotation) | ||
| sudo sed -i 's|http://[a-z.]*.ubuntu.com|http://archive.ubuntu.com|g' /etc/apt/sources.list | ||
| # Install jq (a lightweight and flexible command-line JSON processor) | ||
| sudo apt-get update && sudo apt-get install -y jq | ||
| - name: Setup Node | ||
| uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| with: | ||
| node-version: ${{ inputs.node-version }} | ||
| - name: Configure npm registry | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| run: echo "registry=https://registry.npmjs.org/" > .npmrc | ||
| - name: Install wget | ||
| if: ${{ inputs.runner != 'windows-latest' && !cancelled() && !failure() }} | ||
| uses: step-security/retry@38808b4332b98bb36ca031abff0c100231608288 # v4.0.0 | ||
| with: | ||
| max_attempts: 3 | ||
| timeout_minutes: ${{ env.STEP_TIMEOUT_MINUTES }} | ||
| command: | | ||
| if ! command -v wget >/dev/null 2>&1; then | ||
| sudo apt-get update | ||
| sudo apt-get install -y wget | ||
| fi | ||
| - name: Install Kubectl | ||
| if: ${{ inputs.install-dependencies == true && inputs.kubectl-install == true && !cancelled() && !failure() }} | ||
| uses: step-security/setup-kubectl@b256bc71248a59f1e3a278fc75a65646d6a8a1c4 # pinned to v5.1.0 | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| with: | ||
| version: 'v${{ inputs.kubectl-version }}' | ||
| id: install | ||
| - name: Setup Kind | ||
| if: ${{ inputs.install-dependencies == true && inputs.kind-install == true && !cancelled() && !failure() }} | ||
| uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0 | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| with: | ||
| install_only: true | ||
| node_image: '${{ inputs.kind-node-image }}' | ||
| version: 'v${{ inputs.kind-version }}' | ||
| kubectl_version: 'v${{ inputs.kubectl-version }}' | ||
| verbosity: 3 | ||
| wait: 120s | ||
| - name: Setup Helm | ||
| if: ${{ inputs.install-dependencies == true && inputs.helm-install == true && !cancelled() && !failure() }} | ||
| uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| with: | ||
| version: 'v${{ inputs.helm-version }}' | ||
| - name: Install Dependencies | ||
| id: npm-deps | ||
| uses: step-security/retry@38808b4332b98bb36ca031abff0c100231608288 # v4.0.0 | ||
| with: | ||
| max_attempts: 10 | ||
| timeout_minutes: ${{ env.STEP_TIMEOUT_MINUTES }} | ||
| command: | | ||
| npm ci | ||
| - name: Install Task | ||
| uses: arduino/setup-task@b91d5d2c96a56797b48ac1e0e89220bf64044611 # v2.0.0 | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| with: | ||
| version: 3.49.1 | ||
| repo-token: ${{ secrets.GITHUB_TOKEN }} | ||
| - name: Setup Go | ||
| uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| with: | ||
| go-version: '1.22.3' | ||
| - name: Install grpcurl | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| run: | | ||
| go install github.com/fullstorydev/grpcurl/cmd/grpcurl@latest | ||
| - name: Compile Project | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| run: task build:compile | ||
| - name: Resolve consensus node version and Java version | ||
| id: resolve-consensus-java | ||
| if: ${{ runner.os == 'linux' && inputs.local-java-build && !cancelled() && !failure() }} | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| shell: bash | ||
| run: | | ||
| set -euo pipefail | ||
| source .github/workflows/script/helper.sh | ||
| if [ -z "${{ inputs.consensus-node-version }}" ]; then | ||
| consensus_node_version="$(extract_version HEDERA_PLATFORM_VERSION version.ts)" | ||
| else | ||
| consensus_node_version="${{ inputs.consensus-node-version }}" | ||
| fi | ||
| normalized_version="$(echo "${consensus_node_version}" | sed -E 's/^v//; s/-.*$//')" | ||
| java_version="21" | ||
| if printf '%s\n' '0.73.0' "${normalized_version}" | sort -C -V; then | ||
| java_version="25" | ||
| fi | ||
| echo "consensus-node-version=${consensus_node_version}" >> "${GITHUB_OUTPUT}" | ||
| echo "java-version=${java_version}" >> "${GITHUB_OUTPUT}" | ||
| - name: Setup Java | ||
| if: ${{ runner.os == 'linux' && inputs.local-java-build && !cancelled() && !failure() }} | ||
| uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0 | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| with: | ||
| distribution: temurin | ||
| java-version: ${{ steps.resolve-consensus-java.outputs.java-version }} | ||
| - name: Setup Gradle | ||
| if: ${{ runner.os == 'linux' && inputs.local-java-build && !cancelled() && !failure() }} | ||
| uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| with: | ||
| cache-read-only: false | ||
| - name: Create Diagnostic Script | ||
| if: ${{ !cancelled() && !failure() }} | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| env: | ||
| WRITE_DIAG_SCRIPT: ${{ github.workspace }}/.github/workflows/script/write_diag_script.sh | ||
| run: | | ||
| sudo chmod +x "${WRITE_DIAG_SCRIPT}" | ||
| ${WRITE_DIAG_SCRIPT} | ||
| - name: Build Hedera code locally | ||
| if: ${{ runner.os == 'linux' && inputs.local-java-build && !cancelled() && !failure() }} | ||
| uses: step-security/retry@38808b4332b98bb36ca031abff0c100231608288 # v4.0.0 | ||
| with: | ||
| max_attempts: 3 | ||
| timeout_minutes: 30 | ||
| command: | | ||
| source .github/workflows/script/helper.sh | ||
| if [ -z "${{ inputs.consensus-node-version }}" ]; then | ||
| export CONSENSUS_NODE_VERSION=$(extract_version HEDERA_PLATFORM_VERSION version.ts) | ||
| else | ||
| export CONSENSUS_NODE_VERSION=${{ inputs.consensus-node-version }} | ||
| fi | ||
| cd .. | ||
| git clone https://github.com/hiero-ledger/hiero-consensus-node.git --depth 1 --branch ${CONSENSUS_NODE_VERSION} | ||
| cd hiero-consensus-node | ||
| ls -ltr | ||
| ${{ env.CG_EXEC }} ./gradlew assemble --stacktrace --info || exit 1 | ||
| cd ../solo | ||
| - name: Setup E2E Tests | ||
| if: ${{ inputs.custom-job-label != 'One Shot Single - using Podman' && !cancelled() && !failure() }} | ||
| uses: step-security/retry@38808b4332b98bb36ca031abff0c100231608288 # v4.0.0 | ||
| with: | ||
| max_attempts: 3 | ||
| timeout_minutes: 10 | ||
| shell: bash | ||
| command: | | ||
| npm link | ||
| echo "SOLO_TEST_CLUSTER=${{ inputs.cluster-name }}" | ||
| if [[ "$RUNNER_OS" != "Windows" ]]; then | ||
| SOLO_TEST_CLUSTER=${{ inputs.cluster-name }} ${{ env.CG_EXEC }} ./test/e2e/dual-cluster/setup-dual-e2e.sh | ||
| else | ||
| SOLO_TEST_CLUSTER=${{ inputs.cluster-name }} ./test/e2e/dual-cluster/setup-dual-e2e.sh | ||
| fi | ||
| - name: Run E2E Tests | ||
| id: run-e2e-tests | ||
| if: ${{ inputs.runner != 'windows-latest' && !cancelled() && !failure() }} | ||
| timeout-minutes: ${{ fromJSON(inputs.test-timeout-minutes) }} | ||
| run: | | ||
| export PATH=~/.solo/bin:${PATH} | ||
| echo "PATH: $PATH" | ||
| source .github/workflows/script/helper.sh | ||
| if [[ -z "${{ inputs.consensus-node-version }}" ]]; then | ||
| if [[ "${{ inputs.local-java-build }}" != "true" ]]; then | ||
| echo "No consensus node version provided, using default from version-test.ts" | ||
| else | ||
| if [[ "${{ inputs.test-script }}" == "test-e2e-block-node" ]]; then | ||
| echo "Use default from version.ts for block node test" | ||
| else | ||
| export CONSENSUS_NODE_VERSION=$(extract_version HEDERA_PLATFORM_VERSION version.ts) | ||
| fi | ||
| fi | ||
| else | ||
| export CONSENSUS_NODE_VERSION=${{ inputs.consensus-node-version }} | ||
| fi | ||
| echo SOLO_TEST_CLUSTER=${{ inputs.cluster-name }}-c1 > .env | ||
| echo SOLO_DUAL_CLUSTER_NODE_COUNT=${{ inputs.dual-cluster-node-count || 3 }} >> .env | ||
| if [[ "${{ inputs.custom-job-label }}" != "One Shot Single - using Podman" ]]; then | ||
| # Verify cluster connectivity before running tests | ||
| echo "Verifying cluster connectivity..." | ||
| kubectl cluster-info || { | ||
| echo "Error: Cannot connect to cluster" | ||
| echo "KUBECONFIG: ${KUBECONFIG:-default}" | ||
| kubectl config get-contexts || true | ||
| exit 1 | ||
| } | ||
| else | ||
| echo "SOLO_ONE_SHOT_MINIMAL_SETUP=true" >> .env | ||
| echo "FORCE_PODMAN=true" >> .env | ||
| fi | ||
| if [[ "${{ inputs.custom-job-label }}" == "One Shot Single" ]]; then | ||
| echo "ENABLE_IMAGE_CACHE=true" >> .env | ||
| fi | ||
| cat .env | ||
| # Start resource monitoring in background | ||
| ./.github/workflows/script/monitor-resources.sh start | ||
| # Run the task in the background and trap SIGTERM/SIGINT so that a workflow | ||
| # cancellation (e.g. triggered by a new commit on this PR) terminates the | ||
| # test immediately rather than waiting 30-60 minutes for it to finish on | ||
| # its own. | ||
| TASK_PID="" | ||
| TASK_PGID="" | ||
| TEST_EXIT_CODE=0 | ||
| cleanup() { | ||
| if [[ -n "${TASK_PID}" ]]; then | ||
| echo "Received cancellation signal, stopping task (PID ${TASK_PID}, PGID ${TASK_PGID})..." | ||
| if [[ -n "${TASK_PGID}" ]]; then | ||
| kill -TERM -- "-${TASK_PGID}" 2>/dev/null || true | ||
| sleep 5 | ||
| kill -KILL -- "-${TASK_PGID}" 2>/dev/null || true | ||
| else | ||
| kill -TERM "${TASK_PID}" 2>/dev/null || true | ||
| sleep 5 | ||
| kill -KILL "${TASK_PID}" 2>/dev/null || true | ||
| fi | ||
| wait "${TASK_PID}" 2>/dev/null || true | ||
| fi | ||
| ./.github/workflows/script/monitor-resources.sh stop || true | ||
| exit 130 | ||
| } | ||
| trap cleanup SIGTERM SIGINT | ||
| TASK_COMMAND="${{ env.CG_EXEC }} task ${{ inputs.test-script }}" | ||
| if command -v setsid >/dev/null 2>&1; then | ||
| setsid bash -lc "${TASK_COMMAND}" & | ||
| else | ||
| bash -lc "${TASK_COMMAND}" & | ||
| fi | ||
| TASK_PID=$! | ||
| TASK_PGID="$(ps -o pgid= -p "${TASK_PID}" | tr -d '[:space:]' || true)" | ||
| wait "${TASK_PID}" || TEST_EXIT_CODE=$? | ||
| TASK_PID="" | ||
| TASK_PGID="" | ||
| trap - SIGTERM SIGINT | ||
| # Stop resource monitoring and display summary | ||
| ./.github/workflows/script/monitor-resources.sh stop | ||
| exit $TEST_EXIT_CODE | ||
| - name: Log Metrics | ||
| id: log-metrics | ||
| if: ${{ !cancelled() }} | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| shell: bash | ||
| run: | | ||
| export PATH=~/.solo/bin:${PATH} | ||
| echo "PATH: $PATH" | ||
| echo "json_log={}" >> $GITHUB_OUTPUT | ||
| # Ensure logs directory exists | ||
| mkdir -p ~/.solo/logs | ||
| # Check if any JSON files exist | ||
| shopt -s nullglob | ||
| json_files=(~/.solo/logs/*.json) | ||
| if [[ ${#json_files[@]} -eq 0 ]]; then | ||
| echo "::notice::No JSON log files found" | ||
| else | ||
| for logFile in "${json_files[@]}"; do | ||
| if [[ -z "$logFile" ]]; then | ||
| continue | ||
| fi | ||
| echo "::group::JSON Format" | ||
| cat $logFile | jq | ||
| echo "::endgroup::" | ||
| echo "json_log=$(cat $logFile)" >> $GITHUB_OUTPUT | ||
| echo "::group::Spreadsheet Format" | ||
| echo "logFile=$logFile" | ||
| JSON_FILENAME=$logFile ./.github/workflows/script/metrics.sh | ||
| echo "::endgroup::" | ||
| done | ||
| fi | ||
| echo "solo_log_dir=$(cd ~/.solo/logs && pwd)" >> $GITHUB_OUTPUT | ||
| - name: Generate ASCII Resource Chart | ||
| id: generate-chart | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| run: | | ||
| echo "json_log=${{ steps.log-metrics.outputs.json_log }}" | ||
| METRICS_FILE="$HOME/.solo/logs/runner-metrics.jsonl" | ||
| CHART_FILE="$HOME/.solo/logs/runner-metrics-ascii.txt" | ||
| DATA_POINTS=$(wc -l < "$METRICS_FILE" 2>/dev/null | tr -d ' ' || echo "0") | ||
| if [[ -f "$METRICS_FILE" && "$DATA_POINTS" -gt 0 ]]; then | ||
| echo "Generating ASCII chart from ${DATA_POINTS} data points..." | ||
| ./.github/workflows/script/plot-runner-metrics-ascii.sh "$METRICS_FILE" "$CHART_FILE" | ||
| if [[ -f "$CHART_FILE" && -s "$CHART_FILE" ]]; then | ||
| echo "chart_generated=true" >> $GITHUB_OUTPUT | ||
| echo "chart_file=$CHART_FILE" >> $GITHUB_OUTPUT | ||
| echo "✅ ASCII chart generated successfully" | ||
| else | ||
| echo "chart_generated=false" >> $GITHUB_OUTPUT | ||
| echo "⚠️ ASCII chart generation produced no output" | ||
| fi | ||
| else | ||
| echo "chart_generated=false" >> $GITHUB_OUTPUT | ||
| echo "ℹ️ No metrics data collected (${DATA_POINTS} data points), skipping chart generation" | ||
| fi | ||
| - name: Write Log Metrics Message File | ||
| id: log-metrics-file | ||
| if: ${{ !cancelled() && steps.run-e2e-tests.conclusion == 'success' }} | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| env: | ||
| LOG_FILE: "${{ steps.log-metrics.outputs.solo_log_dir }}/log_file.md" | ||
| JSON_INPUT: "${{ toJSON(fromJSON(steps.log-metrics.outputs.json_log)) }}" | ||
| run: | | ||
| echo "## Log Metrics - ${{ inputs.custom-job-label }}" >> $LOG_FILE | ||
| echo "" >> $LOG_FILE | ||
| # Add ASCII resource chart if available | ||
| if [[ "${{ steps.generate-chart.outputs.chart_generated }}" == "true" ]]; then | ||
| CHART_FILE="${{ steps.generate-chart.outputs.chart_file }}" | ||
| if [[ -f "$CHART_FILE" ]]; then | ||
| echo "<details>" >> $LOG_FILE | ||
| echo "<summary>📊 Runner Resource Usage</summary>" >> $LOG_FILE | ||
| echo "" >> $LOG_FILE | ||
| echo '```' >> $LOG_FILE | ||
| cat "$CHART_FILE" >> $LOG_FILE | ||
| echo '```' >> $LOG_FILE | ||
| echo "" >> $LOG_FILE | ||
| echo "</details>" >> $LOG_FILE | ||
| echo "" >> $LOG_FILE | ||
| fi | ||
| fi | ||
| # Add JSON log details if there's actual data | ||
| if [[ -n "$JSON_INPUT" && "$JSON_INPUT" != "{}" ]]; then | ||
| echo "<details>" >> $LOG_FILE | ||
| echo "<summary>" >> $LOG_FILE | ||
| echo "JSON Log Metrics" >> $LOG_FILE | ||
| echo "</summary>" >> $LOG_FILE | ||
| echo '<pre><code class="JSON">' >> $LOG_FILE | ||
| echo "$JSON_INPUT" >> $LOG_FILE | ||
| echo "</code></pre>" >> $LOG_FILE | ||
| echo "</details>" >> $LOG_FILE | ||
| fi | ||
| echo "log_file=$LOG_FILE" >> $GITHUB_OUTPUT | ||
| - name: Write Log Metrics to Workflow Summary | ||
| if: ${{ !cancelled() && steps.log-metrics-file.conclusion == 'success' }} | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| run: | | ||
| cat "${{ steps.log-metrics-file.outputs.log_file }}" >> $GITHUB_STEP_SUMMARY | ||
| - name: Upload E2E Logs to GitHub | ||
| if: ${{ !cancelled() }} | ||
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| with: | ||
| name: solo-${{ inputs.test-script }}.log | ||
| path: ~/.solo/logs/* | ||
| overwrite: true | ||
| if-no-files-found: warn | ||
| - name: Upload E2E Test Report | ||
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | ||
| if: >- | ||
| ${{ github.event_name != 'workflow_dispatch' && inputs.test-script != 'test-e2e-external-database' && | ||
| steps.npm-deps.conclusion == 'success' && !cancelled() && !failure() }} | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| with: | ||
| name: e2e_test_report_${{ inputs.test-script }} | ||
| path: "junit-${{ inputs.coverage-subdirectory }}.xml" | ||
| overwrite: true | ||
| if-no-files-found: error | ||
| - name: Publish E2E Coverage Report | ||
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | ||
| if: >- | ||
| ${{ github.event_name != 'workflow_dispatch' && inputs.test-script != 'test-e2e-external-database' && | ||
| !cancelled() && !failure() }} | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| with: | ||
| name: ${{ inputs.coverage-report-name }} | ||
| path: 'coverage/${{ inputs.coverage-subdirectory }}' | ||
| - name: Cleanup Kind Clusters | ||
| if: ${{ always() }} | ||
| timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }} | ||
| run: | | ||
| export PATH=~/.solo/bin:${PATH} | ||
| # Helper: collect docker hang diagnostics using non-Docker tools (writes to stderr) | ||
| collect_docker_hang_diagnostics() { | ||
| local hung_command="${1:-unknown command}" | ||
| { | ||
| echo "=== DOCKER HANG DIAGNOSTICS (timed out: '${hung_command}') ===" | ||
| systemctl status docker --no-pager -l 2>/dev/null || service docker status 2>/dev/null || echo "Unable to get docker service status" | ||
| ls -la /var/run/docker.sock 2>/dev/null || echo "Docker socket not found" | ||
| lsof /var/run/docker.sock 2>/dev/null || echo "lsof unavailable or no open handles on docker socket" | ||
| ps aux | grep -E '[d]ocker|[c]ontainerd|[k]ind' 2>/dev/null || true | ||
| free -h 2>/dev/null || true | ||
| journalctl -u docker --no-pager -n 30 2>/dev/null || echo "journalctl unavailable" | ||
| echo "=== END DOCKER HANG DIAGNOSTICS ===" | ||
| } >&2 | ||
| } | ||
| for i in $(seq 1 ${{ env.SOLO_CLUSTER_DUALITY }}); do | ||
| timeout 60 kind delete cluster -n "${{ inputs.cluster-name }}-c${i}" || true | ||
| done | ||
| exit_code=0 | ||
| timeout 30 docker network rm -f kind || exit_code=$? | ||
| if [[ $exit_code -eq 124 ]]; then | ||
| echo "WARNING: 'docker network rm kind' timed out" | ||
| collect_docker_hang_diagnostics "docker network rm kind" | ||
| elif [[ $exit_code -ne 0 ]]; then | ||
| echo "WARNING: 'docker network rm kind' failed with exit code ${exit_code}" | ||
| collect_docker_hang_diagnostics "docker network rm kind" | ||
| fi | ||
| true | ||
