pipeline-perf-test-nightly.yml

name: Pipeline Performance Tests - Nightly
permissions:
  contents: read

on:
  schedule:
    - cron: "0 16 * * *" # runs every day at 8 AM PT (4 PM UTC)
    - cron: "0 0 * * *" # runs every day at 4 PM PT (midnight UTC)
  workflow_dispatch:

jobs:
  pipeline-perf-test:
    runs-on: oracle-bare-metal-64cpu-1024gb-x86-64-ubuntu-24
    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
        with:
          egress-policy: audit

      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Set up Python
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "3.14"

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0

      - name: Set up Python (non-OL8)
        if: ${{ steps.detect_os.outputs.id != 'ol' }}
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "3.14"

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0

      - name: Build dataflow_engine
        run: |
          git submodule init
          git submodule update
          cd rust/otap-dataflow
          docker buildx build --load --build-context otel-arrow=../../ -f Dockerfile -t df_engine .
          cd ../..

      - name: Build otelarrowcol
        run: |
          docker build . -t otelarrowcol

      - name: Install dependencies
        run: |
          python -m pip install --user --require-hashes -r tools/pipeline_perf_test/orchestrator/requirements.lock.txt
          python -m pip install --user --require-hashes -r tools/pipeline_perf_test/load_generator/requirements.lock.txt

      - name: Run syslog performance test log suite
        if: ${{ !cancelled() }}
        run: |
          cd tools/pipeline_perf_test
          python orchestrator/run_orchestrator.py --config test_suites/integration/nightly/syslog-docker.yaml

      - name: Run syslog TCP performance test log suite
        run: |
          cd tools/pipeline_perf_test
          python orchestrator/run_orchestrator.py --config test_suites/integration/nightly/syslog-tcp-docker.yaml

      - name: Run standard load batch size test suite
        if: ${{ !cancelled() }}
        run: |
          cd tools/pipeline_perf_test
          python orchestrator/run_orchestrator.py --config test_suites/integration/nightly/100klrps-batch-sizes-docker.yaml

      - name: Run backpressure performance test log suite
        if: ${{ !cancelled() }}
        run: |
          cd tools/pipeline_perf_test
          python orchestrator/run_orchestrator.py --config test_suites/integration/nightly/backpressure-docker.yaml

      - name: Run filter performance test log suite
        if: ${{ !cancelled() }}
        run: |
          cd tools/pipeline_perf_test
          python orchestrator/run_orchestrator.py --config test_suites/integration/nightly/filter-docker.yaml

      - name: Run filter performance test otelcol log suite
        if: ${{ !cancelled() }}
        run: |
          cd tools/pipeline_perf_test
          python orchestrator/run_orchestrator.py --config test_suites/integration/nightly/otelcol-docker.yaml

      - name: Run syslog TCP performance test otelcol log suite
        if: ${{ !cancelled() }}
        run: |
          cd tools/pipeline_perf_test
          python orchestrator/run_orchestrator.py --config test_suites/integration/nightly/syslog-tcp-otelcol-docker.yaml

      - name: Run batch processor performance test log suite
        if: ${{ !cancelled() }}
        run: |
          cd tools/pipeline_perf_test
          python orchestrator/run_orchestrator.py --config test_suites/integration/nightly/batch-processor-docker.yaml

      - name: Run idle state performance tests
        if: ${{ !cancelled() }}
        run: |
          cd tools/pipeline_perf_test
          ./scripts/run-idle-state-tests.sh --output-json results/idle-memory-scaling.json \
            | tee idle-memory-scaling-report.txt
          # Add memory scaling report to job summary
          echo "### Idle State Memory Scaling Analysis" >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
          # Extract just the analysis section from the report
          sed -n '/^=.*MEMORY SCALING/,/^=\{10,\}$/p' idle-memory-scaling-report.txt >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY

      - name: Run saturation/scaling performance test suite
        if: ${{ !cancelled() }}
        run: |
          cd tools/pipeline_perf_test
          ./scripts/run-saturation-tests.sh --output-json results/scaling-efficiency.json \
            | tee saturation-scaling-report.txt
          # Add scaling report to job summary
          echo "### Saturation/Scaling Test Analysis" >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
          # Extract just the analysis section from the report
          sed -n '/^=.*SCALING ANALYSIS/,/^=\{10,\}$/p' saturation-scaling-report.txt >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY

      - name: Run single-core saturation max throughput test (OTLP)
        if: ${{ !cancelled() }}
        run: |
          cd tools/pipeline_perf_test
          python orchestrator/run_orchestrator.py --config test_suites/integration/continuous/saturation-max-throughput.yaml

      - name: Run single-core saturation max throughput test (OTAP)
        if: ${{ !cancelled() }}
        run: |
          cd tools/pipeline_perf_test
          python orchestrator/run_orchestrator.py --config test_suites/integration/continuous/saturation-max-throughput-otap.yaml

      - name: Upload syslog results for processing
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: syslog-nightly-results
          path: tools/pipeline_perf_test/results/nightly_syslog/gh-actions-benchmark/*.json

      - name: Upload syslog TCP results for processing
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: syslog-tcp-nightly-results
          path: tools/pipeline_perf_test/results/nightly_syslog_tcp/gh-actions-benchmark/*.json

      - name: Upload standard load batch size results for processing
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: standardload-batch-size-nightly-results
          path: tools/pipeline_perf_test/results/nightly_100klrps_batch_sizes/gh-actions-benchmark/*.json

      - name: Upload backpressure results for processing
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: backpressure-nightly-results
          path: tools/pipeline_perf_test/results/nightly_backpressure/gh-actions-benchmark/*.json

      - name: Upload filter results for processing
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: filter-nightly-results
          path: tools/pipeline_perf_test/results/nightly_filter/gh-actions-benchmark/*.json

      - name: Upload batch processor results for processing
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: batch-processor-nightly-results
          path: tools/pipeline_perf_test/results/nightly_batch_processor/gh-actions-benchmark/*.json

      - name: Upload otelcol filter results for processing
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: filter-otelcol-results
          path: tools/pipeline_perf_test/results/otelcol_filter/gh-actions-benchmark/*.json

      - name: Upload otelcol syslog TCP results for processing
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: syslog-tcp-otelcol-results
          path: tools/pipeline_perf_test/results/nightly_syslog_tcp_otelcol/gh-actions-benchmark/*.json

      - name: Upload saturation results for processing
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: saturation-nightly-results
          path: tools/pipeline_perf_test/results/continuous_saturation*/gh-actions-benchmark/*.json

      - name: Upload idle state results for processing
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: idle-state-nightly-results
          path: tools/pipeline_perf_test/results/idle_state*/gh-actions-benchmark/*.json

      - name: Upload idle state memory scaling metrics
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: idle-memory-scaling-nightly-results
          path: tools/pipeline_perf_test/results/idle-memory-scaling.json

      - name: Upload scaling efficiency metrics
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: scaling-efficiency-nightly-results
          path: tools/pipeline_perf_test/results/scaling-efficiency.json

      - name: Add benchmark link to job summary
        if: ${{ !cancelled() }}
        run: |
          echo "### Benchmark Results" >> $GITHUB_STEP_SUMMARY
          echo "[View the benchmark results here](https://open-telemetry.github.io/otel-arrow/benchmarks/nightly/)" >> $GITHUB_STEP_SUMMARY

  update-benchmarks:
    runs-on: ubuntu-24.04
    needs: [pipeline-perf-test]
    if: ${{ !cancelled() }}
    permissions:
      # deployments permission to deploy GitHub pages website
      deployments: write
      # contents permission to update benchmark contents in gh-pages branch
      contents: write
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Download syslog artifacts
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: syslog-nightly-results*
          merge-multiple: true
          path: syslog_results

      - name: Download syslog TCP artifacts
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: syslog-tcp-nightly-results*
          merge-multiple: true
          path: syslog_tcp_results

      - name: Download standard load batch size artifacts
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: standardload-batch-size-nightly-results*
          merge-multiple: true
          path: standardload_batch_size_results

      - name: Download backpressure artifacts
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: backpressure-nightly-results*
          merge-multiple: true
          path: backpressure_results

      - name: Download filter artifacts
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: filter-nightly-results*
          merge-multiple: true
          path: filter_results

      - name: Download batch processor artifacts
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: batch-processor-nightly-results*
          merge-multiple: true
          path: batch_processor_results

      - name: Download otelcol filter artifacts
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: filter-otelcol-results*
          merge-multiple: true
          path: filter_otelcol_results

      - name: Download otelcol syslog TCP artifacts
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: syslog-tcp-otelcol-results*
          merge-multiple: true
          path: syslog_tcp_otelcol_results

      - name: Download saturation artifacts
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: saturation-nightly-results*
          merge-multiple: true
          path: saturation_results

      - name: Download idle state artifacts
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: idle-state-nightly-results*
          merge-multiple: true
          path: idle_state_results

      - name: Download idle memory scaling artifacts
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: idle-memory-scaling-nightly-results*
          merge-multiple: true
          path: idle_memory_scaling_results

      - name: Download scaling efficiency metrics
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: scaling-efficiency-nightly-results*
          merge-multiple: true
          path: scaling_efficiency_results

      - name: Consolidate syslog benchmark data
        if: ${{ !cancelled() }}
        run: |
          bash ./.github/workflows/scripts/consolidate-benchmarks.sh syslog_results syslog_output.json

      - name: Consolidate syslog TCP benchmark data
        if: ${{ !cancelled() }}
        run: |
          bash ./.github/workflows/scripts/consolidate-benchmarks.sh syslog_tcp_results syslog_tcp_output.json

      - name: Consolidate standard load batch size benchmark data
        if: ${{ !cancelled() }}
        run: |
          bash ./.github/workflows/scripts/consolidate-benchmarks.sh standardload_batch_size_results standardload_batch_size_output.json

      - name: Consolidate backpressure benchmark data
        if: ${{ !cancelled() }}
        run: |
          bash ./.github/workflows/scripts/consolidate-benchmarks.sh backpressure_results backpressure_output.json

      - name: Consolidate filter benchmark data
        if: ${{ !cancelled() }}
        run: |
          bash ./.github/workflows/scripts/consolidate-benchmarks.sh filter_results filter_output.json

      - name: Consolidate batch processor benchmark data
        if: ${{ !cancelled() }}
        run: |
          bash ./.github/workflows/scripts/consolidate-benchmarks.sh batch_processor_results batch_processor_output.json

      - name: Consolidate otelcol filter benchmark data
        if: ${{ !cancelled() }}
        run: |
          bash ./.github/workflows/scripts/consolidate-benchmarks.sh filter_otelcol_results filter_otelcol_output.json

      - name: Consolidate otelcol syslog TCP benchmark data
        if: ${{ !cancelled() }}
        run: |
          bash ./.github/workflows/scripts/consolidate-benchmarks.sh syslog_tcp_otelcol_results syslog_tcp_otelcol_output.json

      - name: Consolidate saturation benchmark data
        if: ${{ !cancelled() }}
        run: |
          bash ./.github/workflows/scripts/consolidate-benchmarks.sh saturation_results saturation_output.json

      - name: Consolidate idle state benchmark data
        if: ${{ !cancelled() }}
        run: |
          bash ./.github/workflows/scripts/consolidate-benchmarks.sh idle_state_results idle_state_output.json

      - name: Update benchmark data
        if: ${{ !cancelled() && hashFiles('syslog_output.json') != '' }}
        uses: benchmark-action/github-action-benchmark@a60cea5bc7b49e15c1f58f411161f99e0df48372 # v1.22.0
        with:
          tool: "customSmallerIsBetter"
          output-file-path: syslog_output.json
          gh-pages-branch: benchmarks
          max-items-in-chart: 100
          github-token: ${{ secrets.GITHUB_TOKEN }}
          benchmark-data-dir-path: "docs/benchmarks/nightly/syslog"
          auto-push: true
          save-data-file: true

      - name: Update syslog TCP benchmark data
        if: ${{ !cancelled() && hashFiles('syslog_tcp_output.json') != '' }}
        uses: benchmark-action/github-action-benchmark@a60cea5bc7b49e15c1f58f411161f99e0df48372 # v1.22.0
        with:
          tool: "customSmallerIsBetter"
          output-file-path: syslog_tcp_output.json
          gh-pages-branch: benchmarks
          max-items-in-chart: 100
          github-token: ${{ secrets.GITHUB_TOKEN }}
          benchmark-data-dir-path: "docs/benchmarks/nightly/syslog-tcp"
          auto-push: true
          save-data-file: true

      - name: Update standard load batch size benchmark data
        if: ${{ !cancelled() && hashFiles('standardload_batch_size_output.json') != '' }}
        uses: benchmark-action/github-action-benchmark@a60cea5bc7b49e15c1f58f411161f99e0df48372 # v1.22.0
        with:
          tool: "customSmallerIsBetter"
          output-file-path: standardload_batch_size_output.json
          gh-pages-branch: benchmarks
          max-items-in-chart: 100
          github-token: ${{ secrets.GITHUB_TOKEN }}
          benchmark-data-dir-path: "docs/benchmarks/nightly/standardload-batch-size"
          auto-push: true
          save-data-file: true

      - name: Update benchmark data and deploy to GitHub Pages
        if: ${{ !cancelled() && hashFiles('backpressure_output.json') != '' }}
        uses: benchmark-action/github-action-benchmark@a60cea5bc7b49e15c1f58f411161f99e0df48372 # v1.22.0
        with:
          tool: "customSmallerIsBetter"
          output-file-path: backpressure_output.json
          gh-pages-branch: benchmarks
          max-items-in-chart: 100
          github-token: ${{ secrets.GITHUB_TOKEN }}
          benchmark-data-dir-path: "docs/benchmarks/nightly/backpressure"
          auto-push: true
          save-data-file: true

      - name: Update filter benchmark data
        if: ${{ !cancelled() && hashFiles('filter_output.json') != '' }}
        uses: benchmark-action/github-action-benchmark@a60cea5bc7b49e15c1f58f411161f99e0df48372 # v1.22.0
        with:
          tool: "customSmallerIsBetter"
          output-file-path: filter_output.json
          gh-pages-branch: benchmarks
          max-items-in-chart: 100
          github-token: ${{ secrets.GITHUB_TOKEN }}
          benchmark-data-dir-path: "docs/benchmarks/nightly/filter"
          auto-push: true
          save-data-file: true

      - name: Update batch processor benchmark data
        if: ${{ !cancelled() && hashFiles('batch_processor_output.json') != '' }}
        uses: benchmark-action/github-action-benchmark@a60cea5bc7b49e15c1f58f411161f99e0df48372 # v1.22.0
        with:
          tool: "customSmallerIsBetter"
          output-file-path: batch_processor_output.json
          gh-pages-branch: benchmarks
          max-items-in-chart: 100
          github-token: ${{ secrets.GITHUB_TOKEN }}
          benchmark-data-dir-path: "docs/benchmarks/nightly/batch-processor"
          auto-push: true
          save-data-file: true

      - name: Update filter otelcol benchmark data
        if: ${{ !cancelled() && hashFiles('filter_otelcol_output.json') != '' }}
        uses: benchmark-action/github-action-benchmark@a60cea5bc7b49e15c1f58f411161f99e0df48372 # v1.22.0
        with:
          tool: "customSmallerIsBetter"
          output-file-path: filter_otelcol_output.json
          gh-pages-branch: benchmarks
          max-items-in-chart: 100
          github-token: ${{ secrets.GITHUB_TOKEN }}
          benchmark-data-dir-path: "docs/benchmarks/nightly/filter"
          auto-push: true
          save-data-file: true

      - name: Update syslog TCP otelcol benchmark data
        if: ${{ !cancelled() && hashFiles('syslog_tcp_otelcol_output.json') != '' }}
        uses: benchmark-action/github-action-benchmark@a60cea5bc7b49e15c1f58f411161f99e0df48372 # v1.22.0
        with:
          tool: "customSmallerIsBetter"
          output-file-path: syslog_tcp_otelcol_output.json
          gh-pages-branch: benchmarks
          max-items-in-chart: 100
          github-token: ${{ secrets.GITHUB_TOKEN }}
          benchmark-data-dir-path: "docs/benchmarks/nightly/syslog-tcp-otelcol"
          auto-push: true
          save-data-file: true

      - name: Update saturation benchmark data
        if: ${{ !cancelled() && hashFiles('saturation_output.json') != '' }}
        uses: benchmark-action/github-action-benchmark@a60cea5bc7b49e15c1f58f411161f99e0df48372 # v1.22.0
        with:
          tool: "customSmallerIsBetter"
          output-file-path: saturation_output.json
          gh-pages-branch: benchmarks
          max-items-in-chart: 100
          github-token: ${{ secrets.GITHUB_TOKEN }}
          benchmark-data-dir-path: "docs/benchmarks/nightly/saturation"
          auto-push: true
          save-data-file: true

      - name: Update idle state benchmark data
        if: ${{ !cancelled() && hashFiles('idle_state_output.json') != '' }}
        uses: benchmark-action/github-action-benchmark@a60cea5bc7b49e15c1f58f411161f99e0df48372 # v1.22.0
        with:
          tool: "customSmallerIsBetter"
          output-file-path: idle_state_output.json
          gh-pages-branch: benchmarks
          max-items-in-chart: 100
          github-token: ${{ secrets.GITHUB_TOKEN }}
          benchmark-data-dir-path: "docs/benchmarks/nightly/idle-state"
          auto-push: true
          save-data-file: true

      - name: Update idle state memory scaling data
        if: ${{ !cancelled() && hashFiles('idle_memory_scaling_results/idle-memory-scaling.json') != '' }}
        uses: benchmark-action/github-action-benchmark@a60cea5bc7b49e15c1f58f411161f99e0df48372 # v1.22.0
        with:
          tool: "customSmallerIsBetter"
          output-file-path: idle_memory_scaling_results/idle-memory-scaling.json
          gh-pages-branch: benchmarks
          max-items-in-chart: 100
          github-token: ${{ secrets.GITHUB_TOKEN }}
          benchmark-data-dir-path: "docs/benchmarks/nightly/idle-memory-scaling"
          auto-push: true
          save-data-file: true

      - name: Update scaling efficiency data
        if: ${{ !cancelled() && hashFiles('scaling_efficiency_results/scaling-efficiency.json') != '' }}
        uses: benchmark-action/github-action-benchmark@a60cea5bc7b49e15c1f58f411161f99e0df48372 # v1.22.0
        with:
          tool: "customBiggerIsBetter"
          output-file-path: scaling_efficiency_results/scaling-efficiency.json
          gh-pages-branch: benchmarks
          max-items-in-chart: 100
          github-token: ${{ secrets.GITHUB_TOKEN }}
          benchmark-data-dir-path: "docs/benchmarks/nightly/scaling-efficiency"
          auto-push: true
          save-data-file: true

      - name: Add benchmark link to job summary
        run: |
          echo "### Benchmark Results" >> $GITHUB_STEP_SUMMARY
          echo "[View the benchmark results here](https://open-telemetry.github.io/otel-arrow/benchmarks/nightly/)" >> $GITHUB_STEP_SUMMARY