Merge pull request #822 from Pvsaint/advanced-resource-scheduling #69
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Performance & Benchmark Pipeline | ||
|
Check failure on line 1 in .github/workflows/performance.yml
|
||
| # Unified performance pipeline (#701). | ||
| # Replaces benchmark.yml, performance-regression.yml, and webhook-benchmark.yml | ||
| # with a single matrix-driven workflow using shared composite actions: | ||
| # - setup-rust / setup-perf-env for environment provisioning | ||
| # - compare-benchmarks for baseline comparison | ||
| on: | ||
| push: | ||
| branches: [main] | ||
| paths: | ||
| - "src/**" | ||
| - "benchmarks/**" | ||
| - ".github/workflows/performance.yml" | ||
| - ".github/actions/setup-perf-env/**" | ||
| - ".github/actions/compare-benchmarks/**" | ||
| pull_request: | ||
| branches: [main] | ||
| paths: | ||
| - "src/**" | ||
| - "benchmarks/**" | ||
| - ".github/workflows/performance.yml" | ||
| workflow_dispatch: | ||
| inputs: | ||
| suite: | ||
| description: "Benchmark suite to run (all, operator, regression, webhook)" | ||
| required: false | ||
| default: all | ||
| baseline_version: | ||
| description: Baseline version to compare | ||
| required: false | ||
| default: latest | ||
| regression_threshold: | ||
| description: Regression threshold % | ||
| required: false | ||
| default: "10" | ||
| env: | ||
| CARGO_TERM_COLOR: always | ||
| RUST_BACKTRACE: 1 | ||
| K6_VERSION: "0.54.0" | ||
| jobs: | ||
| # ── Resolve which suites to run ─────────────────────────────────────────── | ||
| resolve-matrix: | ||
| name: Resolve Benchmark Matrix | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| suites: ${{ steps.matrix.outputs.suites }} | ||
| steps: | ||
| - id: matrix | ||
| shell: bash | ||
| run: | | ||
| SUITE="${{ github.event.inputs.suite || 'all' }}" | ||
| case "$SUITE" in | ||
| operator) SUITES='["operator"]' ;; | ||
| regression) SUITES='["regression"]' ;; | ||
| webhook) SUITES='["webhook"]' ;; | ||
| *) SUITES='["operator","regression","webhook"]' ;; | ||
| esac | ||
| echo "suites=$SUITES" >> "$GITHUB_OUTPUT" | ||
| # ── Build once, share across all suites ───────────────────────────────────── | ||
| build: | ||
| name: Build Operator | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| version: ${{ steps.version.outputs.version }} | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| fetch-depth: 0 | ||
| - name: Get version | ||
| id: version | ||
| run: | | ||
| if [[ "${{ github.ref }}" == refs/tags/* ]]; then | ||
| echo "version=${GITHUB_REF#refs/tags/}" >> "$GITHUB_OUTPUT" | ||
| else | ||
| echo "version=sha-$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" | ||
| fi | ||
| name: Continuous Benchmarking | ||
| on: | ||
| push: | ||
| branches: [ main ] | ||
| pull_request: | ||
| branches: [ main ] | ||
| workflow_dispatch: | ||
| env: | ||
| CARGO_TERM_COLOR: always | ||
| CLUSTER_NAME: stellar-perf | ||
| OPERATOR_NAMESPACE: stellar-system | ||
| IMAGE_TAG: perf-test | ||
| jobs: | ||
| performance: | ||
| name: Operator Performance Benchmarks | ||
| runs-on: ubuntu-latest | ||
| timeout-minutes: 30 | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Setup Rust | ||
| uses: ./.github/actions/setup-rust | ||
| with: | ||
| cache-key: "perf-build" | ||
| - name: Build release | ||
| run: cargo build --release --locked | ||
| - name: Build Docker image | ||
| run: docker build -t "stellar-operator:${{ steps.version.outputs.version }}" . | ||
| - name: Save Docker image | ||
| run: | | ||
| docker save "stellar-operator:${{ steps.version.outputs.version }}" \ | ||
| | gzip > stellar-operator-image.tar.gz | ||
| - name: Upload artifacts | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: perf-artifacts-${{ steps.version.outputs.version }} | ||
| path: | | ||
| target/release/stellar-operator | ||
| stellar-operator-image.tar.gz | ||
| retention-days: 7 | ||
| # ── Matrix: operator / regression / webhook ───────────────────────────────── | ||
| benchmark: | ||
| name: "Benchmark (${{ matrix.suite }})" | ||
| runs-on: ubuntu-latest | ||
| needs: [build, resolve-matrix] | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| suite: ${{ fromJSON(needs.resolve-matrix.outputs.suites) }} | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Download artifacts | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| name: perf-artifacts-${{ needs.build.outputs.version }} | ||
| - name: Load Docker image | ||
| if: matrix.suite != 'webhook' | ||
| run: | | ||
| gunzip stellar-operator-image.tar.gz | ||
| docker load < stellar-operator-image.tar | ||
| # ── Operator & regression: kind cluster via setup-perf-env ────────────── | ||
| - name: Setup performance environment | ||
| if: matrix.suite != 'webhook' | ||
| continue-on-error: true | ||
| uses: ./.github/actions/setup-perf-env | ||
| with: | ||
| k6-version: ${{ env.K6_VERSION }} | ||
| cluster-name: perf-${{ matrix.suite }} | ||
| operator-image-tag: ${{ needs.build.outputs.version }} | ||
| worker-nodes: "2" | ||
| # ── Webhook: run server directly (no kind) ────────────────────────────── | ||
| - name: Install k6 (webhook suite) | ||
| if: matrix.suite == 'webhook' | ||
| run: | | ||
| K6_VER="${{ env.K6_VERSION }}" | ||
| curl -fsSL \ | ||
| "https://github.com/grafana/k6/releases/download/v${K6_VER}/k6-v${K6_VER}-linux-amd64.tar.gz" \ | ||
| | tar xvz | ||
| sudo mv "k6-v${K6_VER}-linux-amd64/k6" /usr/local/bin/ | ||
| sudo apt-get update && sudo apt-get install -y jq bc | ||
| - name: Start webhook server | ||
| if: matrix.suite == 'webhook' | ||
| run: | | ||
| chmod +x target/release/stellar-operator | ||
| ./target/release/stellar-operator webhook \ | ||
| --bind 0.0.0.0:8443 --log-level info & | ||
| echo "WEBHOOK_PID=$!" >> "$GITHUB_ENV" | ||
| for i in $(seq 1 30); do | ||
| curl -sf http://localhost:8443/health > /dev/null 2>&1 && break | ||
| sleep 2 | ||
| done | ||
| curl -sf http://localhost:8443/health \ | ||
| || { echo "ERROR: Webhook server failed to start"; exit 1; } | ||
| - name: Determine baseline | ||
| id: baseline | ||
| shell: bash | ||
| run: | | ||
| mkdir -p results | ||
| INPUT_BASELINE="${{ github.event.inputs.baseline_version || 'latest' }}" | ||
| case "${{ matrix.suite }}" in | ||
| webhook) | ||
| DEFAULT="webhook-v0.1.0" | ||
| [[ "$INPUT_BASELINE" != "latest" ]] && BASELINE="benchmarks/baselines/${INPUT_BASELINE}.json" \ | ||
| || BASELINE="benchmarks/baselines/webhook-v0.1.0.json" | ||
| CURRENT="results/webhook-benchmark.json" | ||
| ;; | ||
| *) | ||
| if [[ "$INPUT_BASELINE" == "latest" ]]; then | ||
| BASELINE=$(ls -t benchmarks/baselines/v*.json 2>/dev/null | head -1 || echo "benchmarks/baselines/v0.1.0.json") | ||
| else | ||
| BASELINE="benchmarks/baselines/${INPUT_BASELINE}.json" | ||
| fi | ||
| CURRENT="results/benchmark-summary.json" | ||
| ;; | ||
| esac | ||
| echo "baseline_file=$BASELINE" >> "$GITHUB_OUTPUT" | ||
| echo "current_file=$CURRENT" >> "$GITHUB_OUTPUT" | ||
| [[ -f "$BASELINE" ]] && echo "has_baseline=true" >> "$GITHUB_OUTPUT" \ | ||
| || echo "has_baseline=false" >> "$GITHUB_OUTPUT" | ||
| - name: Run operator benchmark (k6) | ||
| if: matrix.suite == 'operator' | ||
| continue-on-error: true | ||
| run: | | ||
| k6 run \ | ||
| --env BASE_URL=http://localhost:8080 \ | ||
| --env K8S_API_URL=http://localhost:8001 \ | ||
| --env NAMESPACE=stellar-benchmark \ | ||
| --env RUN_ID=${{ github.run_id }} \ | ||
| --env VERSION=${{ needs.build.outputs.version }} \ | ||
| --env GIT_SHA=${{ github.sha }} \ | ||
| --env BASELINE_FILE=${{ steps.baseline.outputs.baseline_file }} \ | ||
| --out json=results/k6-output.json \ | ||
| benchmarks/k6/operator-load-test.js | ||
| python3 -c " | ||
| import json, os | ||
| raw = 'results/k6-output.json' | ||
| data = [] | ||
| if os.path.exists(raw): | ||
| with open(raw) as f: | ||
| data = [json.loads(l) for l in f if l.strip()] | ||
| summary = {'total_requests': len([d for d in data if d.get('type') == 'Point']), | ||
| 'version': '${{ needs.build.outputs.version }}'} | ||
| with open('results/benchmark-summary.json', 'w') as f: | ||
| json.dump(summary, f, indent=2) | ||
| " | ||
| - name: Run regression benchmark (k6) | ||
| if: matrix.suite == 'regression' | ||
| continue-on-error: true | ||
| run: | | ||
| k6 run \ | ||
| --env BASE_URL=http://localhost:8080 \ | ||
| --env K8S_API_URL=http://localhost:8001 \ | ||
| --env NAMESPACE=stellar-benchmark \ | ||
| --env RUN_ID=${{ github.run_id }} \ | ||
| --env VERSION=${{ needs.build.outputs.version }} \ | ||
| --env GIT_SHA=${{ github.sha }} \ | ||
| --env BASELINE_FILE=${{ steps.baseline.outputs.baseline_file }} \ | ||
| --out json=results/operator-benchmark-raw.json \ | ||
| benchmarks/k6/operator-load-test.js | ||
| python3 -c " | ||
| import json, os | ||
| raw = 'results/operator-benchmark-raw.json' | ||
| data = [] | ||
| if os.path.exists(raw): | ||
| with open(raw) as f: | ||
| data = [json.loads(l) for l in f if l.strip()] | ||
| summary = {'total_requests': len([d for d in data if d.get('type') == 'Point']), | ||
| 'version': '${{ needs.build.outputs.version }}'} | ||
| with open('results/benchmark-summary.json', 'w') as f: | ||
| json.dump(summary, f, indent=2) | ||
| " | ||
| - name: Run webhook benchmark (k6) | ||
| if: matrix.suite == 'webhook' | ||
| continue-on-error: true | ||
| run: | | ||
| k6 run \ | ||
| --env WEBHOOK_URL=http://localhost:8443 \ | ||
| --env VERSION=${{ needs.build.outputs.version }} \ | ||
| --env GIT_SHA=${{ github.sha }} \ | ||
| --env RUN_ID=${{ github.run_id }} \ | ||
| --env BASELINE_FILE=${{ steps.baseline.outputs.baseline_file }} \ | ||
| --out json=results/webhook-benchmark-raw.json \ | ||
| benchmarks/k6/webhook-load-test.js | ||
| - name: Stop webhook server | ||
| if: always() && matrix.suite == 'webhook' | ||
| run: '[[ -n "${WEBHOOK_PID:-}" ]] && kill "$WEBHOOK_PID" || true' | ||
| - name: Compare with baseline | ||
| if: steps.baseline.outputs.has_baseline == 'true' && matrix.suite != 'webhook' | ||
| continue-on-error: true | ||
| uses: ./.github/actions/compare-benchmarks | ||
| with: | ||
| current-file: ${{ steps.baseline.outputs.current_file }} | ||
| baseline-file: ${{ steps.baseline.outputs.baseline_file }} | ||
| threshold: ${{ github.event.inputs.regression_threshold || '10' }} | ||
| output-file: results/regression-report-${{ matrix.suite }}.json | ||
| fail-on-regression: "false" | ||
| - name: Collect logs | ||
| if: always() && matrix.suite != 'webhook' | ||
| run: | | ||
| kubectl logs -n stellar-system -l app=stellar-operator \ | ||
| --tail=500 > results/operator-logs-${{ matrix.suite }}.txt || true | ||
| - name: Upload results | ||
| if: always() | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: perf-results-${{ matrix.suite }}-${{ needs.build.outputs.version }} | ||
| path: results/ | ||
| retention-days: 30 | ||
| # ── Aggregate report ──────────────────────────────────────────────────────── | ||
| report: | ||
| name: Performance Report | ||
| runs-on: ubuntu-latest | ||
| needs: [build, benchmark] | ||
| if: always() && needs.build.result == 'success' | ||
| permissions: | ||
| pull-requests: write | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Download all benchmark results | ||
| uses: actions/download-artifact@v4 | ||
| continue-on-error: true | ||
| with: | ||
| pattern: perf-results-* | ||
| merge-multiple: true | ||
| path: results/ | ||
| - name: Write job summary | ||
| if: always() | ||
| run: | | ||
| mkdir -p results | ||
| { | ||
| echo "## Performance & Benchmark Pipeline (#701)" | ||
| echo "" | ||
| echo "**Version:** ${{ needs.build.outputs.version }}" | ||
| echo "**Commit:** ${{ github.sha }}" | ||
| echo "" | ||
| for report in results/regression-report-*.json; do | ||
| [[ -f "$report" ]] || continue | ||
| SUITE=$(basename "$report" .json | sed 's/regression-report-//') | ||
| PASSED=$(jq -r '.overall_passed // "unknown"' "$report") | ||
| echo "### $SUITE: $PASSED" | ||
| done | ||
| } >> "$GITHUB_STEP_SUMMARY" | ||
| - name: Post PR comment | ||
| if: github.event_name == 'pull_request' | ||
| continue-on-error: true | ||
| uses: peter-evans/create-or-update-comment@v5 | ||
| with: | ||
| issue-number: ${{ github.event.pull_request.number }} | ||
| body: | | ||
| ## Performance & Benchmark Pipeline | ||
| **Version:** ${{ needs.build.outputs.version }} | ||
| **Commit:** ${{ github.sha }} | ||
| Unified matrix pipeline completed for operator, regression, and webhook suites. | ||
| See workflow artifacts for detailed results. | ||
| - name: Build operator binary | ||
| run: cargo build --release --locked --bin stellar-operator | ||
| - name: Setup kind cluster | ||
| uses: ./.github/actions/setup-kind-cluster | ||
| with: | ||
| cluster-name: ${{ env.CLUSTER_NAME }} | ||
| image-tag: ${{ env.IMAGE_TAG }} | ||
| operator-namespace: ${{ env.OPERATOR_NAMESPACE }} | ||
| worker-nodes: "1" | ||
| operator-cpu-request: "500m" | ||
| operator-memory-request: "512Mi" | ||
| - name: Install k6 | ||
| run: | | ||
| sudo gpg --no-default-keyring --keyring /usr/share/keyrings/k6-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69 | ||
| echo "deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list | ||
| sudo apt-get update | ||
| sudo apt-get install k6 | ||
| - name: Run Benchmarks | ||
| env: | ||
| OPERATOR_NAMESPACE: ${{ env.OPERATOR_NAMESPACE }} | ||
| RUN_ID: ${{ github.run_id }} | ||
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
| run: | | ||
| # Set up port forwarding to operator | ||
| kubectl port-forward -n ${{ env.OPERATOR_NAMESPACE }} deployment/stellar-operator 8080:8080 & | ||
| PF_PID=$! | ||
| # Set up proxy for K8s API | ||
| kubectl proxy --port=8001 & | ||
| PROXY_PID=$! | ||
| sleep 5 | ||
| mkdir -p results | ||
| # Run k6 benchmarks | ||
| k6 run \ | ||
| --env BASE_URL=http://localhost:8080 \ | ||
| --env K8S_API_URL=http://localhost:8001 \ | ||
| --env NAMESPACE=perf-test \ | ||
| --env RUN_ID=${{ github.run_id }} \ | ||
| --out json=results/benchmarks.json \ | ||
| benchmarks/k6/comprehensive-perf.js | ||
| kill $PF_PID $PROXY_PID | ||
| - name: Compare with Baseline | ||
| id: compare | ||
| run: | | ||
| python3 benchmarks/scripts/compare_benchmarks.py compare \ | ||
| --current results/benchmarks.json \ | ||
| --baseline benchmarks/baselines/v0.1.0.json \ | ||
| --threshold 15 \ | ||
| --output results/comparison.json \ | ||
| --fail-on-regression | ||
| continue-on-error: true | ||
| - name: Generate Performance Report | ||
| run: | | ||
| # Use a simple python script to generate an HTML report from the JSON | ||
| python3 benchmarks/scripts/generate_report.py \ | ||
| --results results/benchmarks.json \ | ||
| --comparison results/comparison.json \ | ||
| --output results/report.html | ||
| - name: Upload Performance Results | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: performance-results-${{ github.run_id }} | ||
| path: results/ | ||
| retention-days: 30 | ||
| - name: Comment PR with Performance Summary | ||
| if: github.event_name == 'pull_request' | ||
| uses: actions/github-script@v7 | ||
| with: | ||
| script: | | ||
| const fs = require('fs'); | ||
| const comparison = JSON.parse(fs.readFileSync('results/comparison.json', 'utf8')); | ||
| let body = `### 📊 Performance Benchmark Results\n\n`; | ||
| body += `${comparison.summary}\n\n`; | ||
| if (comparison.regressions.length > 0) { | ||
| body += `#### ⚠️ Regressions Detected\n\n`; | ||
| body += `| Metric | Current | Baseline | Change |\n`; | ||
| body += `| :--- | :--- | :--- | :--- |\n`; | ||
| comparison.regressions.forEach(r => { | ||
| body += `| ${r.metric} | ${r.current} | ${r.baseline} | ${r.change_pct}% |\n`; | ||
| }); | ||
| body += `\n`; | ||
| } | ||
| if (comparison.improvements.length > 0) { | ||
| body += `#### ✅ Improvements\n\n`; | ||
| body += `| Metric | Current | Baseline | Change |\n`; | ||
| body += `| :--- | :--- | :--- | :--- |\n`; | ||
| comparison.improvements.forEach(i => { | ||
| body += `| ${i.metric} | ${i.current} | ${i.baseline} | ${i.change_pct}% |\n`; | ||
| }); | ||
| body += `\n`; | ||
| } | ||
| github.rest.issues.createComment({ | ||
| issue_number: context.issue.number, | ||
| owner: context.repo.owner, | ||
| repo: context.repo.repo, | ||
| body: body | ||
| }); | ||
