Skip to content

Commit a71e4d2

Browse files
anchapinanchapin
authored
Merge pull request #558 from anchapin/feature/issue-539
[#539] Add performance benchmark CI
2 parents 8f5a4e7 + 80c575a commit a71e4d2

6 files changed

Lines changed: 724 additions & 0 deletions

File tree

‎.github/workflows/performance-benchmarks.yml‎

Lines changed: 332 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,332 @@
1+
name: Performance Benchmarks
2+
3+
on:
4+
push:
5+
branches: [main, develop]
6+
pull_request:
7+
types: [opened, synchronize, reopened]
8+
schedule:
9+
# Run nightly at 2 AM UTC
10+
- cron: '0 2 * * *'
11+
workflow_dispatch:
12+
inputs:
13+
run_baseline:
14+
description: 'Run as baseline (store as reference)'
15+
required: false
16+
default: false
17+
type: boolean
18+
19+
# Cancel in-progress runs for the same branch
20+
concurrency:
21+
group: ${{ github.workflow }}-${{ github.ref }}
22+
cancel-in-progress: true
23+
24+
jobs:
25+
# Rust benchmarks using Criterion
26+
rust-benchmarks:
27+
name: Rust Benchmarks (Criterion)
28+
runs-on: ubuntu-latest
29+
30+
steps:
31+
- name: Checkout code
32+
uses: actions/checkout@v6
33+
with:
34+
fetch-depth: 0 # Full history for comparison
35+
36+
- name: Install Rust
37+
uses: dtolnay/rust-toolchain@4be9e76fd7c4901c61fb841f559994984270fce7
38+
with:
39+
toolchain: stable
40+
41+
- name: Install Criterion comparison tool
42+
run: cargo install cargo-criterion
43+
44+
- name: Cache cargo registry
45+
uses: actions/cache@v5
46+
with:
47+
path: ~/.cargo/registry
48+
key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
49+
50+
- name: Cache cargo index
51+
uses: actions/cache@v5
52+
with:
53+
path: ~/.cargo/git
54+
key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}
55+
56+
- name: Cache cargo build
57+
uses: actions/cache@v5
58+
with:
59+
path: orchestrator/target
60+
key: ${{ runner.os }}-cargo-bench-build-${{ hashFiles('**/Cargo.lock') }}
61+
62+
- name: Run Rust benchmarks
63+
working-directory: orchestrator
64+
run: |
65+
echo "Running Rust benchmarks with Criterion..."
66+
cargo bench --bench performance_baseline
67+
cargo bench --bench mcp_startup
68+
cargo bench --bench snapshot_pool
69+
cargo bench --bench concurrent_agents
70+
71+
- name: Compare with baseline (if not baseline run)
72+
if: github.event.inputs.run_baseline != 'true'
73+
working-directory: orchestrator
74+
run: |
75+
echo "Comparing with baseline..."
76+
# Download baseline if available from artifacts or compare with main
77+
git fetch origin main
78+
# Run criterion to generate comparison report
79+
cargo criterion --message-format short || true
80+
81+
- name: Upload benchmark results
82+
if: always()
83+
uses: actions/upload-artifact@v4
84+
with:
85+
name: rust-benchmark-results-${{ github.sha }}
86+
path: |
87+
orchestrator/target/criterion/
88+
retention-days: 30
89+
90+
- name: Generate benchmark report
91+
if: always()
92+
run: |
93+
echo "## Rust Benchmark Results" > $GITHUB_STEP_SUMMARY
94+
echo "" >> $GITHUB_STEP_SUMMARY
95+
echo "Benchmark results are saved to \`target/criterion/\`" >> $GITHUB_STEP_SUMMARY
96+
echo "" >> $GITHUB_STEP_SUMMARY
97+
echo "### Key Metrics" >> $GITHUB_STEP_SUMMARY
98+
echo "" >> $GITHUB_STEP_SUMMARY
99+
echo "| Metric | Target | Current | Status |" >> $GITHUB_STEP_SUMMARY
100+
echo "|--------|--------|---------|--------|" >> $GITHUB_STEP_SUMMARY
101+
# Extract results from Criterion JSON output if available
102+
find orchestrator/target/criterion -name "new.json" -exec sh -c '
103+
echo "Found benchmark results"
104+
' \; || echo "No detailed results found"
105+
106+
- name: Check for performance regressions (>10%)
107+
if: github.event_name == 'pull_request'
108+
working-directory: orchestrator
109+
run: |
110+
echo "Checking for performance regressions..."
111+
# Use criterion to check for regressions
112+
cargo criterion --fail-threshold 10 || {
113+
echo "::warning::Performance regression detected (>10% degradation)"
114+
echo "Please review benchmark results in the artifacts"
115+
exit 1
116+
}
117+
118+
# Python benchmarks using pytest-benchmark
119+
python-benchmarks:
120+
name: Python Benchmarks (pytest-benchmark)
121+
runs-on: ubuntu-latest
122+
123+
steps:
124+
- name: Checkout code
125+
uses: actions/checkout@v6
126+
with:
127+
fetch-depth: 0
128+
129+
- name: Set up Python
130+
uses: actions/setup-python@v6
131+
with:
132+
python-version: '3.11'
133+
134+
- name: Cache pip packages
135+
uses: actions/cache@v5
136+
with:
137+
path: agent/.venv
138+
key: ${{ runner.os }}-pip-bench-${{ hashFiles('agent/pyproject.toml') }}
139+
140+
- name: Install dependencies
141+
working-directory: agent
142+
run: |
143+
python -m venv .venv
144+
.venv/bin/pip install --upgrade pip
145+
.venv/bin/pip install -e ".[dev]"
146+
147+
- name: Run Python benchmarks
148+
working-directory: agent
149+
run: |
150+
echo "Running Python benchmarks with pytest-benchmark..."
151+
.venv/bin/pytest tests/benchmarks/ \
152+
--benchmark-only \
153+
--benchmark-autosave \
154+
--benchmark-save-data \
155+
--benchmark-json benchmark-results.json \
156+
--benchmark-columns=mean,stddev,ops,rounds
157+
158+
- name: Compare with baseline
159+
if: github.event.inputs.run_baseline != 'true'
160+
working-directory: agent
161+
run: |
162+
echo "Comparing Python benchmarks with baseline..."
163+
.venv/bin/pytest tests/benchmarks/ \
164+
--benchmark-only \
165+
--benchmark-compare \
166+
--benchmark-json benchmark-comparison.json || true
167+
168+
- name: Upload benchmark results
169+
if: always()
170+
uses: actions/upload-artifact@v4
171+
with:
172+
name: python-benchmark-results-${{ github.sha }}
173+
path: |
174+
agent/.benchmarks/
175+
agent/benchmark-results.json
176+
agent/benchmark-comparison.json
177+
retention-days: 30
178+
179+
- name: Generate benchmark report
180+
if: always()
181+
run: |
182+
echo "## Python Benchmark Results" > $GITHUB_STEP_SUMMARY
183+
echo "" >> $GITHUB_STEP_SUMMARY
184+
echo "Benchmark results saved to artifacts" >> $GITHUB_STEP_SUMMARY
185+
186+
- name: Check for Python performance regressions (>10%)
187+
if: github.event_name == 'pull_request'
188+
working-directory: agent
189+
run: |
190+
echo "Checking Python benchmarks for regressions..."
191+
if [ -f benchmark-comparison.json ]; then
192+
.venv/bin/python -c "
193+
import json
194+
with open('benchmark-comparison.json') as f:
195+
data = json.load(f)
196+
for bench in data.get('benchmarks', []):
197+
name = bench.get('name', 'unknown')
198+
current = bench.get('stats', {}).get('mean', 0)
199+
baseline = bench.get('baseline', {}).get('mean', 0)
200+
if baseline > 0:
201+
change = ((current - baseline) / baseline) * 100
202+
if change > 10:
203+
print(f'::warning::Regression in {name}: {change:.1f}% increase')
204+
exit(1)
205+
" || {
206+
echo "::warning::Python benchmark regression detected (>10% degradation)"
207+
exit 1
208+
}
209+
fi
210+
211+
# Store baseline results (for historical tracking)
212+
store-baseline:
213+
name: Store Baseline Results
214+
runs-on: ubuntu-latest
215+
needs: [rust-benchmarks, python-benchmarks]
216+
if: github.event.inputs.run_baseline == 'true' || github.event_name == 'push' && github.ref == 'refs/heads/main'
217+
218+
steps:
219+
- name: Download Rust benchmark results
220+
uses: actions/download-artifact@v4
221+
with:
222+
name: rust-benchmark-results-${{ github.sha }}
223+
path: .baseline/rust
224+
225+
- name: Download Python benchmark results
226+
uses: actions/download-artifact@v4
227+
with:
228+
name: python-benchmark-results-${{ github.sha }}
229+
path: .baseline/python
230+
231+
- name: Store baseline as GitHub Release asset
232+
uses: softprops/action-gh-release@v2
233+
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
234+
with:
235+
tag_name: baseline-${{ github.sha }}
236+
name: Performance Baseline - ${{ github.sha }}
237+
files: |
238+
.baseline/rust/**/*
239+
.baseline/python/**/*
240+
draft: false
241+
prerelease: true
242+
generate_release_notes: true
243+
244+
- name: Generate summary
245+
run: |
246+
echo "## Baseline Results Stored" > $GITHUB_STEP_SUMMARY
247+
echo "" >> $GITHUB_STEP_SUMMARY
248+
echo "Baseline commit: \`${{ github.sha }}\`" >> $GITHUB_STEP_SUMMARY
249+
echo "" >> $GITHUB_STEP_SUMMARY
250+
echo "Results stored for future comparison" >> $GITHUB_STEP_SUMMARY
251+
252+
# Performance dashboard (optional - generates HTML report)
253+
performance-dashboard:
254+
name: Generate Performance Dashboard
255+
runs-on: ubuntu-latest
256+
needs: [rust-benchmarks, python-benchmarks]
257+
if: always()
258+
259+
steps:
260+
- name: Checkout code
261+
uses: actions/checkout@v6
262+
263+
- name: Download all benchmark results
264+
uses: actions/download-artifact@v4
265+
with:
266+
path: all-results
267+
268+
- name: Generate dashboard HTML
269+
run: |
270+
cat > performance-dashboard.html << 'EOF'
271+
<!DOCTYPE html>
272+
<html>
273+
<head>
274+
<title>LuminaGuard Performance Dashboard</title>
275+
<style>
276+
body { font-family: Arial, sans-serif; margin: 20px; }
277+
.metric { margin: 20px 0; padding: 15px; background: #f5f5f5; border-radius: 5px; }
278+
.target { color: green; }
279+
.warning { color: orange; }
280+
.error { color: red; }
281+
table { border-collapse: collapse; width: 100%; }
282+
th, td { border: 1px solid #ddd; padding: 8px; text-align: left; }
283+
th { background-color: #4CAF50; color: white; }
284+
</style>
285+
</head>
286+
<body>
287+
<h1>LuminaGuard Performance Dashboard</h1>
288+
<p><strong>Commit:</strong> ${{ github.sha }}</p>
289+
<p><strong>Branch:</strong> ${{ github.ref_name }}</p>
290+
<p><strong>Date:</strong> $(date -u)</p>
291+
292+
<h2>Performance Targets</h2>
293+
<div class="metric">
294+
<h3>Startup Time</h3>
295+
<p>Target: <span class="target">&lt; 500ms</span></p>
296+
</div>
297+
<div class="metric">
298+
<h3>VM Spawn Time</h3>
299+
<p>Target: <span class="target">&lt; 200ms</span></p>
300+
</div>
301+
<div class="metric">
302+
<h3>Memory Footprint</h3>
303+
<p>Target: <span class="target">&lt; 200MB baseline</span></p>
304+
</div>
305+
<div class="metric">
306+
<h3>Tool Call Latency</h3>
307+
<p>Target: <span class="target">&lt; 100ms</span></p>
308+
</div>
309+
310+
<h2>Benchmark Results</h2>
311+
<p>Detailed results available in workflow artifacts</p>
312+
</body>
313+
</html>
314+
EOF
315+
316+
- name: Upload dashboard
317+
uses: actions/upload-artifact@v4
318+
with:
319+
name: performance-dashboard
320+
path: performance-dashboard.html
321+
retention-days: 30
322+
323+
- name: Deploy to GitHub Pages (if enabled)
324+
if: github.ref == 'refs/heads/main'
325+
uses: peaceiris/actions-gh-pages@v4
326+
with:
327+
github_token: ${{ secrets.GITHUB_TOKEN }}
328+
publish_dir: .
329+
publish_branch: gh-pages
330+
keep_files: true
331+
destination_dir: performance
332+
continue-on-error: true

‎agent/pyproject.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ dev = [
2929
"pytest-cov>=4.0",
3030
"pytest-asyncio>=0.21",
3131
"pytest-timeout>=2.1",
32+
"pytest-benchmark>=4.0",
3233
"hypothesis>=6.91",
3334
"black>=23.12",
3435
"mypy>=1.8",

‎agent/tests/benchmarks/__init__.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
"""
2+
Performance benchmarks for LuminaGuard Agent.
3+
4+
This module contains benchmarks measuring key performance metrics:
5+
- Startup time
6+
- Tool call latency
7+
- Memory footprint
8+
- Message processing latency
9+
10+
Usage:
11+
pytest tests/benchmarks/ --benchmark-only
12+
13+
Performance Targets (from CLAUDE.md):
14+
- Startup time: <500ms
15+
- Tool call latency: <100ms
16+
- Memory footprint: <200MB baseline
17+
"""

0 commit comments

Comments
 (0)