test(server): add Compose capacity qualification scenarios

.github/workflows/ci-quality-gates.yml

@@ -120,6 +120,10 @@ jobs:
           fi
           bun run check:agents \
             || { TOOLING_OK=false; ISSUES_FOUND+=("Repository tooling checks failed. Run: bun run check:agents"); }
+          bun run format:load:check \
+            || { TOOLING_OK=false; ISSUES_FOUND+=("Load scenarios are not formatted. Run: bun run format:load"); }
+          bun run test:load:syntax \
+            || { TOOLING_OK=false; ISSUES_FOUND+=("k6 rejected a load scenario. Run: bun run test:load:syntax"); }
           bun run test:agents \
             || { TESTS_OK=false; ISSUES_FOUND+=("Agent runtime test(s) failed. Run: bun run test:agents"); }
           bun run check:contracts || { CONTRACTS_OK=false; ISSUES_FOUND+=("Artifact-source contract validation failed. Run: bun run check:contracts"); }

.github/workflows/cicd.yml

@@ -78,6 +78,7 @@ jobs:
               - 'bunfig.toml'
             application-server:
               - 'server/**'
+              - 'load-tests/**'
               # Repository tooling runs on the App Server or Database leg.
               - 'scripts/**'
               # test:agents and typecheck:agents cover the precompute tree.

.gitignore

@@ -127,6 +127,7 @@ lib64/
 # local .env backups (deploy scratch) — must never be committed
 local_settings.py
 *.log
+/load-results/
 *.manifest
 MANIFEST
 MISSION.md

docs/.markdownlint-cli2.jsonc

@@ -16,10 +16,9 @@
 {
 	"$schema": "https://raw.githubusercontent.com/DavidAnson/markdownlint-cli2/main/schema/markdownlint-cli2-config-schema.json",
 
-	// Lint EVERY markdown file in docs/, not just the three plugin-mounted guides.
 	// `decisions/` (33 ADRs) and `runbooks/` are mounted by no Docusaurus plugin, so the site
 	// build never compiles them and never link-checks them — the linter is their only gate.
-	"globs": ["**/*.md", "**/*.mdx"],
+	"globs": ["**/*.md", "**/*.mdx", "../load-tests/**/*.md"],
 	"ignores": ["node_modules", "build", ".docusaurus"],
 
 	"config": {

docs/admin/install.mdx

@@ -38,11 +38,17 @@ Each release is qualified on both matrix cells before publication.
 Only the listed native hosts and the unmodified self-hosted Compose topology are supported. All other
 hosts, runtimes, and topologies are outside release qualification.
 
-- **4 vCPUs, 8 GB RAM, 40 GB SSD** recommended. Absolute floor: 2 vCPUs / **8 GB RAM** — the two JVMs
+- **4 vCPUs, 8 GB RAM, 40 GB SSD** is the conservative recommended starting point. Absolute floor:
+  2 vCPUs / **8 GB RAM** — the two JVMs
   ship with container memory limits of 5 GB (`application-server`) and 2 GB (`webhook-server`), so a
   smaller host does not swap, it OOM-kills. On a host below that, lower both limits before the first
   start: set `APPLICATION_SERVER_MEM_LIMIT` and `WEBHOOK_SERVER_MEM_LIMIT` in `.env` (each JVM sizes
   its heap from its own limit, so lowering the limit lowers the heap with it).
+- These are deployment limits, not a claim that every workload fits. Before raising webhook volume,
+  mentor concurrency, or `SANDBOX_MAX_CONCURRENT`, reproduce the release's
+  [Compose capacity qualification](https://github.com/ls1intum/Hephaestus/tree/main/load-tests) on
+  your host shape. A release baseline is valid only when it includes the raw k6 and container-resource
+  evidence described there; absent that evidence, keep the conservative defaults above.
 - **AI practice review adds real memory**: each concurrent review sandbox may use up to
   4 GiB. The default caps it at 1 concurrent sandbox; raise
   `SANDBOX_MAX_CONCURRENT` only with RAM to match.

load-tests/README.md

@@ -0,0 +1,94 @@
+# Compose capacity qualification
+
+These [Grafana k6](https://grafana.com/docs/k6/latest/) scenarios qualify the supported single-host
+Compose topology. They spend LLM budget and create durable rows; run them only on an isolated host.
+
+Webhook traffic uses a [constant arrival rate](https://grafana.com/docs/k6/latest/using-k6/scenarios/concepts/arrival-rate-vu-allocation/).
+Pass/fail uses [k6 thresholds](https://grafana.com/docs/k6/latest/using-k6/thresholds/). Sizing requires
+both service objectives and saturation, following Google's SLO guidance
+([SRE Workbook](https://sre.google/workbook/implementing-slos/)).
+
+## Safety and prerequisites
+
+- Deploy the exact release from `docker/self-host/compose.yaml` on a dedicated Linux host. Run k6
+  elsewhere.
+- Populate a representative workspace before testing. `ARTIFACT_IDS` must contain distinct, reviewable
+  pull requests so each request performs work rather than hitting deduplication/cooldown.
+- Bind practice review and mentor to a deterministic OpenAI-compatible test provider. Record its
+  response latency separately: provider time is not host capacity.
+- Use a short-lived workspace-admin token. Never commit it or the webhook secret.
+- Stop background sync and other tenants, then reset PostgreSQL and JetStream to the same snapshot
+  before every candidate. Run three times; publish the median and retain every raw result.
+
+## Run
+
+Use the digest-pinned k6 image:
+
+```bash
+bun run test:load:syntax
+```
+
+```bash
+export BASE_URL=https://hephaestus-load.example.test
+export WEBHOOK_SECRET=...
+export K6_IMAGE=grafana/k6:1.2.3@sha256:4f82892217f3110cb233e2b2622bcc97fabc70f14bd241fbfbfe7305105c68aa
+mkdir -p load-results
+docker run --rm -i \
+  -e BASE_URL -e WEBHOOK_SECRET -e WEBHOOK_RATE=100 -e DURATION=2m \
+  -v "$PWD/load-tests:/tests:ro" -v "$PWD/load-results:/results" \
+  "$K6_IMAGE" \
+  run --summary-export=/results/webhook-summary.json /tests/webhook-burst.js
+```
+
+The mixed scenario runs long-lived mentor HTTP responses while distinct practice reviews execute. It
+requires every review request and job to complete:
+
+```bash
+export API_BASE_URL=https://hephaestus-load.example.test/api
+export AUTH_TOKEN=... WORKSPACE_SLUG=capacity-test
+export ARTIFACT_IDS=101,102,103,104,105
+docker run --rm -i \
+  -e API_BASE_URL -e AUTH_TOKEN -e WORKSPACE_SLUG -e ARTIFACT_IDS \
+  -e MENTOR_VUS=5 -e REVIEW_VUS=2 -e REVIEW_REQUESTS=5 -e DURATION=10m \
+  -v "$PWD/load-tests:/tests:ro" -v "$PWD/load-results:/results" \
+  "$K6_IMAGE" \
+  run --summary-export=/results/mixed-summary.json /tests/detection-mentor.js
+```
+
+The pinned k6 client buffers each `text/event-stream` response. This scenario measures concurrent
+full-response completion, not time to first event or per-event latency.
+
+In separate terminals collect container and host samples. Docker defines the container fields and
+their semantics in [`docker stats`](https://docs.docker.com/reference/cli/docker/container/stats/);
+in particular, its Linux CLI memory figure subtracts cache, so retain the raw output rather than
+copying only a peak percentage:
+
+```bash
+(cd docker/self-host && \
+  docker compose --env-file .env --env-file release-lock.env stats --format json \
+    application-server webhook-server postgres nats-server) \
+  > load-results/container-stats.jsonl
+```
+
+```bash
+vmstat 1 > load-results/host-vmstat.txt
+```
+
+Stop both collectors when k6 exits. Capture `df -h` before and after the run. Retain `docker compose
+ps`, `docker info`, the release lock, k6 summaries, provider latency, PostgreSQL size, JetStream state,
+and application metrics/logs. The increase in `webhook.publish{outcome="success"}` must equal accepted
+webhook iterations while the failure counter remains unchanged. Reject a run with a reconciliation
+mismatch, dropped iterations, provider throttling, a container restart/OOM, or swap activity.
+
+## Qualification matrix and sizing rule
+
+Run both scenarios at the **minimum** and **recommended** host shapes, then increase webhook rate and
+mentor/review concurrency independently until the first threshold or saturation failure. Do not add
+the maxima together: that invents a workload never tested.
+
+The documented size qualifies only when all three repeated runs meet thresholds, the busiest rolling
+60-second host-CPU window remains below 80%, host available memory stays above 15% in every sample,
+the data filesystem has at least 20% free before and after, no swap/OOM/restart occurs, and the review
+queue drains within five minutes after load stops. Publish the first failing step as the ceiling.
+
+Copy `baseline-template.md` for every release. Record every repetition and link its raw artifacts.

load-tests/baseline-template.md

@@ -0,0 +1,35 @@
+# Hephaestus capacity baseline template
+
+## Reproduction identity
+
+| Field | Value |
+| --- | --- |
+| Date / operator | |
+| Commit, release-lock path and digest | |
+| Release-lock signature verification result | |
+| Rendered Compose configuration digest | |
+| Host provider / machine / region | |
+| CPU model, vCPUs, RAM, storage | |
+| Linux, Docker, Compose, k6 versions | |
+| Application, webhook, Postgres, NATS image digests | |
+| Database snapshot / row counts / JetStream state | |
+| Test-provider model and latency distribution | |
+
+## Results
+
+| Host | Run | Scenario | Offered load | p50 / p95 / p99 | Error rate | CPU 60s max | Minimum available memory | Filesystem free | Drain time | Result |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| | 1 | Webhook burst | | | | | | | | |
+| | 2 | Webhook burst | | | | | | | | |
+| | 3 | Webhook burst | | | | | | | | |
+| | 1 | Detection + mentor | | | | | | | | |
+| | 2 | Detection + mentor | | | | | | | | |
+| | 3 | Detection + mentor | | | | | | | | |
+
+## Ceiling and conclusion
+
+- First failing step and failure mode:
+- Minimum qualified host:
+- Recommended qualified host and measured headroom:
+- Limits outside the qualified envelope:
+- Links to k6 JSON, container stats, metrics and logs:

load-tests/detection-mentor.js

@@ -0,0 +1,119 @@
+import { check, sleep } from "k6";
+import crypto from "k6/crypto";
+import exec from "k6/execution";
+import http from "k6/http";
+import { Rate, Trend } from "k6/metrics";
+
+import {
+	apiBaseUrl,
+	authHeaders,
+	integer,
+	jsonField,
+	required,
+	sharedThresholds,
+} from "./lib/config.js";
+
+const workspace = required("WORKSPACE_SLUG");
+const mentorVUs = integer("MENTOR_VUS", 5);
+const reviewVUs = integer("REVIEW_VUS", 2);
+const reviewRequests = integer("REVIEW_REQUESTS", reviewVUs);
+const reviewJobDuration = new Trend("review_job_duration", true);
+const reviewJobsCompleted = new Rate("review_jobs_completed");
+const terminalStatuses = new Set(["COMPLETED", "FAILED", "TIMED_OUT", "CANCELLED"]);
+const api = apiBaseUrl();
+const headers = authHeaders();
+
+export const options = {
+	discardResponseBodies: false,
+	scenarios: {
+		mentor_sessions: {
+			exec: "mentor",
+			executor: "constant-vus",
+			vus: mentorVUs,
+			duration: __ENV.DURATION ?? "10m",
+		},
+		practice_detection: {
+			exec: "detection",
+			executor: "shared-iterations",
+			vus: reviewVUs,
+			iterations: reviewRequests,
+			maxDuration: __ENV.REVIEW_MAX_DURATION ?? "20m",
+			startTime: __ENV.REVIEW_START_TIME ?? "15s",
+		},
+	},
+	thresholds: {
+		...sharedThresholds,
+		"checks{scenario:mentor_sessions}": ["rate>0.99"],
+		"checks{scenario:practice_detection}": ["rate==1"],
+		"http_req_duration{operation:mentor_turn}": ["p(95)<120000"],
+		"http_req_duration{operation:review_request}": ["p(95)<1000"],
+		review_job_duration: ["p(95)<900000"],
+		review_jobs_completed: ["rate==1"],
+	},
+};
+
+export function setup() {
+	const artifactIds = required("ARTIFACT_IDS")
+		.split(",")
+		.map((value) => Number(value.trim()));
+	if (artifactIds.some((value) => !Number.isSafeInteger(value) || value < 1))
+		throw new Error("ARTIFACT_IDS must be comma-separated positive integers");
+	if (reviewRequests > artifactIds.length)
+		throw new Error("ARTIFACT_IDS must contain at least REVIEW_REQUESTS distinct ids");
+	return { artifactIds };
+}
+
+export function mentor() {
+	const messageId = crypto.randomUUID();
+	const response = http.post(
+		`${api}/workspaces/${encodeURIComponent(workspace)}/mentor/chat`,
+		JSON.stringify({
+			message: {
+				id: messageId,
+				role: "user",
+				parts: [{ type: "text", text: "Summarize my current practice priorities." }],
+			},
+		}),
+		{ headers, tags: { operation: "mentor_turn" }, timeout: "10m" },
+	);
+	check(response, {
+		"mentor stream completed": (result) => result.status === 200,
+		"mentor emitted completion": (result) => result.body?.includes("[DONE]") === true,
+	});
+}
+
+export function detection(data) {
+	const artifactId = data.artifactIds[exec.scenario.iterationInTest];
+	const startedAt = Date.now();
+	const response = http.post(
+		`${api}/workspaces/${encodeURIComponent(workspace)}/practices/review-requests`,
+		JSON.stringify({ artifactKind: "scm.pull-request", artifactId }),
+		{ headers, tags: { operation: "review_request" } },
+	);
+	const submitted = response.status === 200 && jsonField(response, "status") === "SUBMITTED";
+	const jobId = submitted ? jsonField(response, "jobId") : null;
+	if (
+		!check(response, { "review request submitted": () => submitted && typeof jobId === "string" })
+	) {
+		reviewJobsCompleted.add(false);
+		return;
+	}
+
+	const deadline = startedAt + integer("REVIEW_TIMEOUT_SECONDS", 1200) * 1000;
+	while (Date.now() < deadline) {
+		sleep(2);
+		const job = http.get(
+			`${api}/workspaces/${encodeURIComponent(workspace)}/agents/jobs/${jobId}`,
+			{ headers, tags: { operation: "review_status" } },
+		);
+		if (job.status !== 200) continue;
+		const status = jsonField(job, "status");
+		if (terminalStatuses.has(status)) {
+			reviewJobDuration.add(Date.now() - startedAt);
+			reviewJobsCompleted.add(status === "COMPLETED");
+			check(job, { "review job completed": () => status === "COMPLETED" });
+			return;
+		}
+	}
+	reviewJobsCompleted.add(false);
+}

load-tests/lib/config.js

@@ -0,0 +1,40 @@
+import exec from "k6/execution";
+
+export function required(name) {
+	const value = __ENV[name];
+	if (!value) exec.test.abort(`${name} is required`);
+	return value;
+}
+
+export function integer(name, fallback) {
+	const raw = __ENV[name] ?? String(fallback);
+	if (!/^\d+$/.test(raw) || Number(raw) < 1) exec.test.abort(`${name} must be a positive integer`);
+	return Number(raw);
+}
+
+export function baseUrl() {
+	return required("BASE_URL").replace(/\/$/, "");
+}
+
+export function apiBaseUrl() {
+	return required("API_BASE_URL").replace(/\/$/, "");
+}
+
+export function authHeaders() {
+	return {
+		Authorization: `Bearer ${required("AUTH_TOKEN")}`,
+		"Content-Type": "application/json",
+	};
+}
+
+export function jsonField(response, field) {
+	try {
+		return response.json(field);
+	} catch {
+		return undefined;
+	}
+}
+
+export const sharedThresholds = {
+	http_req_failed: ["rate<0.01"],
+};