perf(qc): add benchmarks (#5714)

Closes:
https://linear.app/prisma-company/issue/TML-1732/add-query-compiler-benchmarking-and-profiling-infrastructure

Author: aqrln

.gitignore

@@ -65,3 +65,6 @@ package-lock.json
 *.pyc
 __pycache__
 .venv
+
+# Performance profiles written by `samply record`
+profile.json.gz

Cargo.lock

@@ -139,6 +139,34 @@ check-schema-wasm-package: build-schema-wasm
 	NODE=$(shell which node) \
 	./prisma-schema-wasm/scripts/check.sh
 
+######################
+# Benchmark commands #
+######################
+
+# Run query compiler benchmarks
+bench-qc:
+	cargo bench -p query-compiler --profile profiling
+
+# Run query graph building benchmarks
+bench-qc-graph:
+	cargo bench -p core-tests --profile profiling --bench query_graph_bench
+
+# Run schema building benchmarks
+bench-schema:
+	cargo bench -p schema --profile profiling --bench schema_builder_bench
+
+# Save benchmark baseline (usage: make bench-baseline NAME=main)
+bench-qc-baseline:
+	cargo bench -p query-compiler --profile profiling -- --save-baseline $(NAME)
+
+# Compare against baseline (usage: make bench-compare NAME=main)
+bench-qc-compare:
+	cargo bench -p query-compiler --profile profiling -- --baseline $(NAME)
+
+# Run profile_query example for profiling
+profile-qc:
+	cargo run -p query-compiler --example profile_query --profile profiling
+
 ###########################
 # Database setup commands #
 ###########################

Makefile

@@ -20,11 +20,21 @@ pretty = { workspace = true, features = ["termcolor"] }
 indexmap = { workspace = true, features = ["serde"] }
 
 [dev-dependencies]
+codspeed-criterion-compat.workspace = true
 insta = { workspace = true, features = ["glob"] }
-request-handlers.workspace = true
+request-handlers = { workspace = true, features = ["all"] }
+schema.workspace = true
 # pull all connectors for testing
 psl = { workspace = true, features = ["sqlite", "mysql", "postgresql", "mssql", "cockroachdb"] }
 quaint = { workspace = true, features = ["sqlite", "mysql", "postgresql", "mssql"] }
+sql-query-builder = { workspace = true, features = ["relation_joins"] }
+
+[[example]]
+name = "profile_query"
+
+[[bench]]
+name = "compilation_bench"
+harness = false
 
 [features]
 default = ["all"]

query-compiler/query-compiler/Cargo.toml

@@ -0,0 +1,114 @@
+//! End-to-end compilation benchmarks for the query compiler.
+//!
+//! This benchmark uses the same schema and query fixtures as the unit tests
+//! in `tests/data/` to measure full compilation pipeline performance.
+//!
+//! New benchmarks are automatically discovered when new query JSON files
+//! are added to the `tests/data/` directory.
+//!
+//! ## Running benchmarks
+//!
+//! ```bash
+//! cargo bench -p query-compiler
+//! ```
+//!
+//! ## Filtering benchmarks
+//!
+//! ```bash
+//! cargo bench -p query-compiler -- "create"
+//! cargo bench -p query-compiler -- "query-m2o"
+//! ```
+
+use codspeed_criterion_compat::{Criterion, black_box, criterion_group, criterion_main};
+use itertools::Itertools;
+use quaint::prelude::{ConnectionInfo, ExternalConnectionInfo, SqlFamily};
+use query_compiler::compile;
+use request_handlers::{JsonBody, JsonSingleQuery, RequestBody};
+use schema::QuerySchema;
+use std::sync::Arc;
+use std::{fs, path::PathBuf};
+
+fn get_test_data_dir() -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/data")
+}
+
+struct BenchContext {
+    query_schema: Arc<QuerySchema>,
+    connection_info: ConnectionInfo,
+}
+
+impl BenchContext {
+    fn new() -> Self {
+        let data_dir = get_test_data_dir();
+        let schema_path = data_dir.join("schema.prisma");
+        let schema_str = fs::read_to_string(&schema_path)
+            .unwrap_or_else(|e| panic!("Failed to read {}: {}", schema_path.display(), e));
+
+        let validated_schema = psl::parse_schema_without_extensions(&schema_str).unwrap();
+        let query_schema = Arc::new(schema::build(Arc::new(validated_schema), true));
+        let connection_info = ConnectionInfo::External(ExternalConnectionInfo::new(
+            SqlFamily::Postgres,
+            Some("public".to_string()),
+            None,
+            true,
+        ));
+        Self {
+            query_schema,
+            connection_info,
+        }
+    }
+
+    fn compile(&self, query_json: &str) {
+        let query: JsonSingleQuery = serde_json::from_str(query_json).unwrap();
+        let request = RequestBody::Json(JsonBody::Single(query));
+        let doc = request.into_doc(&self.query_schema).unwrap();
+
+        let query_core::QueryDocument::Single(operation) = doc else {
+            panic!("expected single query");
+        };
+
+        let mut expr = compile(&self.query_schema, operation, &self.connection_info).unwrap();
+        expr.simplify();
+        black_box(expr);
+    }
+}
+
+fn discover_query_files() -> impl IntoIterator<Item = (String, String)> {
+    let data_dir = get_test_data_dir();
+
+    fs::read_dir(&data_dir)
+        .expect("failed to read data directory")
+        .map(|entry| entry.expect("failed to read directory entry").path())
+        .filter(|path| path.extension().is_some_and(|ext| ext == "json"))
+        .sorted()
+        .map(|path| {
+            let name = path
+                .file_stem()
+                .expect("file should have a stem")
+                .to_string_lossy()
+                .into_owned();
+
+            let content =
+                fs::read_to_string(&path).unwrap_or_else(|e| panic!("failed to read {}: {}", path.display(), e));
+
+            (name, content)
+        })
+}
+
+fn compilation_benchmarks(c: &mut Criterion) {
+    let ctx = BenchContext::new();
+    let queries = discover_query_files();
+
+    let mut group = c.benchmark_group("compile");
+
+    for (name, query_json) in queries {
+        group.bench_function(name, |b| {
+            b.iter(|| ctx.compile(&query_json));
+        });
+    }
+
+    group.finish();
+}
+
+criterion_group!(benches, compilation_benchmarks);
+criterion_main!(benches);

query-compiler/query-compiler/benches/compilation_bench.rs

@@ -0,0 +1,191 @@
+# Query Compiler Benchmarking and Profiling Guide
+
+This document describes how to run benchmarks and profile the query compiler to identify performance bottlenecks.
+
+## Quick Start
+
+```bash
+# Run all query-compiler benchmarks
+cargo bench -p query-compiler
+
+# Or use the Makefile target
+make bench-qc
+
+# Run benchmarks matching a pattern
+cargo bench -p query-compiler -- "create"
+cargo bench -p query-compiler -- "query-m2o"
+```
+
+## How Benchmarks Work
+
+The compilation benchmarks automatically discover all `.json` query files in `tests/data/` and create a benchmark for each one. This means:
+
+- **No code changes needed** when adding new test queries
+- Benchmarks stay in sync with unit tests
+- Each benchmark measures full end-to-end compilation (JSON → Expression)
+
+The benchmarks use the same schema (`tests/data/schema.prisma`) as the unit tests.
+
+## Running Benchmarks
+
+### Basic Usage
+
+```bash
+# Run all benchmarks
+cargo bench -p query-compiler
+
+# Run benchmarks matching a pattern
+cargo bench -p query-compiler -- "compile/create"
+cargo bench -p query-compiler -- "compile/query-m2o"
+
+# Save baseline for comparison
+cargo bench -p query-compiler -- --save-baseline main
+
+# Compare against baseline
+cargo bench -p query-compiler -- --baseline main
+```
+
+### Makefile Targets
+
+```bash
+make bench-qc                    # Run query compiler benchmarks
+make bench-qc-graph              # Run query graph benchmarks
+make bench-schema                # Run schema building benchmarks
+make bench-baseline NAME=main    # Save baseline
+make bench-compare NAME=main     # Compare against baseline
+make profile-qc                  # Run profiling example
+```
+
+### Benchmark Options
+
+```bash
+# Run with more iterations for accuracy
+cargo bench -p query-compiler -- --sample-size 200
+
+# Run for a specific duration
+cargo bench -p query-compiler -- --measurement-time 30
+
+# Skip warmup (useful for debugging)
+cargo bench -p query-compiler -- --warm-up-time 0
+
+# List all available benchmarks
+cargo bench -p query-compiler -- --list
+```
+
+## Adding New Benchmarks
+
+Simply add a new `.json` query file to `tests/data/`. The benchmark will be automatically discovered on the next run.
+
+For example, adding `tests/data/my-new-query.json` will create a benchmark named `compile/my-new-query`.
+
+## Profiling
+
+### Using the Profiling Example
+
+The `profile_query` example is designed for use with profilers:
+
+```bash
+# Run with default settings (10,000 iterations)
+cargo run -p query-compiler --example profile_query --profile profiling
+
+# Customize via environment variables
+PROFILE_ITERATIONS=50000 PROFILE_QUERY=nested cargo run -p query-compiler --example profile_query --profile profiling
+```
+
+Environment variables:
+- `PROFILE_ITERATIONS`: Number of iterations (default: 10000)
+- `PROFILE_QUERY`: Which query to profile (`simple`, `nested`, `mutation`, or `all`)
+- `PROFILE_WARMUP`: Number of warmup iterations (default: 100)
+
+### Using samply (Recommended, Cross-platform)
+
+```bash
+cargo install samply
+
+# Profile the example
+samply record cargo run -p query-compiler --example profile_query --profiling
+
+# Opens Firefox Profiler with results
+```
+
+### Using Instruments.app (macOS)
+
+```bash
+cargo build -p query-compiler --example profile_query --profile profiling
+
+xcrun xctrace record --template 'Time Profiler' --launch -- \
+    ./target/profiling/examples/profile_query
+```
+
+### Using perf (Linux)
+
+```bash
+cargo build -p query-compiler --example profile_query --profile profiling
+
+perf record -g ./target/profiling/examples/profile_query
+perf report
+```
+
+### Using flamegraph
+
+```bash
+# macOS
+brew install flamegraph
+
+# Linux
+cargo install flamegraph
+
+# Generate flamegraph
+cargo flamegraph -p query-compiler --example profile_query
+```
+
+## Interpreting Results
+
+### Criterion Output
+
+```
+compile/query-m2o       time:   [143.02 µs 156.65 µs 173.14 µs]
+                        change: [-2.12% -0.57% +1.01%] (p = 0.42 > 0.05)
+                        No change in performance detected.
+```
+
+- **time**: [lower bound, estimate, upper bound] at 95% confidence
+- **change**: Comparison to previous run or baseline
+- **p-value**: Statistical significance (< 0.05 means significant change)
+
+### Performance Regression Detection
+
+- Changes > 5% warrant investigation
+- Changes > 10% are likely regressions
+- Always verify with multiple runs
+
+## Other Benchmark Suites
+
+- `query-compiler/core-tests/benches/query_graph_bench.rs` - Query graph building
+- `query-compiler/schema/benches/schema_builder_bench.rs` - Schema building
+
+## Troubleshooting
+
+### Benchmarks are too slow
+
+```bash
+# Use fewer samples for quick runs
+cargo bench -p query-compiler -- --sample-size 10
+
+# Filter to specific benchmarks
+cargo bench -p query-compiler -- "compile/query-m2o"
+```
+
+### High variance in results
+
+- Close other applications
+- Use `--warm-up-time 5` for longer warmup
+- Use `--sample-size 500` for more samples
+
+### Missing debug symbols in profiles
+
+Use the `profiling` profile defined in the workspace `Cargo.toml`:
+
+```bash
+cargo build -p query-compiler --profile profiling
+```