DRAFT: reuse pre-computed data for 3343

.github/workflows/test-rust.yml

@@ -18,16 +18,8 @@ jobs:
           - os: ubuntu-24.04
             test-args: --features full,unstable --workspace
             cargo-profile: ci-tests
-          - os: ubuntu-24.04
-            test-args: --features future_snark -p mithril-stm -p mithril-common -p mithril-aggregator -p mithril-signer -p mithril-relay
-            artifact-suffix: -future-snark
-            # Default to the fast profile; the `prepare` step upgrades to `slow-cargo-profile`
-            # only when the slow SNARK tests are actually in scope for this run (so we don't pay
-            # the opt-level 3 build cost on PRs that filter those tests out).
-            cargo-profile: ci-tests
-            # Optimizes (opt-level 3) the crypto deps and `mithril-stm` so the slow SNARK tests
-            # run faster, at the cost of a longer (cache-amortized) build.
-            slow-cargo-profile: ci-tests-snark
+          # NOTE: the `future_snark` (SNARK / Halo2) leg lives in its own `test-future-snark` job
+          # below, because it is by far the heaviest and is sharded across runners.
           - os: ubuntu-24.04-arm
             test-args: --features full,unstable --workspace
             cargo-profile: ci-tests
@@ -129,11 +121,142 @@ jobs:
           name: test-results-${{ runner.os }}-${{ runner.arch }}${{ matrix.artifact-suffix }}
           path: ./test-results-*.xml
 
+  # Decide, once, whether the heavy SNARK slow tests are in scope and how many shards to fan out to:
+  # a single shard for the common fast path (so non-SNARK PRs aren't penalized with extra builds),
+  # and several shards when the slow recursive-proof tests actually run.
+  future-snark-gate:
+    runs-on: ubuntu-24.04
+    outputs:
+      cargo-profile: ${{ steps.prepare.outputs.cargo-profile }}
+      filterset: ${{ steps.prepare.outputs.filterset }}
+      shards: ${{ steps.prepare.outputs.shards }}
+      shard-count: ${{ steps.prepare.outputs.shard-count }}
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 2 # Needed to get the diff of the last commit, which is used to filter out slow tests
+
+      - id: prepare
+        name: Prepare
+        shell: bash
+        run: |
+          if [[ "${{ inputs.include-slow-tests }}" == "true" ]]; then
+            FILTERSET="$(.github/workflows/scripts/filter-slow-tests.sh --all)"
+            SLOW_TESTS=true
+          else
+            FILTERSET="$(.github/workflows/scripts/filter-slow-tests.sh --commit-ref "HEAD^1")"
+            # The filter script only appends a `package(mithril-stm)` slow-test allowance when the
+            # changed files touch a slow SNARK test path, so its presence tells us whether the slow
+            # tests are in scope for this run.
+            if [[ "$FILTERSET" == *"package(mithril-stm)"* ]]; then
+              SLOW_TESTS=true
+            else
+              SLOW_TESTS=false
+            fi
+          fi
+
+          # Only the slow path is worth sharding (and worth the opt-level 3 build). The fast path
+          # runs as a single shard so non-SNARK PRs don't pay extra builds.
+          if [[ "$SLOW_TESTS" == "true" ]]; then
+            CARGO_PROFILE="ci-tests-snark"
+            SHARDS="[1, 2, 3, 4]"
+            SHARD_COUNT=4
+          else
+            CARGO_PROFILE="ci-tests"
+            SHARDS="[1]"
+            SHARD_COUNT=1
+          fi
+
+          echo "Tests filterset: $FILTERSET"
+          echo "Cargo profile: $CARGO_PROFILE / shards: $SHARDS"
+          echo "filterset=$FILTERSET" >> $GITHUB_OUTPUT
+          echo "cargo-profile=$CARGO_PROFILE" >> $GITHUB_OUTPUT
+          echo "shards=$SHARDS" >> $GITHUB_OUTPUT
+          echo "shard-count=$SHARD_COUNT" >> $GITHUB_OUTPUT
+
+  # The `future_snark` (SNARK / Halo2) tests are the heaviest in the workspace. Run them on their own
+  # runner(s), sharded with `nextest --partition` so the slow recursive-proof tests are spread across
+  # machines instead of serialized on one. Each shard builds independently (simple and robust; the
+  # extra build compute on heavy runs could later be removed with `cargo nextest archive`). Doc tests
+  # and the examples build run in the separate `future-snark-checks` job, off these shards' critical path.
+  test-future-snark:
+    needs: future-snark-gate
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: ${{ fromJSON(needs.future-snark-gate.outputs.shards) }}
+    runs-on: ubuntu-24.04
+    env:
+      SNARK_TEST_ARGS: --features future_snark -p mithril-stm -p mithril-common -p mithril-aggregator -p mithril-signer -p mithril-relay
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 2
+
+      - name: Install stable toolchain, tools, and restore cache
+        uses: ./.github/workflows/actions/toolchain-and-cache
+        with:
+          cache-version: ${{ vars.CACHE_VERSION }}
+          cargo-tools: cargo-nextest
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          never-save-cache: ${{ inputs.disable-cache-save }}
+
+      - name: Build tests
+        run: cargo nextest run --cargo-profile ${{ needs.future-snark-gate.outputs.cargo-profile }} --no-run $SNARK_TEST_ARGS
+
+      - name: Run tests (shard ${{ matrix.shard }}/${{ needs.future-snark-gate.outputs.shard-count }})
+        run: |
+          cargo nextest run --profile ci \
+            --cargo-profile ${{ needs.future-snark-gate.outputs.cargo-profile }} \
+            --partition hash:${{ matrix.shard }}/${{ needs.future-snark-gate.outputs.shard-count }} \
+            --filterset "${{ needs.future-snark-gate.outputs.filterset }}" \
+            $SNARK_TEST_ARGS
+
+      - name: Rename junit file to include runner info
+        shell: bash
+        if: success() || failure()
+        run: |
+          mv target/nextest/ci/tests-result.junit.xml test-results-${{ runner.os }}-${{ runner.arch }}-future-snark-shard${{ matrix.shard }}.xml
+
+      - name: Upload Tests Results
+        uses: actions/upload-artifact@v7
+        if: success() || failure()
+        with:
+          name: test-results-${{ runner.os }}-${{ runner.arch }}-future-snark-shard${{ matrix.shard }}
+          path: ./test-results-*.xml
+
+  # Doc tests and the examples build don't do heavy proving, so they don't need the opt-level 3
+  # profile. Run them once here in parallel with the shards, on the fast `ci-tests` profile, so they
+  # never sit on a test shard's critical path.
+  future-snark-checks:
+    runs-on: ubuntu-24.04
+    env:
+      SNARK_TEST_ARGS: --features future_snark -p mithril-stm -p mithril-common -p mithril-aggregator -p mithril-signer -p mithril-relay
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v6
+
+      - name: Install stable toolchain, tools, and restore cache
+        uses: ./.github/workflows/actions/toolchain-and-cache
+        with:
+          cache-version: ${{ vars.CACHE_VERSION }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          never-save-cache: ${{ inputs.disable-cache-save }}
+
+      - name: Run doc tests
+        run: cargo test --profile ci-tests --doc $SNARK_TEST_ARGS
+
+      - name: Ensure examples build
+        run: cargo build --profile ci-tests --examples $SNARK_TEST_ARGS
+
   send-tests-results:
     if: success() || failure()
     runs-on: ubuntu-24.04
     needs:
       - test
+      - test-future-snark
     steps:
       - name: Download Tests Results
         if: success() || failure()

mithril-stm/Cargo.toml

@@ -87,6 +87,7 @@ rug = { version = "1.30.0", optional = true }
 [dev-dependencies]
 blake2b_simd = "1.0.4"
 criterion = { version = "0.8.2", features = ["html_reports"] }
+fs4 = { version = "0.13", features = ["sync"] }
 httpmock = "0.8.3"
 num-bigint = "0.4.6"
 num-rational = "0.4.2"

mithril-stm/src/circuits/halo2/tests/golden/helpers.rs

@@ -748,6 +748,8 @@ pub(crate) fn compute_unsafe_circuit_verification_key(
     const RNG_SEED: u64 = 42;
     let circuit = StmCertificateCircuit::try_new(params, merkle_tree_depth).unwrap();
     let circuit_degree = MidnightCircuit::from_relation(&circuit).min_k();
+    // Generated locally at this circuit's (small) degree: cheaper than loading/downsizing the
+    // shared k=19 file, and avoids cold-start contention on its one-time generation lock.
     let srs: ParamsKZG<Bls12> =
         ParamsKZG::unsafe_setup(circuit_degree, ChaCha20Rng::seed_from_u64(RNG_SEED));
 

mithril-stm/src/circuits/halo2/tests/test_helpers.rs

@@ -39,6 +39,8 @@ where
     R: Relation,
 {
     let circuit = MidnightCircuit::from_relation(relation);
+    // These gadget relations need only a small SRS (k ~ 10); generate it locally rather than
+    // loading/downsizing the shared k=19 file, which would add I/O and cold-start lock contention.
     let srs = ParamsKZG::<Bls12>::unsafe_setup(circuit.min_k(), ChaCha20Rng::seed_from_u64(42));
     let vk = zk::setup_vk(&srs, relation);
     let pk = zk::setup_pk(relation, &vk);

mithril-stm/src/circuits/halo2_ivc/tests/common/generators/setup.rs

@@ -167,8 +167,12 @@ fn merkle_tree_commitment_from_stm_tree(merkle_tree: &SignerRegistrationMerkleTr
 }
 
 /// Builds the shared universal KZG parameters that both circuits derive from.
+///
+/// Backed by the process-shared on-disk SRS cache (see
+/// [`crate::circuits::trusted_setup::shared_unsafe_srs`], seed 42 == `ASSET_SEED`), so the
+/// expensive degree-19 SRS is generated once per CI run instead of once per test process.
 pub(crate) fn build_deterministic_params(circuit_degree: u32) -> ParamsKZG<Bls12> {
-    ParamsKZG::<Bls12>::unsafe_setup(circuit_degree, ChaCha20Rng::seed_from_u64(ASSET_SEED))
+    crate::circuits::trusted_setup::shared_unsafe_srs(circuit_degree)
 }
 
 /// Derives circuit-specific commitment parameters from a shared universal SRS.

mithril-stm/src/circuits/trusted_setup.rs

@@ -8,6 +8,9 @@ use std::{
 use anyhow::Context;
 use midnight_curves::Bls12;
 use midnight_proofs::{poly::kzg::params::ParamsKZG, utils::SerdeFormat};
+// `max_k()` (used by the shared test-SRS helper) lives on the `Params` trait.
+#[cfg(test)]
+use midnight_proofs::poly::commitment::Params;
 use sha2::{Digest, Sha256};
 
 use crate::{StmResult, circuits::MITHRIL_CIRCUIT_CACHE_FOLDER};
@@ -177,19 +180,132 @@ impl Default for TrustedSetupProvider {
     }
 }
 
-/// Builds a `TrustedSetupProvider` backed by a freshly generated unsafe SRS of
-/// degree `k`, written to `base_dir/srs/srs-parameters` with a matching SHA256
-/// hash so the provider's hash check passes. For tests only.
+/// Maximum SRS degree required by any test circuit. A single unsafe SRS generated at this degree
+/// downsizes to every smaller circuit, so all tests can share one cached file.
+///
+/// Derived from the circuit-degree constants so it tracks them automatically: if a circuit's
+/// degree changes, this updates with no separate pin to maintain. (`Ord::max` is not const-stable,
+/// so the maximum is taken with a `const`-safe comparison.)
+#[cfg(test)]
+pub(crate) const MAX_TEST_SRS_DEGREE: u32 = {
+    use crate::circuits::halo2_ivc::tests::common::{
+        CERTIFICATE_CIRCUIT_DEGREE, RECURSIVE_CIRCUIT_DEGREE,
+    };
+    if RECURSIVE_CIRCUIT_DEGREE > CERTIFICATE_CIRCUIT_DEGREE {
+        RECURSIVE_CIRCUIT_DEGREE
+    } else {
+        CERTIFICATE_CIRCUIT_DEGREE
+    }
+};
+
+/// Folder under `temp_dir()` holding the shared unsafe test SRS.
+#[cfg(test)]
+const SHARED_TEST_SRS_FOLDER: &str = "mithril-stm-test-srs";
+
+/// Loads the shared seed-42 unsafe SRS at [`MAX_TEST_SRS_DEGREE`], generating it once and caching
+/// it on disk, then downsizes the result to degree `k`.
+///
+/// The cache lives at `temp_dir()/mithril-stm-test-srs/srs-unsafe-k{MAX_TEST_SRS_DEGREE}` and is
+/// shared across every nextest process on the runner. nextest runs each test in its own process,
+/// so an on-disk cache is the only way to avoid regenerating the (~2^19-element) SRS per test.
+/// Generation is serialized behind an exclusive file lock so concurrent process startup builds the
+/// file exactly once instead of every process racing to generate it.
+///
+/// Every caller still requests the degree it needs; this loads the single max-degree file and
+/// downsizes. For callers requesting `MAX_TEST_SRS_DEGREE` the result is byte-identical to a fresh
+/// `unsafe_setup(MAX_TEST_SRS_DEGREE, seed 42)`.
+#[cfg(test)]
+pub(crate) fn shared_unsafe_srs(k: u32) -> ParamsKZG<Bls12> {
+    use fs4::fs_std::FileExt;
+    use rand_chacha::ChaCha20Rng;
+    use rand_core::SeedableRng;
+    use std::fs::{File, create_dir_all};
+
+    assert!(
+        k <= MAX_TEST_SRS_DEGREE,
+        "requested SRS degree {k} exceeds MAX_TEST_SRS_DEGREE {MAX_TEST_SRS_DEGREE}"
+    );
+
+    let cache_dir = std::env::temp_dir().join(SHARED_TEST_SRS_FOLDER);
+    let srs_path = cache_dir.join(format!("srs-unsafe-k{MAX_TEST_SRS_DEGREE}"));
+
+    // Steady state: load the cached file without taking the lock.
+    if let Some(srs) = try_load_shared_srs(&srs_path) {
+        return downsize_srs(srs, k);
+    }
+
+    // Cache miss: serialize generation behind an exclusive lock so only one process builds the SRS.
+    create_dir_all(&cache_dir).expect("Failed to create shared SRS cache directory.");
+    let lock_path = cache_dir.join(format!("srs-unsafe-k{MAX_TEST_SRS_DEGREE}.lock"));
+    let lock_file = File::create(&lock_path).expect("Failed to create shared SRS lock file.");
+    FileExt::lock_exclusive(&lock_file).expect("Failed to acquire exclusive shared SRS lock.");
+    // The OS lock is released when `lock_file` is dropped, including on panic / unwind.
+
+    // Re-check under the lock: another process may have generated the file while we waited.
+    let srs = try_load_shared_srs(&srs_path).unwrap_or_else(|| {
+        let srs =
+            ParamsKZG::<Bls12>::unsafe_setup(MAX_TEST_SRS_DEGREE, ChaCha20Rng::seed_from_u64(42));
+        write_shared_srs_atomically(&srs, &cache_dir, &srs_path);
+        srs
+    });
+
+    downsize_srs(srs, k)
+}
+
+/// Reads the cached shared SRS, returning `None` if the file is absent or unreadable. Asserts the
+/// loaded degree matches [`MAX_TEST_SRS_DEGREE`] so a corrupt/mismatched file can never be used.
+#[cfg(test)]
+fn try_load_shared_srs(srs_path: &std::path::Path) -> Option<ParamsKZG<Bls12>> {
+    let file = File::open(srs_path).ok()?;
+    let mut reader = BufReader::new(file);
+    let srs = ParamsKZG::<Bls12>::read_custom(&mut reader, SerdeFormat::RawBytesUnchecked).ok()?;
+    assert_eq!(
+        srs.max_k(),
+        MAX_TEST_SRS_DEGREE,
+        "Cached shared SRS at {srs_path:?} has degree {} but expected {MAX_TEST_SRS_DEGREE}.",
+        srs.max_k(),
+    );
+    Some(srs)
+}
+
+/// Serializes `srs` to a temporary file in `cache_dir`, then atomically renames it onto `srs_path`.
+#[cfg(test)]
+fn write_shared_srs_atomically(
+    srs: &ParamsKZG<Bls12>,
+    cache_dir: &std::path::Path,
+    srs_path: &std::path::Path,
+) {
+    let mut srs_bytes = Vec::new();
+    srs.write_custom(&mut srs_bytes, SerdeFormat::RawBytesUnchecked)
+        .expect("Failed to serialize shared SRS.");
+    let mut tmp = tempfile::NamedTempFile::new_in(cache_dir)
+        .expect("Failed to create temporary file for the shared SRS.");
+    tmp.write_all(&srs_bytes)
+        .expect("Failed to write the shared SRS to its temporary file.");
+    tmp.persist(srs_path)
+        .expect("Failed to atomically rename the shared SRS file.");
+}
+
+/// Downsizes `srs` to degree `k` in place when `k` is smaller than the loaded degree.
+#[cfg(test)]
+fn downsize_srs(mut srs: ParamsKZG<Bls12>, k: u32) -> ParamsKZG<Bls12> {
+    if k < srs.max_k() {
+        srs.downsize(k);
+    }
+    srs
+}
+
+/// Builds a `TrustedSetupProvider` backed by the shared unsafe SRS (see [`shared_unsafe_srs`])
+/// downsized to degree `k`, written to `base_dir/srs/srs-parameters` with a matching SHA256 hash
+/// so the provider's hash check passes. For tests only.
 #[cfg(test)]
 pub(crate) fn build_provider_with_unsafe_srs(
     base_dir: &std::path::Path,
     k: u32,
 ) -> TrustedSetupProvider {
-    use rand_chacha::ChaCha20Rng;
-    use rand_core::SeedableRng;
     use std::fs::{File, create_dir_all};
 
-    let srs = ParamsKZG::<Bls12>::unsafe_setup(k, ChaCha20Rng::seed_from_u64(42));
+    let srs = shared_unsafe_srs(k);
     let mut srs_bytes = Vec::new();
     srs.write_custom(&mut srs_bytes, SerdeFormat::RawBytes).unwrap();