feat: MsgpackTagged serialization (#12730)

Co-authored-by: Ary Borenszweig <asterite@gmail.com>

Author: aakoshh

.github/workflows/test-rust-workspace-arm64.yml

@@ -49,6 +49,15 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+      # `compile_fail` runs trybuild, which spawns its own `cargo build` and
+      # needs the cargo registry to resolve transitive proc-macro deps. The
+      # run-tests job runs from the archive without registry access, so the
+      # test detects the nextest run via NEXTEST_TEST_GROUP and skips itself.
+      # We exercise it here via plain `cargo test`, where the registry cache
+      # restored by Swatinem/rust-cache is intact.
+      - name: Run trybuild compile_fail tests
+        run: cargo test -p msgpack_tagged --test compile_fail
+
       - name: Upload archive to workflow
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
@@ -89,6 +98,12 @@ jobs:
           cargo nextest run --archive-file nextest-archive-arm64.tar.zst \
             --partition count:${{ matrix.partition }}/8 \
             --profile ci-master
+        env:
+          # Tests that spawn their own `cargo build` (notably trybuild's
+          # compile_fail in `msgpack_tagged`) detect this and skip — see
+          # acvm-repo/msgpack_tagged/tests/compile_fail.rs. They run instead
+          # in the build-test-artifacts job where the registry cache is intact.
+          NOIR_NEXTEST_ARCHIVED: 1
 
   # This is a job which depends on all test jobs and reports the overall status.
   # This allows us to add/remove test jobs without having to update the required workflows.

.github/workflows/test-rust-workspace-msrv.yml

@@ -63,6 +63,15 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+      # `compile_fail` runs trybuild, which spawns its own `cargo build` and
+      # needs the cargo registry to resolve transitive proc-macro deps. The
+      # run-tests job runs from the archive without registry access, so the
+      # test detects the nextest run via NEXTEST_TEST_GROUP and skips itself.
+      # We exercise it here via plain `cargo test`, where the registry cache
+      # restored by Swatinem/rust-cache is intact.
+      - name: Run trybuild compile_fail tests
+        run: cargo test -p msgpack_tagged --test compile_fail
+
       - name: Upload archive to workflow
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
@@ -103,6 +112,12 @@ jobs:
           cargo nextest run --archive-file nextest-archive.tar.zst \
             --partition count:${{ matrix.partition }}/8 \
             --profile ci-master
+        env:
+          # Tests that spawn their own `cargo build` (notably trybuild's
+          # compile_fail in `msgpack_tagged`) detect this and skip — see
+          # acvm-repo/msgpack_tagged/tests/compile_fail.rs. They run instead
+          # in the build-test-artifacts job where the registry cache is intact.
+          NOIR_NEXTEST_ARCHIVED: 1
 
   # This is a job which depends on all test jobs and reports the overall status.
   # This allows us to add/remove test jobs without having to update the required workflows.

.github/workflows/test-rust-workspace.yml

@@ -50,6 +50,15 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+      # `compile_fail` runs trybuild, which spawns its own `cargo build` and
+      # needs the cargo registry to resolve transitive proc-macro deps. The
+      # run-tests job runs from the archive without registry access, so the
+      # test detects the nextest run via NEXTEST_TEST_GROUP and skips itself.
+      # We exercise it here via plain `cargo test`, where the registry cache
+      # restored by Swatinem/rust-cache is intact.
+      - name: Run trybuild compile_fail tests
+        run: cargo test -p msgpack_tagged --test compile_fail
+
       - name: Upload archive to workflow
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
@@ -93,6 +102,11 @@ jobs:
             --profile $NEXTEST_PROFILE
         env:
           NEXTEST_PROFILE: ${{ (github.event_name == 'merge_group' && 'merge-queue') || (github.ref == 'refs/heads/master' && 'ci-master') || 'ci' }}
+          # Tests that spawn their own `cargo build` (notably trybuild's
+          # compile_fail in `msgpack_tagged`) detect this and skip — see
+          # acvm-repo/msgpack_tagged/tests/compile_fail.rs. They run instead
+          # in the build-test-artifacts job where the registry cache is intact.
+          NOIR_NEXTEST_ARCHIVED: 1
 
   check-pending-snapshots:
     name: Check for pending snapshots

.rustfmt.toml

@@ -1,3 +1,4 @@
 edition = "2024"
 style_edition = "2024"
 use_small_heuristics = "Max"
+merge_derives = false

Cargo.lock

@@ -42,6 +42,8 @@ members = [
     "acvm-repo/brillig_vm",
     "acvm-repo/blackbox_solver",
     "acvm-repo/bn254_blackbox_solver",
+    "acvm-repo/msgpack_tagged",
+    "acvm-repo/msgpack_tagged_derive",
     # Utility crates
     "utils/iter-extended",
 
@@ -122,6 +124,8 @@ brillig = { version = "1.0.0-beta.21", path = "acvm-repo/brillig", default-featu
 brillig_vm = { version = "1.0.0-beta.21", path = "acvm-repo/brillig_vm", default-features = false }
 acvm_blackbox_solver = { version = "1.0.0-beta.21", path = "acvm-repo/blackbox_solver", default-features = false }
 bn254_blackbox_solver = { version = "1.0.0-beta.21", path = "acvm-repo/bn254_blackbox_solver", default-features = false }
+msgpack_tagged = { version = "1.0.0-beta.20", path = "acvm-repo/msgpack_tagged", default-features = false }
+msgpack_tagged_derive = { version = "1.0.0-beta.20", path = "acvm-repo/msgpack_tagged_derive" }
 
 # Noir compiler workspace dependencies
 fm = { path = "compiler/fm" }
@@ -190,8 +194,10 @@ criterion = "^0.5.0"
 # Note that using the "frame-pointer" feature breaks framegraphs on linux
 # https://github.com/tikv/pprof-rs/pull/172
 pprof = { version = "^0.15.0", features = ["flamegraph", "criterion"] }
+rmp = "0.8.14"
 rmp-serde = "1.3.0"
 rmpv = "1.3.0"
+smallvec = "1.15.1"
 
 arbitrary = "1.4.1"
 arbtest = "0.3.2"
@@ -201,6 +207,7 @@ dirs = "^6.0.0"
 env_logger = "0.11.6"
 log = "0.4"
 serde = { version = "1.0.136", features = ["derive", "rc"] }
+serde_bytes = "0.11"
 serde_json = "1.0"
 smol_str = { version = "0.3.2", features = ["serde"] }
 thiserror = "2.0.18"
@@ -248,6 +255,10 @@ tracing-subscriber = { version = "0.3.18", features = ["env-filter", "json"] }
 rust-embed = "^8.7.2"
 petgraph = "^0.8.2"
 insta = "1.47.2"
+trybuild = "1.0.116"
+proc-macro2 = "1.0"
+quote = "1.0"
+syn = "2.0"
 
 [profile.dev]
 # This is required to be able to run `cargo test` in acvm_js due to the `locals exceeds maximum` error.

Cargo.toml

@@ -6,6 +6,7 @@ This folder is organized into several key components:
 - `acvm/` - Implementation of ACVM that executes ACIR
 - `brillig/` - Definition and implementation of unconstrained Brillig opcodes
 - `brillig_vm/` - Implementation of Brillig VM that executes Brillig
+- `msgpack_tagged/` - Tagged-map serialization format for evolvable bytecode (paired with the `msgpack_tagged_derive` proc-macro)
 
 Click into each folder to learn more from their READMEs.
 

acvm-repo/README.md

@@ -18,6 +18,7 @@ workspace = true
 [dependencies]
 acir_field.workspace = true
 brillig.workspace = true
+msgpack_tagged.workspace = true
 
 serde.workspace = true
 thiserror.workspace = true