Extract descriptor types into buffa-descriptor crate

iainmcgin · iainmcgin · commit e1e09492582f · 2026-03-24T20:53:28.000Z
Moves the generated FileDescriptorProto/DescriptorProto types and
vendored descriptor.proto sources from buffa-codegen into a new
buffa-descriptor crate that depends only on buffa. This makes the
descriptor types available without pulling in quote/syn/prettyplease,
laying the foundation for runtime reflection.

buffa-codegen re-exports buffa_descriptor::generated so all existing
crate::generated:: paths resolve unchanged. gen_descriptor_types now
takes an explicit output directory argument, eliminating the previous
cwd-sensitivity.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -98,6 +98,9 @@ jobs:
       - name: Check buffa-types no_std
         run: cargo check -p buffa-types --no-default-features
 
+      - name: Check buffa-descriptor no_std
+        run: cargo check -p buffa-descriptor --no-default-features
+
       - name: Check workspace 32-bit
         run: cargo check --workspace --target i686-unknown-linux-gnu
 
@@ -118,7 +121,7 @@ jobs:
       - uses: Swatinem/rust-cache@v2
 
       - name: Regenerate bootstrap descriptor types
-        # Proto sources are vendored in buffa-codegen/protos/ (pinned),
+        # Proto sources are vendored in buffa-descriptor/protos/ (pinned),
         # so this is independent of the installed protoc's bundled includes.
         run: scripts/gen-bootstrap-types.sh
 
@@ -143,7 +146,7 @@ jobs:
 
       - name: Check for differences
         run: |
-          if ! git diff --exit-code buffa-codegen/src/generated/ buffa-types/src/generated/; then
+          if ! git diff --exit-code buffa-descriptor/src/generated/ buffa-types/src/generated/; then
             echo "::error::Checked-in generated code is stale. Run 'task gen-wkt-types' and/or regenerate bootstrap types, then commit."
             exit 1
           fi
diff --git a/.github/workflows/publish-crates.yml b/.github/workflows/publish-crates.yml
@@ -83,6 +83,9 @@ jobs:
       - name: Publish buffa-types
         run: /tmp/publish.sh buffa-types $DRY_RUN_FLAG
 
+      - name: Publish buffa-descriptor
+        run: /tmp/publish.sh buffa-descriptor $DRY_RUN_FLAG
+
       - name: Publish buffa-codegen
         run: /tmp/publish.sh buffa-codegen $DRY_RUN_FLAG
 
diff --git a/CLAUDE.md b/CLAUDE.md
@@ -10,7 +10,7 @@ If a change to `buffa-codegen` (notably `message.rs`, `impl_message.rs`, `view.r
 
 ```bash
 task gen-wkt-types          # buffa-types/src/generated/ — WKTs for consumer use
-task gen-bootstrap-types    # buffa-codegen/src/generated/ — only if the change affects descriptor types
+task gen-bootstrap-types    # buffa-descriptor/src/generated/ — only if the change affects descriptor types
 ```
 
 Most codegen changes don't touch descriptor-specific paths, so `gen-wkt-types` is usually sufficient.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -69,17 +69,17 @@ docker run --rm -v /tmp/conf:/out -e CONFORMANCE_OUT=/out buffa-conformance
 
 ```bash
 task tools-image               # rebuild and push the multi-arch tools image
-task vendor-bootstrap-protos   # re-fetch buffa-codegen/protos/ from the new release tag
+task vendor-bootstrap-protos   # re-fetch buffa-descriptor/protos/ from the new release tag
 task gen-bootstrap-types       # regenerate checked-in descriptor types
 ```
 
-Commit the refreshed `buffa-codegen/protos/` and `buffa-codegen/src/generated/` alongside the version bump.
+Commit the refreshed `buffa-descriptor/protos/` and `buffa-descriptor/src/generated/` alongside the version bump.
 
 ## Checked-In Generated Code
 
 Three sets of generated code are checked into the repo and **must be regenerated** whenever codegen output changes (e.g. changes to `imports.rs`, `message.rs`, `oneof.rs`, etc.):
 
-1. **Bootstrap descriptor types** (`buffa-codegen/src/generated/`): Used by codegen itself to parse `.proto` descriptors. Regenerate with `task gen-bootstrap-types`. The source protos are vendored in `buffa-codegen/protos/` (pinned; refresh with `task vendor-bootstrap-protos` when bumping the protobuf version), so output is independent of your local protoc's bundled includes — only a protoc binary ≥ v27 is needed. Only needs regeneration when a codegen change affects the descriptor types themselves — most changes don't.
+1. **Bootstrap descriptor types** (`buffa-descriptor/src/generated/`): Used by codegen itself to parse `.proto` descriptors. Regenerate with `task gen-bootstrap-types`. The source protos are vendored in `buffa-descriptor/protos/` (pinned; refresh with `task vendor-bootstrap-protos` when bumping the protobuf version), so output is independent of your local protoc's bundled includes — only a protoc binary ≥ v27 is needed. Only needs regeneration when a codegen change affects the descriptor types themselves — most changes don't.
 
 2. **Well-known types** (`buffa-types/src/generated/`): `Timestamp`, `Duration`, `Any`, `Struct`/`Value`, `FieldMask`, `Empty`, wrappers. Checked in (rather than generated at build time) so that consumers of `buffa-types` don't need `protoc` or the `buffa-build`/`buffa-codegen` toolchain. Regenerate with `task gen-wkt-types`. The WKT `.proto` sources are vendored in `buffa-types/protos/` (not read from the protoc installation) so the output is pinned. **This is the one most likely to need regeneration** — WKTs use views, unknown-field preservation, and the `arbitrary` derive, so almost any codegen output-format change touches them. If in doubt, run it and check `git status`.
 
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -3,6 +3,7 @@ resolver = "2"
 members = [
     "buffa",
     "buffa-types",
+    "buffa-descriptor",
     "buffa-codegen",
     "buffa-build",
     "buffa-test",
@@ -32,6 +33,7 @@ categories = ["encoding", "no-std"]
 [workspace.dependencies]
 buffa = { path = "buffa", version = "0.2.0", default-features = false }
 buffa-types = { path = "buffa-types", version = "0.2.0" }
+buffa-descriptor = { path = "buffa-descriptor", version = "0.2.0" }
 buffa-codegen = { path = "buffa-codegen", version = "0.2.0" }
 buffa-build = { path = "buffa-build", version = "0.2.0" }
 buffa-test = { path = "buffa-test", version = "0.2.0" }
diff --git a/DESIGN.md b/DESIGN.md
@@ -67,6 +67,12 @@ The WKT wire format is completely vanilla — two varints for `Timestamp`, etc.
 
 Both layer on top of the generated `Message` impl via `include!()` + sibling modules; the checked-in code and the hand-written extensions coexist cleanly.
 
+### `buffa-descriptor` — Protobuf Descriptor Types
+
+Self-hosted Rust types for `google/protobuf/descriptor.proto` and `google/protobuf/compiler/plugin.proto`, generated by `buffa-codegen` itself. These are the types that `buffa-codegen` uses to parse protoc's `CodeGeneratorRequest`, and the foundation for future runtime reflection.
+
+The generated code is checked in (regenerate via `task gen-bootstrap-types`). The only runtime dependency is `buffa` — no quote/syn/prettyplease — so the crate is `no_std`-capable and dependency-light enough to depend on from the runtime without pulling in the codegen toolchain.
+
 ### `buffa-codegen` — Shared Code Generation Logic
 
 The code generation library, shared between `protoc-gen-buffa` and `buffa-build`. Takes protobuf descriptors (from protoc's `FileDescriptorProto`) and emits Rust source code.
@@ -101,7 +107,7 @@ protoc --buffa_out=. --plugin=protoc-gen-buffa my_service.proto
 
 Reads a `CodeGeneratorRequest` from stdin, passes the file descriptors to `buffa-codegen`, writes a `CodeGeneratorResponse` to stdout.
 
-**Bootstrapping:** The `CodeGeneratorRequest` and `CodeGeneratorResponse` messages are themselves protobuf — we decode/encode them using buffa's own generated descriptor and compiler types (checked into `buffa-codegen/src/generated/`), eliminating any external protobuf library dependency from the build graph.
+**Bootstrapping:** The `CodeGeneratorRequest` and `CodeGeneratorResponse` messages are themselves protobuf — we decode/encode them using buffa's own generated descriptor and compiler types (checked into `buffa-descriptor/src/generated/`), eliminating any external protobuf library dependency from the build graph.
 
 ### `buffa-build` — Build Script Integration
 
diff --git a/README.md b/README.md
@@ -132,6 +132,7 @@ let decoded: MyMessage = serde_json::from_str(&json).unwrap();
 |---|---|
 | `buffa` | Core runtime: `Message` trait, wire format codec, `no_std` support |
 | `buffa-types` | Well-known types: Timestamp, Duration, Any, Struct, wrappers, etc. |
+| `buffa-descriptor` | Protobuf descriptor types (`FileDescriptorProto`, `DescriptorProto`, ...) |
 | `buffa-codegen` | Code generation from protobuf descriptors |
 | `buffa-build` | `build.rs` helper for invoking codegen via `protoc` |
 | `protoc-gen-buffa` | `protoc` plugin binary |
diff --git a/Taskfile.yml b/Taskfile.yml
@@ -385,23 +385,23 @@ tasks:
   # ── Checked-in generated code ─────────────────────────────────────────
   #
   # Two sets of generated code are checked into the repo:
-  #   - buffa-codegen/src/generated/  (bootstrap descriptor types)
+  #   - buffa-descriptor/src/generated/  (bootstrap descriptor types)
   #   - buffa-types/src/generated/    (well-known types)
   # CI (check-generated-code) fails if either is stale.
 
   gen-bootstrap-types:
     desc: >-
-      Regenerate buffa-codegen/src/generated/ (descriptor.proto + plugin.proto
+      Regenerate buffa-descriptor/src/generated/ (descriptor.proto + plugin.proto
       types used by codegen itself to parse descriptors). Only needed when a
       codegen change affects the descriptor types themselves — most codegen
-      changes don't. Proto sources are vendored in buffa-codegen/protos/;
+      changes don't. Proto sources are vendored in buffa-descriptor/protos/;
       only a protoc binary (v27+) is required.
     cmds:
       - scripts/gen-bootstrap-types.sh
 
   vendor-bootstrap-protos:
     desc: >-
-      Re-fetch buffa-codegen/protos/ from the protobuf release matching
+      Re-fetch buffa-descriptor/protos/ from the protobuf release matching
       TOOLS_IMAGE. Run this when bumping the protobuf version, then follow
       with `task gen-bootstrap-types` and commit both. Uses gh (authenticated
       API) rather than raw curl to avoid GitHub's anonymous rate limit.
@@ -416,15 +416,15 @@ tasks:
           gh CLI must be authenticated. Run: gh auth login
     cmds:
       - echo "Vendoring descriptor.proto + plugin.proto from protobuf v{{.PROTO_VER}}"
-      - mkdir -p buffa-codegen/protos/google/protobuf/compiler
+      - mkdir -p buffa-descriptor/protos/google/protobuf/compiler
       - >-
         {{.GH_RAW}} '{{.SRC}}/descriptor.proto?ref=v{{.PROTO_VER}}'
-        > buffa-codegen/protos/google/protobuf/descriptor.proto
+        > buffa-descriptor/protos/google/protobuf/descriptor.proto
       - >-
         {{.GH_RAW}} '{{.SRC}}/compiler/plugin.proto?ref=v{{.PROTO_VER}}'
-        > buffa-codegen/protos/google/protobuf/compiler/plugin.proto
-      - wc -l buffa-codegen/protos/google/protobuf/descriptor.proto
-          buffa-codegen/protos/google/protobuf/compiler/plugin.proto
+        > buffa-descriptor/protos/google/protobuf/compiler/plugin.proto
+      - wc -l buffa-descriptor/protos/google/protobuf/descriptor.proto
+          buffa-descriptor/protos/google/protobuf/compiler/plugin.proto
 
   gen-wkt-types:
     desc: >-
diff --git a/benchmarks/Dockerfile.bench-buffa b/benchmarks/Dockerfile.bench-buffa
@@ -17,6 +17,7 @@ COPY Cargo.toml Cargo.lock ./
 COPY buffa/             buffa/
 COPY buffa-build/       buffa-build/
 COPY buffa-codegen/     buffa-codegen/
+COPY buffa-descriptor/  buffa-descriptor/
 COPY buffa-test/        buffa-test/
 COPY buffa-types/       buffa-types/
 COPY protoc-gen-buffa/  protoc-gen-buffa/
diff --git a/buffa-codegen/Cargo.toml b/buffa-codegen/Cargo.toml
@@ -12,6 +12,7 @@ categories = ["development-tools::build-utils"]
 [dependencies]
 # `path` wins for local workspace development; `version` is used on publish.
 buffa = { path = "../buffa", version = "0.2.0" }
+buffa-descriptor = { workspace = true }
 prettyplease = { workspace = true }
 proc-macro2 = { workspace = true }
 quote = { workspace = true }
diff --git a/buffa-codegen/src/bin/gen_descriptor_types.rs b/buffa-codegen/src/bin/gen_descriptor_types.rs
@@ -3,16 +3,17 @@
 //! This is the bootstrap step: it reads a binary FileDescriptorSet (produced
 //! by `protoc --descriptor_set_out --include_imports`) and generates Rust
 //! source using buffa-codegen.  The output is checked into the repo at
-//! `buffa-codegen/src/generated/`.
+//! `buffa-descriptor/src/generated/`.
 //!
 //! Usage:
 //!
 //! ```text
 //!   protoc --descriptor_set_out=descriptor_set.pb --include_imports \
-//!       -I <protobuf-src>/src \
+//!       -I buffa-descriptor/protos \
 //!       google/protobuf/descriptor.proto \
 //!       google/protobuf/compiler/plugin.proto
-//!   cargo run --bin gen_descriptor_types -- descriptor_set.pb
+//!   cargo run -p buffa-codegen --bin gen_descriptor_types -- \
+//!       descriptor_set.pb buffa-descriptor/src/generated
 //! ```
 
 use buffa::Message;
@@ -21,8 +22,8 @@ use std::fs;
 
 fn main() {
     let args: Vec<String> = std::env::args().collect();
-    if args.len() != 2 {
-        eprintln!("Usage: gen_descriptor_types <descriptor_set.pb>");
+    if args.len() != 3 {
+        eprintln!("Usage: gen_descriptor_types <descriptor_set.pb> <output_dir>");
         std::process::exit(1);
     }
 
@@ -49,7 +50,7 @@ fn main() {
     let generated = buffa_codegen::generate(&descriptor_set.file, &files_to_generate, &config)
         .expect("code generation failed");
 
-    let out_dir = std::path::Path::new("src/generated");
+    let out_dir = std::path::Path::new(&args[2]);
     fs::create_dir_all(out_dir).expect("failed to create output dir");
 
     for file in &generated {
diff --git a/buffa-codegen/src/lib.rs b/buffa-codegen/src/lib.rs
@@ -27,7 +27,7 @@ pub(crate) mod enumeration;
 pub(crate) mod extension;
 pub(crate) mod features;
 #[doc(hidden)]
-pub mod generated;
+pub use buffa_descriptor::generated;
 pub mod idents;
 pub(crate) mod impl_message;
 pub(crate) mod imports;
diff --git a/buffa-descriptor/Cargo.toml b/buffa-descriptor/Cargo.toml
@@ -0,0 +1,17 @@
+[package]
+name = "buffa-descriptor"
+description = "Protobuf descriptor types (FileDescriptorProto, DescriptorProto, ...) for buffa"
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+repository.workspace = true
+keywords = ["protobuf", "protocol-buffers", "descriptor", "reflection"]
+categories = ["encoding", "no-std"]
+
+[dependencies]
+buffa = { workspace = true }
+
+[features]
+default = ["std"]
+std = ["buffa/std"]
diff --git a/buffa-descriptor/README.md b/buffa-descriptor/README.md
@@ -0,0 +1,9 @@
+# buffa-descriptor
+
+Protobuf descriptor types for [buffa](https://crates.io/crates/buffa).
+
+This crate provides self-hosted Rust types for `google/protobuf/descriptor.proto`
+and `google/protobuf/compiler/plugin.proto`, generated by buffa-codegen itself.
+It underpins both compile-time code generation and runtime reflection.
+
+The only runtime dependency is `buffa` — no external protobuf library required.
diff --git a/buffa-descriptor/protos/google/protobuf/compiler/plugin.proto b/buffa-descriptor/protos/google/protobuf/compiler/plugin.proto
diff --git a/buffa-descriptor/protos/google/protobuf/descriptor.proto b/buffa-descriptor/protos/google/protobuf/descriptor.proto
diff --git a/buffa-descriptor/src/generated/google.protobuf.compiler.plugin.rs b/buffa-descriptor/src/generated/google.protobuf.compiler.plugin.rs
diff --git a/buffa-descriptor/src/generated/google.protobuf.descriptor.rs b/buffa-descriptor/src/generated/google.protobuf.descriptor.rs
diff --git a/buffa-descriptor/src/generated/mod.rs b/buffa-descriptor/src/generated/mod.rs
@@ -1,19 +1,12 @@
-//! Generated protobuf descriptor types for bootstrapping.
+//! Generated protobuf descriptor types.
 //!
 //! These types are generated from `google/protobuf/descriptor.proto` and
 //! `google/protobuf/compiler/plugin.proto` using buffa-codegen itself.
-//! This makes buffa-codegen fully self-hosted — no external protobuf
-//! library is needed to decode protoc's `CodeGeneratorRequest` — and gives
-//! direct access to edition features (`FeatureSet`, `Edition`, etc.).
+//! This makes buffa fully self-hosted — no external protobuf library is
+//! needed to decode descriptors — and gives direct access to edition
+//! features (`FeatureSet`, `Edition`, etc.).
 //!
-//! To regenerate:
-//! ```sh
-//! protoc --descriptor_set_out=/tmp/descriptor_set.pb --include_imports \
-//!     -I <protobuf-src>/src \
-//!     google/protobuf/descriptor.proto \
-//!     google/protobuf/compiler/plugin.proto
-//! cargo run --bin gen_descriptor_types -- /tmp/descriptor_set.pb
-//! ```
+//! To regenerate, run `task gen-bootstrap-types` from the repo root.
 
 #[allow(
     clippy::all,
diff --git a/buffa-descriptor/src/lib.rs b/buffa-descriptor/src/lib.rs
@@ -0,0 +1,33 @@
+//! Protobuf descriptor types for buffa.
+//!
+//! This crate provides buffa-generated Rust types for the protobuf descriptor
+//! schema (`google/protobuf/descriptor.proto`) and the protoc plugin protocol
+//! (`google/protobuf/compiler/plugin.proto`).  It is the foundation for both
+//! compile-time code generation (`buffa-codegen`) and runtime reflection.
+//!
+//! The types are self-hosted — generated by buffa-codegen itself — so there
+//! is no dependency on an external protobuf library.  The only runtime
+//! dependency is `buffa`.
+//!
+//! # Modules
+//!
+//! - [`generated::descriptor`] — `FileDescriptorProto`, `DescriptorProto`,
+//!   `FieldDescriptorProto`, `FeatureSet`, `Edition`, and the rest of
+//!   `descriptor.proto`.
+//! - [`generated::compiler`] — `CodeGeneratorRequest`, `CodeGeneratorResponse`
+//!   from `plugin.proto`.
+//!
+//! # Regenerating
+//!
+//! The generated code is checked in.  To regenerate (after a codegen output
+//! change or a protobuf version bump):
+//!
+//! ```sh
+//! task gen-bootstrap-types
+//! ```
+
+#![cfg_attr(not(feature = "std"), no_std)]
+
+extern crate alloc;
+
+pub mod generated;
diff --git a/conformance/Dockerfile b/conformance/Dockerfile
@@ -29,6 +29,7 @@ COPY Cargo.toml Cargo.lock ./
 COPY buffa/             buffa/
 COPY buffa-build/       buffa-build/
 COPY buffa-codegen/     buffa-codegen/
+COPY buffa-descriptor/  buffa-descriptor/
 COPY buffa-test/        buffa-test/
 COPY buffa-types/       buffa-types/
 COPY protoc-gen-buffa/  protoc-gen-buffa/
diff --git a/protoc-gen-buffa/Dockerfile b/protoc-gen-buffa/Dockerfile
@@ -24,6 +24,7 @@ COPY Cargo.toml Cargo.lock ./
 COPY buffa/           buffa/
 COPY buffa-build/     buffa-build/
 COPY buffa-codegen/   buffa-codegen/
+COPY buffa-descriptor/ buffa-descriptor/
 COPY buffa-test/      buffa-test/
 COPY buffa-types/     buffa-types/
 COPY protoc-gen-buffa/ protoc-gen-buffa/
diff --git a/scripts/gen-bootstrap-types.sh b/scripts/gen-bootstrap-types.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
-# Regenerate buffa-codegen/src/generated/ (bootstrap descriptor types).
+# Regenerate buffa-descriptor/src/generated/ (bootstrap descriptor types).
 #
-# The source protos are vendored in buffa-codegen/protos/ (pinned to a
+# The source protos are vendored in buffa-descriptor/protos/ (pinned to a
 # specific protobuf release) so the generated output does not depend on
 # which protoc is installed locally — only the protoc binary is needed,
 # not its bundled includes.
@@ -37,10 +37,10 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd)"
 
 DESC=/tmp/buffa-descriptor-set.pb
 "$PROTOC" --descriptor_set_out="$DESC" --include_imports --include_source_info \
-    -I "$ROOT/buffa-codegen/protos" \
+    -I "$ROOT/buffa-descriptor/protos" \
     google/protobuf/descriptor.proto \
     google/protobuf/compiler/plugin.proto
 
-# gen_descriptor_types writes to "src/generated" relative to cwd.
-cd "$ROOT/buffa-codegen"
-cargo run --bin gen_descriptor_types -- "$DESC"
+cd "$ROOT"
+cargo run -p buffa-codegen --bin gen_descriptor_types -- \
+    "$DESC" buffa-descriptor/src/generated
diff --git a/stress/googleapis/Dockerfile b/stress/googleapis/Dockerfile
@@ -27,6 +27,7 @@ COPY Cargo.toml Cargo.lock ./
 COPY buffa/             buffa/
 COPY buffa-build/       buffa-build/
 COPY buffa-codegen/     buffa-codegen/
+COPY buffa-descriptor/  buffa-descriptor/
 COPY buffa-test/        buffa-test/
 COPY buffa-types/       buffa-types/
 COPY protoc-gen-buffa/  protoc-gen-buffa/
@@ -59,6 +60,7 @@ COPY Cargo.toml Cargo.lock ./
 COPY buffa/             buffa/
 COPY buffa-build/       buffa-build/
 COPY buffa-codegen/     buffa-codegen/
+COPY buffa-descriptor/  buffa-descriptor/
 COPY buffa-test/        buffa-test/
 COPY buffa-types/       buffa-types/
 COPY protoc-gen-buffa/  protoc-gen-buffa/
