abgen: asset-reuse parity — canonical glb names, shared assets layout, probe (#1)

Scene glb/gltf bundles now carry the upstream converter's deps digest in
their canonical name ({hash}_{depsdigest}_{platform}) in both the live/JIT
server and abgen-corpus, matching the asset-reuse flow the ab-cdn
deployment has run since v49. On by default; ABGEN_ASSET_REUSE=0 opts out
for parity runs against pre-v49 reference trees.

- naming: filter deps digests to GLB_DEP_EXTENSIONS (.bin + textures),
  mirroring upstream computeDepsDigest — fixes digest divergence for glbs
  referencing non-texture uris (also corrects the wearables path);
  add split_bundle_stem for parsing canonical names
- live: per-entity deps-digest map (magenta-tolerant when enabled),
  canonical naming + digest validation in the JIT build path, shared
  {version}/assets/ space layout with entity-scoped read fallback,
  pre-build HEAD probe that lists space hits in the manifest without
  rebuilding
- space: signed HEAD support for the probe
- abcdn: flat lane strips the digest for owner-entity resolution
- corpus: canonical naming in derive_one_entity behind
  EffectiveToggles.asset_reuse; strict mode skips digest-less glbs
  (upstream skipped-assets semantics)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Author: dalkia

README.md

@@ -158,6 +158,23 @@ Enabled when `ABGEN_S3_BUCKET` is non-empty (or `ABGEN_USE_SPACE=1`):
 - `ABGEN_FALLBACK_VERSION` (default `v41`) - extra version prefix tried on space-cache lookups
 - `ABGEN_WORLDS_CONTENT_URL` - worlds-content-server fallback for entity/content fetches that miss the primary source (default public worlds server; `0`/`off`/empty disables)
 
+### Asset-reuse mode (upstream converter parity)
+ON by default — the ab-cdn deployment has run asset-reuse since v49. Scene glb/gltf bundles use
+the upstream converter's canonical naming and shared bucket layout (applies to the JIT server
+and `abgen-corpus`). Set `ABGEN_ASSET_REUSE=0` to fall back to legacy `{hash}_{platform}` names
+and entity-scoped space keys — needed only for parity runs against pre-v49 reference trees
+(e.g. `--live-mode` sampling of v15–v41 vintages, whose manifests list non-digest names).
+- glb/gltf bundles are named `{hash}_{depsdigest}_{platform}` — the digest is a 128-bit hash of
+  the glb's resolved `(file, hash)` dependency pairs (`.bin` + textures), so a glb whose
+  dependency set changes lands at a new name/key; textures stay `{hash}_{platform}`
+- space keys move from entity-scoped `{version}/{entity}/{bundle}` to the shared
+  `{version}/assets/{bundle}` layout (entity-scoped keys remain a read fallback)
+- before building, the space is HEAD-probed at the canonical key; hits are listed in the entity
+  manifest without rebuilding, so bundles are converted once across entities
+- a glb whose deps can't be resolved is skipped (upstream skipped-assets semantics) unless
+  `ABGEN_MAGENTA_MISSING` is on, in which case unresolvable deps are dropped from the digest and
+  the build substitutes placeholder textures
+
 ### Registry index eager build
 On `POST /entities/active|versions`, entities missing a converted bundle are queued for conversion; the
 request waits up to the deadline, builds finish in the background. Knobs: `ABGEN_INDEX_EAGER_BUILD`

crate/src/abcdn/handlers/jit.rs

@@ -547,10 +547,13 @@ pub(super) fn flat_target(path: &str) -> Option<(String, String)> {
         return None;
     }
     let raw = segs[1].strip_suffix(".br").unwrap_or(segs[1]);
-    let (bare, platform) = raw.rsplit_once('_')?;
-    if bare.is_empty() || !resolver::is_platform(platform) {
+    let (stem, platform) = raw.rsplit_once('_')?;
+    if stem.is_empty() || !resolver::is_platform(platform) {
         return None;
     }
+    // Canonical glb names are `{hash}_{depsdigest}_{platform}` — owner-entity
+    // resolution needs the bare content hash, not the digest-qualified stem.
+    let (bare, _digest) = crate::naming::split_bundle_stem(stem);
     Some((bare.to_string(), platform.to_string()))
 }
 

crate/src/abcdn/handlers/tests.rs

@@ -253,6 +253,16 @@ fn flat_target_table() {
         super::flat_target("v41/Qmhash_webgl"),
         Some(("Qmhash".to_string(), "webgl".to_string()))
     );
+    // Canonical glb names carry a deps digest — owner resolution gets the
+    // bare content hash.
+    assert_eq!(
+        super::flat_target("v41/Qmhash_0123abcd_windows"),
+        Some(("Qmhash".to_string(), "windows".to_string()))
+    );
+    assert_eq!(
+        super::flat_target("v41/Qmhash_0123abcd_mac.br"),
+        Some(("Qmhash".to_string(), "mac".to_string()))
+    );
     assert_eq!(super::flat_target("v41/Qmhash"), None);
     assert_eq!(super::flat_target("v41/_windows"), None);
     assert_eq!(super::flat_target("manifest/Qmhash_windows"), None);

crate/src/abcdn/mod.rs

@@ -90,6 +90,18 @@ pub async fn build_state(cfg: &Config) -> Result<AppState> {
             "ab-cdn S3 space cache ENABLED (read-through + write-back)"
         );
     }
+    if crate::clihelp::env_bool("ABGEN_ASSET_REUSE", true) {
+        tracing::info!(
+            "asset-reuse mode ON (default): canonical glb names \
+             ({{hash}}_{{depsdigest}}_{{platform}}), shared {{version}}/assets/ space layout, \
+             pre-build probe"
+        );
+    } else {
+        tracing::warn!(
+            "asset-reuse mode DISABLED (ABGEN_ASSET_REUSE=0): legacy {{hash}}_{{platform}} \
+             names + entity-scoped space keys — only for parity against pre-v49 references"
+        );
+    }
 
     let live_proxy = crate::live::Proxy::new(pcfg);
     let ab_date = live_proxy.date().to_string();

crate/src/bin/abgen-corpus/build.rs

@@ -349,6 +349,7 @@ pub(crate) fn derive_one_entity(
     ent_id: &str,
     platform: &str,
     uri_cache: &abgen::glbscan::UriCache,
+    toggles: EffectiveToggles,
 ) -> Option<EntityEntry> {
     let entity = load_entity_json(store, ent_id)?;
     let entity_type = entity
@@ -391,7 +392,28 @@ pub(crate) fn derive_one_entity(
         if !store.exists(&c.hash) {
             continue;
         }
-        let bundle_name = format!("{}_{platform}", c.hash);
+        let bundle_name = if toggles.asset_reuse && is_glb {
+            // Magenta-tolerant runs drop unresolvable deps from the digest
+            // (the build substitutes placeholders); strict runs skip the glb
+            // entirely, mirroring the upstream converter's skipped-assets
+            // handling for missing deps.
+            let digest = store.fetch_mmap(&c.hash).ok().and_then(|bytes| {
+                abgen::naming::deps_digest_for_glb(
+                    &bytes,
+                    &c.file,
+                    &content_by_file,
+                    toggles.magenta_missing,
+                )
+                .map_err(|e| eprintln!("skip {ent_id}/{}: deps digest: {e:#}", c.file))
+                .ok()
+            });
+            match digest {
+                Some(d) => format!("{}_{d}_{platform}", c.hash),
+                None => continue,
+            }
+        } else {
+            format!("{}_{platform}", c.hash)
+        };
         if !local_seen.insert(bundle_name.clone()) {
             continue;
         }
@@ -476,7 +498,7 @@ pub(crate) fn run_fused_entity_ids(
                 errs.load(Ordering::Relaxed),
             );
         }
-        let entry = match derive_one_entity(store, ent_id, primary, &uri_cache) {
+        let entry = match derive_one_entity(store, ent_id, primary, &uri_cache, toggles) {
             Some(e) => e,
             None => return,
         };
@@ -628,3 +650,115 @@ pub(crate) fn load_entity_json(store: &LocalContentStore, cid: &str) -> Option<s
     let bytes = store.fetch(cid).ok()?;
     serde_json::from_slice(&bytes).ok()
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn toggles(asset_reuse: bool, magenta_missing: bool) -> EffectiveToggles {
+        EffectiveToggles {
+            collection_mode: false,
+            real_textures: false,
+            v38_compat: false,
+            v38_timestamp: 0,
+            magenta_missing,
+            asset_reuse,
+        }
+    }
+
+    fn store_with_entity(tag: &str, content: serde_json::Value) -> LocalContentStore {
+        let dir = std::env::temp_dir().join(format!(
+            "abgen-corpus-derive-{tag}-{}",
+            std::process::id()
+        ));
+        let _ = std::fs::remove_dir_all(&dir);
+        let store = LocalContentStore::new(&dir);
+        let entity = serde_json::json!({"type": "scene", "content": content});
+        store
+            .write("bafyentity", entity.to_string().as_bytes())
+            .unwrap();
+        store
+    }
+
+    fn glb_names(entry: &EntityEntry) -> Vec<String> {
+        entry
+            .bundles
+            .iter()
+            .filter(|b| b.cid == "Qmglb")
+            .map(|b| b.bundle_name.clone())
+            .collect()
+    }
+
+    const GLTF_JSON: &str = r#"{"asset":{"version":"2.0"},
+        "images":[{"uri":"t.png"}],"buffers":[{"uri":"a.bin"}]}"#;
+
+    #[test]
+    fn derive_names_glbs_canonically_in_asset_reuse_mode() {
+        let store = store_with_entity(
+            "canonical",
+            serde_json::json!([
+                {"file": "m.gltf", "hash": "Qmglb"},
+                {"file": "a.bin", "hash": "Qmbin"},
+                {"file": "t.png", "hash": "Qmtex"},
+            ]),
+        );
+        store.write("Qmglb", GLTF_JSON.as_bytes()).unwrap();
+        store.write("Qmbin", b"BIN").unwrap();
+        store.write("Qmtex", b"PNG").unwrap();
+        let cache = abgen::glbscan::UriCache::new();
+
+        let legacy =
+            derive_one_entity(&store, "bafyentity", "windows", &cache, toggles(false, false))
+                .unwrap();
+        assert_eq!(glb_names(&legacy), vec!["Qmglb_windows".to_string()]);
+
+        let reuse =
+            derive_one_entity(&store, "bafyentity", "windows", &cache, toggles(true, false))
+                .unwrap();
+        let digest = abgen::naming::compute_deps_digest(&[
+            ("a.bin".to_string(), "Qmbin".to_string()),
+            ("t.png".to_string(), "Qmtex".to_string()),
+        ]);
+        assert_eq!(glb_names(&reuse), vec![format!("Qmglb_{digest}_windows")]);
+        // Textures stay hash-keyed: leaves have no inbound dep refs.
+        assert!(reuse
+            .bundles
+            .iter()
+            .any(|b| b.bundle_name == "Qmtex_windows"));
+    }
+
+    #[test]
+    fn derive_skips_glb_with_missing_dep_unless_magenta_tolerant() {
+        let store = store_with_entity(
+            "missing-dep",
+            serde_json::json!([
+                {"file": "m.gltf", "hash": "Qmglb"},
+                {"file": "t.png", "hash": "Qmtex"},
+            ]),
+        );
+        store.write("Qmglb", GLTF_JSON.as_bytes()).unwrap();
+        store.write("Qmtex", b"PNG").unwrap();
+        let cache = abgen::glbscan::UriCache::new();
+
+        // Strict: unresolvable "a.bin" skips the glb (upstream skipped-assets
+        // semantics) but leaves the rest of the entity intact.
+        let strict =
+            derive_one_entity(&store, "bafyentity", "windows", &cache, toggles(true, false))
+                .unwrap();
+        assert!(glb_names(&strict).is_empty());
+        assert!(strict
+            .bundles
+            .iter()
+            .any(|b| b.bundle_name == "Qmtex_windows"));
+
+        // Magenta-tolerant: digest over the resolvable subset.
+        let tolerant =
+            derive_one_entity(&store, "bafyentity", "windows", &cache, toggles(true, true))
+                .unwrap();
+        let digest = abgen::naming::compute_deps_digest(&[(
+            "t.png".to_string(),
+            "Qmtex".to_string(),
+        )]);
+        assert_eq!(glb_names(&tolerant), vec![format!("Qmglb_{digest}_windows")]);
+    }
+}

crate/src/bin/abgen-corpus/dedup.rs

@@ -422,7 +422,7 @@ fn build_candidate(
                     .iter()
                     .min()
                     .ok_or_else(|| anyhow::anyhow!("variant has no claimants"))?;
-                let entry = derive_one_entity(store, rep, platform, uri_cache)
+                let entry = derive_one_entity(store, rep, platform, uri_cache, toggles)
                     .ok_or_else(|| anyhow::anyhow!("re-derive of {rep} failed"))?;
                 let spec = entry
                     .bundles

crate/src/bin/abgen-corpus/main.rs

@@ -80,6 +80,12 @@ pub(crate) struct EffectiveToggles {
     pub(crate) v38_compat: bool,
     pub(crate) v38_timestamp: i64,
     pub(crate) magenta_missing: bool,
+
+    /// Canonical `{hash}_{depsdigest}_{platform}` naming for glb/gltf
+    /// bundles (upstream asset-reuse parity). Default ON (ab-cdn runs
+    /// asset-reuse since v49); ABGEN_ASSET_REUSE=0 opts out for parity runs
+    /// against pre-v49 reference trees.
+    pub(crate) asset_reuse: bool,
 }
 
 const BIN_NAME: &str = "abgen-corpus";
@@ -495,6 +501,7 @@ fn run() -> Result<()> {
                 v38_compat: set_v38 || (!parity_mode && BuildOpts::env_v38_compat()),
                 v38_timestamp: BuildOpts::env_v38_timestamp(),
                 magenta_missing: BuildOpts::env_magenta_missing(),
+                asset_reuse: abgen::clihelp::env_bool("ABGEN_ASSET_REUSE", true),
             }
         }
         Err(msg) => {
@@ -526,7 +533,8 @@ fn run() -> Result<()> {
         let cdir = content_dir
             .or_else(|| std::env::var(ABGEN_CONTENT_ROOT_ENV).ok())
             .unwrap_or_else(|| DEFAULT_CONTENT_ROOT.to_string());
-        let (m, summary) = from_live_reference(Path::new(live_ref), &cdir, &platform, per_vintage)?;
+        let (m, summary) =
+            from_live_reference(Path::new(live_ref), &cdir, &platform, per_vintage, toggles)?;
         live_sample_summary = Some(summary);
         (m, out_root)
     } else if let Some(ids_path) = entity_ids_path {
@@ -592,6 +600,7 @@ fn run() -> Result<()> {
             &platform,
             cdn_layout,
             fetch_missing.then_some(content_server_url.as_str()),
+            toggles,
         )?;
         (m, out_root)
     } else if !worlds.is_empty() {
@@ -625,7 +634,7 @@ fn run() -> Result<()> {
         if ids.is_empty() {
             return Err(anyhow!("--world resolved no scene entities"));
         }
-        let m = manifest_from_ids(&ids, &cdir, &platform, cdn_layout)?;
+        let m = manifest_from_ids(&ids, &cdir, &platform, cdn_layout, toggles)?;
         (m, out_root)
     } else if let Some(urn) = collection_urn {
         if positional.len() != 1 {

crate/src/bin/abgen-corpus/sources.rs

@@ -1,5 +1,5 @@
 use crate::build::{derive_one_entity, load_entity_json, IMAGE_EXTS};
-use crate::{BundleSpec, ContentItem, EntityEntry, Manifest};
+use crate::{BundleSpec, ContentItem, EffectiveToggles, EntityEntry, Manifest};
 use abgen::glbscan::file_ext_lower;
 use abgen::hashes;
 use abgen::local_store::LocalContentStore;
@@ -221,6 +221,7 @@ pub(crate) fn from_entity_ids(
     platform: &str,
     cdn_layout: bool,
     fetch_from: Option<&str>,
+    toggles: EffectiveToggles,
 ) -> Result<Manifest> {
     let raw = std::fs::read_to_string(ids_path).with_context(|| format!("read {ids_path}"))?;
     let ids: Vec<String> = raw
@@ -232,7 +233,7 @@ pub(crate) fn from_entity_ids(
     if let Some(csu) = fetch_from {
         fetch_ids_into_store(&LocalContentStore::new(content_dir), csu, &ids);
     }
-    manifest_from_ids(&ids, content_dir, platform, cdn_layout)
+    manifest_from_ids(&ids, content_dir, platform, cdn_layout, toggles)
 }
 
 pub(crate) fn contents_base_url(content_server_url: &str) -> String {
@@ -302,6 +303,7 @@ pub(crate) fn manifest_from_ids(
     content_dir: &str,
     platform: &str,
     keep_shared_bundles: bool,
+    toggles: EffectiveToggles,
 ) -> Result<Manifest> {
     let store = LocalContentStore::new(content_dir);
     let missing = AtomicUsize::new(0);
@@ -323,7 +325,7 @@ pub(crate) fn manifest_from_ids(
                     secs,
                 );
             }
-            let entry = derive_one_entity(&store, ent_id, platform, &uri_cache);
+            let entry = derive_one_entity(&store, ent_id, platform, &uri_cache, toggles);
             if entry.is_none() {
                 missing.fetch_add(1, Ordering::Relaxed);
             }
@@ -456,6 +458,7 @@ pub(crate) fn from_live_reference(
     content_dir: &str,
     platform: &str,
     per_vintage: usize,
+    toggles: EffectiveToggles,
 ) -> Result<(Manifest, serde_json::Value)> {
     let ents = scan_live_reference(ref_dir, platform)?;
     if ents.is_empty() {
@@ -499,7 +502,7 @@ pub(crate) fn from_live_reference(
         .map(|e| (e.entity_id.as_str(), &e.files))
         .collect();
 
-    let mut m = manifest_from_ids(&ids, content_dir, platform, true)?;
+    let mut m = manifest_from_ids(&ids, content_dir, platform, true, toggles)?;
     let mut dropped_entities = 0usize;
     for e in &mut m.entities {
         if let Some(files) = allowed.get(e.entity_id.as_str()) {