abgen-corpus: --pointer mode, progress heartbeats, deps-digest naming controls

Local-workflow ergonomics on top of the asset-reuse parity change:

- --pointer <x,y | urn | entityId>: single-target convenience mode —
  resolves the pointer via the content server (CatalystClient::
  resolve_scene already handled all three forms), then runs the same
  fused derive+build path as --entity-ids without needing an ids file.
  With --cdn-layout it also copies manifests to the production CDN
  naming (<out>/manifest/<entity>_<platform>.json). from_entity_ids
  folded into the shared ids-vector path (now dead) and removed.
- fetch heartbeat (worlds.rs): scenes with 1000+ content files sat
  silent for minutes during --fetch-missing/--world; now prints
  progress at most every 2s while downloads are actually happening.
- build heartbeat (corpus build.rs): the per-entity progress line only
  fires every 5000 entities, so a single big scene (hundreds of
  bundles) looked hung; now prints completed-bundle counts every 2s.
- naming controls renamed to say what they do: corpus flag
  --no-deps-digest (legacy {hash}_{platform} glb names, for consumers
  that request bundles by bare content hash) and env
  ABGEN_DEPS_DIGEST=0 (was ABGEN_ASSET_REUSE; also disables the shared
  assets/ space layout + pre-build probe on the server).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: dalkia

.gitignore

@@ -27,6 +27,7 @@ crate/third_party/**/build/
 /data
 /runs
 /content
+/out
 *.log
 *.env
 .env

README.md

@@ -161,7 +161,7 @@ Enabled when `ABGEN_S3_BUCKET` is non-empty (or `ABGEN_USE_SPACE=1`):
 ### Asset-reuse mode (upstream converter parity)
 ON by default — the ab-cdn deployment has run asset-reuse since v49. Scene glb/gltf bundles use
 the upstream converter's canonical naming and shared bucket layout (applies to the JIT server
-and `abgen-corpus`). Set `ABGEN_ASSET_REUSE=0` to fall back to legacy `{hash}_{platform}` names
+and `abgen-corpus`). Set `ABGEN_DEPS_DIGEST=0` to fall back to legacy `{hash}_{platform}` names
 and entity-scoped space keys — needed only for parity runs against pre-v49 reference trees
 (e.g. `--live-mode` sampling of v15–v41 vintages, whose manifests list non-digest names).
 - glb/gltf bundles are named `{hash}_{depsdigest}_{platform}` — the digest is a 128-bit hash of

crate/src/abcdn/mod.rs

@@ -90,15 +90,15 @@ pub async fn build_state(cfg: &Config) -> Result<AppState> {
             "ab-cdn S3 space cache ENABLED (read-through + write-back)"
         );
     }
-    if crate::clihelp::env_bool("ABGEN_ASSET_REUSE", true) {
+    if crate::clihelp::env_bool("ABGEN_DEPS_DIGEST", true) {
         tracing::info!(
             "asset-reuse mode ON (default): canonical glb names \
              ({{hash}}_{{depsdigest}}_{{platform}}), shared {{version}}/assets/ space layout, \
              pre-build probe"
         );
     } else {
         tracing::warn!(
-            "asset-reuse mode DISABLED (ABGEN_ASSET_REUSE=0): legacy {{hash}}_{{platform}} \
+            "asset-reuse mode DISABLED (ABGEN_DEPS_DIGEST=0): legacy {{hash}}_{{platform}} \
              names + entity-scoped space keys — only for parity against pre-v49 references"
         );
     }

crate/src/bin/abgen-corpus/build.rs

@@ -483,6 +483,28 @@ pub(crate) fn run_fused_entity_ids(
         errs: &errs,
         skipped: &skipped,
     };
+    // Per-bundle heartbeat: the per-entity progress line above only fires
+    // every 5000 entities, so a single big scene (hundreds of bundles) sits
+    // silent for minutes. Prints at most every 2s while bundles complete.
+    let last_print_ms = std::sync::atomic::AtomicU64::new(0);
+    let heartbeat = || {
+        let elapsed_ms = t0.elapsed().as_millis() as u64;
+        let last = last_print_ms.load(Ordering::Relaxed);
+        if elapsed_ms.saturating_sub(last) >= 2000
+            && last_print_ms
+                .compare_exchange(last, elapsed_ms, Ordering::Relaxed, Ordering::Relaxed)
+                .is_ok()
+        {
+            let b = built.load(Ordering::Relaxed);
+            let s = skipped.load(Ordering::Relaxed);
+            let e = errs.load(Ordering::Relaxed);
+            eprintln!(
+                "  build: {} bundles done (built={b} skipped={s} errs={e}, {:.0}s)",
+                b + s + e,
+                t0.elapsed().as_secs_f64()
+            );
+        }
+    };
     let primary = &platforms[0];
 
     ids.par_iter().for_each(|ent_id| {
@@ -565,6 +587,7 @@ pub(crate) fn run_fused_entity_ids(
                     &counters,
                 );
             }
+            heartbeat();
         });
         for (pi, plat) in platforms.iter().enumerate() {
             match write_cdn_manifest(

crate/src/bin/abgen-corpus/main.rs

@@ -19,8 +19,8 @@ mod sources;
 
 use build::{build_bundle_at, run_fused_entity_ids, write_cdn_manifest, BuildCounters};
 use sources::{
-    fetch_ids_into_store, from_collection_urn, from_entity_ids, from_live_reference,
-    from_reference, manifest_from_ids,
+    fetch_ids_into_store, from_collection_urn, from_live_reference, from_reference,
+    manifest_from_ids,
 };
 
 const DEFAULT_LAMBDAS_URL: &str = "http://localhost:5141/lambdas";
@@ -83,7 +83,7 @@ pub(crate) struct EffectiveToggles {
 
     /// Canonical `{hash}_{depsdigest}_{platform}` naming for glb/gltf
     /// bundles (upstream asset-reuse parity). Default ON (ab-cdn runs
-    /// asset-reuse since v49); ABGEN_ASSET_REUSE=0 opts out for parity runs
+    /// asset-reuse since v49); ABGEN_DEPS_DIGEST=0 opts out for parity runs
     /// against pre-v49 reference trees.
     pub(crate) asset_reuse: bool,
 }
@@ -103,6 +103,13 @@ fn usage_text() -> &'static str {
          abgen-corpus --entity-ids <ids.txt> <out-dir> \\\n               \
              [--platform windows|mac] [--content-dir <dir>] [--cdn-layout] \\\n               \
              [--fetch-missing] [--content-server-url <url>] [-j JOBS]\n  \
+         abgen-corpus --pointer <x,y | urn | entityId> <out-dir> \\\n               \
+             [--platform windows|mac] [--content-dir <dir>] [--cdn-layout] \\\n               \
+             [--fetch-missing] [--no-deps-digest] [-j JOBS]\n               \
+             (single-target convenience: resolves the pointer via the content\n               \
+              server, then behaves like --entity-ids with that one entity;\n               \
+              with --cdn-layout also copies manifests to the production CDN\n               \
+              naming <out>/manifest/<entity>_<platform>.json)\n  \
          abgen-corpus --collection-urn <urn> <out-dir> \\\n               \
              [--lambdas-url <url>] [--platform windows|mac] [--content-dir <dir>] \\\n               \
              [--fetch-missing] [-j JOBS]\n  \
@@ -209,6 +216,11 @@ fn usage_text() -> &'static str {
          default rebuilds/overwrites every bundle (golden/determinism workflows\n  \
          rely on that).\n\
          \n\
+         --no-deps-digest: legacy {hash}_{platform} glb names instead of the\n  \
+         canonical {hash}_{depsdigest}_{platform} (for consumers that request\n  \
+         bundles by bare content hash, or pre-v49 parity). Equivalent to\n  \
+         ABGEN_DEPS_DIGEST=0.\n\
+         \n\
          --gpu: enable the GPU BC7/BC5 encode path (needs a binary built with\n  \
          --features gpu; exits 2 otherwise).\n\
          \n\
@@ -279,6 +291,8 @@ fn run() -> Result<()> {
     let mut skip_existing = false;
     let mut force = false;
     let mut fetch_missing = false;
+    let mut no_deps_digest = false;
+    let mut pointer_target: Option<String> = None;
     let mut i = 0;
     while i < argv.len() {
         match argv[i].as_str() {
@@ -363,6 +377,10 @@ fn run() -> Result<()> {
                 i += 1;
                 entity_ids_path = Some(argv.get(i).cloned().unwrap_or_else(|| usage()));
             }
+            "--pointer" => {
+                i += 1;
+                pointer_target = Some(argv.get(i).cloned().unwrap_or_else(|| usage()));
+            }
             "--worlds-url" => {
                 i += 1;
                 worlds_url_flag = Some(argv.get(i).cloned().unwrap_or_else(|| usage()));
@@ -383,6 +401,9 @@ fn run() -> Result<()> {
             "--skip-existing" => {
                 skip_existing = true;
             }
+            "--no-deps-digest" => {
+                no_deps_digest = true;
+            }
             "--fetch-missing" => {
                 fetch_missing = true;
             }
@@ -423,13 +444,17 @@ fn run() -> Result<()> {
         eprintln!("error: --cdn-layout is incompatible with --flat / --collection-urn");
         usage();
     }
-    if cdn_layout && entity_ids_path.is_none() && worlds.is_empty() {
-        eprintln!("error: --cdn-layout currently requires --entity-ids or --world");
+    if cdn_layout && entity_ids_path.is_none() && pointer_target.is_none() && worlds.is_empty() {
+        eprintln!("error: --cdn-layout currently requires --entity-ids, --pointer or --world");
         usage();
     }
-    if fetch_missing && entity_ids_path.is_none() && collection_urn.is_none() {
+    if fetch_missing
+        && entity_ids_path.is_none()
+        && pointer_target.is_none()
+        && collection_urn.is_none()
+    {
         eprintln!(
-            "error: --fetch-missing requires --entity-ids or --collection-urn \
+            "error: --fetch-missing requires --entity-ids, --pointer or --collection-urn \
              (--world always fetches)"
         );
         usage();
@@ -456,10 +481,10 @@ fn run() -> Result<()> {
             );
             usage();
         }
-        if !(cdn_layout && entity_ids_path.is_some()) {
+        if !(cdn_layout && (entity_ids_path.is_some() || pointer_target.is_some())) {
             eprintln!(
-                "error: --platform with a comma list requires --entity-ids --cdn-layout \
-                 (the fused encode-once pass)"
+                "error: --platform with a comma list requires --entity-ids/--pointer \
+                 --cdn-layout (the fused encode-once pass)"
             );
             usage();
         }
@@ -470,6 +495,16 @@ fn run() -> Result<()> {
         eprintln!("error: --world conflicts with --entity-ids/--from-reference/--collection-urn");
         usage();
     }
+    if pointer_target.is_some()
+        && (entity_ids_path.is_some()
+            || !worlds.is_empty()
+            || from_ref.is_some()
+            || collection_urn.is_some()
+            || live_mode.is_some())
+    {
+        eprintln!("error: --pointer conflicts with the other entity-source modes");
+        usage();
+    }
     if cdn_layout {
         let valid_version = ab_version.len() >= 2
             && ab_version.as_bytes().first() == Some(&b'v')
@@ -501,7 +536,8 @@ fn run() -> Result<()> {
                 v38_compat: set_v38 || (!parity_mode && BuildOpts::env_v38_compat()),
                 v38_timestamp: BuildOpts::env_v38_timestamp(),
                 magenta_missing: BuildOpts::env_magenta_missing(),
-                asset_reuse: abgen::clihelp::env_bool("ABGEN_ASSET_REUSE", true),
+                asset_reuse: !no_deps_digest
+                    && abgen::clihelp::env_bool("ABGEN_DEPS_DIGEST", true),
             }
         }
         Err(msg) => {
@@ -537,23 +573,32 @@ fn run() -> Result<()> {
             from_live_reference(Path::new(live_ref), &cdir, &platform, per_vintage, toggles)?;
         live_sample_summary = Some(summary);
         (m, out_root)
-    } else if let Some(ids_path) = entity_ids_path {
+    } else if entity_ids_path.is_some() || pointer_target.is_some() {
         if positional.len() != 1 {
             usage();
         }
         let out_root = PathBuf::from(&positional[0]);
         let cdir = content_dir
             .or_else(|| std::env::var(ABGEN_CONTENT_ROOT_ENV).ok())
             .unwrap_or_else(|| DEFAULT_CONTENT_ROOT.to_string());
-        if cdn_layout {
+        let ids: Vec<String> = if let Some(ids_path) = &entity_ids_path {
             let raw =
-                std::fs::read_to_string(&ids_path).with_context(|| format!("read {ids_path}"))?;
-            let ids: Vec<String> = raw
-                .lines()
+                std::fs::read_to_string(ids_path).with_context(|| format!("read {ids_path}"))?;
+            raw.lines()
                 .map(|l| l.trim())
                 .filter(|l| !l.is_empty() && !l.starts_with('#'))
                 .map(|l| l.to_string())
-                .collect();
+                .collect()
+        } else {
+            let target = pointer_target.as_deref().unwrap();
+            let client = abgen::catalyst::CatalystClient::from_args(&content_server_url, None);
+            let scene = client
+                .resolve_scene(target)
+                .with_context(|| format!("resolve --pointer {target:?}"))?;
+            eprintln!("pointer {target:?} -> entity {}", scene.entity_id);
+            vec![scene.entity_id]
+        };
+        if cdn_layout {
             let store = LocalContentStore::new(&cdir);
             if fetch_missing {
                 fetch_ids_into_store(&store, &content_server_url, &ids);
@@ -583,6 +628,23 @@ fn run() -> Result<()> {
                 "reconcile: divergent={} rebuilt={} relinked={} errs={}",
                 o.reconcile.divergent, o.reconcile.rebuilt, o.reconcile.relinked, o.reconcile.errs
             );
+            // Single-target convenience: also emit the manifests under the
+            // production CDN naming (manifest/<entity>_<platform>.json).
+            if pointer_target.is_some() {
+                let man_dir = out_root.join("manifest");
+                std::fs::create_dir_all(&man_dir)?;
+                for id in &ids {
+                    for plat in &platforms {
+                        let src = out_root.join(id).join(format!("{plat}.manifest.json"));
+                        if src.is_file() {
+                            let dst = man_dir.join(format!("{id}_{plat}.json"));
+                            std::fs::copy(&src, &dst)?;
+                            eprintln!("manifest: {}", dst.display());
+                            eprintln!("bundles:  {}", out_root.join(id).join(plat).display());
+                        }
+                    }
+                }
+            }
             let n_errs = o.errs + o.manifest_errs + o.reconcile.errs;
             let total = o.built + o.skipped + o.errs;
             println!(
@@ -594,14 +656,11 @@ fn run() -> Result<()> {
             }
             return Ok(());
         }
-        let m = from_entity_ids(
-            &ids_path,
-            &cdir,
-            &platform,
-            cdn_layout,
-            fetch_missing.then_some(content_server_url.as_str()),
-            toggles,
-        )?;
+        let store = LocalContentStore::new(&cdir);
+        if fetch_missing {
+            fetch_ids_into_store(&store, &content_server_url, &ids);
+        }
+        let m = manifest_from_ids(&ids, &cdir, &platform, cdn_layout, toggles)?;
         (m, out_root)
     } else if !worlds.is_empty() {
         if positional.len() != 1 {

crate/src/bin/abgen-corpus/sources.rs

@@ -215,26 +215,6 @@ fn metadata_deps_for_glb(
     out
 }
 
-pub(crate) fn from_entity_ids(
-    ids_path: &str,
-    content_dir: &str,
-    platform: &str,
-    cdn_layout: bool,
-    fetch_from: Option<&str>,
-    toggles: EffectiveToggles,
-) -> Result<Manifest> {
-    let raw = std::fs::read_to_string(ids_path).with_context(|| format!("read {ids_path}"))?;
-    let ids: Vec<String> = raw
-        .lines()
-        .map(|l| l.trim())
-        .filter(|l| !l.is_empty() && !l.starts_with('#'))
-        .map(|l| l.to_string())
-        .collect();
-    if let Some(csu) = fetch_from {
-        fetch_ids_into_store(&LocalContentStore::new(content_dir), csu, &ids);
-    }
-    manifest_from_ids(&ids, content_dir, platform, cdn_layout, toggles)
-}
 
 pub(crate) fn contents_base_url(content_server_url: &str) -> String {
     format!("{}/contents/", content_server_url.trim_end_matches('/'))

crate/src/live.rs

@@ -730,7 +730,7 @@ pub struct ProxyConfig {
 
     /// Canonical glb naming + shared assets space layout + build probe
     /// (upstream asset-reuse parity). Default ON — the ab-cdn deployment has
-    /// run asset-reuse since v49. ABGEN_ASSET_REUSE=0 opts out for parity
+    /// run asset-reuse since v49. ABGEN_DEPS_DIGEST=0 opts out for parity
     /// runs against pre-v49 reference trees.
     pub asset_reuse: bool,
 
@@ -766,7 +766,7 @@ impl Proxy {
         let v38_compat = !cfg.parity || BuildOpts::env_v38_compat();
         let v38_timestamp = BuildOpts::env_v38_timestamp();
         let magenta_missing = cfg.magenta_missing || BuildOpts::env_magenta_missing();
-        let asset_reuse = crate::clihelp::env_bool("ABGEN_ASSET_REUSE", cfg.asset_reuse);
+        let asset_reuse = crate::clihelp::env_bool("ABGEN_DEPS_DIGEST", cfg.asset_reuse);
         if let Some(root) = cfg.template_root.as_deref().filter(|s| !s.is_empty()) {
             let env_root = std::env::var("ABGEN_ROOT").unwrap_or_default();
             if env_root.trim() != root {

crate/src/worlds.rs

@@ -228,14 +228,43 @@ pub fn fetch_scene_into_store(
                 .collect()
         })
         .unwrap_or_default();
+    // Progress heartbeat: large scenes (1000+ content files) otherwise sit
+    // silent for minutes and look hung. Prints at most every 2s, and only
+    // while downloads are actually happening — fully-cached entities stay
+    // quiet.
+    let t0 = std::time::Instant::now();
+    let total = hashes.len();
+    let done = std::sync::atomic::AtomicUsize::new(0);
+    let downloaded = std::sync::atomic::AtomicUsize::new(0);
+    let last_print_ms = std::sync::atomic::AtomicU64::new(0);
+    use std::sync::atomic::Ordering;
     let fetched: usize = hashes
         .par_iter()
-        .map(|hash| match fetch_to_store(store, &scene.base_url, hash) {
-            Ok(new) => usize::from(new),
-            Err(e) => {
-                eprintln!("{}: {hash}: {e:#}", scene.entity_id);
-                0
+        .map(|hash| {
+            let new = match fetch_to_store(store, &scene.base_url, hash) {
+                Ok(new) => usize::from(new),
+                Err(e) => {
+                    eprintln!("{}: {hash}: {e:#}", scene.entity_id);
+                    0
+                }
+            };
+            let d = done.fetch_add(1, Ordering::Relaxed) + 1;
+            let dl = downloaded.fetch_add(new, Ordering::Relaxed) + new;
+            let elapsed_ms = t0.elapsed().as_millis() as u64;
+            let last = last_print_ms.load(Ordering::Relaxed);
+            if dl > 0
+                && elapsed_ms.saturating_sub(last) >= 2000
+                && last_print_ms
+                    .compare_exchange(last, elapsed_ms, Ordering::Relaxed, Ordering::Relaxed)
+                    .is_ok()
+            {
+                eprintln!(
+                    "fetch-missing: {}: {d}/{total} content files ({dl} downloaded, {:.0}s)",
+                    scene.entity_id,
+                    t0.elapsed().as_secs_f64()
+                );
             }
+            new
         })
         .sum();
     Ok((fetched, hashes.len()))