Skip to content

Commit e15afa7

Browse files
eordanodalkiaclaude
authored
fix: digest tooling + mergeable (#4)
* abgen: asset-reuse parity — canonical glb names, shared assets layout, probe Scene glb/gltf bundles now carry the upstream converter's deps digest in their canonical name ({hash}_{depsdigest}_{platform}) in both the live/JIT server and abgen-corpus, matching the asset-reuse flow the ab-cdn deployment has run since v49. On by default; ABGEN_ASSET_REUSE=0 opts out for parity runs against pre-v49 reference trees. - naming: filter deps digests to GLB_DEP_EXTENSIONS (.bin + textures), mirroring upstream computeDepsDigest — fixes digest divergence for glbs referencing non-texture uris (also corrects the wearables path); add split_bundle_stem for parsing canonical names - live: per-entity deps-digest map (magenta-tolerant when enabled), canonical naming + digest validation in the JIT build path, shared {version}/assets/ space layout with entity-scoped read fallback, pre-build HEAD probe that lists space hits in the manifest without rebuilding - space: signed HEAD support for the probe - abcdn: flat lane strips the digest for owner-entity resolution - corpus: canonical naming in derive_one_entity behind EffectiveToggles.asset_reuse; strict mode skips digest-less glbs (upstream skipped-assets semantics) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * abgen-corpus: --pointer mode, progress heartbeats, deps-digest naming controls Local-workflow ergonomics on top of the asset-reuse parity change: - --pointer <x,y | urn | entityId>: single-target convenience mode — resolves the pointer via the content server (CatalystClient:: resolve_scene already handled all three forms), then runs the same fused derive+build path as --entity-ids without needing an ids file. With --cdn-layout it also copies manifests to the production CDN naming (<out>/manifest/<entity>_<platform>.json). from_entity_ids folded into the shared ids-vector path (now dead) and removed. - fetch heartbeat (worlds.rs): scenes with 1000+ content files sat silent for minutes during --fetch-missing/--world; now prints progress at most every 2s while downloads are actually happening. - build heartbeat (corpus build.rs): the per-entity progress line only fires every 5000 entities, so a single big scene (hundreds of bundles) looked hung; now prints completed-bundle counts every 2s. - naming controls renamed to say what they do: corpus flag --no-deps-digest (legacy {hash}_{platform} glb names, for consumers that request bundles by bare content hash) and env ABGEN_DEPS_DIGEST=0 (was ABGEN_ASSET_REUSE; also disables the shared assets/ space layout + pre-build probe on the server). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * style: rustfmt Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Juan Molteni <juanignaciomolteni@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
1 parent b996bf3 commit e15afa7

8 files changed

Lines changed: 182 additions & 82 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ crate/third_party/**/build/
2727
/data
2828
/runs
2929
/content
30+
/out
3031
*.log
3132
*.env
3233
.env

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,7 @@ Enabled when `ABGEN_S3_BUCKET` is non-empty (or `ABGEN_USE_SPACE=1`):
161161
### Asset-reuse mode (upstream converter parity)
162162
ON by default — the ab-cdn deployment has run asset-reuse since v49. Scene glb/gltf bundles use
163163
the upstream converter's canonical naming and shared bucket layout (applies to the JIT server
164-
and `abgen-corpus`). Set `ABGEN_ASSET_REUSE=0` to fall back to legacy `{hash}_{platform}` names
164+
and `abgen-corpus`). Set `ABGEN_DEPS_DIGEST=0` to fall back to legacy `{hash}_{platform}` names
165165
and entity-scoped space keys — needed only for parity runs against pre-v49 reference trees
166166
(e.g. `--live-mode` sampling of v15–v41 vintages, whose manifests list non-digest names).
167167
- glb/gltf bundles are named `{hash}_{depsdigest}_{platform}` — the digest is a 128-bit hash of

‎crate/src/abcdn/mod.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,15 +90,15 @@ pub async fn build_state(cfg: &Config) -> Result<AppState> {
9090
"ab-cdn S3 space cache ENABLED (read-through + write-back)"
9191
);
9292
}
93-
if crate::clihelp::env_bool("ABGEN_ASSET_REUSE", true) {
93+
if crate::clihelp::env_bool("ABGEN_DEPS_DIGEST", true) {
9494
tracing::info!(
9595
"asset-reuse mode ON (default): canonical glb names \
9696
({{hash}}_{{depsdigest}}_{{platform}}), shared {{version}}/assets/ space layout, \
9797
pre-build probe"
9898
);
9999
} else {
100100
tracing::warn!(
101-
"asset-reuse mode DISABLED (ABGEN_ASSET_REUSE=0): legacy {{hash}}_{{platform}} \
101+
"asset-reuse mode DISABLED (ABGEN_DEPS_DIGEST=0): legacy {{hash}}_{{platform}} \
102102
names + entity-scoped space keys — only for parity against pre-v49 references"
103103
);
104104
}

‎crate/src/bin/abgen-corpus/build.rs‎

Lines changed: 55 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -483,6 +483,28 @@ pub(crate) fn run_fused_entity_ids(
483483
errs: &errs,
484484
skipped: &skipped,
485485
};
486+
// Per-bundle heartbeat: the per-entity progress line above only fires
487+
// every 5000 entities, so a single big scene (hundreds of bundles) sits
488+
// silent for minutes. Prints at most every 2s while bundles complete.
489+
let last_print_ms = std::sync::atomic::AtomicU64::new(0);
490+
let heartbeat = || {
491+
let elapsed_ms = t0.elapsed().as_millis() as u64;
492+
let last = last_print_ms.load(Ordering::Relaxed);
493+
if elapsed_ms.saturating_sub(last) >= 2000
494+
&& last_print_ms
495+
.compare_exchange(last, elapsed_ms, Ordering::Relaxed, Ordering::Relaxed)
496+
.is_ok()
497+
{
498+
let b = built.load(Ordering::Relaxed);
499+
let s = skipped.load(Ordering::Relaxed);
500+
let e = errs.load(Ordering::Relaxed);
501+
eprintln!(
502+
" build: {} bundles done (built={b} skipped={s} errs={e}, {:.0}s)",
503+
b + s + e,
504+
t0.elapsed().as_secs_f64()
505+
);
506+
}
507+
};
486508
let primary = &platforms[0];
487509

488510
ids.par_iter().for_each(|ent_id| {
@@ -565,6 +587,7 @@ pub(crate) fn run_fused_entity_ids(
565587
&counters,
566588
);
567589
}
590+
heartbeat();
568591
});
569592
for (pi, plat) in platforms.iter().enumerate() {
570593
match write_cdn_manifest(
@@ -667,10 +690,8 @@ mod tests {
667690
}
668691

669692
fn store_with_entity(tag: &str, content: serde_json::Value) -> LocalContentStore {
670-
let dir = std::env::temp_dir().join(format!(
671-
"abgen-corpus-derive-{tag}-{}",
672-
std::process::id()
673-
));
693+
let dir =
694+
std::env::temp_dir().join(format!("abgen-corpus-derive-{tag}-{}", std::process::id()));
674695
let _ = std::fs::remove_dir_all(&dir);
675696
let store = LocalContentStore::new(&dir);
676697
let entity = serde_json::json!({"type": "scene", "content": content});
@@ -707,14 +728,24 @@ mod tests {
707728
store.write("Qmtex", b"PNG").unwrap();
708729
let cache = abgen::glbscan::UriCache::new();
709730

710-
let legacy =
711-
derive_one_entity(&store, "bafyentity", "windows", &cache, toggles(false, false))
712-
.unwrap();
731+
let legacy = derive_one_entity(
732+
&store,
733+
"bafyentity",
734+
"windows",
735+
&cache,
736+
toggles(false, false),
737+
)
738+
.unwrap();
713739
assert_eq!(glb_names(&legacy), vec!["Qmglb_windows".to_string()]);
714740

715-
let reuse =
716-
derive_one_entity(&store, "bafyentity", "windows", &cache, toggles(true, false))
717-
.unwrap();
741+
let reuse = derive_one_entity(
742+
&store,
743+
"bafyentity",
744+
"windows",
745+
&cache,
746+
toggles(true, false),
747+
)
748+
.unwrap();
718749
let digest = abgen::naming::compute_deps_digest(&[
719750
("a.bin".to_string(), "Qmbin".to_string()),
720751
("t.png".to_string(), "Qmtex".to_string()),
@@ -742,9 +773,14 @@ mod tests {
742773

743774
// Strict: unresolvable "a.bin" skips the glb (upstream skipped-assets
744775
// semantics) but leaves the rest of the entity intact.
745-
let strict =
746-
derive_one_entity(&store, "bafyentity", "windows", &cache, toggles(true, false))
747-
.unwrap();
776+
let strict = derive_one_entity(
777+
&store,
778+
"bafyentity",
779+
"windows",
780+
&cache,
781+
toggles(true, false),
782+
)
783+
.unwrap();
748784
assert!(glb_names(&strict).is_empty());
749785
assert!(strict
750786
.bundles
@@ -755,10 +791,11 @@ mod tests {
755791
let tolerant =
756792
derive_one_entity(&store, "bafyentity", "windows", &cache, toggles(true, true))
757793
.unwrap();
758-
let digest = abgen::naming::compute_deps_digest(&[(
759-
"t.png".to_string(),
760-
"Qmtex".to_string(),
761-
)]);
762-
assert_eq!(glb_names(&tolerant), vec![format!("Qmglb_{digest}_windows")]);
794+
let digest =
795+
abgen::naming::compute_deps_digest(&[("t.png".to_string(), "Qmtex".to_string())]);
796+
assert_eq!(
797+
glb_names(&tolerant),
798+
vec![format!("Qmglb_{digest}_windows")]
799+
);
763800
}
764801
}

‎crate/src/bin/abgen-corpus/main.rs‎

Lines changed: 83 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@ mod sources;
1919

2020
use build::{build_bundle_at, run_fused_entity_ids, write_cdn_manifest, BuildCounters};
2121
use sources::{
22-
fetch_ids_into_store, from_collection_urn, from_entity_ids, from_live_reference,
23-
from_reference, manifest_from_ids,
22+
fetch_ids_into_store, from_collection_urn, from_live_reference, from_reference,
23+
manifest_from_ids,
2424
};
2525

2626
const DEFAULT_LAMBDAS_URL: &str = "http://localhost:5141/lambdas";
@@ -83,7 +83,7 @@ pub(crate) struct EffectiveToggles {
8383

8484
/// Canonical `{hash}_{depsdigest}_{platform}` naming for glb/gltf
8585
/// bundles (upstream asset-reuse parity). Default ON (ab-cdn runs
86-
/// asset-reuse since v49); ABGEN_ASSET_REUSE=0 opts out for parity runs
86+
/// asset-reuse since v49); ABGEN_DEPS_DIGEST=0 opts out for parity runs
8787
/// against pre-v49 reference trees.
8888
pub(crate) asset_reuse: bool,
8989
}
@@ -103,6 +103,13 @@ fn usage_text() -> &'static str {
103103
abgen-corpus --entity-ids <ids.txt> <out-dir> \\\n \
104104
[--platform windows|mac] [--content-dir <dir>] [--cdn-layout] \\\n \
105105
[--fetch-missing] [--content-server-url <url>] [-j JOBS]\n \
106+
abgen-corpus --pointer <x,y | urn | entityId> <out-dir> \\\n \
107+
[--platform windows|mac] [--content-dir <dir>] [--cdn-layout] \\\n \
108+
[--fetch-missing] [--no-deps-digest] [-j JOBS]\n \
109+
(single-target convenience: resolves the pointer via the content\n \
110+
server, then behaves like --entity-ids with that one entity;\n \
111+
with --cdn-layout also copies manifests to the production CDN\n \
112+
naming <out>/manifest/<entity>_<platform>.json)\n \
106113
abgen-corpus --collection-urn <urn> <out-dir> \\\n \
107114
[--lambdas-url <url>] [--platform windows|mac] [--content-dir <dir>] \\\n \
108115
[--fetch-missing] [-j JOBS]\n \
@@ -209,6 +216,11 @@ fn usage_text() -> &'static str {
209216
default rebuilds/overwrites every bundle (golden/determinism workflows\n \
210217
rely on that).\n\
211218
\n\
219+
--no-deps-digest: legacy {hash}_{platform} glb names instead of the\n \
220+
canonical {hash}_{depsdigest}_{platform} (for consumers that request\n \
221+
bundles by bare content hash, or pre-v49 parity). Equivalent to\n \
222+
ABGEN_DEPS_DIGEST=0.\n\
223+
\n\
212224
--gpu: enable the GPU BC7/BC5 encode path (needs a binary built with\n \
213225
--features gpu; exits 2 otherwise).\n\
214226
\n\
@@ -279,6 +291,8 @@ fn run() -> Result<()> {
279291
let mut skip_existing = false;
280292
let mut force = false;
281293
let mut fetch_missing = false;
294+
let mut no_deps_digest = false;
295+
let mut pointer_target: Option<String> = None;
282296
let mut i = 0;
283297
while i < argv.len() {
284298
match argv[i].as_str() {
@@ -363,6 +377,10 @@ fn run() -> Result<()> {
363377
i += 1;
364378
entity_ids_path = Some(argv.get(i).cloned().unwrap_or_else(|| usage()));
365379
}
380+
"--pointer" => {
381+
i += 1;
382+
pointer_target = Some(argv.get(i).cloned().unwrap_or_else(|| usage()));
383+
}
366384
"--worlds-url" => {
367385
i += 1;
368386
worlds_url_flag = Some(argv.get(i).cloned().unwrap_or_else(|| usage()));
@@ -383,6 +401,9 @@ fn run() -> Result<()> {
383401
"--skip-existing" => {
384402
skip_existing = true;
385403
}
404+
"--no-deps-digest" => {
405+
no_deps_digest = true;
406+
}
386407
"--fetch-missing" => {
387408
fetch_missing = true;
388409
}
@@ -423,13 +444,17 @@ fn run() -> Result<()> {
423444
eprintln!("error: --cdn-layout is incompatible with --flat / --collection-urn");
424445
usage();
425446
}
426-
if cdn_layout && entity_ids_path.is_none() && worlds.is_empty() {
427-
eprintln!("error: --cdn-layout currently requires --entity-ids or --world");
447+
if cdn_layout && entity_ids_path.is_none() && pointer_target.is_none() && worlds.is_empty() {
448+
eprintln!("error: --cdn-layout currently requires --entity-ids, --pointer or --world");
428449
usage();
429450
}
430-
if fetch_missing && entity_ids_path.is_none() && collection_urn.is_none() {
451+
if fetch_missing
452+
&& entity_ids_path.is_none()
453+
&& pointer_target.is_none()
454+
&& collection_urn.is_none()
455+
{
431456
eprintln!(
432-
"error: --fetch-missing requires --entity-ids or --collection-urn \
457+
"error: --fetch-missing requires --entity-ids, --pointer or --collection-urn \
433458
(--world always fetches)"
434459
);
435460
usage();
@@ -456,10 +481,10 @@ fn run() -> Result<()> {
456481
);
457482
usage();
458483
}
459-
if !(cdn_layout && entity_ids_path.is_some()) {
484+
if !(cdn_layout && (entity_ids_path.is_some() || pointer_target.is_some())) {
460485
eprintln!(
461-
"error: --platform with a comma list requires --entity-ids --cdn-layout \
462-
(the fused encode-once pass)"
486+
"error: --platform with a comma list requires --entity-ids/--pointer \
487+
--cdn-layout (the fused encode-once pass)"
463488
);
464489
usage();
465490
}
@@ -470,6 +495,16 @@ fn run() -> Result<()> {
470495
eprintln!("error: --world conflicts with --entity-ids/--from-reference/--collection-urn");
471496
usage();
472497
}
498+
if pointer_target.is_some()
499+
&& (entity_ids_path.is_some()
500+
|| !worlds.is_empty()
501+
|| from_ref.is_some()
502+
|| collection_urn.is_some()
503+
|| live_mode.is_some())
504+
{
505+
eprintln!("error: --pointer conflicts with the other entity-source modes");
506+
usage();
507+
}
473508
if cdn_layout {
474509
let valid_version = ab_version.len() >= 2
475510
&& ab_version.as_bytes().first() == Some(&b'v')
@@ -501,7 +536,7 @@ fn run() -> Result<()> {
501536
v38_compat: set_v38 || (!parity_mode && BuildOpts::env_v38_compat()),
502537
v38_timestamp: BuildOpts::env_v38_timestamp(),
503538
magenta_missing: BuildOpts::env_magenta_missing(),
504-
asset_reuse: abgen::clihelp::env_bool("ABGEN_ASSET_REUSE", true),
539+
asset_reuse: !no_deps_digest && abgen::clihelp::env_bool("ABGEN_DEPS_DIGEST", true),
505540
}
506541
}
507542
Err(msg) => {
@@ -537,23 +572,32 @@ fn run() -> Result<()> {
537572
from_live_reference(Path::new(live_ref), &cdir, &platform, per_vintage, toggles)?;
538573
live_sample_summary = Some(summary);
539574
(m, out_root)
540-
} else if let Some(ids_path) = entity_ids_path {
575+
} else if entity_ids_path.is_some() || pointer_target.is_some() {
541576
if positional.len() != 1 {
542577
usage();
543578
}
544579
let out_root = PathBuf::from(&positional[0]);
545580
let cdir = content_dir
546581
.or_else(|| std::env::var(ABGEN_CONTENT_ROOT_ENV).ok())
547582
.unwrap_or_else(|| DEFAULT_CONTENT_ROOT.to_string());
548-
if cdn_layout {
583+
let ids: Vec<String> = if let Some(ids_path) = &entity_ids_path {
549584
let raw =
550-
std::fs::read_to_string(&ids_path).with_context(|| format!("read {ids_path}"))?;
551-
let ids: Vec<String> = raw
552-
.lines()
585+
std::fs::read_to_string(ids_path).with_context(|| format!("read {ids_path}"))?;
586+
raw.lines()
553587
.map(|l| l.trim())
554588
.filter(|l| !l.is_empty() && !l.starts_with('#'))
555589
.map(|l| l.to_string())
556-
.collect();
590+
.collect()
591+
} else {
592+
let target = pointer_target.as_deref().unwrap();
593+
let client = abgen::catalyst::CatalystClient::from_args(&content_server_url, None);
594+
let scene = client
595+
.resolve_scene(target)
596+
.with_context(|| format!("resolve --pointer {target:?}"))?;
597+
eprintln!("pointer {target:?} -> entity {}", scene.entity_id);
598+
vec![scene.entity_id]
599+
};
600+
if cdn_layout {
557601
let store = LocalContentStore::new(&cdir);
558602
if fetch_missing {
559603
fetch_ids_into_store(&store, &content_server_url, &ids);
@@ -583,6 +627,23 @@ fn run() -> Result<()> {
583627
"reconcile: divergent={} rebuilt={} relinked={} errs={}",
584628
o.reconcile.divergent, o.reconcile.rebuilt, o.reconcile.relinked, o.reconcile.errs
585629
);
630+
// Single-target convenience: also emit the manifests under the
631+
// production CDN naming (manifest/<entity>_<platform>.json).
632+
if pointer_target.is_some() {
633+
let man_dir = out_root.join("manifest");
634+
std::fs::create_dir_all(&man_dir)?;
635+
for id in &ids {
636+
for plat in &platforms {
637+
let src = out_root.join(id).join(format!("{plat}.manifest.json"));
638+
if src.is_file() {
639+
let dst = man_dir.join(format!("{id}_{plat}.json"));
640+
std::fs::copy(&src, &dst)?;
641+
eprintln!("manifest: {}", dst.display());
642+
eprintln!("bundles: {}", out_root.join(id).join(plat).display());
643+
}
644+
}
645+
}
646+
}
586647
let n_errs = o.errs + o.manifest_errs + o.reconcile.errs;
587648
let total = o.built + o.skipped + o.errs;
588649
println!(
@@ -594,14 +655,11 @@ fn run() -> Result<()> {
594655
}
595656
return Ok(());
596657
}
597-
let m = from_entity_ids(
598-
&ids_path,
599-
&cdir,
600-
&platform,
601-
cdn_layout,
602-
fetch_missing.then_some(content_server_url.as_str()),
603-
toggles,
604-
)?;
658+
let store = LocalContentStore::new(&cdir);
659+
if fetch_missing {
660+
fetch_ids_into_store(&store, &content_server_url, &ids);
661+
}
662+
let m = manifest_from_ids(&ids, &cdir, &platform, cdn_layout, toggles)?;
605663
(m, out_root)
606664
} else if !worlds.is_empty() {
607665
if positional.len() != 1 {

‎crate/src/bin/abgen-corpus/sources.rs‎

Lines changed: 0 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -215,27 +215,6 @@ fn metadata_deps_for_glb(
215215
out
216216
}
217217

218-
pub(crate) fn from_entity_ids(
219-
ids_path: &str,
220-
content_dir: &str,
221-
platform: &str,
222-
cdn_layout: bool,
223-
fetch_from: Option<&str>,
224-
toggles: EffectiveToggles,
225-
) -> Result<Manifest> {
226-
let raw = std::fs::read_to_string(ids_path).with_context(|| format!("read {ids_path}"))?;
227-
let ids: Vec<String> = raw
228-
.lines()
229-
.map(|l| l.trim())
230-
.filter(|l| !l.is_empty() && !l.starts_with('#'))
231-
.map(|l| l.to_string())
232-
.collect();
233-
if let Some(csu) = fetch_from {
234-
fetch_ids_into_store(&LocalContentStore::new(content_dir), csu, &ids);
235-
}
236-
manifest_from_ids(&ids, content_dir, platform, cdn_layout, toggles)
237-
}
238-
239218
pub(crate) fn contents_base_url(content_server_url: &str) -> String {
240219
format!("{}/contents/", content_server_url.trim_end_matches('/'))
241220
}

0 commit comments

Comments
 (0)