Skip to content

Commit 13250a9

Browse files
eordanoeordano
committed
abgen: standalone Decentraland asset-bundle converter + ab-cdn parity compare pipeline
Release v0.9.9.
0 parents  commit 13250a9

1,053 files changed

Lines changed: 330122 additions & 0 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/ci.yml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
name: ci
2+
3+
on:
4+
push:
5+
branches: [main]
6+
pull_request:
7+
8+
concurrency:
9+
group: ${{ github.workflow }}-${{ github.ref }}
10+
cancel-in-progress: true
11+
12+
env:
13+
CARGO_TERM_COLOR: always
14+
CARGO_INCREMENTAL: "0"
15+
CARGO_PROFILE_DEV_DEBUG: "0"
16+
CARGO_PROFILE_TEST_DEBUG: "0"
17+
RUSTFLAGS: -D warnings
18+
19+
jobs:
20+
fmt:
21+
name: rustfmt
22+
runs-on: ubuntu-latest
23+
steps:
24+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
25+
- uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable @ 2026-07-08
26+
with:
27+
toolchain: "1.97.0"
28+
components: rustfmt
29+
- run: cargo fmt --all -- --check
30+
31+
check:
32+
name: clippy + test
33+
runs-on: ubuntu-latest
34+
steps:
35+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
36+
- uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable @ 2026-07-08
37+
with:
38+
toolchain: "1.97.0"
39+
components: clippy
40+
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
41+
# --test-threads=1: the lib tests share the ABGEN_ROOT env (lodgen vs emote-template race).
42+
- run: cargo clippy --workspace --all-targets -- -D warnings
43+
- run: ABGEN_ROOT="$GITHUB_WORKSPACE" cargo test --workspace --lib -- --test-threads=1

‎.github/workflows/image.yml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
name: image
2+
3+
on:
4+
push:
5+
tags: ["v*"]
6+
workflow_dispatch:
7+
8+
permissions:
9+
contents: read
10+
packages: write
11+
12+
concurrency:
13+
group: image-${{ github.ref }}
14+
cancel-in-progress: true
15+
16+
jobs:
17+
image:
18+
name: build + push (nix)
19+
runs-on: ubuntu-22.04
20+
steps:
21+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
22+
23+
- uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 # v22
24+
with:
25+
extra-conf: |
26+
max-jobs = auto
27+
cores = 0
28+
29+
- uses: nix-community/cache-nix-action@7df957e333c1e5da7721f60227dbba6d06080569 # v7
30+
with:
31+
primary-key: nix-${{ runner.os }}-${{ hashFiles('flake.lock', 'Cargo.lock') }}
32+
restore-prefixes-first-match: nix-${{ runner.os }}-
33+
34+
- name: build image
35+
run: |
36+
ref="${GITHUB_REF_NAME:-dev}"
37+
sha="$(git rev-parse --short HEAD)"
38+
echo "::group::nix build .#dockerImage — ${GITHUB_REPOSITORY} @ ${ref} (${sha})"
39+
nix build .#dockerImage --print-build-logs
40+
echo "store path : $(readlink -f result)"
41+
echo "compressed : $(ls -lLh result | awk '{print $5}')"
42+
echo "layers : $(zcat result | tar -xO manifest.json | grep -o '"[a-f0-9]*/layer.tar"' | wc -l)"
43+
echo "::endgroup::"
44+
{
45+
echo "REF=${ref}"
46+
echo "IMG_SIZE=$(ls -lLh result | awk '{print $5}')"
47+
} >> "$GITHUB_ENV"
48+
49+
- name: push to ghcr
50+
run: |
51+
repo="${GITHUB_REPOSITORY,,}"
52+
conf="$(mktemp)"
53+
echo 'unqualified-search-registries = []' > "$conf"
54+
for tag in "${REF}" latest; do
55+
echo "::group::push ghcr.io/${repo}:${tag}"
56+
nix run nixpkgs#skopeo -- --registries-conf "$conf" --insecure-policy copy \
57+
--dest-creds "${{ github.actor }}:${{ github.token }}" \
58+
"docker-archive:$(readlink -f result)" \
59+
"docker://ghcr.io/${repo}:${tag}"
60+
echo "::endgroup::"
61+
done
62+
{
63+
echo "### abgen container image"
64+
echo ""
65+
echo "| | |"
66+
echo "|---|---|"
67+
echo "| image | \`ghcr.io/${repo}:${REF}\` (and \`:latest\`) |"
68+
echo "| size | ${IMG_SIZE} compressed, no base OS |"
69+
echo "| build | \`nix build .#dockerImage\` — deterministic, pinned \`flake.lock\` |"
70+
} >> "$GITHUB_STEP_SUMMARY"

‎.github/workflows/release.yml‎

Lines changed: 237 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,237 @@
1+
name: release
2+
3+
on:
4+
push:
5+
tags: ["v*"]
6+
workflow_dispatch:
7+
8+
permissions:
9+
contents: write
10+
11+
env:
12+
CARGO_TERM_COLOR: always
13+
CARGO_INCREMENTAL: "0"
14+
15+
jobs:
16+
build:
17+
name: ${{ matrix.name }}
18+
runs-on: ${{ matrix.runner }}
19+
strategy:
20+
fail-fast: false
21+
matrix:
22+
include:
23+
- name: x86_64-unknown-linux-gnu
24+
runner: ubuntu-22.04
25+
target: x86_64-unknown-linux-gnu
26+
bin: abgen
27+
- name: aarch64-unknown-linux-gnu
28+
runner: ubuntu-22.04-arm
29+
target: aarch64-unknown-linux-gnu
30+
bin: abgen
31+
- name: x86_64-pc-windows-gnu
32+
runner: ubuntu-22.04
33+
target: x86_64-pc-windows-gnu
34+
bin: abgen.exe
35+
- name: aarch64-pc-windows-gnullvm
36+
runner: ubuntu-22.04
37+
target: aarch64-pc-windows-gnullvm
38+
bin: abgen.exe
39+
- name: aarch64-apple-darwin
40+
runner: macos-14
41+
target: aarch64-apple-darwin
42+
bin: abgen
43+
- name: x86_64-apple-darwin
44+
# the last free Intel image; scheduled to live through August 2027
45+
runner: macos-15-intel
46+
target: x86_64-apple-darwin
47+
bin: abgen
48+
49+
steps:
50+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
51+
52+
- uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable @ 2026-07-08
53+
with:
54+
toolchain: "1.97.0"
55+
targets: ${{ matrix.target }}
56+
57+
- name: install mingw cross toolchain
58+
if: matrix.target == 'x86_64-pc-windows-gnu'
59+
run: |
60+
sudo apt-get update
61+
sudo apt-get install -y --no-install-recommends g++-mingw-w64-x86-64-posix cmake
62+
cat > /tmp/mingw-toolchain.cmake <<'EOF'
63+
set(CMAKE_SYSTEM_NAME Windows)
64+
set(CMAKE_SYSTEM_PROCESSOR x86_64)
65+
set(CMAKE_C_COMPILER x86_64-w64-mingw32-gcc-posix)
66+
set(CMAKE_CXX_COMPILER x86_64-w64-mingw32-g++-posix)
67+
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
68+
EOF
69+
{
70+
echo "CMAKE_TOOLCHAIN_FILE=/tmp/mingw-toolchain.cmake"
71+
echo "CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER=x86_64-w64-mingw32-gcc-posix"
72+
echo "CC_x86_64_pc_windows_gnu=x86_64-w64-mingw32-gcc-posix"
73+
echo "CXX_x86_64_pc_windows_gnu=x86_64-w64-mingw32-g++-posix"
74+
echo "AR_x86_64_pc_windows_gnu=x86_64-w64-mingw32-ar"
75+
} >> "$GITHUB_ENV"
76+
77+
- name: install llvm-mingw cross toolchain
78+
if: matrix.target == 'aarch64-pc-windows-gnullvm'
79+
run: |
80+
curl -fsSL -o llvm-mingw.tar.xz https://github.com/mstorsjo/llvm-mingw/releases/download/20260616/llvm-mingw-20260616-ucrt-ubuntu-22.04-x86_64.tar.xz
81+
echo "534b92e067b22a6b4441f48ae9240a3341b17825d04d577eab0cf85c44b4deda llvm-mingw.tar.xz" | sha256sum -c
82+
sudo tar -xJf llvm-mingw.tar.xz -C /opt && rm llvm-mingw.tar.xz
83+
cat > /tmp/aarch64-mingw-toolchain.cmake <<'EOF'
84+
set(CMAKE_SYSTEM_NAME Windows)
85+
set(CMAKE_SYSTEM_PROCESSOR aarch64)
86+
set(CMAKE_C_COMPILER aarch64-w64-mingw32-clang)
87+
set(CMAKE_CXX_COMPILER aarch64-w64-mingw32-clang++)
88+
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
89+
EOF
90+
# drop the import libs so lld can only resolve libc++/libunwind statically:
91+
# the explicit -lc++ the cc crate emits would otherwise bind to the DLLs
92+
sudo rm /opt/llvm-mingw-20260616-ucrt-ubuntu-22.04-x86_64/aarch64-w64-mingw32/lib/libc++.dll.a \
93+
/opt/llvm-mingw-20260616-ucrt-ubuntu-22.04-x86_64/aarch64-w64-mingw32/lib/libunwind.dll.a
94+
echo "/opt/llvm-mingw-20260616-ucrt-ubuntu-22.04-x86_64/bin" >> "$GITHUB_PATH"
95+
{
96+
echo "CMAKE_TOOLCHAIN_FILE=/tmp/aarch64-mingw-toolchain.cmake"
97+
echo "CARGO_TARGET_AARCH64_PC_WINDOWS_GNULLVM_LINKER=aarch64-w64-mingw32-clang"
98+
echo "CC_aarch64_pc_windows_gnullvm=aarch64-w64-mingw32-clang"
99+
echo "CXX_aarch64_pc_windows_gnullvm=aarch64-w64-mingw32-clang++"
100+
echo "AR_aarch64_pc_windows_gnullvm=aarch64-w64-mingw32-ar"
101+
} >> "$GITHUB_ENV"
102+
103+
- name: reproducibility environment
104+
run: |
105+
rustflags="--remap-path-prefix $PWD=/build --remap-path-prefix $HOME=/home"
106+
if [ "${{ matrix.target }}" = "x86_64-pc-windows-gnu" ]; then
107+
rustflags="$rustflags -C link-arg=-Wl,--no-insert-timestamp"
108+
fi
109+
if [ "${{ matrix.target }}" = "aarch64-pc-windows-gnullvm" ]; then
110+
# crt-static folds the unwinder in; lld links deterministically
111+
rustflags="$rustflags -C target-feature=+crt-static -C link-arg=-Wl,--no-insert-timestamp"
112+
fi
113+
{
114+
echo "SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)"
115+
echo "RUSTFLAGS=$rustflags"
116+
echo "CFLAGS=-ffile-prefix-map=$PWD=/build"
117+
echo "CXXFLAGS=-ffile-prefix-map=$PWD=/build"
118+
} >> "$GITHUB_ENV"
119+
120+
- name: build (twice + bit-identical; windows-x64 once)
121+
run: |
122+
cargo build --release --locked --target ${{ matrix.target }} --bin abgen
123+
if [ "${{ matrix.target }}" = "x86_64-pc-windows-gnu" ]; then
124+
echo "windows-gnu: built once; mingw ld has residual link-order non-determinism the bit-identical gate cannot pass"
125+
exit 0
126+
fi
127+
cp "target/${{ matrix.target }}/release/${{ matrix.bin }}" /tmp/build-1
128+
rm -rf target
129+
cargo build --release --locked --target ${{ matrix.target }} --bin abgen
130+
cp "target/${{ matrix.target }}/release/${{ matrix.bin }}" /tmp/build-2
131+
h1=$(shasum -a 256 /tmp/build-1 | cut -d' ' -f1)
132+
h2=$(shasum -a 256 /tmp/build-2 | cut -d' ' -f1)
133+
echo "build-1: $h1"
134+
echo "build-2: $h2"
135+
test "$h1" = "$h2"
136+
137+
- name: assemble archive (binary + template + shader)
138+
run: |
139+
dist="abgen-${GITHUB_REF_NAME:-dev}-${{ matrix.name }}"
140+
mkdir -p "$dist/shader"
141+
cp "target/${{ matrix.target }}/release/${{ matrix.bin }}" "$dist/"
142+
cp -R template "$dist/template"
143+
cp crate/shader/scene_ignore_windows "$dist/shader/"
144+
cp LICENSE README.md "$dist/"
145+
# windows loads these DLLs from the exe directory, keeping the archive self-contained
146+
if [ "${{ matrix.target }}" = "x86_64-pc-windows-gnu" ]; then
147+
for dll in libstdc++-6.dll libgcc_s_seh-1.dll libwinpthread-1.dll; do
148+
cp "$(x86_64-w64-mingw32-g++-posix -print-file-name=$dll)" "$dist/"
149+
done
150+
fi
151+
# deterministic tar needs GNU tar; macOS runners ship it as gtar
152+
TAR=tar; command -v gtar >/dev/null && TAR=gtar
153+
"$TAR" --sort=name --owner=0 --group=0 --numeric-owner \
154+
--mtime="@${SOURCE_DATE_EPOCH}" -czf "$dist.tar.gz" "$dist"
155+
shasum -a 256 "$dist.tar.gz" > "$dist.tar.gz.sha256"
156+
157+
- name: smoke test the unpacked archive
158+
if: ${{ !contains(matrix.target, 'windows') }}
159+
run: |
160+
dist="abgen-${GITHUB_REF_NAME:-dev}-${{ matrix.name }}"
161+
rm -rf /tmp/smoke && mkdir /tmp/smoke
162+
tar -xzf "$dist.tar.gz" -C /tmp/smoke
163+
cd "/tmp/smoke/$dist"
164+
./abgen --version
165+
HTTP_SERVER_PORT=5199 ./abgen &
166+
server=$!
167+
for _ in $(seq 1 40); do
168+
curl -sf http://127.0.0.1:5199/readyz && ok=1 && break
169+
sleep 0.5
170+
done
171+
kill "$server" || true
172+
test "${ok:-0}" = 1
173+
174+
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
175+
with:
176+
name: ${{ matrix.name }}
177+
path: |
178+
abgen-*.tar.gz
179+
abgen-*.tar.gz.sha256
180+
if-no-files-found: error
181+
182+
release:
183+
needs: build
184+
runs-on: ubuntu-22.04
185+
if: startsWith(github.ref, 'refs/tags/v')
186+
steps:
187+
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
188+
with:
189+
path: dist
190+
merge-multiple: true
191+
192+
- name: checksums
193+
run: |
194+
cd dist
195+
cat ./*.sha256 > SHA256SUMS.txt
196+
cat SHA256SUMS.txt
197+
198+
- name: create release
199+
env:
200+
GH_TOKEN: ${{ github.token }}
201+
run: |
202+
gh release create "$GITHUB_REF_NAME" \
203+
--repo "$GITHUB_REPOSITORY" \
204+
--title "abgen $GITHUB_REF_NAME" \
205+
--notes "Reproducible builds: the linux and macOS binaries were each built twice from a clean tree in CI and required bit-identical before publishing; the windows binary is built once (mingw's linker has a residual link-order non-determinism). Toolchain: rust 1.97.0 (pinned), \`--locked\` against the committed Cargo.lock, \`SOURCE_DATE_EPOCH\` from the tagged commit. Each archive is self-contained: the \`abgen\` server binary plus the \`template/\` and \`shader/\` assets it needs, resolved from the binary's own directory (\`ABGEN_ROOT\`/\`ABGEN_SHADER_BUNDLE\` override). Runtime requirements - Linux (x64 and arm64): glibc >= 2.35 and libstdc++6/libgcc_s (present on any machine that runs node). Windows x64: none, the mingw runtime DLLs ship in the archive next to the exe. Windows arm64: none, a single static exe (llvm-mingw/libc++). macOS (Apple Silicon and Intel): system libraries only. Optional everywhere: libturbojpeg (bit-parity JPEG decode with the upstream ab-cdn pipeline; without it JPEG textures decode through a pure-Rust fallback - valid output, not guaranteed byte-identical to upstream bundles). The same binaries publish to npm as \`@dcl/abgen\` (per-platform packages as optionalDependencies; \`npx @dcl/abgen\` runs the server)." \
206+
dist/*.tar.gz dist/SHA256SUMS.txt
207+
208+
npm:
209+
needs: build
210+
runs-on: ubuntu-22.04
211+
if: startsWith(github.ref, 'refs/tags/v')
212+
permissions:
213+
contents: read
214+
# npm --provenance attests the packages against this workflow run
215+
id-token: write
216+
steps:
217+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
218+
219+
- uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
220+
with:
221+
node-version: "22"
222+
registry-url: "https://registry.npmjs.org"
223+
224+
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
225+
with:
226+
path: dist
227+
merge-multiple: true
228+
229+
- name: publish npm packages
230+
env:
231+
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
232+
run: |
233+
if [ -z "$NODE_AUTH_TOKEN" ]; then
234+
echo "NPM_TOKEN secret not configured - skipping npm publish"
235+
exit 0
236+
fi
237+
bash npm/publish.sh "${GITHUB_REF_NAME#v}" dist

‎.gitignore‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# Rust
2+
/target
3+
/crate/target
4+
/crate/wasm-poc/target/
5+
.cargo-home/
6+
7+
# wasm lab build artifact — crate/wasm-poc/build.sh regenerates it
8+
/site/wasm/abgen_poc.wasm
9+
10+
# Generated C/C++ build artifacts (cmake out-of-source builds in the vendored
11+
# third_party crates). Must never be committed: the cmake cache bakes an
12+
# absolute path, so a stale copy breaks the build in a new location (incl. the
13+
# nix sandbox). cmake regenerates these from source.
14+
crate/third_party/draco_decoder/third_party/draco/build/
15+
crate/third_party/**/build/
16+
**/CMakeCache.txt
17+
**/CMakeFiles/
18+
*.a
19+
*.o
20+
21+
# Per-checkout tool config (machine-local unity paths; see
22+
# pipeline/abgencompare/toolconfig.py)
23+
/abgen-compare.json
24+
25+
# Runtime caches / output / env
26+
/abgen-serve-cache
27+
/data
28+
/runs
29+
/content
30+
*.log
31+
*.env
32+
.env
33+
.env.*
34+
35+
# Nix
36+
/result
37+
/result-*
38+
39+
# Editor / OS noise
40+
__pycache__/
41+
.DS_Store
42+
*.swp
43+
node_modules/

0 commit comments

Comments
 (0)