[anneal][release] Add exocrate archive metadata helpers

joshlf · joshlf · commit 3fc9363811bd · 2026-06-06T23:38:39.000Z
gherrit-pr-id: Gra3t2uewe7dw5cfr6mu3gnvggplikenc
diff --git a/anneal/tools/collect-release-archive-metadata.py b/anneal/tools/collect-release-archive-metadata.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+#
+# Copyright 2026 The Fuchsia Authors
+#
+# Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0
+# <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT
+# license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+"""Emit JSON metadata for one published Anneal toolchain archive."""
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+
+
+def sha256_file(path: Path) -> str:
+    hasher = hashlib.sha256()
+    with path.open("rb") as f:
+        for chunk in iter(lambda: f.read(1024 * 1024), b""):
+            hasher.update(chunk)
+    return hasher.hexdigest()
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--archive", required=True, type=Path)
+    parser.add_argument("--target", required=True)
+    parser.add_argument("--os", required=True)
+    parser.add_argument("--arch", required=True)
+    parser.add_argument("--url", required=True)
+    parser.add_argument("--out", required=True, type=Path)
+    args = parser.parse_args()
+
+    archive = args.archive.resolve()
+    if not archive.is_file():
+        raise SystemExit(f"archive does not exist: {archive}")
+
+    metadata = {
+        "target": args.target,
+        "os": args.os,
+        "arch": args.arch,
+        "filename": archive.name,
+        "sha256": sha256_file(archive),
+        "url": args.url,
+    }
+
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/anneal/tools/test_exocrate_metadata_helpers.py b/anneal/tools/test_exocrate_metadata_helpers.py
@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+#
+# Copyright 2026 The Fuchsia Authors
+#
+# Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0
+# <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT
+# license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+"""Unit tests for Anneal exocrate metadata helper scripts."""
+
+from __future__ import annotations
+
+import hashlib
+import importlib.util
+import tempfile
+import unittest
+from pathlib import Path
+
+
+TOOLS = Path(__file__).resolve().parent
+
+
+def load_script(name: str):
+    spec = importlib.util.spec_from_file_location(name.replace("-", "_"), TOOLS / f"{name}.py")
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+collect_metadata = load_script("collect-release-archive-metadata")
+update_metadata = load_script("update-exocrate-metadata")
+
+
+def exocrate_section(os_name: str, arch: str, sha256: str, url: str) -> str:
+    return f"""[package.metadata.exocrate.{os_name}.{arch}]
+sha256 = "{sha256}"
+url = "{url}"
+"""
+
+
+class ExocrateMetadataHelperTests(unittest.TestCase):
+    def test_collect_archive_metadata_hashes_bytes(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            archive = Path(tmp) / "archive.tar.zst"
+            archive.write_bytes(b"archive contents")
+
+            self.assertEqual(
+                collect_metadata.sha256_file(archive),
+                hashlib.sha256(b"archive contents").hexdigest(),
+            )
+
+    def test_update_manifest_requires_complete_platform_metadata(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            cargo_toml = root / "Cargo.toml"
+            original_sha = "0" * 64
+            cargo_toml.write_text(
+                "[package]\nname = \"cargo-anneal\"\n\n"
+                + "".join(
+                    exocrate_section(os_name, arch, original_sha, f"https://example.com/{os_name}-{arch}.tar.zst")
+                    for os_name, arch in sorted(update_metadata.EXPECTED_PLATFORMS)
+                ),
+                encoding="utf-8",
+            )
+
+            metadata = {
+                platform: {
+                    "sha256": f"{i + 1:064x}",
+                    "url": f"https://github.com/google/zerocopy/releases/download/tag/{target}.tar.zst",
+                }
+                for i, (platform, target) in enumerate(sorted(update_metadata.EXPECTED_TARGETS.items()))
+            }
+            update_metadata.update_manifest(cargo_toml, metadata)
+            updated = cargo_toml.read_text(encoding="utf-8")
+
+            for platform, values in metadata.items():
+                os_name, arch = platform
+                self.assertIn(f"[package.metadata.exocrate.{os_name}.{arch}]", updated)
+                self.assertIn(f'sha256 = "{values["sha256"]}"', updated)
+                self.assertIn(f'url = "{values["url"]}"', updated)
+            self.assertNotIn(original_sha, updated)
+
+    def test_load_metadata_rejects_wrong_target_duplicate_and_wrong_tag(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            linux = root / "linux.json"
+            linux.write_text(
+                """{
+  "target": "linux-x86_64",
+  "os": "linux",
+  "arch": "x86_64",
+  "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+  "url": "https://github.com/google/zerocopy/releases/download/tag/archive.tar.zst"
+}
+""",
+                encoding="utf-8",
+            )
+            duplicate = root / "duplicate.json"
+            duplicate.write_text(linux.read_text(encoding="utf-8"), encoding="utf-8")
+            wrong_target = root / "wrong-target.json"
+            wrong_target.write_text(
+                linux.read_text(encoding="utf-8").replace("linux-x86_64", "macos-x86_64"),
+                encoding="utf-8",
+            )
+            wrong_tag = root / "wrong-tag.json"
+            wrong_tag.write_text(
+                linux.read_text(encoding="utf-8").replace("/tag/", "/other-tag/"),
+                encoding="utf-8",
+            )
+
+            self.assertEqual(
+                update_metadata.load_metadata([linux], expected_release_tag="tag"),
+                {
+                    ("linux", "x86_64"): {
+                        "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+                        "url": "https://github.com/google/zerocopy/releases/download/tag/archive.tar.zst",
+                    }
+                },
+            )
+            with self.assertRaises(SystemExit):
+                update_metadata.load_metadata([linux, duplicate], expected_release_tag="tag")
+            with self.assertRaises(SystemExit):
+                update_metadata.load_metadata([wrong_target], expected_release_tag="tag")
+            with self.assertRaises(SystemExit):
+                update_metadata.load_metadata([wrong_tag], expected_release_tag="tag")
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/anneal/tools/update-exocrate-metadata.py b/anneal/tools/update-exocrate-metadata.py
@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+#
+# Copyright 2026 The Fuchsia Authors
+#
+# Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0
+# <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT
+# license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+"""Update anneal/Cargo.toml's exocrate archive URLs and hashes."""
+
+import argparse
+import json
+import re
+from pathlib import Path
+
+EXPECTED_PLATFORMS = {
+    ("linux", "x86_64"),
+    ("linux", "aarch64"),
+    ("macos", "x86_64"),
+    ("macos", "aarch64"),
+}
+
+EXPECTED_TARGETS = {
+    ("linux", "x86_64"): "linux-x86_64",
+    ("linux", "aarch64"): "linux-aarch64",
+    ("macos", "x86_64"): "macos-x86_64",
+    ("macos", "aarch64"): "macos-aarch64",
+}
+
+SECTION_RE = re.compile(r"^\[package\.metadata\.exocrate\.([^.]+)\.([^\]]+)\]$")
+SHA_RE = re.compile(r'^\s*sha256\s*=\s*"[0-9a-fA-F]{64}"\s*$')
+URL_RE = re.compile(r'^\s*url\s*=\s*".*"\s*$')
+
+
+def load_metadata(
+    paths: list[Path], expected_release_tag: str | None
+) -> dict[tuple[str, str], dict[str, str]]:
+    metadata = {}
+    for path in paths:
+        value = json.loads(path.read_text())
+        try:
+            target = value["target"]
+            os_name = value["os"]
+            arch = value["arch"]
+            sha256 = value["sha256"]
+            url = value["url"]
+        except KeyError as e:
+            raise SystemExit(f"{path}: missing required key {e.args[0]!r}") from e
+        if not all(isinstance(field, str) for field in (target, os_name, arch, sha256, url)):
+            raise SystemExit(f"{path}: target, os, arch, sha256, and url must all be strings")
+        if not re.fullmatch(r"[0-9a-f]{64}", sha256):
+            raise SystemExit(f"{path}: sha256 must be 64 lowercase hex characters")
+        platform = (os_name, arch)
+        expected_target = EXPECTED_TARGETS.get(platform)
+        if expected_target is not None and target != expected_target:
+            raise SystemExit(
+                f"{path}: target {target!r} does not match platform "
+                f"{os_name}.{arch}; expected {expected_target!r}"
+            )
+        if expected_release_tag is not None:
+            expected_url_component = f"/releases/download/{expected_release_tag}/"
+            if expected_url_component not in url:
+                raise SystemExit(
+                    f"{path}: url does not point at release tag "
+                    f"{expected_release_tag!r}: {url}"
+                )
+        if platform in metadata:
+            raise SystemExit(f"duplicate metadata for {os_name}.{arch}")
+        metadata[platform] = {"sha256": sha256, "url": url}
+    return metadata
+
+
+def metadata_files(metadata_dir: Path) -> list[Path]:
+    return sorted(path for path in metadata_dir.glob("*.json") if path.is_file())
+
+
+def update_manifest(cargo_toml: Path, metadata: dict[tuple[str, str], dict[str, str]]) -> None:
+    lines = cargo_toml.read_text().splitlines()
+    seen = set()
+    updated = {platform: set() for platform in metadata}
+    current_platform = None
+
+    for i, line in enumerate(lines):
+        section = SECTION_RE.match(line)
+        if section:
+            platform = (section.group(1), section.group(2))
+            current_platform = platform if platform in metadata else None
+            if current_platform is not None:
+                seen.add(current_platform)
+            continue
+
+        if current_platform is None:
+            continue
+
+        values = metadata[current_platform]
+        if SHA_RE.match(line):
+            lines[i] = f'sha256 = "{values["sha256"]}"'
+            updated[current_platform].add("sha256")
+        elif URL_RE.match(line):
+            lines[i] = f'url = "{values["url"]}"'
+            updated[current_platform].add("url")
+
+    missing_sections = set(metadata) - seen
+    if missing_sections:
+        formatted = ", ".join(f"{os_name}.{arch}" for os_name, arch in sorted(missing_sections))
+        raise SystemExit(f"Cargo.toml is missing exocrate sections for: {formatted}")
+
+    incomplete = {
+        platform: {"sha256", "url"} - fields
+        for platform, fields in updated.items()
+        if {"sha256", "url"} - fields
+    }
+    if incomplete:
+        formatted = ", ".join(
+            f"{os_name}.{arch} missing {','.join(sorted(fields))}"
+            for (os_name, arch), fields in sorted(incomplete.items())
+        )
+        raise SystemExit(f"Cargo.toml has incomplete exocrate sections: {formatted}")
+
+    cargo_toml.write_text("\n".join(lines) + "\n")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--cargo-toml", default="anneal/Cargo.toml", type=Path)
+    parser.add_argument("--metadata-dir", type=Path)
+    parser.add_argument("--metadata", action="append", default=[], type=Path)
+    parser.add_argument("--expected-release-tag")
+    parser.add_argument("--require-all", action="store_true")
+    args = parser.parse_args()
+
+    paths = list(args.metadata)
+    if args.metadata_dir is not None:
+        paths.extend(metadata_files(args.metadata_dir))
+    if not paths:
+        raise SystemExit("no metadata JSON files provided")
+
+    metadata = load_metadata(paths, args.expected_release_tag)
+    if args.require_all and set(metadata) != EXPECTED_PLATFORMS:
+        missing = EXPECTED_PLATFORMS - set(metadata)
+        extra = set(metadata) - EXPECTED_PLATFORMS
+        messages = []
+        if missing:
+            messages.append("missing " + ", ".join(f"{os_name}.{arch}" for os_name, arch in sorted(missing)))
+        if extra:
+            messages.append("unexpected " + ", ".join(f"{os_name}.{arch}" for os_name, arch in sorted(extra)))
+        raise SystemExit("; ".join(messages))
+
+    update_manifest(args.cargo_toml, metadata)
+
+
+if __name__ == "__main__":
+    main()
