From 69f9854de647b89cf4dc5f5018db92d3b0c7e80d Mon Sep 17 00:00:00 2001 From: Joshua Liebow-Feeser Date: Sat, 6 Jun 2026 01:18:28 +0000 Subject: [PATCH] [anneal][release] Add exocrate archive metadata helpers gherrit-pr-id: Gra3t2uewe7dw5cfr6mu3gnvggplikenc --- .../tools/collect-release-archive-metadata.py | 55 +++++++ .../tools/test_exocrate_metadata_helpers.py | 132 +++++++++++++++ anneal/tools/update-exocrate-metadata.py | 155 ++++++++++++++++++ 3 files changed, 342 insertions(+) create mode 100755 anneal/tools/collect-release-archive-metadata.py create mode 100644 anneal/tools/test_exocrate_metadata_helpers.py create mode 100755 anneal/tools/update-exocrate-metadata.py diff --git a/anneal/tools/collect-release-archive-metadata.py b/anneal/tools/collect-release-archive-metadata.py new file mode 100755 index 0000000000..8c5b376c7a --- /dev/null +++ b/anneal/tools/collect-release-archive-metadata.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# +# Copyright 2026 The Fuchsia Authors +# +# Licensed under a BSD-style license , Apache License, Version 2.0 +# , or the MIT +# license , at your option. +# This file may not be copied, modified, or distributed except according to +# those terms. + +"""Emit JSON metadata for one published Anneal toolchain archive.""" + +import argparse +import hashlib +import json +from pathlib import Path + + +def sha256_file(path: Path) -> str: + hasher = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + hasher.update(chunk) + return hasher.hexdigest() + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--archive", required=True, type=Path) + parser.add_argument("--target", required=True) + parser.add_argument("--os", required=True) + parser.add_argument("--arch", required=True) + parser.add_argument("--url", required=True) + parser.add_argument("--out", required=True, type=Path) + args = parser.parse_args() + + archive = args.archive.resolve() + if not archive.is_file(): + raise SystemExit(f"archive does not exist: {archive}") + + metadata = { + "target": args.target, + "os": args.os, + "arch": args.arch, + "filename": archive.name, + "sha256": sha256_file(archive), + "url": args.url, + } + + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n") + + +if __name__ == "__main__": + main() diff --git a/anneal/tools/test_exocrate_metadata_helpers.py b/anneal/tools/test_exocrate_metadata_helpers.py new file mode 100644 index 0000000000..19d5184cef --- /dev/null +++ b/anneal/tools/test_exocrate_metadata_helpers.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +# +# Copyright 2026 The Fuchsia Authors +# +# Licensed under a BSD-style license , Apache License, Version 2.0 +# , or the MIT +# license , at your option. +# This file may not be copied, modified, or distributed except according to +# those terms. + +"""Unit tests for Anneal exocrate metadata helper scripts.""" + +from __future__ import annotations + +import hashlib +import importlib.util +import tempfile +import unittest +from pathlib import Path + + +TOOLS = Path(__file__).resolve().parent + + +def load_script(name: str): + spec = importlib.util.spec_from_file_location(name.replace("-", "_"), TOOLS / f"{name}.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +collect_metadata = load_script("collect-release-archive-metadata") +update_metadata = load_script("update-exocrate-metadata") + + +def exocrate_section(os_name: str, arch: str, sha256: str, url: str) -> str: + return f"""[package.metadata.exocrate.{os_name}.{arch}] +sha256 = "{sha256}" +url = "{url}" +""" + + +class ExocrateMetadataHelperTests(unittest.TestCase): + def test_collect_archive_metadata_hashes_bytes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + archive = Path(tmp) / "archive.tar.zst" + archive.write_bytes(b"archive contents") + + self.assertEqual( + collect_metadata.sha256_file(archive), + hashlib.sha256(b"archive contents").hexdigest(), + ) + + def test_update_manifest_requires_complete_platform_metadata(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + cargo_toml = root / "Cargo.toml" + original_sha = "0" * 64 + cargo_toml.write_text( + "[package]\nname = \"cargo-anneal\"\n\n" + + "".join( + exocrate_section(os_name, arch, original_sha, f"https://example.com/{os_name}-{arch}.tar.zst") + for os_name, arch in sorted(update_metadata.EXPECTED_PLATFORMS) + ), + encoding="utf-8", + ) + + metadata = { + platform: { + "sha256": f"{i + 1:064x}", + "url": f"https://github.com/google/zerocopy/releases/download/tag/{target}.tar.zst", + } + for i, (platform, target) in enumerate(sorted(update_metadata.EXPECTED_TARGETS.items())) + } + update_metadata.update_manifest(cargo_toml, metadata) + updated = cargo_toml.read_text(encoding="utf-8") + + for platform, values in metadata.items(): + os_name, arch = platform + self.assertIn(f"[package.metadata.exocrate.{os_name}.{arch}]", updated) + self.assertIn(f'sha256 = "{values["sha256"]}"', updated) + self.assertIn(f'url = "{values["url"]}"', updated) + self.assertNotIn(original_sha, updated) + + def test_load_metadata_rejects_wrong_target_duplicate_and_wrong_tag(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + linux = root / "linux.json" + linux.write_text( + """{ + "target": "linux-x86_64", + "os": "linux", + "arch": "x86_64", + "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "url": "https://github.com/google/zerocopy/releases/download/tag/archive.tar.zst" +} +""", + encoding="utf-8", + ) + duplicate = root / "duplicate.json" + duplicate.write_text(linux.read_text(encoding="utf-8"), encoding="utf-8") + wrong_target = root / "wrong-target.json" + wrong_target.write_text( + linux.read_text(encoding="utf-8").replace("linux-x86_64", "macos-x86_64"), + encoding="utf-8", + ) + wrong_tag = root / "wrong-tag.json" + wrong_tag.write_text( + linux.read_text(encoding="utf-8").replace("/tag/", "/other-tag/"), + encoding="utf-8", + ) + + self.assertEqual( + update_metadata.load_metadata([linux], expected_release_tag="tag"), + { + ("linux", "x86_64"): { + "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "url": "https://github.com/google/zerocopy/releases/download/tag/archive.tar.zst", + } + }, + ) + with self.assertRaises(SystemExit): + update_metadata.load_metadata([linux, duplicate], expected_release_tag="tag") + with self.assertRaises(SystemExit): + update_metadata.load_metadata([wrong_target], expected_release_tag="tag") + with self.assertRaises(SystemExit): + update_metadata.load_metadata([wrong_tag], expected_release_tag="tag") + + +if __name__ == "__main__": + unittest.main() diff --git a/anneal/tools/update-exocrate-metadata.py b/anneal/tools/update-exocrate-metadata.py new file mode 100755 index 0000000000..f8e2fc4516 --- /dev/null +++ b/anneal/tools/update-exocrate-metadata.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +# +# Copyright 2026 The Fuchsia Authors +# +# Licensed under a BSD-style license , Apache License, Version 2.0 +# , or the MIT +# license , at your option. +# This file may not be copied, modified, or distributed except according to +# those terms. + +"""Update anneal/Cargo.toml's exocrate archive URLs and hashes.""" + +import argparse +import json +import re +from pathlib import Path + +EXPECTED_PLATFORMS = { + ("linux", "x86_64"), + ("linux", "aarch64"), + ("macos", "x86_64"), + ("macos", "aarch64"), +} + +EXPECTED_TARGETS = { + ("linux", "x86_64"): "linux-x86_64", + ("linux", "aarch64"): "linux-aarch64", + ("macos", "x86_64"): "macos-x86_64", + ("macos", "aarch64"): "macos-aarch64", +} + +SECTION_RE = re.compile(r"^\[package\.metadata\.exocrate\.([^.]+)\.([^\]]+)\]$") +SHA_RE = re.compile(r'^\s*sha256\s*=\s*"[0-9a-fA-F]{64}"\s*$') +URL_RE = re.compile(r'^\s*url\s*=\s*".*"\s*$') + + +def load_metadata( + paths: list[Path], expected_release_tag: str | None +) -> dict[tuple[str, str], dict[str, str]]: + metadata = {} + for path in paths: + value = json.loads(path.read_text()) + try: + target = value["target"] + os_name = value["os"] + arch = value["arch"] + sha256 = value["sha256"] + url = value["url"] + except KeyError as e: + raise SystemExit(f"{path}: missing required key {e.args[0]!r}") from e + if not all(isinstance(field, str) for field in (target, os_name, arch, sha256, url)): + raise SystemExit(f"{path}: target, os, arch, sha256, and url must all be strings") + if not re.fullmatch(r"[0-9a-f]{64}", sha256): + raise SystemExit(f"{path}: sha256 must be 64 lowercase hex characters") + platform = (os_name, arch) + expected_target = EXPECTED_TARGETS.get(platform) + if expected_target is not None and target != expected_target: + raise SystemExit( + f"{path}: target {target!r} does not match platform " + f"{os_name}.{arch}; expected {expected_target!r}" + ) + if expected_release_tag is not None: + expected_url_component = f"/releases/download/{expected_release_tag}/" + if expected_url_component not in url: + raise SystemExit( + f"{path}: url does not point at release tag " + f"{expected_release_tag!r}: {url}" + ) + if platform in metadata: + raise SystemExit(f"duplicate metadata for {os_name}.{arch}") + metadata[platform] = {"sha256": sha256, "url": url} + return metadata + + +def metadata_files(metadata_dir: Path) -> list[Path]: + return sorted(path for path in metadata_dir.glob("*.json") if path.is_file()) + + +def update_manifest(cargo_toml: Path, metadata: dict[tuple[str, str], dict[str, str]]) -> None: + lines = cargo_toml.read_text().splitlines() + seen = set() + updated = {platform: set() for platform in metadata} + current_platform = None + + for i, line in enumerate(lines): + section = SECTION_RE.match(line) + if section: + platform = (section.group(1), section.group(2)) + current_platform = platform if platform in metadata else None + if current_platform is not None: + seen.add(current_platform) + continue + + if current_platform is None: + continue + + values = metadata[current_platform] + if SHA_RE.match(line): + lines[i] = f'sha256 = "{values["sha256"]}"' + updated[current_platform].add("sha256") + elif URL_RE.match(line): + lines[i] = f'url = "{values["url"]}"' + updated[current_platform].add("url") + + missing_sections = set(metadata) - seen + if missing_sections: + formatted = ", ".join(f"{os_name}.{arch}" for os_name, arch in sorted(missing_sections)) + raise SystemExit(f"Cargo.toml is missing exocrate sections for: {formatted}") + + incomplete = { + platform: {"sha256", "url"} - fields + for platform, fields in updated.items() + if {"sha256", "url"} - fields + } + if incomplete: + formatted = ", ".join( + f"{os_name}.{arch} missing {','.join(sorted(fields))}" + for (os_name, arch), fields in sorted(incomplete.items()) + ) + raise SystemExit(f"Cargo.toml has incomplete exocrate sections: {formatted}") + + cargo_toml.write_text("\n".join(lines) + "\n") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--cargo-toml", default="anneal/Cargo.toml", type=Path) + parser.add_argument("--metadata-dir", type=Path) + parser.add_argument("--metadata", action="append", default=[], type=Path) + parser.add_argument("--expected-release-tag") + parser.add_argument("--require-all", action="store_true") + args = parser.parse_args() + + paths = list(args.metadata) + if args.metadata_dir is not None: + paths.extend(metadata_files(args.metadata_dir)) + if not paths: + raise SystemExit("no metadata JSON files provided") + + metadata = load_metadata(paths, args.expected_release_tag) + if args.require_all and set(metadata) != EXPECTED_PLATFORMS: + missing = EXPECTED_PLATFORMS - set(metadata) + extra = set(metadata) - EXPECTED_PLATFORMS + messages = [] + if missing: + messages.append("missing " + ", ".join(f"{os_name}.{arch}" for os_name, arch in sorted(missing))) + if extra: + messages.append("unexpected " + ", ".join(f"{os_name}.{arch}" for os_name, arch in sorted(extra))) + raise SystemExit("; ".join(messages)) + + update_manifest(args.cargo_toml, metadata) + + +if __name__ == "__main__": + main()