From ca2b7551bf0f81ee97522d63b5ef0574fbd3a2e6 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Mon, 21 Sep 2026 15:17:56 +0800 Subject: [PATCH] Benchmark state scratch cleanup with verified Formula workloads --- .github/workflows/benchmark.yml | 100 +++++++++++++ benchmarks/.gitignore | 1 + benchmarks/Dockerfile | 12 ++ benchmarks/README.md | 34 +++++ benchmarks/build.sh | 28 ++++ benchmarks/engine_memory.py | 132 ++++++++++++++++++ benchmarks/guest-init.sh | 18 +++ benchmarks/prepare.py | 39 ++++++ benchmarks/reboot.c | 12 ++ benchmarks/report.py | 47 +++++++ benchmarks/run.py | 240 ++++++++++++++++++++++++++++++++ benchmarks/setup-firecracker.sh | 39 ++++++ benchmarks/workload.py | 53 +++++++ testdata/llar/benchmark/main.go | 193 +++++++++++++++++++++++++ 14 files changed, 948 insertions(+) create mode 100644 .github/workflows/benchmark.yml create mode 100644 benchmarks/.gitignore create mode 100644 benchmarks/Dockerfile create mode 100644 benchmarks/README.md create mode 100644 benchmarks/build.sh create mode 100644 benchmarks/engine_memory.py create mode 100644 benchmarks/guest-init.sh create mode 100644 benchmarks/prepare.py create mode 100644 benchmarks/reboot.c create mode 100644 benchmarks/report.py create mode 100644 benchmarks/run.py create mode 100644 benchmarks/setup-firecracker.sh create mode 100644 benchmarks/workload.py create mode 100644 testdata/llar/benchmark/main.go diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..a893289 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,100 @@ +name: Formula benchmark + +on: + workflow_dispatch: + push: + branches: [codex/benchmark-rooted-accounting] + paths: + - 'benchmarks/**' + - 'testdata/llar/benchmark/**' + - '.github/workflows/benchmark.yml' + +permissions: + contents: read + +jobs: + benchmark: + runs-on: ubuntu-latest + timeout-minutes: 60 + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v4 + - name: Check native Linux and KVM + run: | + uname -a + lscpu + test -c /dev/kvm + ls -l /dev/kvm + docker info + - name: Build common payload and toolchain image + id: build + run: docker build --label "org.opencontainers.image.revision=$(git rev-parse HEAD)" -f benchmarks/Dockerfile -t sandbox-formula-bench:ci . + - name: Prepare Firecracker wrapper and identical root filesystem + id: firecracker + run: sudo bash benchmarks/setup-firecracker.sh sandbox-formula-bench:ci /tmp/firecracker-benchmark + - name: Measure three independent cold starts per backend + if: ${{ !cancelled() && steps.firecracker.outcome == 'success' }} + run: | + mkdir -p results/cold + git rev-parse HEAD > results/source-revision.txt + for trial in 1 2 3; do + case "$trial" in + 1) order=firecracker,docker,sandbox ;; + 2) order=docker,sandbox,firecracker ;; + 3) order=sandbox,firecracker,docker ;; + esac + mkdir -p "results/cold/trial-$trial" + sudo python3 benchmarks/engine_memory.py --output "$PWD/results/cold/trial-$trial/engine-memory.jsonl" -- python3 benchmarks/run.py --image sandbox-formula-bench:ci --output "$PWD/results/cold/trial-$trial" --backends "$order" --firecracker-assets /tmp/firecracker-benchmark --concurrency 1 --builds 1 | tee "results/cold/trial-$trial/summary.jsonl" + done + - uses: actions/upload-artifact@v4 + if: always() + with: + name: formula-benchmark-cold-amd64 + path: results/cold/ + - name: Run Firecracker Formula benchmarks + if: ${{ !cancelled() && steps.firecracker.outcome == 'success' }} + run: | + mkdir -p results/firecracker + sudo python3 benchmarks/engine_memory.py --output "$PWD/results/firecracker/engine-memory.jsonl" -- python3 benchmarks/run.py --image sandbox-formula-bench:ci --output "$PWD/results/firecracker" --backends firecracker --firecracker-assets /tmp/firecracker-benchmark --concurrency 1,2,4,8 --builds 8 | tee results/firecracker/summary.jsonl + - uses: actions/upload-artifact@v4 + if: always() + with: + name: formula-benchmark-firecracker-amd64 + path: results/firecracker/ + - name: Run Docker Formula benchmarks + if: ${{ !cancelled() && steps.build.outcome == 'success' }} + run: | + mkdir -p results/docker + sudo python3 benchmarks/engine_memory.py --output "$PWD/results/docker/engine-memory.jsonl" -- python3 benchmarks/run.py --image sandbox-formula-bench:ci --output "$PWD/results/docker" --backends docker --concurrency 1,2,4,8 --builds 8 | tee results/docker/summary.jsonl + - uses: actions/upload-artifact@v4 + if: always() + with: + name: formula-benchmark-docker-amd64 + path: results/docker/ + - name: Run sandbox Formula benchmarks + if: ${{ !cancelled() && steps.build.outcome == 'success' }} + run: | + mkdir -p results/sandbox + sudo python3 benchmarks/engine_memory.py --output "$PWD/results/sandbox/engine-memory.jsonl" -- python3 benchmarks/run.py --image sandbox-formula-bench:ci --output "$PWD/results/sandbox" --backends sandbox --concurrency 1,2,4,8 --builds 8 | tee results/sandbox/summary.jsonl + - name: Save exact guest kernel and VMM identities + if: always() + run: | + mkdir -p results + if test -f /tmp/firecracker-benchmark/kernel.json; then cp /tmp/firecracker-benchmark/kernel.json results/; fi + if test -f /tmp/firecracker-benchmark/checksums.txt; then cp /tmp/firecracker-benchmark/checksums.txt results/; fi + docker run --rm --entrypoint cat sandbox-formula-bench:ci /opt/benchmark/toolchain.txt > results/toolchain.txt + docker run --rm --entrypoint cat sandbox-formula-bench:ci /opt/benchmark/binaries.sha256 > results/binaries.sha256 + docker run --rm --entrypoint cat sandbox-formula-bench:ci /opt/benchmark/input/manifest.json > results/input-manifest.json + for trial in 1 2 3; do + if test -f "results/cold/trial-$trial/environment.json"; then python3 benchmarks/report.py "results/cold/trial-$trial" >> "$GITHUB_STEP_SUMMARY"; fi + done + for backend in firecracker docker sandbox; do + if test -f "results/$backend/environment.json"; then python3 benchmarks/report.py "results/$backend" >> "$GITHUB_STEP_SUMMARY"; fi + done + - uses: actions/upload-artifact@v4 + if: always() + with: + name: formula-benchmark-amd64 + path: results/ diff --git a/benchmarks/.gitignore b/benchmarks/.gitignore new file mode 100644 index 0000000..c18dd8d --- /dev/null +++ b/benchmarks/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/benchmarks/Dockerfile b/benchmarks/Dockerfile new file mode 100644 index 0000000..c9680e2 --- /dev/null +++ b/benchmarks/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:1.26.6-bookworm AS build +WORKDIR /src +COPY . . +RUN bash benchmarks/build.sh /out + +FROM golang:1.26.6-bookworm +COPY --from=build /out /opt/benchmark +COPY benchmarks/guest-init.sh /sbin/benchmark-init +RUN chmod 755 /sbin/benchmark-init && mkdir /work && chmod 1777 /work +ENV GLIBC_TUNABLES=glibc.pthread.rseq=0 LANG=C LC_ALL=C HOME=/work TMPDIR=/tmp MAKEFLAGS=-j1 +USER 1000:1000 +ENTRYPOINT ["/opt/benchmark/formula-bench"] diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..944ff4d --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,34 @@ +# LLAR Formula benchmarks + +The payload is the unmodified `madler/zlib/v1.3.1/zlib_llar.gox` from xgo-dev/llarhub commit `1c2b4666ef598ed51afeb44221a73f53eea7e65c`, building zlib commit `51b7f2abdade71cd9bb0e7a373ef2610ec6f9daf`. Preparation records input hashes. Every successful sample must install a nonempty static library, header, license and pkg-config file, and return `-lz` metadata. Downloads and image construction happen before timing. + +All backends call the loaded Formula's `OnBuild` directly; the `llar make` build-cache lookup is not on this path. Every job copies pristine source into its own directory on a fresh `/work` tmpfs. The controller also requires one callback start, one configure, all 19 expected source/object compilation pairs and one archive with the 15 zlib members per build. Missing or repeated work fails the run even if artifacts already exist. Normalized compiler-command hashes are retained for comparison between jobs and backends. This verifies build work; it does not claim a cold host filesystem page cache. + +```sh +docker build -f benchmarks/Dockerfile -t sandbox-formula-bench:local . +python3 benchmarks/run.py --image sandbox-formula-bench:local \ + --output /tmp/formula-results --backends sandbox,docker \ + --concurrency 1,2,4,8 --builds 8 +``` + +For all three backends, use a native Linux x86_64 machine with Docker, Python 3, e2fsprogs and readable/writable `/dev/kvm`: + +```sh +sudo bash benchmarks/setup-firecracker.sh sandbox-formula-bench:local /tmp/firecracker-assets +sudo python3 benchmarks/run.py --image sandbox-formula-bench:local \ + --output /tmp/formula-results --firecracker-assets /tmp/firecracker-assets +``` + +The `Formula benchmark` workflow provisions this environment on `ubuntu-latest`. Firecracker v1.17.0 and the pinned CI guest kernel 6.18.48 boot a read-only ext4 export of the exact Docker payload image, including its environment variables. It mounts private writable work/tmp directories and runs the same executable as UID/GID 1000. Kernel URL and SHA-256 are retained. No network, swap or build cache is available during execution. Every backend shares the same selected host CPU affinity, toolchain and `MAKEFLAGS=-j1`. The total memory budget defaults to 4096 MiB: sandbox shares it across its workers; Docker and Firecracker split it equally across concurrent containers. Firecracker guest RAM matches its container share; actual memory usage is measured separately. + +## Measurements + +- The payload comparison includes LLAR and its build tools for Docker; host LLAR, Sentry, guest and build tools for sandbox; and VMM, guest Linux, LLAR and build tools for Firecracker. The CI wrapper additionally samples `docker.service` and `containerd.service`, including their shims, and reports their sum with payload memory at each sample. These shared services are counted once per sample, not once per concurrent build. All three backends currently use an outer Docker container, so this total includes Docker services for all three. The benchmark controller and its Docker CLI clients remain excluded. Source revision, image identity, binary hashes and input hashes are retained. +- Three independent cold trials create a fresh execution environment and complete one build each. Backend order rotates across trials. These are process/VM cold starts; the host page cache is not globally flushed. `launch_to_callback_ns` uses the same callback-start boundary for all backends. The batch matrix separately measures eight builds at each concurrency with the lifecycle described below. +- `launch_to_entry_ns` is host-observed time from `docker start -a` to the entry event. Docker/Firecracker emit it at `main.main`; sandbox emits it at the redirected guest entry. All three include their outer Docker launch. For sandbox this also includes host Formula preparation and export. Marker delivery overhead is included; these are not bare-kernel boot times. +- Serial sandbox samples additionally report `entry_ns`: host-observed Run-start to guest-entry markers, after Go initialization but before `state.Load`, and `callback_ns`: Run-start to the first closure instruction after restoration. The benchmark-only Go overlay adds a stdout marker at guest entry. Library sources and production behavior are unchanged; no syscall inspector is enabled. All backends send markers through stdout, so delivery overhead is included. Parallel sandbox runs do not guess which Run an entry belongs to. Direct workers also emit a callback-start event so application readiness can be distinguished from reaching main. +- `duration_ns` times `OnBuild` plus validation; sandbox includes state export, guest execution and writeback. The batch throughput includes Formula preparation, container startup, execution, exit and collection, but ends before container deletion. Raw events distinguish these phases. +- The controller samples raw Docker API `memory_stats.usage` with a 100 ms wait between rounds and sums active containers. The optional `engine_memory.py` wrapper reads disjoint cgroup-v2 `memory.current` counters for payloads and Docker services on the native systemd runner, preserving baseline, per-sample breakdowns and runtime-process membership. Totals come from paired samples, not the sum of separate peaks. Both metrics include page cache and remain separate from cache-subtracted Docker CLI displays and Go heap counters. Short peaks can be missed and reads across cgroups are not atomic. State Save/Load and exit GC remain inside the measurement window. +- Every concurrency level completes the same requested number of builds. Each sandbox worker owns one interpreter, reused sequentially with fresh build contexts and directories; workers share one Kernel. Docker and Firecracker create one execution environment per build. Interpreter construction finishes before concurrent sandbox transfers, as required by the current type-cache contract. Failures remain failures and are excluded from successful latency percentiles. + +Each run writes environment metadata, container logs, per-task JSON, raw memory samples and batch summaries. Do not compare local Docker Desktop ARM64 results with native amd64 runner results as if they came from the same machine. diff --git a/benchmarks/build.sh b/benchmarks/build.sh new file mode 100644 index 0000000..88aa230 --- /dev/null +++ b/benchmarks/build.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail +repo=$(cd "$(dirname "$0")/.." && pwd) +mkdir -p "$1" +output=$(cd "$1" && pwd) +python3 "$repo/benchmarks/prepare.py" "$output/input" +bash "$repo/sentry/build-linux.sh" "$output" +# Instrument only the benchmark build. Production guest entry stays unchanged. +python3 - "$repo" "$output" <<'PY' +import json, pathlib, sys +repo, output = map(pathlib.Path, sys.argv[1:]) +source = repo / "guest_linux.go" +text = source.read_text() +entry = "func guestEntry() {\n" +assert text.count(entry) == 1 +replacement = output / "guest_linux.go" +replacement.write_text(text.replace(entry, entry + '\tif _, err := os.Stdout.WriteString("BENCH:{\\"event\\":\\"guest_entry\\"}\\n"); err != nil { panic(err) }\n')) +(output / "overlay.json").write_text(json.dumps({"Replace": {str(source): str(replacement)}})) +PY +go -C "$repo/testdata/llar" build -mod=readonly -overlay="$output/overlay.json" \ + -ldflags='-checklinkname=0 -s=false -w=false -extldflags=-Wl,-z,separate-code' -o "$output/formula-bench" ./benchmark +gcc -O2 -o "$output/reboot" "$repo/benchmarks/reboot.c" +chmod 755 "$output" "$output/formula-bench" +go version > "$output/toolchain.txt" +gcc --version >> "$output/toolchain.txt" +make --version >> "$output/toolchain.txt" +pkg-config --version >> "$output/toolchain.txt" +sha256sum "$output/formula-bench" "$output/sentrylib.so" > "$output/binaries.sha256" diff --git a/benchmarks/engine_memory.py b/benchmarks/engine_memory.py new file mode 100644 index 0000000..768bc6a --- /dev/null +++ b/benchmarks/engine_memory.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Include the shared Docker services without changing the measured payload.""" +import argparse +import json +import pathlib +import subprocess +import threading +import time + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--output", required=True) + parser.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args() + command = args.command + if command and command[0] == "--": + command = command[1:] + if not command: + parser.error("a benchmark command is required") + + info = json.loads(subprocess.check_output(["docker", "info", "--format", "{{json .}}"], text=True)) + if info["CgroupDriver"] != "systemd" or info["CgroupVersion"] != "2" or info["Containers"] != 0: + raise RuntimeError("engine accounting requires an idle native Linux systemd/cgroup-v2 Docker daemon") + root = pathlib.Path("/sys/fs/cgroup") + services = {} + for service in ("docker.service", "containerd.service"): + group = subprocess.check_output(["systemctl", "show", "--property=ControlGroup", "--value", service], text=True).strip() + if not group or group == "/": + raise RuntimeError(f"missing service cgroup: {service}") + services[service] = root / group.lstrip("/") + paths = list(services.values()) + if paths[0] == paths[1] or paths[0] in paths[1].parents or paths[1] in paths[0].parents: + raise RuntimeError("Docker service cgroups overlap") + + output = pathlib.Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + samples, errors = [], [] + finished = threading.Event() + + def memory(path): + return {"bytes": int((path / "memory.current").read_text()), + "stats": dict((key, int(value)) for key, value in + (line.split() for line in (path / "memory.stat").read_text().splitlines()))} + + def sample(): + begin = time.perf_counter_ns() + engine = {name: memory(path) for name, path in services.items()} + containers = {} + for path in (root / "system.slice").glob("docker-*.scope"): + cid = path.name.removeprefix("docker-").removesuffix(".scope") + if len(cid) != 64: + raise RuntimeError(f"unexpected container cgroup: {path}") + if any(service == path or service in path.parents or path in service.parents for service in paths): + raise RuntimeError(f"container and service cgroups overlap: {path}") + try: + containers[cid] = memory(path) + except FileNotFoundError: + # A completed container can disappear between the directory + # listing and the read. Service cgroups must remain available. + if path.exists(): + raise + runtime_processes = [] + for proc in pathlib.Path("/proc").iterdir(): + if not proc.name.isdecimal(): + continue + try: + name = (proc / "comm").read_text().strip() + if name not in ("dockerd", "containerd") and not name.startswith("containerd-shim"): + continue + group = (proc / "cgroup").read_text().strip().removeprefix("0::") + except FileNotFoundError: + continue + path = root / group.lstrip("/") + if not any(service == path or service in path.parents for service in paths): + raise RuntimeError(f"unaccounted Docker runtime process: {proc.name} {name} {group}") + runtime_processes.append({"pid": int(proc.name), "name": name, "cgroup": group}) + entry = {"begin_ns": begin, "end_ns": time.perf_counter_ns(), + "services": engine, "containers": containers, "runtime_processes": runtime_processes, + "engine_bytes": sum(value["bytes"] for value in engine.values()), + "payload_bytes": sum(value["bytes"] for value in containers.values())} + entry["total_bytes"] = entry["engine_bytes"] + entry["payload_bytes"] + samples.append(entry) + log.write(json.dumps(entry) + "\n") + log.flush() + + def observe(): + try: + while not finished.wait(0.1): + sample() + except Exception as error: + errors.append(str(error)) + + with output.open("w") as log: + sample() + observer = threading.Thread(target=observe) + observer.start() + try: + result = subprocess.run(command) + finally: + finished.set() + observer.join() + sample() + + for path in sorted(output.parent.glob("*-c*.json")): + data = json.loads(path.read_text()) + if "records" not in data: + continue + start = min(record["start_ns"] - record["create_ns"] for record in data["records"]) + end = max(record["start_ns"] + record["wall_ns"] for record in data["records"]) + ids = {record["container"] for record in data["records"]} + window = [entry for entry in samples if entry["begin_ns"] <= end and entry["end_ns"] >= start] + if not window or not any(ids.intersection(entry["containers"]) for entry in window): + errors.append(f"no payload cgroup samples for {path.name}") + data["engine_accounting"] = { + "services": {name: str(path) for name, path in services.items()}, + "baseline_bytes": samples[0]["engine_bytes"], + "engine_peak_bytes": max((entry["engine_bytes"] for entry in window), default=0), + "payload_peak_bytes": max((sum(value["bytes"] for cid, value in entry["containers"].items() if cid in ids) for entry in window), default=0), + "total_peak_bytes": max((entry["engine_bytes"] + sum(value["bytes"] for cid, value in entry["containers"].items() if cid in ids) for entry in window), default=0), + "samples": len(window), "errors": list(errors), + "metric": "Sum of disjoint cgroup-v2 memory.current reads in each sample; includes cache and shared Docker services once", + "excluded": "benchmark controller and its Docker CLI clients; host kernel outside these cgroups", + } + path.write_text(json.dumps(data, indent=2) + "\n") + if errors: + raise SystemExit("engine memory sampling failed: " + "; ".join(errors)) + raise SystemExit(result.returncode) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/guest-init.sh b/benchmarks/guest-init.sh new file mode 100644 index 0000000..bfe4366 --- /dev/null +++ b/benchmarks/guest-init.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# Firecracker starts the same image payload as Docker, as the same UID. +mount -t proc proc /proc +mount -t sysfs sysfs /sys +mount -t tmpfs -o mode=1777,size=256m,exec tmpfs /tmp +mount -t tmpfs -o mode=1777,size=256m,exec tmpfs /work +# docker export contains files, not Config.Env. setup-firecracker.sh materializes it. +. /opt/benchmark/environment.sh +setpriv --reuid=1000 --regid=1000 --clear-groups /opt/benchmark/formula-bench -backend=direct +status=$? +if test "$status" -ne 0; then + printf 'PATH=%s\n' "$PATH" + gcc -print-search-dirs + ls -l /usr/lib/gcc/*/*/cc1 + find /work -name configure.log -print -exec cat '{}' ';' +fi +echo "BENCH:{\"event\":\"guest_exit\",\"code\":$status}" +exec /opt/benchmark/reboot diff --git a/benchmarks/prepare.py b/benchmarks/prepare.py new file mode 100644 index 0000000..a69d9f1 --- /dev/null +++ b/benchmarks/prepare.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""Prepare pinned upstream inputs outside the timed region.""" +import hashlib +import io +import json +import pathlib +import sys +import tarfile +import urllib.request + +HUB = "1c2b4666ef598ed51afeb44221a73f53eea7e65c" +ZLIB = "51b7f2abdade71cd9bb0e7a373ef2610ec6f9daf" +out = pathlib.Path(sys.argv[1]) +formula = out / "formula" / "v1.3.1" +formula.mkdir(parents=True) +manifest = {"llarhub_commit": HUB, "zlib_commit": ZLIB, "sha256": {}} +for name in ("zlib_llar.gox", "consumer.c"): + url = f"https://raw.githubusercontent.com/xgo-dev/llarhub/{HUB}/madler/zlib/v1.3.1/{name}" + data = urllib.request.urlopen(url, timeout=60).read() + (formula / name).write_bytes(data) + manifest["sha256"][name] = hashlib.sha256(data).hexdigest() +data = urllib.request.urlopen(f"https://api.github.com/repos/madler/zlib/tarball/{ZLIB}", timeout=60).read() +manifest["sha256"]["zlib.tar.gz"] = hashlib.sha256(data).hexdigest() +with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as archive: + prefix = archive.getmembers()[0].name.split("/")[0] + for member in archive.getmembers(): + parts = pathlib.PurePosixPath(member.name).parts + if parts[0] != prefix or ".." in parts or member.issym() or member.islnk(): + raise ValueError(f"unexpected archive entry {member.name}") + target = out / "source" / pathlib.Path(*parts[1:]) + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + elif member.isfile(): + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(archive.extractfile(member).read()) + target.chmod(member.mode & 0o777) + else: + raise ValueError(f"unsupported archive entry {member.name}") +(out / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") diff --git a/benchmarks/reboot.c b/benchmarks/reboot.c new file mode 100644 index 0000000..823cf92 --- /dev/null +++ b/benchmarks/reboot.c @@ -0,0 +1,12 @@ +#include +#include +#include +#include + +int main(void) { + sync(); + tcdrain(STDOUT_FILENO); + reboot(RB_AUTOBOOT); + perror("reboot"); + return 1; +} diff --git a/benchmarks/report.py b/benchmarks/report.py new file mode 100644 index 0000000..16917f5 --- /dev/null +++ b/benchmarks/report.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Summarize retained samples without hiding unsuccessful builds.""" +import csv +import json +import math +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]) +rows = [] +for path in sorted(root.glob("*-c*.json")): + data = json.loads(path.read_text()) + if "records" not in data: + continue + starts = [r["launch_to_entry_ns"]/1e6 for r in data["records"] if r.get("launch_to_entry_ns") is not None] + ready = [r["launch_to_callback_ns"]/1e6 for r in data["records"] if r.get("launch_to_callback_ns") is not None] + entries = [e["entry_ns"]/1e6 for r in data["records"] for e in r["events"] if e.get("entry_ns")] + callbacks = [e["callback_ns"]/1e6 for r in data["records"] for e in r["events"] if e.get("callback_ns")] + def percentile(values, p): + return round(sorted(values)[math.ceil(len(values)*p/100)-1], 3) if values else "" + rows.append({"backend": data["backend"], "concurrency": data["concurrency"], + "success": f"{data['successful']}/{data['count']}", + "builds_per_second": round(data["builds_per_second"], 4), + "build_p50_ms": round(data["build_latency"]["p50_ns"]/1e6, 3) if data["build_latency"] else "N/A", + "build_p95_ms": round(data["build_latency"]["p95_ns"]/1e6, 3) if data["build_latency"] else "N/A", + "sampled_full_environment_peak_mib": round(data["memory_peak_bytes"]/1048576, 2), + "sampled_with_docker_services_peak_mib": round(data["engine_accounting"]["total_peak_bytes"]/1048576, 2) if data.get("engine_accounting") and not data["engine_accounting"]["errors"] else "N/A", + "launch_to_entry_p50_ms": percentile(starts, 50), + "launch_to_entry_p95_ms": percentile(starts, 95), + "launch_to_entry_p99_ms": percentile(starts, 99), + "launch_to_callback_p50_ms": percentile(ready, 50), + "run_to_entry_p50_ms": percentile(entries, 50), + "run_to_callback_p50_ms": percentile(callbacks, 50), + "run_to_entry_first_ms": entries[0] if entries else "", + "run_to_entry_warm_p50_ms": percentile(entries[1:], 50), + "oom_killed": data.get("oom_killed", "unrecorded"), + "event_errors": data.get("event_errors", 0)}) +if not rows: + raise SystemExit("no batch result files") +with (root/"summary.csv").open("w") as f: + writer = csv.DictWriter(f, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) +print("| Backend | Concurrency | Success | Builds/s | Build p50 ms | Build p95 ms | Payload cgroup peak MiB | With Docker services peak MiB |") +print("| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |") +for row in rows: + print("| " + " | ".join(str(row[k]) for k in list(row)[:8]) + " |") diff --git a/benchmarks/run.py b/benchmarks/run.py new file mode 100644 index 0000000..d111e11 --- /dev/null +++ b/benchmarks/run.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +"""Run real Formula builds and retain raw timings and cgroup samples.""" +import argparse +import concurrent.futures +import http.client +import json +import math +import os +import pathlib +import socket +import subprocess +import threading +import time + +from workload import audit_workload + + +class Engine(http.client.HTTPConnection): + def __init__(self, path): + super().__init__("localhost", timeout=30) + self.path = path + + def connect(self): + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.sock.settimeout(self.timeout) + self.sock.connect(self.path) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--image", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--backends", default="sandbox,docker,firecracker") + parser.add_argument("--concurrency", default="1,2,4,8") + parser.add_argument("--builds", type=int, default=8, help="same total build count at every concurrency") + parser.add_argument("--cpuset", default="0,1") + parser.add_argument("--memory-mib", type=int, default=4096, help="total memory budget for each backend case") + parser.add_argument("--firecracker-assets") + args = parser.parse_args() + output = pathlib.Path(args.output).resolve() + output.mkdir(parents=True, exist_ok=True) + context = json.loads(subprocess.check_output(["docker", "context", "inspect"]))[0] + endpoint = context["Endpoints"]["docker"]["Host"] + if not endpoint.startswith("unix://"): + raise ValueError("run the controller beside the Docker daemon using its Unix socket") + engine_socket = endpoint[7:] + + def api(method, path, data=None): + conn = Engine(engine_socket) + try: + conn.request(method, path, json.dumps(data) if data is not None else None, + {"Content-Type": "application/json"}) + response = conn.getresponse() + body = response.read() + if response.status >= 300: + raise RuntimeError(f"Docker {method} {path}: {response.status} {body.decode()}") + return json.loads(body) if body else None + finally: + conn.close() + + (output / "environment.json").write_text(json.dumps({ + "arguments": vars(args), "engine": api("GET", "/info"), + "image": api("GET", f"/images/{args.image}/json"), + "source_revision": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=pathlib.Path(__file__).resolve().parents[1], text=True).strip(), + "memory_metric": "Docker API memory_stats.usage, including cache; shared Docker daemon excluded for all backends", + "memory_window": "Full execution environment lifetime, including preparation, state Save/Load, guest exit and GC; no host-memory subtraction", + "sample_interval_seconds": 0.1, + }, indent=2) + "\n") + lock = threading.Lock() + samples = [] + active = set() + finished = threading.Event() + + def sample(): + while not finished.is_set(): + with lock: + ids = list(active) + current = [] + for cid in ids: + begin = time.perf_counter_ns() + stats = api("GET", f"/containers/{cid}/stats?stream=false&one-shot=true") + memory = stats.get("memory_stats", {}) + current.append({"id": cid, "bytes": memory.get("usage", 0), + "begin_ns": begin, "end_ns": time.perf_counter_ns(), + "memory_stats": memory.get("stats", {}), + "cpu_ns": stats.get("cpu_stats", {}).get("cpu_usage", {}).get("total_usage", 0)}) + if current: + samples.append({"time_ns": time.perf_counter_ns(), "containers": current, + "memory_bytes": sum(s["bytes"] for s in current)}) + finished.wait(0.1) + + def task(backend, concurrency, count, task_id): + name = f"{backend}-c{concurrency}-{task_id}" + memory_mib = args.memory_mib if backend == "sandbox" else args.memory_mib // concurrency + host = {"ReadonlyRootfs": True, "NetworkMode": "none", "CapDrop": ["ALL"], + "SecurityOpt": ["seccomp=unconfined"], "CpusetCpus": args.cpuset, + "Memory": memory_mib * 1024 * 1024, + "MemorySwap": memory_mib * 1024 * 1024, + "Tmpfs": {"/work": "rw,exec,nosuid,nodev,mode=1777,size=256m", + "/tmp": "rw,exec,nosuid,nodev,mode=1777,size=256m"}} + config = {"Image": args.image, "HostConfig": host, + "Cmd": [f"-backend={'sandbox' if backend == 'sandbox' else 'direct'}", + f"-count={count}", f"-concurrency={concurrency if backend == 'sandbox' else 1}"], + "AttachStdout": True, "AttachStderr": True} + if backend == "firecracker": + assets = pathlib.Path(args.firecracker_assets).resolve() + fc = {"boot-source": {"kernel_image_path": "/assets/vmlinux", + "boot_args": "console=ttyS0 reboot=k panic=1 pci=off root=/dev/vda ro init=/sbin/benchmark-init"}, + "drives": [{"drive_id": "rootfs", "path_on_host": "/assets/rootfs.ext4", + "is_root_device": True, "is_read_only": True}], + "machine-config": {"vcpu_count": len(args.cpuset.split(",")), "mem_size_mib": memory_mib}} + directory = output / name + directory.mkdir() + (directory / "config.json").write_text(json.dumps(fc)) + host["Binds"] = [f"{assets}:/assets:ro", f"{directory}:/config:ro"] + host["Devices"] = [{"PathOnHost": "/dev/kvm", "PathInContainer": "/dev/kvm", "CgroupPermissions": "rwm"}] + host["GroupAdd"] = [str(os.stat("/dev/kvm").st_gid)] + config["Entrypoint"] = ["/assets/firecracker"] + config["Cmd"] = ["--no-api", "--config-file", "/config/config.json"] + t0 = time.perf_counter_ns() + cid = api("POST", "/containers/create", config)["Id"] + record = {"backend": backend, "concurrency": concurrency, "count": count, "task": task_id, + "container": cid, "create_ns": time.perf_counter_ns()-t0, "events": [], "event_errors": []} + with lock: + active.add(cid) + t0 = time.perf_counter_ns() + record["start_ns"] = t0 + print(json.dumps({"event": "task_start", "backend": backend, "concurrency": concurrency, "task": task_id}), flush=True) + with (output / f"{name}.log").open("w") as log: + proc = subprocess.Popen(["docker", "start", "-a", cid], stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True) + for line in proc.stdout: + now = time.perf_counter_ns() + log.write(line) + log.flush() + if "BENCH:" in line: + try: + event = json.loads(line.split("BENCH:", 1)[1]) + except json.JSONDecodeError as error: + # A guest panic can interleave kernel output with serial + # JSON. Retain the failed measurement and finish cleanup. + record["event_errors"].append(str(error)) + continue + event["observed_ns"] = now-t0 + if event["event"] in ("main", "ready", "guest_entry"): + stats = api("GET", f"/containers/{cid}/stats?stream=false&one-shot=true") + event["memory_bytes"] = stats.get("memory_stats", {}).get("usage", 0) + event["memory_sample_lag_ns"] = time.perf_counter_ns()-now + record["events"].append(event) + if event["event"] in ("main", "ready", "result", "guest_exit"): + print(json.dumps({"backend": backend, "task": task_id, **event}), flush=True) + proc.wait() + record["wall_ns"] = time.perf_counter_ns()-t0 + record["state"] = api("GET", f"/containers/{cid}/json")["State"] + record["exit_code"] = record["state"]["ExitCode"] + print(json.dumps({"event": "task_exit", "backend": backend, "task": task_id, "exit_code": record["exit_code"]}), flush=True) + expected = "guest_entry" if backend == "sandbox" else "main" + entries = [e["observed_ns"] for e in record["events"] if e["event"] == expected] + record["launch_to_entry_ns"] = entries[0] if entries else None + callbacks = [e["observed_ns"] for e in record["events"] if e["event"] == "callback_start"] + record["launch_to_callback_ns"] = callbacks[0] if callbacks else None + outcomes = [e for e in record["events"] if e["event"] == "result"] + if backend == "sandbox" and concurrency == 1: + start = None + phase_times = {} + for event in record["events"]: + if event["event"] == "run_start": + start = event["observed_ns"] + phase_times = {} + elif start is not None and event["event"] in ("guest_entry", "callback_start"): + phase_times[event["event"]] = event["observed_ns"] - start + elif event["event"] == "result": + if "guest_entry" in phase_times: + event["entry_ns"] = phase_times["guest_entry"] + if "callback_start" in phase_times: + event["callback_ns"] = phase_times["callback_start"] + record["workload"] = audit_workload((output / f"{name}.log").read_text(), count, record["events"]) + record["success"] = record["exit_code"] == 0 and not record["event_errors"] and not record["workload"]["errors"] and len(outcomes) == count and all(not e.get("error") for e in outcomes) + with lock: + active.remove(cid) + # An exited container keeps its logs/state until its measurements are retained. + (output / f"{name}.json").write_text(json.dumps(record, indent=2)+"\n") + return record + + failures = 0 + for backend in args.backends.split(","): + for concurrency in map(int, args.concurrency.split(",")): + if backend == "firecracker" and not args.firecracker_assets: + raise ValueError("Firecracker needs --firecracker-assets; run setup-firecracker.sh first") + count = args.builds + if concurrency > count: + raise ValueError("--builds must be at least every requested concurrency") + samples.clear() + finished.clear() + sampler_errors = [] + + def observe(): + try: + sample() + except Exception as error: + sampler_errors.append(str(error)) + + observer = threading.Thread(target=observe) + observer.start() + t0 = time.perf_counter_ns() + try: + if backend == "sandbox": + records = [task(backend, concurrency, count, 0)] + else: + with concurrent.futures.ThreadPoolExecutor(concurrency) as workers: + records = list(workers.map(lambda i: task(backend, concurrency, 1, i), range(count))) + finally: + finished.set() + observer.join() + wall = time.perf_counter_ns()-t0 + success = sum(1 for r in records for e in r["events"] + if e["event"] == "result" and not e.get("error")) + durations = sorted(e["duration_ns"] for r in records for e in r["events"] + if e["event"] == "result" and not e.get("error")) + percentiles = {f"p{p}_ns": durations[min(len(durations)-1, math.ceil(len(durations)*p/100)-1)] + for p in (50, 95, 99)} if durations else {} + summary = {"backend": backend, "concurrency": concurrency, "count": count, + "successful": success, "wall_ns": wall, "builds_per_second": success/(wall/1e9), + "memory_peak_bytes": max((s["memory_bytes"] for s in samples), default=0), + "build_latency": percentiles, + "oom_killed": sum(r["state"].get("OOMKilled", False) for r in records), + "event_errors": sum(len(r["event_errors"]) for r in records), + "workload_errors": sum(len(r["workload"]["errors"]) for r in records), + "sampler_errors": sampler_errors, "records": records, "samples": list(samples)} + (output / f"{backend}-c{concurrency}.json").write_text(json.dumps(summary, indent=2)+"\n") + print(json.dumps({k: v for k, v in summary.items() if k not in ("records", "samples")}), flush=True) + for record in records: + api("DELETE", f"/containers/{record['container']}") + failures += count-success+bool(sampler_errors)+sum(r["exit_code"] != 0 for r in records)+summary["event_errors"]+summary["workload_errors"] + if failures: + raise SystemExit(f"{failures} failed builds/measurements; see raw results") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/setup-firecracker.sh b/benchmarks/setup-firecracker.sh new file mode 100644 index 0000000..eee54b2 --- /dev/null +++ b/benchmarks/setup-firecracker.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail +image=$1 +mkdir -p "$2" +output=$(cd "$2" && pwd) +test -c /dev/kvm +test "$(uname -m)" = x86_64 +curl -fL --retry 3 https://github.com/firecracker-microvm/firecracker/releases/download/v1.17.0/firecracker-v1.17.0-x86_64.tgz -o "$output/firecracker.tgz" +tar -xzf "$output/firecracker.tgz" -C "$output" +cp "$output/release-v1.17.0-x86_64/firecracker-v1.17.0-x86_64" "$output/firecracker" +python3 - "$output" <<'PY' +import hashlib, json, pathlib, sys, urllib.request +out = pathlib.Path(sys.argv[1]) +base = "https://s3.amazonaws.com/spec.ccfc.min" +key = "firecracker-ci/20260916-dcfc69b625d0-0/x86_64/vmlinux-6.18.48" +data = urllib.request.urlopen(base+"/"+key, timeout=120).read() +(out/"vmlinux").write_bytes(data) +(out/"kernel.json").write_text(json.dumps({"url": base+"/"+key, "sha256": hashlib.sha256(data).hexdigest()}, indent=2)+"\n") +PY +mkdir "$output/rootfs" +container=$(docker create "$image") +docker export "$container" | tar --numeric-owner -xf - -C "$output/rootfs" +docker rm "$container" +docker image inspect "$image" > "$output/image.json" +python3 - "$output" <<'PY' +import json, pathlib, re, shlex, sys +out = pathlib.Path(sys.argv[1]) +image = json.loads((out/"image.json").read_text())[0] +lines = [] +for entry in image["Config"]["Env"]: + key, value = entry.split("=", 1) + assert re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key) + lines.append(f"export {key}={shlex.quote(value)}") +(out/"rootfs/opt/benchmark/environment.sh").write_text("\n".join(lines)+"\n") +PY +truncate -s 3G "$output/rootfs.ext4" +mkfs.ext4 -q -F -d "$output/rootfs" "$output/rootfs.ext4" +chmod -R a+rX "$output" +sha256sum "$output/firecracker" "$output/vmlinux" "$output/rootfs.ext4" > "$output/checksums.txt" diff --git a/benchmarks/workload.py b/benchmarks/workload.py new file mode 100644 index 0000000..ef689d2 --- /dev/null +++ b/benchmarks/workload.py @@ -0,0 +1,53 @@ +"""Verify the compilation workload of the pinned zlib Formula.""" +import collections +import hashlib +import json +import pathlib +import shlex + + +def audit_workload(log, count, events): + library = "adler32 crc32 deflate infback inffast inflate inftrees trees zutil compress uncompr gzclose gzlib gzread gzwrite".split() + expected = collections.Counter((name + ".o", name + ".c") for name in library) + expected.update({("example.o", "test/example.c"): 1, + ("minigzip.o", "test/minigzip.c"): 1, + ("example64.o", "test/example.c"): 1, + ("minigzip64.o", "test/minigzip.c"): 1}) + jobs = {str(i): collections.Counter() for i in range(count)} + commands = {str(i): [] for i in range(count)} + configure, archives, errors = 0, 0, [] + for line in log.splitlines(): + if line == "Building static library libz.a version 1.3.1 with gcc.": + configure += 1 + if not line.startswith(("gcc ", "ar ")): + continue + args = shlex.split(line) + if args[:3] == ["ar", "rc", "libz.a"]: + archives += 1 + if args[3:] != [name + ".o" for name in library]: + errors.append("archive members differ from the pinned zlib workload") + if args[0] != "gcc" or "-c" not in args: + continue + source = pathlib.PurePosixPath(args[-1]) + parts = source.parts + if len(parts) < 5 or parts[:2] != ("/", "work") or not parts[2].startswith("job-") or parts[3] != "source": + errors.append("unexpected compilation source: " + str(source)) + continue + job = parts[2].removeprefix("job-") + if job not in jobs or "-o" not in args or args.index("-o") + 1 >= len(args): + errors.append("unexpected compilation command: " + line) + continue + jobs[job][(args[args.index("-o") + 1], "/".join(parts[4:]))] += 1 + commands[job].append([arg.replace("/work/job-" + job + "/", "/work/job/", 1) for arg in args]) + callbacks = sum(event["event"] == "callback_start" for event in events) + if callbacks != count or configure != count or archives != count: + errors.append(f"expected {count} callbacks/configures/archives, got {callbacks}/{configure}/{archives}") + for job, compiled in jobs.items(): + if compiled != expected: + errors.append(f"job {job}: missing {list((expected-compiled).elements())}, extra {list((compiled-expected).elements())}") + digests = {job: hashlib.sha256(json.dumps(sorted(args)).encode()).hexdigest() for job, args in commands.items()} + if len(set(digests.values())) != 1: + errors.append("compiler commands differ between jobs") + return {"callbacks": callbacks, "configures": configure, "archives": archives, + "compilations": {job: sum(compiled.values()) for job, compiled in jobs.items()}, + "compiler_commands_sha256": digests, "errors": errors} diff --git a/testdata/llar/benchmark/main.go b/testdata/llar/benchmark/main.go new file mode 100644 index 0000000..f1738e4 --- /dev/null +++ b/testdata/llar/benchmark/main.go @@ -0,0 +1,193 @@ +//go:build linux && (amd64 || arm64) && cgo + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io/fs" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "time" + + formulapkg "github.com/goplus/llar/formula" + "github.com/goplus/llar/internal/formula" + "github.com/xgo-dev/sandbox" +) + +var outputMu sync.Mutex + +func emit(value any) { + data, err := json.Marshal(value) + if err != nil { + panic(err) + } + outputMu.Lock() + fmt.Printf("BENCH:%s\n", data) + outputMu.Unlock() +} + +type job struct { + build func(*formulapkg.Context) + ctx *formulapkg.Context + out string +} + +type result struct { + Event string `json:"event"` + ID int `json:"id"` + DurationNS int64 `json:"duration_ns"` + Metadata string `json:"metadata,omitempty"` + Error string `json:"error,omitempty"` +} + +func main() { + started := time.Now() + emit(map[string]any{"event": "main", "unix_ns": started.UnixNano()}) + if err := run(started); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(started time.Time) error { + backend := flag.String("backend", "direct", "direct or sandbox") + input := flag.String("input", "/opt/benchmark/input", "prepared Formula and source directory") + work := flag.String("work", "/work", "empty writable build directory") + library := flag.String("library", "/opt/benchmark/sentrylib.so", "matching Sentry library") + count := flag.Int("count", 1, "number of complete builds") + concurrency := flag.Int("concurrency", 1, "simultaneous builds") + flag.Parse() + if *count < 1 || *concurrency < 1 || (*backend != "direct" && *backend != "sandbox") { + return fmt.Errorf("invalid backend, count or concurrency") + } + if err := os.MkdirAll(*work, 0755); err != nil { + return err + } + jobs := make([]job, *count) + formulaFS := os.DirFS(filepath.Join(*input, "formula")).(fs.ReadFileFS) + builds := make([]func(*formulapkg.Context), min(*concurrency, *count)) + for i := range builds { + loaded, err := formula.LoadFS(formulaFS, "v1.3.1/zlib_llar.gox") + if err != nil { + return fmt.Errorf("load Formula: %w", err) + } + if loaded.ModPath != "madler/zlib" || loaded.OnBuild == nil { + return fmt.Errorf("unexpected Formula %q", loaded.ModPath) + } + builds[i] = loaded.OnBuild + } + for i := range jobs { + dir := filepath.Join(*work, fmt.Sprintf("job-%d", i)) + source, out := filepath.Join(dir, "source"), filepath.Join(dir, "install") + if err := os.CopyFS(source, os.DirFS(filepath.Join(*input, "source"))); err != nil { + return err + } + if err := os.MkdirAll(out, 0755); err != nil { + return err + } + jobs[i] = job{ + ctx: formulapkg.NewContext(&formulapkg.Project{SourceFS: formulaFS}, source, out, "linux/"+runtime.GOARCH, nil), + out: out, + } + } + // Each interpreter is prepared before concurrent transfers: constructing + // reflectx types while Save/Load enumerates their caches is unsupported. + s := sandbox.Sandbox{ + Library: *library, + Mounts: []sandbox.Mount{ + {Type: "bind", Source: "/", Target: "/", Options: []string{"ro"}}, + {Type: "bind", Source: *work, Target: *work, Options: []string{"rw"}}, + {Type: "tmpfs", Target: "/tmp", Options: []string{"mode=1777"}}, + {Type: "proc", Target: "/proc"}, + }, + Env: os.Environ(), + } + defer s.Close() + emit(map[string]any{"event": "ready", "backend": *backend, "count": *count, "concurrency": *concurrency, "prepare_ns": time.Since(started).Nanoseconds()}) + queue := make(chan int) + results := make(chan result, *count) + var workers sync.WaitGroup + begin := time.Now() + for worker := range builds { + workers.Add(1) + go func() { + defer workers.Done() + for id := range queue { + t0 := time.Now() + if *concurrency == 1 { + emit(map[string]any{"event": "run_start", "id": id}) + } + r := result{Event: "result", ID: id} + j := jobs[id] + j.build = builds[worker] + err := execute(j, *backend, &s) + r.DurationNS = time.Since(t0).Nanoseconds() + if err != nil { + r.Error = err.Error() + } else { + r.Metadata = jobs[id].ctx.Out.Metadata() + } + emit(r) + results <- r + } + }() + } + for i := range jobs { + queue <- i + } + close(queue) + workers.Wait() + close(results) + failed := 0 + for r := range results { + if r.Error != "" { + failed++ + } + } + emit(map[string]any{"event": "summary", "backend": *backend, "count": *count, "concurrency": *concurrency, "failed": failed, "wall_ns": time.Since(begin).Nanoseconds(), "total_ns": time.Since(started).Nanoseconds()}) + if failed != 0 { + return fmt.Errorf("%d/%d builds failed", failed, *count) + } + return nil +} + +func execute(j job, backend string, s *sandbox.Sandbox) (err error) { + defer func() { + if p := recover(); p != nil { + err = fmt.Errorf("Formula panic: %v", p) + } + }() + build, ctx := j.build, j.ctx + if backend == "sandbox" { + err = s.Run(func() { + if _, err := os.Stdout.WriteString("BENCH:{\"event\":\"callback_start\"}\n"); err != nil { + panic(err) + } + build(ctx) + }) + } else { + emit(map[string]any{"event": "callback_start"}) + build(ctx) + } + if err != nil { + return err + } + if len(ctx.Errs) != 0 { + return ctx.Errs.ToError() + } + if !strings.Contains(ctx.Out.Metadata(), "-lz") { + return fmt.Errorf("missing zlib metadata: %q", ctx.Out.Metadata()) + } + for _, name := range []string{"lib/libz.a", "include/zlib.h", "lib/pkgconfig/zlib.pc", "licenses/LICENSE"} { + info, err := os.Stat(filepath.Join(j.out, name)) + if err != nil || info.Size() == 0 { + return fmt.Errorf("missing/empty artifact %s: %v", name, err) + } + } + return nil +}