Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# Preserve the immutable archive's bytes even with core.autocrlf=true.
benchmark/deepswe-gptxhigh-v1/** text eol=lf
benchmark/frontiercs2-single-host-v1/** text eol=lf

# Published study archives bind sha256 of their own LF bytes in index.json, so
# a CRLF checkout must not rewrite them either.
Expand Down
5 changes: 5 additions & 0 deletions benchmark/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,11 @@ preserved for inspection; the companion notice governs their current evidence
status. Neither archive category is the architecture for new work or grants
runner, verifier, submission, or publication authority.

The [FrontierCS-2 single-host v1 archive note](frontiercs2-single-host-versions.md)
registers the frozen native-client, HTTP bridge, Docker transport and verifier
wrapper used by a one-task four-arm pilot. The companion note links the separate
data release, dependencies, read-only identity check and qualification limits.

## Matched model provider routing

For a custom Responses-compatible model endpoint, set `OPENAI_BASE_URL` and
Expand Down
77 changes: 77 additions & 0 deletions benchmark/check_frontiercs2_v1.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
#!/usr/bin/env python3
"""Verify frozen compatibility sources without importing or executing them."""

from __future__ import annotations

import argparse
import ast
import hashlib
import json
from pathlib import Path


ARCHIVE = Path(__file__).resolve().with_name("frontiercs2-single-host-v1")
EXPECTED = {
"docker_environment.py": "d6e22a5f894e801d3736657a69b700f16b83f008240235121f5ff40361914b64",
"frontier_submit.py": "f4cbc551acd10a729e5cade5cc3a70b653d9aee1dee4b1cbfeb20b3208df6f00",
"local_runner.py": "5f407c8c9d800eb110cc453a9815b571861d0d2fca6412ffe1c83e9918938cad",
"source-manifest.json": "d0c096b7e1b67630174990f9813e16f7904d27b11be9edf7892c9fe8ae579cbc",
"submission_bridge.py": "cd1be73411bbb94f1a7ee0df9e49f0d26aa5d6e23412be233c5e483a1728b639",
}


def check_archive(archive: Path) -> dict:
observed = set()
python_count = embedded_count = 0
for path in archive.iterdir():
if path.name == "__pycache__" and path.is_dir() and not path.is_symlink():
if any(p.is_symlink() or not p.is_file() or p.suffix != ".pyc" for p in path.iterdir()):
raise ValueError("unexpected non-bytecode cache entry")
continue
if path.is_symlink() or not path.is_file() or path.name not in EXPECTED:
raise ValueError(f"unexpected archive entry: {path.name}")
if path.stat().st_mode & 0o111:
raise ValueError(f"executable mode changed: {path.name}")
raw = path.read_bytes()
if hashlib.sha256(raw).hexdigest() != EXPECTED[path.name]:
raise ValueError(f"archive bytes changed: {path.name}")
observed.add(path.name)
if path.suffix == ".py":
compile(raw, path.name, "exec")
python_count += 1
for node in ast.parse(raw).body:
if (isinstance(node, ast.Assign) and isinstance(node.value, ast.Constant)
and isinstance(node.value.value, str)
and any(isinstance(t, ast.Name) and t.id == "NNP_BOOTSTRAP" for t in node.targets)):
compile(node.value.value, f"{path.name}:NNP_BOOTSTRAP", "exec")
embedded_count += 1
else:
json.loads(raw)
if observed != set(EXPECTED):
raise ValueError("archive source files are missing")
return {
"original_snapshot_preserved": True,
"files_checked": len(observed),
"python_syntax_checks": python_count,
"embedded_python_checks": embedded_count,
"benchmark_executed": False,
"standalone_runnable": False,
"runtime_validation": "not_performed",
}


def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--archive", type=Path, default=ARCHIVE)
args = parser.parse_args()
try:
result = check_archive(args.archive)
except (OSError, ValueError, SyntaxError) as error:
print(json.dumps({"ok": False, "error": str(error)}))
return 1
print(json.dumps({"ok": True, **result}, indent=2))
return 0


if __name__ == "__main__":
raise SystemExit(main())
76 changes: 76 additions & 0 deletions benchmark/frontiercs2-single-host-v1/docker_environment.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
"""Private Docker transport for the existing shared benchmark installer."""
import io
from pathlib import Path
import shlex
import subprocess
import tarfile
from types import SimpleNamespace


class DockerEnvironment:
default_user = '10001:10001'

def __init__(self, name, security_bootstrap, *, sealed=False):
self.name = name
self.security_bootstrap = security_bootstrap
self.sealed = sealed

async def exec(self, command, *, user=None, env=None, cwd=None, timeout_sec=None):
effective_user = self.default_user if self.sealed else user or self.default_user
argv = ['docker', 'exec', '--user', effective_user]
if cwd:
argv += ['--workdir', cwd]
for key, value in (env or {}).items():
argv += ['--env', key + '=' + str(value)]
# Trusted installation runs as root. Every solver-side process enters
# the verified no-new-privileges bootstrap before executing its argv.
if not self.sealed and user in ('root', '0', '0:0', 0):
argv += [self.name, '/bin/bash', '-c', command]
else:
argv += [self.name, '/usr/bin/python3', '-c', self.security_bootstrap,
'/bin/bash', '-c', command]
result = subprocess.run(argv, capture_output=True, text=True, timeout=timeout_sec or 300)
if result.returncode:
print('environment command exit', result.returncode, 'user', user,
'command', command[:200], 'stderr', result.stderr[-1200:], flush=True)
return SimpleNamespace(return_code=result.returncode, stdout=result.stdout, stderr=result.stderr)

async def upload_file(self, source, target):
if self.sealed:
code = 'import pathlib,sys; p=pathlib.Path(sys.argv[1]); p.write_bytes(sys.stdin.buffer.read()); p.chmod(0o644)'
result = subprocess.run(['docker', 'exec', '-i', '--user', self.default_user,
self.name, '/usr/bin/python3', '-c', self.security_bootstrap,
'/usr/bin/python3', '-c', code, target], input=Path(source).read_bytes(),
capture_output=True, timeout=120)
if result.returncode:
raise RuntimeError(result.stderr.decode(errors='replace')[-1500:])
return
# Snap Docker has its own /tmp. Stream archive bytes rather than
# asking its client to open a host temporary pathname.
stream = io.BytesIO()
with tarfile.open(fileobj=stream, mode='w') as archive:
def owned(info):
info.uid = info.gid = 10001
info.uname = info.gname = ''
return info
archive.add(source, arcname=Path(target).name, filter=owned)
result = subprocess.run(['docker', 'cp', '-', self.name + ':' + str(Path(target).parent)],
input=stream.getvalue(), capture_output=True, timeout=120)
if result.returncode:
raise RuntimeError(result.stderr.decode(errors='replace')[-1500:])

async def upload_dir(self, source, target):
result = await self.exec('mkdir -p ' + shlex.quote(target), user='root')
if result.return_code:
raise RuntimeError(result.stderr)
stream = io.BytesIO()
with tarfile.open(fileobj=stream, mode='w') as archive:
def owned(info):
info.uid = info.gid = 10001
info.uname = info.gname = ''
return info
archive.add(source, arcname='.', filter=owned)
result = subprocess.run(['docker', 'cp', '-', self.name + ':' + target],
input=stream.getvalue(), capture_output=True, timeout=240)
if result.returncode:
raise RuntimeError(result.stderr.decode(errors='replace')[-1500:])
Loading
Loading