From 7bda3ba430ca085d7eaf2d4407dba51b9f4e4062 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 21:51:08 +0000 Subject: [PATCH 01/16] Checkpoint incomplete node generation protocol and retained payload work --- deploy/install/install.py | 108 +++++-- deploy/install/node_generations.py | 233 +++++++++++++ deploy/install/node_install.py | 20 +- deploy/install/node_spec.py | 11 +- scripts/build-core-distribution.sh | 2 +- scripts/core-distribution-manifest.py | 7 +- .../cmd/sandbox-node/generations.go | 166 ++++++++++ services/agents-api/cmd/sandbox-node/main.go | 11 + .../agents-api/cmd/server/managed_nodes.go | 19 +- .../agents-api/cmd/server/managed_setup.go | 6 +- .../cmd/server/managed_setup_test.go | 4 + .../api/sandbox_node_configuration.go | 18 +- .../internal/db/queries/node_generations.sql | 35 ++ .../internal/db/queries/runtime_nodes.sql | 7 +- .../internal/db/queries/sandbox_reset.sql | 11 +- .../agents-api/internal/db/sqlc/models.go | 12 + .../internal/db/sqlc/node_generations.sql.go | 144 +++++++++ .../internal/db/sqlc/runtime_nodes.sql.go | 33 +- .../internal/db/sqlc/sandbox_reset.sql.go | 11 +- .../agents-api/internal/sandbox/generation.go | 49 +++ .../internal/sandbox/microsandbox/process.go | 11 +- .../internal/sandbox/microsandbox/provider.go | 6 + .../agents-api/internal/sandbox/node/agent.go | 159 ++++++--- .../internal/sandbox/node/generation_agent.go | 76 +++++ .../internal/sandbox/node/generation_wire.go | 109 +++++++ .../internal/sandbox/node/generations.go | 306 ++++++++++++++++++ .../agents-api/internal/sandbox/node/hub.go | 159 +++++++-- .../agents-api/internal/sandbox/node/proxy.go | 25 +- .../agents-api/internal/sandbox/node/wire.go | 65 ++-- .../internal/sandbox/node_diagnostic.go | 3 + .../store/runtime_node_configuration.go | 27 +- .../store/runtime_node_generations.go | 138 ++++++++ .../internal/store/runtime_node_presence.go | 33 +- .../internal/store/runtime_nodes.go | 22 +- .../internal/store/runtime_placements.go | 27 +- .../internal/store/runtime_suspension.go | 2 +- .../migrations/000081_node_generations.sql | 29 ++ services/core-console/node_installation.go | 81 ++++- 38 files changed, 1970 insertions(+), 215 deletions(-) create mode 100644 deploy/install/node_generations.py create mode 100644 services/agents-api/cmd/sandbox-node/generations.go create mode 100644 services/agents-api/internal/db/queries/node_generations.sql create mode 100644 services/agents-api/internal/db/sqlc/node_generations.sql.go create mode 100644 services/agents-api/internal/sandbox/generation.go create mode 100644 services/agents-api/internal/sandbox/node/generation_agent.go create mode 100644 services/agents-api/internal/sandbox/node/generation_wire.go create mode 100644 services/agents-api/internal/sandbox/node/generations.go create mode 100644 services/agents-api/internal/store/runtime_node_generations.go create mode 100644 services/agents-api/migrations/000081_node_generations.sql diff --git a/deploy/install/install.py b/deploy/install/install.py index 99a3f6074..2460aa65c 100644 --- a/deploy/install/install.py +++ b/deploy/install/install.py @@ -323,36 +323,90 @@ def prepare_node_payload(root, state, bundle, replace=False): if state["mode"] == "core-only": return destination = root / "node-payload" - # Public distribution files only. Never copy the private installation config. - names = ["node-install.pyz", "self-hosted-install.pyz", "manifest.json", "SHA256SUMS", "runtime/seccomp.json"] - manifest = json.loads((bundle / "manifest.json").read_text()) - for logical in manifest.get("artifacts", {}): - entry = artifact(manifest, logical) - name = "artifacts/" + entry["filename"] - source = bundle / name - if source.exists(): - if (source.is_symlink() or not source.is_file() - or not source.resolve().is_relative_to(bundle.resolve()) - or source.stat().st_size != entry["size"] or digest(source) != entry["sha256"]): - raise InstallError("Offline artifact verification failed: " + logical) - names.append(name) - if replace and destination.is_dir() and not destination.is_symlink(): - shutil.rmtree(destination) - for name in names: - source, target = bundle / name, destination / name + # Each release remains immutable and addressable while old nodes retain it. + # The only mutable publication is a small, atomically replaced active pointer. + def publish(source): + manifest = json.loads((source / "manifest.json").read_text()) + revision = manifest.get("source_commit", "") + if not re.fullmatch(r"[0-9a-f]{40}", revision): + raise InstallError("Invalid node payload release identity") + names = ["node-install.pyz", "self-hosted-install.pyz", "manifest.json", "SHA256SUMS", "runtime/seccomp.json"] + for logical in manifest.get("artifacts", {}): + entry = artifact(manifest, logical) + name = "artifacts/" + entry["filename"] + path = source / name + if path.exists(): + if (path.is_symlink() or not path.is_file() + or not path.resolve().is_relative_to(source.resolve()) + or path.stat().st_size != entry["size"] or digest(path) != entry["sha256"]): + raise InstallError("Offline artifact verification failed: " + logical) + names.append(name) + target = destination / "releases" / revision + if target.is_symlink(): + raise InstallError("Installed node payload differs; preserve it and inspect the distribution") target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) if target.exists(): - if target.is_symlink() or not target.is_file() or digest(target) != digest(source): - raise InstallError("Installed node payload differs; preserve it and inspect the distribution") - else: - descriptor, temporary = tempfile.mkstemp(prefix=".payload-", dir=target.parent) + for name in names: + previous = target / name + if previous.is_symlink() or not previous.is_file() or digest(previous) != digest(source / name): + raise InstallError("Installed node payload differs; preserve it and inspect the distribution") + return revision + with tempfile.TemporaryDirectory(prefix=".payload-", dir=target.parent) as temporary: + stage = Path(temporary) / "release" + stage.mkdir(mode=0o700) + for name in names: + path = source / name + if path.is_symlink() or not path.is_file(): + raise InstallError("Invalid node payload source file") + copied = stage / name + copied.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + with path.open("rb") as incoming, copied.open("xb") as outgoing: + os.chmod(copied, 0o600) + shutil.copyfileobj(incoming, outgoing) + outgoing.flush() + os.fsync(outgoing.fileno()) + os.rename(stage, target) + return revision + + if destination.is_symlink(): + raise InstallError("Invalid node payload directory") + destination.mkdir(mode=0o700, exist_ok=True) + # Preserve the legacy flat release before first publication. Its checksum + # list authenticates the files we copy; unrelated private files are excluded. + if not (destination / "active.json").exists() and (destination / "manifest.json").exists(): + sums = {} + for line in (destination / "SHA256SUMS").read_text().splitlines(): + value, name = line.split(" ", 1) + sums[name] = value + for name in ("node-install.pyz", "self-hosted-install.pyz", "manifest.json", "runtime/seccomp.json"): + path = destination / name + if path.is_symlink() or not path.is_file() or digest(path) != sums.get(name): + raise InstallError("Legacy node payload checksum differs; preserve it before upgrade") + publish(destination) + revision = publish(bundle) + pointer = destination / "active.json" + if pointer.is_symlink(): + raise InstallError("Invalid active node payload pointer") + if pointer.exists() and not replace: + if json.loads(pointer.read_text()) != {"source_commit": revision}: + raise InstallError("Installed node payload differs; preserve it and inspect the distribution") + return + descriptor, temporary = tempfile.mkstemp(prefix=".active-", dir=destination) + try: + with os.fdopen(descriptor, "w") as stream: + json.dump({"source_commit": revision}, stream) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, pointer) + descriptor = os.open(destination, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: os.close(descriptor) - try: - shutil.copyfile(source, temporary) - os.replace(temporary, target) - finally: - if os.path.exists(temporary): - os.unlink(temporary) + finally: + if os.path.exists(temporary): + os.unlink(temporary) def install_oac(root, bundle): diff --git a/deploy/install/node_generations.py b/deploy/install/node_generations.py new file mode 100644 index 000000000..4ddd06e12 --- /dev/null +++ b/deploy/install/node_generations.py @@ -0,0 +1,233 @@ +"""Private node generation preparation and collection; Core owns all authorization.""" +import copy +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +from types import SimpleNamespace +import zipapp + + +def atomic_json(path, value): + descriptor, temporary = tempfile.mkstemp(prefix=".generation-", dir=path.parent) + try: + with os.fdopen(descriptor, "w") as stream: + json.dump(value, stream, sort_keys=True) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + descriptor = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + +def install_helper(root, args, installer): + """Keep the executed, trusted installer available to the unprivileged node.""" + settings = root / "preparation.json" + value = {"source_url": args.source_url or args.core_url} + if settings.exists() and installer.private_json(settings) != value: + raise installer.InstallError("The retained node artifact origin differs; preserve its configuration") + target = root / "generation-preparer.pyz" + installer.existing_file(target) + source = Path(sys.argv[0]) + if getattr(args, "bundle", None) is not None: + source = args.bundle / "node-install.pyz" + with tempfile.TemporaryDirectory(dir=root) as directory: + staged = Path(directory) / "helper.pyz" + if source.suffix == ".pyz" and source.is_file() and not source.is_symlink(): + shutil.copyfile(source, staged) + else: + package = Path(directory) / "package" + package.mkdir(mode=0o700) + source_dir = Path(installer.__file__).parent + for name in ("node_install.py", "node_spec.py", "distribution.py", "node_generations.py"): + shutil.copyfile(source_dir / name, package / ("__main__.py" if name == "node_install.py" else name)) + zipapp.create_archive(package, staged, compressed=True) + os.chmod(staged, 0o600) + os.replace(staged, target) + atomic_json(settings, value) + + +def retained_configs(root, installer): + directory = root / "state/node/generations" + result = {} + base = installer.private_json(root / "provider.json") + if base and not (directory / (str(base["generation"]) + ".dropped")).exists(): + result[base["generation"]] = base + if directory.exists(): + for path in sorted(directory.glob("*.json")): + if not re.fullmatch(r"[1-9][0-9]*\.json", path.name): + raise installer.InstallError("Invalid retained generation filename") + value = installer.private_json(path) + if value is None or value.get("generation") != int(path.stem): + raise installer.InstallError("Invalid retained generation configuration") + if not (directory / (path.stem + ".dropped")).exists(): + result[value["generation"]] = value + return result + + +def owned_root(args, installer): + if os.getuid() == 0: + raise installer.InstallError("Generation operations run only as the node service user") + root = Path.home() / ".oac/nodes" / args.installation_id + if not root.is_dir(): + raise installer.InstallError("Retained node installation is missing") + installer.safe_directory(root) + identity = installer.private_json(root / "state/node/identity.json") + if not identity or identity["identity"]["installation_id"] != args.installation_id: + raise installer.InstallError("Retained node identity differs") + return root, identity + + +def generation_home(root, configuration, base, installer): + runtime = configuration["specification"]["runtime"] + previous = base["microsandbox"] + if (runtime["runtime_sha256"], runtime["firmware_sha256"]) == (previous["runtime_sha256"], previous["firmware_sha256"]): + return Path(previous["runtime_home"]) + material = ":".join((configuration["installation_id"], runtime["runtime_sha256"], runtime["firmware_sha256"])) + home = Path.home() / ".oac/m" / hashlib.sha256(material.encode()).hexdigest()[:12] + if len(os.fsencode(home)) > 48: + raise installer.InstallError("HOME is too long for versioned microsandbox socket paths") + return home + + +def image_available(value, installer): + try: + if value["provider"] == "docker": + raw = installer.checked(list(installer.DOCKER) + ["image", "inspect", value["docker"]["image"], "--format", "{{.Id}} {{.Os}}/{{.Architecture}}"], "Cannot inspect pinned image") + return raw.strip() == value["docker"]["image"] + " linux/amd64" + micro = value["microsandbox"] + env = dict(os.environ, MSB_BACKEND="local", MSB_HOME=micro["runtime_home"], MSB_PATH=micro["runtime_path"], MSB_LIBKRUNFW_PATH=micro["firmware_path"]) + image = json.loads(installer.checked([micro["runtime_path"], "image", "inspect", micro["image"], "--format", "json"], "Cannot inspect pinned image", env=env)) + return image.get("digest") == micro["image"].split("@", 1)[1] and image.get("architecture") == "amd64" and image.get("os") == "linux" + except (installer.InstallError, OSError, ValueError): + return False + + +def prepare(args, installer): + root, identity = owned_root(args, installer) + with installer.install_lock(root): + directory = root / "state/node/generations" + installer.safe_directory(directory) + if (directory / (str(args.generation) + ".dropped")).exists(): + raise installer.InstallError("A dropped generation cannot be adopted again") + args.core_url = identity["core_url"] + args.configuration = installer.node_spec.fetch(args, "", identity, installer.open_request, + generation=args.generation, allow_selection_change=True) + if args.configuration["specification_digest"] != args.specification_digest: + raise installer.InstallError("Core generation identity differs from its authorization") + args.provider = args.configuration["provider"] + configurations = retained_configs(root, installer) + base = installer.private_json(root / "provider.json") + runtime = args.configuration["specification"]["runtime"] + value = None + for candidate in configurations.values(): + if candidate["specification"]["runtime"] == runtime and image_available(candidate, installer): + value = copy.deepcopy(candidate) + value["generation"] = args.generation + value["specification"] = args.configuration["specification"] + if args.provider == "microsandbox": + value["microsandbox"].update(value["specification"]["resources"]) + break + if value is None: + settings = installer.private_json(root / "preparation.json") + args.source_url = installer.origin(settings["source_url"]) + args.bundle = None + prefix = "releases/" + runtime["source_commit"] + "/" + manifest, sums = installer.metadata(args.source_url, prefix=prefix) + installer.node_spec.verify_release(args.configuration, manifest) + release = root / "releases" / runtime["source_commit"] + installer.safe_directory(release.parent) + names = ("runtime/seccomp.json",) + (installer.MICRO if args.provider == "microsandbox" else ()) + if not release.exists(): + with tempfile.TemporaryDirectory(prefix=".release-", dir=release.parent) as temporary: + stage = Path(temporary) + for name in names: + installer.safe_directory((stage / name).parent) + if name == "runtime/seccomp.json": + with installer.fetch(args.source_url, prefix + name) as stream: + raw = stream.read(1024 * 1024 + 1) + if hashlib.sha256(raw).hexdigest() != sums.get(name): + raise installer.InstallError("Runtime seccomp checksum differs") + (stage / name).write_bytes(raw) + os.chmod(stage / name, 0o600) + else: + installer.distribution.obtain_artifact(manifest, name, stage / name) + os.chmod(stage / name, 0o700) + atomic_json(stage / "manifest.json", manifest) + os.rename(stage, release) + saved = installer.private_json(release / "manifest.json") + # The origin URL is transport metadata, not immutable release identity. + installer.node_spec.verify_release(args.configuration, saved) + for name in names: + installer.existing_file(release / name) + expected = sums[name] if name == "runtime/seccomp.json" else installer.distribution.artifact(manifest, name)["sha256"] + if installer.file_digest(release / name) != expected: + raise installer.InstallError("Retained Runtime artifact checksum differs") + if args.provider == "microsandbox": + args.runtime_home = generation_home(root, args.configuration, base, installer) + installer.safe_directory(args.runtime_home) + owner = args.runtime_home / "oac-installation.json" + if not owner.exists() and any(args.runtime_home.iterdir()): + raise installer.InstallError("Versioned microsandbox store contains unowned state") + installer.write_once(owner, installer.json_text({"installation_id": args.installation_id})) + runtime_image = installer.prepare_runtime(release, args, manifest) + value = installer.provider_config(release, args, manifest, runtime_image) + target = directory / (str(args.generation) + ".json") + if installer.existing_file(target) and installer.private_json(target) != value: + raise installer.InstallError("Immutable generation configuration differs") + atomic_json(target, value) + print(json.dumps(value)) + + +def collect(args, installer): + root, _ = owned_root(args, installer) + with installer.install_lock(root): + configurations = retained_configs(root, installer) + value = configurations.get(args.generation) + if value is None: + return + if installer.node_spec.digest(value["provider"], value["specification"]) != args.specification_digest: + raise installer.InstallError("Collection grant does not match the local generation") + others = [item for generation, item in configurations.items() if generation != args.generation] + if value["provider"] == "microsandbox": + micro = value["microsandbox"] + shared = [item for item in others if item["microsandbox"]["runtime_home"] == micro["runtime_home"]] + env = dict(os.environ, MSB_BACKEND="local", MSB_HOME=micro["runtime_home"], MSB_PATH=micro["runtime_path"], MSB_LIBKRUNFW_PATH=micro["firmware_path"]) + if not any(item["microsandbox"]["image"] == micro["image"] for item in shared): + # No force: native ownership can still refuse removal after Core's grant. + installer.checked([micro["runtime_path"], "image", "remove", micro["image"], "--quiet"], "Runtime image is still in use", env=env) + if not shared: + raw = installer.checked([micro["runtime_path"], "sandbox", "list", "--format", "json"], "Cannot verify empty microsandbox store", env=env) + if json.loads(raw) != []: + raise installer.InstallError("Microsandbox store still contains native sandboxes") + home = Path(micro["runtime_home"]) + installer.safe_directory(home) + if installer.private_json(home / "oac-installation.json") != {"installation_id": args.installation_id}: + raise installer.InstallError("Microsandbox store ownership differs") + # Keep receipt/lock history conservatively; an empty VM list is not + # authority to discard unknown provider receipts or shared host state. + else: + image = value["docker"]["image"] + if not any(item["docker"]["image"] == image for item in others): + installer.checked(list(installer.DOCKER) + ["image", "rm", image], "Docker image is still in use") + directory = root / "state/node/generations" + installer.safe_directory(directory) + atomic_json(directory / (str(args.generation) + ".dropped"), {"specification_digest": args.specification_digest}) + (directory / (str(args.generation) + ".json")).unlink(missing_ok=True) + source = value["specification"]["runtime"]["source_commit"] + if not any(item["specification"]["runtime"]["source_commit"] == source for item in others): + release = root / "releases" / source + if release.exists() and not release.is_symlink() and release.resolve().parent == (root / "releases").resolve(): + shutil.rmtree(release) diff --git a/deploy/install/node_install.py b/deploy/install/node_install.py index abc5d1d0e..a1a8d5532 100644 --- a/deploy/install/node_install.py +++ b/deploy/install/node_install.py @@ -34,6 +34,7 @@ import distribution import node_spec +import node_generations class InstallError(Exception): @@ -197,7 +198,7 @@ def fetch(source, name): time.sleep(attempt + 1) -def metadata(source, bundle=None): +def metadata(source, bundle=None, prefix=""): if bundle is not None: def read(name): path = bundle / name @@ -205,7 +206,7 @@ def read(name): raise InstallError("Invalid local distribution metadata") return path.open("rb") else: - read = lambda name: fetch(source, name) + read = lambda name: fetch(source, prefix + name) with read("SHA256SUMS") as response: raw = response.read(1024 * 1024 + 1) if len(raw) > 1024 * 1024: @@ -228,7 +229,7 @@ def read(name): for name in (COMMON[0], "images/runtime.tar.gz") + MICRO: distribution.artifact(manifest, name) # Nodes download only from their console, never from a release URL the build recorded. - manifest["artifact_base_url"] = source + "/node-install/artifacts" if source else "" + manifest["artifact_base_url"] = source + "/node-install/" + prefix + "artifacts" if source else "" return manifest, sums @@ -315,7 +316,7 @@ def provider_config(root, args, manifest, runtime_image): result["microsandbox"] = { "helper_path": str(root / MICRO[0]), "runtime_path": str(root / MICRO[1]), "firmware_path": str(root / MICRO[2]), "runtime_sha256": manifest["microsandbox"]["runtime_sha256"], "firmware_sha256": manifest["microsandbox"]["firmware_sha256"], - "runtime_home": str(micro_home(args.installation_id)), "image": manifest["runtime_ref"], + "runtime_home": str(getattr(args, "runtime_home", micro_home(args.installation_id))), "image": manifest["runtime_ref"], **args.configuration["specification"]["resources"], "network": {"default_egress": "deny", "default_ingress": "deny", "rules": core_rules + [ {"action": "allow", "direction": "egress", "destination": "public"}, @@ -342,7 +343,7 @@ def prepare_runtime(root, args, manifest): output = result.stdout.decode() if "not found" in output or (result.returncode and "statically linked" not in output and "not a dynamic executable" not in output): raise InstallError("Install the microsandbox host shared-library prerequisites") - env = dict(os.environ, MSB_BACKEND="local", MSB_HOME=str(micro_home(args.installation_id)), + env = dict(os.environ, MSB_BACKEND="local", MSB_HOME=str(getattr(args, "runtime_home", micro_home(args.installation_id))), MSB_PATH=str(root / MICRO[1]), MSB_LIBKRUNFW_PATH=str(root / MICRO[2])) inspect = [str(root / MICRO[1]), "image", "inspect", manifest["runtime_ref"], "--format", "json"] def matches(): @@ -438,6 +439,7 @@ def register_node(root, args, token): existing_file(target) distribution.obtain_artifact(manifest, name, target, getattr(args, "bundle", None)) os.chmod(target, 0o700) + node_generations.install_helper(root, args, sys.modules[__name__]) safe_directory(root / "state/node") print("Checking the sandbox runtime...", flush=True) runtime_image = prepare_runtime(root, args, manifest) @@ -1425,6 +1427,9 @@ def main(argv=None): parser.add_argument("--provider", choices=("docker", "microsandbox"), help="Optional assertion; Core owns provider selection") parser.add_argument("--installation-id", required=True) parser.add_argument("--enrollment-token-stdin", action="store_true", help="Read the one-time enrollment token from standard input") + parser.add_argument("--generation-action", choices=("prepare", "collect"), help=argparse.SUPPRESS) + parser.add_argument("--generation", type=int, help=argparse.SUPPRESS) + parser.add_argument("--specification-digest", help=argparse.SUPPRESS) parser.add_argument("--uninstall", action="store_true", help="Remove this host's node after it was removed on the Nodes page") parser.add_argument("--force", action="store_true", help="With --uninstall: skip the Core check, for a Core that no longer exists") args = parser.parse_args(argv) @@ -1433,6 +1438,11 @@ def main(argv=None): "--enrollment-token-stdin.\n") if str(uuid.UUID(args.installation_id)) != args.installation_id: raise InstallError("Installation ID must be a canonical UUID") + if args.generation_action: + if args.generation is None or not 1 <= args.generation <= 9223372036854775807 or not re.fullmatch(r"[0-9a-f]{64}", args.specification_digest or ""): + parser.error("Invalid generation authorization") + (node_generations.prepare if args.generation_action == "prepare" else node_generations.collect)(args, sys.modules[__name__]) + return if args.uninstall: if args.source_url or args.bundle or args.core_url or args.provider or args.enrollment_token_stdin: parser.error("--uninstall takes only --installation-id and --force") diff --git a/deploy/install/node_spec.py b/deploy/install/node_spec.py index a7363c6ba..2165e20cb 100644 --- a/deploy/install/node_spec.py +++ b/deploy/install/node_spec.py @@ -78,7 +78,7 @@ def validate(data, args): return data -def fetch(args, token, retained, open_request, allow_enrollment=False): +def fetch(args, token, retained, open_request, allow_enrollment=False, generation=None, allow_selection_change=False): headers = {"Authorization": "Bearer " + token} if retained is not None: try: @@ -92,7 +92,8 @@ def fetch(args, token, retained, open_request, allow_enrollment=False): raise SpecificationError("Retained node identity differs or is invalid; preserve its state") from None elif not token: raise SpecificationError("A new node needs its one-time enrollment token on standard input (--enrollment-token-stdin); copy the command from Add node") - request = urllib.request.Request(args.core_url + "/api/v1/sandbox-node/configuration", headers=headers) + query = "?generation=" + str(generation) if generation is not None else "" + request = urllib.request.Request(args.core_url + "/api/v1/sandbox-node/configuration" + query, headers=headers) try: try: response = open_request(request) @@ -126,9 +127,11 @@ def fetch(args, token, retained, open_request, allow_enrollment=False): if retained is not None: identity = retained["identity"] if (identity.get("provider") != data["provider"] - or identity.get("specification_digest") != data["specification_digest"] - or identity.get("deployment_generation") != data["generation"]): + or not allow_selection_change and (identity.get("specification_digest") != data["specification_digest"] + or identity.get("deployment_generation") != data["generation"])): raise SpecificationError("Retained node specification differs from Core; preserve its state and follow the deployment change procedure") + if generation is not None and data["generation"] != generation: + raise SpecificationError("Core returned a different generation") return data diff --git a/scripts/build-core-distribution.sh b/scripts/build-core-distribution.sh index 17854b802..5cb6c0a0f 100755 --- a/scripts/build-core-distribution.sh +++ b/scripts/build-core-distribution.sh @@ -99,7 +99,7 @@ if [[ "$(go env GOVERSION)" != "$required_go" ]]; then exit 1 fi for file in install.sh install.py configuration.py config_model.py config.schema.json oac_cli.py convert.py rename.py \ - native_service.py node_install.py node_spec.py sandbox_setup.py distribution.py self_hosted_install.py \ + native_service.py node_install.py node_spec.py node_generations.py sandbox_setup.py distribution.py self_hosted_install.py \ model_provider_sessions.py; do cp "deploy/install/$file" "$bundle/$file" done diff --git a/scripts/core-distribution-manifest.py b/scripts/core-distribution-manifest.py index 858e5498b..560f5fa33 100644 --- a/scripts/core-distribution-manifest.py +++ b/scripts/core-distribution-manifest.py @@ -253,9 +253,10 @@ def bootstraps(bundle, epoch): shutil.copyfile(bundle / original, target) os.utime(target, (int(epoch), int(epoch))) if source == "node_install.py": - target = pathlib.Path(directory) / "node_spec.py" - shutil.copyfile(bundle / "node_spec.py", target) - os.utime(target, (int(epoch), int(epoch))) + for name in ("node_spec.py", "node_generations.py"): + target = pathlib.Path(directory) / name + shutil.copyfile(bundle / name, target) + os.utime(target, (int(epoch), int(epoch))) zipapp.create_archive(directory, bundle / output, compressed=True) with tempfile.TemporaryDirectory(dir=bundle.parent) as directory: for name in OAC_CLI_MODULES: diff --git a/services/agents-api/cmd/sandbox-node/generations.go b/services/agents-api/cmd/sandbox-node/generations.go new file mode 100644 index 000000000..9111fd0ea --- /dev/null +++ b/services/agents-api/cmd/sandbox-node/generations.go @@ -0,0 +1,166 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + providerconfig "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/config" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/node" +) + +func runGenerations(ctx context.Context, configFile, stateDir string) error { + root := filepath.Dir(configFile) + if stateDir != filepath.Join(root, "state", "node") { + return errors.New("generation state must belong to the installed node root") + } + stored, err := node.RefreshIdentity(ctx, stateDir) + if err != nil { + return err + } + base, err := providerconfig.Load(configFile) + if err != nil { + return err + } + if base.InstallationID != stored.Identity.InstallationID || base.Provider != stored.Identity.Provider || base.Generation != stored.Identity.DeploymentGeneration || base.Specification.Digest(base.Provider) != stored.Identity.SpecificationDigest { + return sandbox.ErrOwnership + } + paths, err := filepath.Glob(filepath.Join(stateDir, "generations", "*.json")) + if err != nil { + return err + } + paths = append([]string{configFile}, paths...) + values := map[uint64]node.GenerationProvider{} + closeValues := func() { + for _, v := range values { + if v.Close != nil { + v.Close() + } + } + } + for _, path := range paths { + config, err := providerconfig.Load(path) + if err != nil { + closeValues() + return err + } + if config.InstallationID != base.InstallationID || config.Provider != base.Provider { + closeValues() + return sandbox.ErrOwnership + } + if path != configFile && filepath.Base(path) != strconv.FormatUint(config.Generation, 10)+".json" { + closeValues() + return sandbox.ErrOwnership + } + if _, err := os.Lstat(filepath.Join(stateDir, "generations", strconv.FormatUint(config.Generation, 10)+".dropped")); err == nil { + continue + } else if !os.IsNotExist(err) { + closeValues() + return err + } + if _, ok := values[config.Generation]; ok { + continue + } + value, err := buildGeneration(config) + if err != nil { + closeValues() + return err + } + values[config.Generation] = value + } + helper := filepath.Join(root, "generation-preparer.pyz") + runHelper := func(ctx context.Context, action string, generation uint64, digest string) error { + command := exec.CommandContext(ctx, "python3", helper, "--installation-id", base.InstallationID, "--generation-action", action, "--generation", strconv.FormatUint(generation, 10), "--specification-digest", digest) + command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + // Only the preparation/collection process group is canceled. Native sandbox + // helpers retain their independent allocation-lock completion semantics. + command.Cancel = func() error { return syscall.Kill(-command.Process.Pid, syscall.SIGKILL) } + command.Stdout = os.Stderr + command.Stderr = os.Stderr + if err := command.Run(); err != nil { + return fmt.Errorf("%w: local generation %s did not complete", sandbox.ErrRuntimeDownloadFailed, action) + } + return nil + } + initial := make([]node.GenerationProvider, 0, len(values)) + for _, v := range values { + initial = append(initial, v) + } + manager, err := node.NewGenerationManager(ctx, node.GenerationManagerOptions{Initial: initial, + Prepare: func(ctx context.Context, generation uint64, digest string) (node.GenerationProvider, error) { + if err := runHelper(ctx, "prepare", generation, digest); err != nil { + return node.GenerationProvider{}, err + } + config, err := providerconfig.Load(filepath.Join(stateDir, "generations", strconv.FormatUint(generation, 10)+".json")) + if err != nil { + return node.GenerationProvider{}, err + } + if config.InstallationID != base.InstallationID || config.Provider != base.Provider || config.Generation != generation || config.Specification.Digest(config.Provider) != digest { + return node.GenerationProvider{}, sandbox.ErrOwnership + } + value, err := buildGeneration(config) + if err != nil { + return value, err + } + probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + return value, value.Probe(probeCtx) + }, + Remove: func(ctx context.Context, value node.GenerationProvider) error { + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + return runHelper(ctx, "collect", value.Generation, value.SpecificationDigest) + }, + }) + if err != nil { + closeValues() + return err + } + defer manager.Close() + return node.Run(ctx, node.AgentConfig{CoreURL: stored.CoreURL, StateDirectory: stateDir, Identity: stored.Identity, Credential: stored.Credential, Generations: manager}) +} + +func buildGeneration(config providerconfig.Config) (node.GenerationProvider, error) { + built, closeProvider, err := providerconfig.Build(config) + if err != nil { + return node.GenerationProvider{}, err + } + probe := built.Probe + if micro := config.Microsandbox; micro != nil { + probe = func(ctx context.Context) error { + if err := built.Probe(ctx); err != nil { + return err + } + command := exec.CommandContext(ctx, micro.RuntimePath, "image", "inspect", micro.Image, "--format", "json") + command.Env = append([]string{"PATH=/usr/local/bin:/usr/bin:/bin", "HOME=" + os.Getenv("HOME")}, "MSB_BACKEND=local", "MSB_HOME="+micro.RuntimeHome, "MSB_PATH="+micro.RuntimePath, "MSB_LIBKRUNFW_PATH="+micro.FirmwarePath) + reader, err := command.StdoutPipe() + if err != nil { + return sandbox.ErrRuntimeImageUnavailable + } + if err := command.Start(); err != nil { + return sandbox.ErrRuntimeImageUnavailable + } + raw, readErr := io.ReadAll(io.LimitReader(reader, 64*1024+1)) + if len(raw) > 64*1024 { + _ = command.Process.Kill() + } + waitErr := command.Wait() + var image struct{ Digest, Architecture, OS string } + if readErr != nil || waitErr != nil || len(raw) > 64*1024 || json.Unmarshal(raw, &image) != nil || image.Digest != strings.SplitN(micro.Image, "@", 2)[1] || image.Architecture != "amd64" || image.OS != "linux" { + return sandbox.ErrRuntimeImageUnavailable + } + return nil + } + } + return node.GenerationProvider{Generation: config.Generation, SpecificationDigest: built.SpecificationDigest, Provider: built.Provider, Probe: probe, Close: closeProvider}, nil +} diff --git a/services/agents-api/cmd/sandbox-node/main.go b/services/agents-api/cmd/sandbox-node/main.go index cd0ccc9f0..1cb80b83e 100644 --- a/services/agents-api/cmd/sandbox-node/main.go +++ b/services/agents-api/cmd/sandbox-node/main.go @@ -65,6 +65,17 @@ func run(ctx context.Context, args []string) error { if !filepath.IsAbs(*configFile) || !filepath.IsAbs(*stateDir) { return errors.New("config and state-dir must be absolute paths") } + if args[0] == "run" { + helper := filepath.Join(filepath.Dir(*configFile), "generation-preparer.pyz") + if info, err := os.Lstat(helper); err == nil { + if !info.Mode().IsRegular() || info.Mode().Perm() != 0600 { + return errors.New("generation preparer must be a private regular file") + } + return runGenerations(ctx, *configFile, *stateDir) + } else if !os.IsNotExist(err) { + return err + } + } config, err := providerconfig.Load(*configFile) if err != nil { return err diff --git a/services/agents-api/cmd/server/managed_nodes.go b/services/agents-api/cmd/server/managed_nodes.go index daa669867..8450b765b 100644 --- a/services/agents-api/cmd/server/managed_nodes.go +++ b/services/agents-api/cmd/server/managed_nodes.go @@ -8,6 +8,7 @@ import ( "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/api" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/execution" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/node" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" "github.com/google/uuid" @@ -52,6 +53,18 @@ func configureManagedNodes(s *store.Store, publicURL string, owner func(context. return nil, errors.New("Web sandbox setup requires OAC_CORE_KEY_DIGESTS_FILE with the Core key digest") } result.hub = node.NewHub(node.HubOptions{ + Generations: func(ctx context.Context, n node.Identity, connection string, epoch uint64, health node.Health) error { + if err := owner(ctx); err != nil { + return err + } + return s.HeartbeatRuntimeNodeGenerations(ctx, n.NodeID, connection, epoch, nodeHealthRecord(health), health.Generations) + }, + Retention: func(ctx context.Context, n node.Identity, connection string, epoch uint64, refs []sandbox.GenerationReference) (sandbox.NodeDeployment, []sandbox.GenerationRetention, error) { + if err := owner(ctx); err != nil { + return sandbox.NodeDeployment{}, nil, err + } + return s.RuntimeNodeRetention(ctx, n.NodeID, connection, epoch, refs) + }, Authenticate: func(ctx context.Context, id, credential string) (node.Identity, error) { n, err := s.AuthenticateRuntimeNode(ctx, id, credential) if errors.Is(err, store.ErrRuntimeNodeCredential) { @@ -78,7 +91,7 @@ func configureManagedNodes(s *store.Store, publicURL string, owner func(context. if err := owner(ctx); err != nil { return err } - return s.HeartbeatRuntimeNode(ctx, n.NodeID, connection, epoch, store.RuntimeNodeHealth{Host: &store.RuntimeNodeHost{EffectiveCPUCores: health.EffectiveCPUCores, CPUUtilization: health.CPUUtilization, TotalMemoryBytes: health.TotalMemoryBytes, AvailableMemoryBytes: health.AvailableMemoryBytes, AvailableDiskBytes: health.AvailableDiskBytes, ObservedAt: &health.ObservedAt}, ProviderReady: health.ProviderReady, Diagnostic: health.Diagnostic, CPUCount: health.CPUCount, AvailableMemoryBytes: health.AvailableMemoryBytes, AvailableDiskBytes: health.AvailableDiskBytes}) + return s.HeartbeatRuntimeNode(ctx, n.NodeID, connection, epoch, nodeHealthRecord(health)) }, }) result.setup = &managedSetup{store: s, hub: result.hub, installationID: setupID, publicURL: publicURL} @@ -117,3 +130,7 @@ func serverAddress() string { } return "127.0.0.1:8091" } + +func nodeHealthRecord(health node.Health) store.RuntimeNodeHealth { + return store.RuntimeNodeHealth{Host: &store.RuntimeNodeHost{EffectiveCPUCores: health.EffectiveCPUCores, CPUUtilization: health.CPUUtilization, TotalMemoryBytes: health.TotalMemoryBytes, AvailableMemoryBytes: health.AvailableMemoryBytes, AvailableDiskBytes: health.AvailableDiskBytes, ObservedAt: &health.ObservedAt}, ProviderReady: health.ProviderReady, Diagnostic: health.Diagnostic, CPUCount: health.CPUCount, AvailableMemoryBytes: health.AvailableMemoryBytes, AvailableDiskBytes: health.AvailableDiskBytes} +} diff --git a/services/agents-api/cmd/server/managed_setup.go b/services/agents-api/cmd/server/managed_setup.go index b362c09a1..df7f21c4e 100644 --- a/services/agents-api/cmd/server/managed_setup.go +++ b/services/agents-api/cmd/server/managed_setup.go @@ -23,7 +23,7 @@ import ( type managedSetup struct { store interface { GetSandboxSetup(context.Context) (store.SandboxSetup, error) - ResolveRuntimeNode(context.Context, string, string) (string, error) + ResolveRuntimeGeneration(context.Context, sandbox.Reference) (string, uint64, error) } hub *node.Hub installationID string @@ -182,9 +182,7 @@ func (s *managedSetup) provider(setup store.SandboxSetup) (sandbox.Provider, err if s.hub == nil { return nil, errors.New("sandbox node transport is unavailable") } - return s.hub.Provider(setup.Provider, func(ctx context.Context, ref sandbox.Reference) (string, error) { - return s.store.ResolveRuntimeNode(ctx, ref.TenantID, ref.EnvironmentID) - }), nil + return s.hub.GenerationProvider(setup.Provider, s.store.ResolveRuntimeGeneration), nil case "e2b": if setup.E2B == nil { return nil, errors.New("E2B deployment configuration is unavailable") diff --git a/services/agents-api/cmd/server/managed_setup_test.go b/services/agents-api/cmd/server/managed_setup_test.go index 81dac915f..1bb0cd09c 100644 --- a/services/agents-api/cmd/server/managed_setup_test.go +++ b/services/agents-api/cmd/server/managed_setup_test.go @@ -205,3 +205,7 @@ func (s *setupStore) SandboxGenerationPage(context.Context, int64) ([]store.Sand func (s *setupStore) SandboxCredentialAllocationPage(context.Context, string) ([]store.RuntimeAllocation, error) { return nil, nil } + +func (s *setupStore) ResolveRuntimeGeneration(context.Context, sandbox.Reference) (string, uint64, error) { + return "", 0, errors.New("unexpected node generation lookup") +} diff --git a/services/agents-api/internal/api/sandbox_node_configuration.go b/services/agents-api/internal/api/sandbox_node_configuration.go index aa35f67af..cdd6370e1 100644 --- a/services/agents-api/internal/api/sandbox_node_configuration.go +++ b/services/agents-api/internal/api/sandbox_node_configuration.go @@ -2,7 +2,9 @@ package api import ( "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "math" "net/http" + "strconv" ) // @Summary Read the active configuration for node installation @@ -11,6 +13,7 @@ import ( // @Produce json // @Security NodeEnrollmentAuth // @Param X-OAC-Node-ID header string false "Retained node UUID" +// @Param generation query integer false "Exact kept generation (registered nodes only); omitted reads the current target" // @Success 200 {object} store.RuntimeNodeConfiguration // @Failure 400,401,409,500,503 {object} v1.ErrorResponse // @Router /api/v1/sandbox-node/configuration [get] @@ -25,7 +28,20 @@ func (h *Handler) sandboxNodeConfiguration(w http.ResponseWriter, r *http.Reques writeStoreError(w, r, store.ErrRuntimeNodeCredential) return } - value, err := h.sandboxStore.RuntimeNodeConfiguration(r.Context(), r.Header.Get("X-OAC-Node-ID"), token) + var generation uint64 + if values, present := r.URL.Query()["generation"]; present { + var err error + if len(values) != 1 { + writeStoreError(w, r, store.ErrInvalidInput) + return + } + generation, err = strconv.ParseUint(values[0], 10, 63) + if err != nil || generation == 0 || generation > math.MaxInt64 || strconv.FormatUint(generation, 10) != values[0] { + writeStoreError(w, r, store.ErrInvalidInput) + return + } + } + value, err := h.sandboxStore.RuntimeNodeGenerationConfiguration(r.Context(), r.Header.Get("X-OAC-Node-ID"), token, generation) if err != nil { writeStoreError(w, r, err) return diff --git a/services/agents-api/internal/db/queries/node_generations.sql b/services/agents-api/internal/db/queries/node_generations.sql new file mode 100644 index 000000000..7b917e529 --- /dev/null +++ b/services/agents-api/internal/db/queries/node_generations.sql @@ -0,0 +1,35 @@ +-- name: NodeGenerationKept :one +SELECT (EXISTS(SELECT 1 FROM runtime_deployment d WHERE d.generation=sqlc.arg(generation)) + OR EXISTS(SELECT 1 FROM runtime_nodes n WHERE n.id=sqlc.arg(node_id) AND n.removed_at IS NULL AND n.ready_generation=sqlc.arg(generation)) + OR EXISTS(SELECT 1 FROM runtime_placements p WHERE p.node_id=sqlc.arg(node_id) AND p.released_at IS NULL AND p.deployment_generation=sqlc.arg(generation)) + OR EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.node_id=sqlc.arg(node_id) AND a.state<>'released' AND a.deployment_generation=sqlc.arg(generation)))::boolean AS kept; + +-- name: GetNodeGenerationSpecification :one +SELECT d.provider_kind,d.specification FROM runtime_deployment d WHERE d.generation=$1 +UNION ALL SELECT g.provider_kind,g.specification FROM runtime_deployment_generations g WHERE g.generation=$1 AND g.generation<>(SELECT d.generation FROM runtime_deployment d) +LIMIT 1; + +-- name: UpsertNodeGenerationStatus :exec +INSERT INTO runtime_node_generation_status(node_id,generation,specification_digest,connection_id,owner_epoch,state,diagnostic) +VALUES($1,$2,$3,$4,$5,$6,$7) +ON CONFLICT(node_id,generation) DO UPDATE SET specification_digest=EXCLUDED.specification_digest,connection_id=EXCLUDED.connection_id, + owner_epoch=EXCLUDED.owner_epoch,state=EXCLUDED.state,diagnostic=EXCLUDED.diagnostic,observed_at=clock_timestamp(); + +-- name: PromoteNodeServingGeneration :exec +UPDATE runtime_nodes SET ready_generation=$2 WHERE id=$1 AND removed_at IS NULL +AND $2=(SELECT generation FROM runtime_deployment) AND (ready_generation IS NULL OR ready_generation<$2); + +-- name: RefreshNodeServingReadiness :exec +UPDATE runtime_nodes n SET protocol_version=$2, provider_ready=EXISTS( + SELECT 1 FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=n.ready_generation + AND g.connection_id=n.connection_id AND g.owner_epoch=n.connected_epoch AND g.state='ready') +WHERE n.id=$1 AND n.removed_at IS NULL; + +-- name: NodeGenerationReady :one +SELECT EXISTS(SELECT 1 FROM runtime_node_generation_status g JOIN runtime_nodes n ON n.id=g.node_id CROSS JOIN runtime_deployment d + WHERE n.id=sqlc.arg(node_id) AND g.generation=sqlc.arg(generation) AND n.removed_at IS NULL + AND n.connection_id=g.connection_id AND n.connected_epoch=g.owner_epoch AND g.owner_epoch=d.owner_epoch + AND n.last_seen_at>clock_timestamp()-interval '45 seconds' AND g.state='ready')::boolean AS ready; + +-- name: DeleteNodeGenerationStatus :exec +DELETE FROM runtime_node_generation_status WHERE node_id=$1 AND generation=$2; diff --git a/services/agents-api/internal/db/queries/runtime_nodes.sql b/services/agents-api/internal/db/queries/runtime_nodes.sql index efe1503c8..c3df72dff 100644 --- a/services/agents-api/internal/db/queries/runtime_nodes.sql +++ b/services/agents-api/internal/db/queries/runtime_nodes.sql @@ -14,6 +14,9 @@ SELECT * FROM runtime_nodes WHERE id=$1 AND removed_at IS NULL; -- name: ListRuntimeNodes :many SELECT n.*, (n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND n.last_seen_at > clock_timestamp()-interval '45 seconds')::boolean AS online, d.provider_kind, d.generation AS target_generation, + EXISTS(SELECT 1 FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=n.ready_generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch AND g.state='ready')::boolean AS serving_ready, + COALESCE((SELECT g.state FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=d.generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch),'')::text AS target_state, + COALESCE((SELECT g.diagnostic FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=d.generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch),'')::text AS target_diagnostic, (SELECT count(*) FROM runtime_placements p LEFT JOIN runtime_allocations a ON a.environment_id=p.environment_id WHERE p.node_id=n.id AND p.released_at IS NULL AND (a.id IS NULL OR a.compute_phase <> 'suspended'))::bigint AS active, (SELECT count(*) FROM runtime_placements p WHERE p.node_id=n.id AND p.released_at IS NULL)::bigint AS retained, (SELECT count(*) FROM runtime_placements p WHERE p.node_id=n.id AND p.released_at IS NULL AND NOT EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.environment_id=p.environment_id))::bigint AS reserved, @@ -60,7 +63,7 @@ SELECT * FROM runtime_node_enrollments WHERE token_sha256=$1; INSERT INTO runtime_placements(environment_id,node_id,deployment_generation) VALUES($1,$2,(SELECT generation FROM runtime_deployment)); -- name: GetRuntimePlacement :one -SELECT p.*, n.name, (n.provider_ready AND n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND n.last_seen_at>clock_timestamp()-interval '45 seconds' AND n.removed_at IS NULL)::boolean AS available, +SELECT p.*, n.name, (EXISTS(SELECT 1 FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=p.deployment_generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch AND g.state='ready') AND n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND n.last_seen_at>clock_timestamp()-interval '45 seconds' AND n.removed_at IS NULL)::boolean AS available, COALESCE(a.observation_error,'')::text AS observation_error, COALESCE(a.state,'reserved')::text AS state, COALESCE(a.compute_phase,'disabled')::text AS compute_phase FROM runtime_placements p JOIN runtime_nodes n ON n.id=p.node_id CROSS JOIN runtime_deployment d LEFT JOIN runtime_allocations a ON a.environment_id=p.environment_id WHERE p.environment_id=$1; @@ -96,7 +99,7 @@ WHERE a.node_id=$1 AND a.state<>'released' ORDER BY a.created_at,a.id LIMIT 1000 -- name: CreateSessionRuntimePlacement :exec INSERT INTO runtime_placements(environment_id,node_id,deployment_generation) -SELECT id,$2,(SELECT generation FROM runtime_deployment) FROM environments WHERE session_id=$1; +SELECT id,$2,sqlc.arg(generation)::bigint FROM environments WHERE session_id=$1; -- name: SetRuntimeObservation :exec UPDATE runtime_allocations SET observation_error=$4 WHERE id=$1 AND compute_revision=$2 AND state=$3 AND state<>'released'; diff --git a/services/agents-api/internal/db/queries/sandbox_reset.sql b/services/agents-api/internal/db/queries/sandbox_reset.sql index 77f30e4a2..27e1cc52a 100644 --- a/services/agents-api/internal/db/queries/sandbox_reset.sql +++ b/services/agents-api/internal/db/queries/sandbox_reset.sql @@ -84,11 +84,10 @@ held AS ( ), rollout_nodes AS ( SELECT CASE WHEN n.connection_id IS NULL OR n.connected_epoch <> d.owner_epoch OR n.last_seen_at IS NULL OR n.last_seen_at <= observed.as_of - interval '45 seconds' THEN 'unknown' - WHEN n.deployment_generation <> d.generation THEN 'update_required' - WHEN n.provider_ready AND n.ready_generation = d.generation THEN 'ready' - WHEN COALESCE(n.health->>'diagnostic','') <> '' THEN 'failed' - ELSE 'unknown' END AS state + WHEN n.protocol_version=1 AND n.deployment_generation <> d.generation THEN 'update_required' + ELSE COALESCE(g.state,'unknown') END AS state FROM runtime_nodes n CROSS JOIN runtime_deployment d CROSS JOIN observed + LEFT JOIN runtime_node_generation_status g ON g.node_id=n.id AND g.generation=d.generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id ), offline AS ( SELECT node_id, name, count(*)::bigint AS resources FROM classified WHERE offline GROUP BY node_id, name @@ -103,11 +102,11 @@ SELECT sqlc.embed(d), 'on_offline_nodes', (SELECT count(*) FROM classified WHERE offline), 'offline_nodes', COALESCE((SELECT jsonb_agg(jsonb_build_object('node_id', node_id, 'name', name, 'resources', resources) ORDER BY node_id) FROM offline), '[]'::jsonb) )::jsonb AS remaining, - jsonb_build_object('state','settled', + jsonb_build_object('state',CASE WHEN EXISTS(SELECT 1 FROM rollout_nodes WHERE state='preparing') THEN 'preparing' ELSE 'settled' END, 'previous_generation_sandboxes',(SELECT count(*) FROM held h WHERE h.deployment_generation <> d.generation), 'nodes',CASE WHEN d.mode='nodes' THEN jsonb_build_object( 'ready',(SELECT count(*) FROM rollout_nodes WHERE state='ready'), - 'preparing',0, + 'preparing',(SELECT count(*) FROM rollout_nodes WHERE state='preparing'), 'failed',(SELECT count(*) FROM rollout_nodes WHERE state='failed'), 'update_required',(SELECT count(*) FROM rollout_nodes WHERE state='update_required'), 'unknown',(SELECT count(*) FROM rollout_nodes WHERE state='unknown')) ELSE NULL END diff --git a/services/agents-api/internal/db/sqlc/models.go b/services/agents-api/internal/db/sqlc/models.go index 90f61391c..b093fcecf 100644 --- a/services/agents-api/internal/db/sqlc/models.go +++ b/services/agents-api/internal/db/sqlc/models.go @@ -331,6 +331,7 @@ type RuntimeNode struct { CoreUrl string `json:"core_url"` EnrollmentID pgtype.UUID `json:"enrollment_id"` ReadyGeneration pgtype.Int8 `json:"ready_generation"` + ProtocolVersion int32 `json:"protocol_version"` } type RuntimeNodeEnrollment struct { @@ -344,6 +345,17 @@ type RuntimeNodeEnrollment struct { ID pgtype.UUID `json:"id"` } +type RuntimeNodeGenerationStatus struct { + NodeID pgtype.UUID `json:"node_id"` + Generation int64 `json:"generation"` + SpecificationDigest string `json:"specification_digest"` + ConnectionID pgtype.UUID `json:"connection_id"` + OwnerEpoch int64 `json:"owner_epoch"` + State string `json:"state"` + Diagnostic string `json:"diagnostic"` + ObservedAt pgtype.Timestamptz `json:"observed_at"` +} + type RuntimePlacement struct { EnvironmentID pgtype.UUID `json:"environment_id"` NodeID pgtype.UUID `json:"node_id"` diff --git a/services/agents-api/internal/db/sqlc/node_generations.sql.go b/services/agents-api/internal/db/sqlc/node_generations.sql.go new file mode 100644 index 000000000..ec512ca60 --- /dev/null +++ b/services/agents-api/internal/db/sqlc/node_generations.sql.go @@ -0,0 +1,144 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: node_generations.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const deleteNodeGenerationStatus = `-- name: DeleteNodeGenerationStatus :exec +DELETE FROM runtime_node_generation_status WHERE node_id=$1 AND generation=$2 +` + +type DeleteNodeGenerationStatusParams struct { + NodeID pgtype.UUID `json:"node_id"` + Generation int64 `json:"generation"` +} + +func (q *Queries) DeleteNodeGenerationStatus(ctx context.Context, arg DeleteNodeGenerationStatusParams) error { + _, err := q.db.Exec(ctx, deleteNodeGenerationStatus, arg.NodeID, arg.Generation) + return err +} + +const getNodeGenerationSpecification = `-- name: GetNodeGenerationSpecification :one +SELECT d.provider_kind,d.specification FROM runtime_deployment d WHERE d.generation=$1 +UNION ALL SELECT g.provider_kind,g.specification FROM runtime_deployment_generations g WHERE g.generation=$1 AND g.generation<>(SELECT d.generation FROM runtime_deployment d) +LIMIT 1 +` + +type GetNodeGenerationSpecificationRow struct { + ProviderKind string `json:"provider_kind"` + Specification []byte `json:"specification"` +} + +func (q *Queries) GetNodeGenerationSpecification(ctx context.Context, generation int64) (GetNodeGenerationSpecificationRow, error) { + row := q.db.QueryRow(ctx, getNodeGenerationSpecification, generation) + var i GetNodeGenerationSpecificationRow + err := row.Scan(&i.ProviderKind, &i.Specification) + return i, err +} + +const nodeGenerationKept = `-- name: NodeGenerationKept :one +SELECT (EXISTS(SELECT 1 FROM runtime_deployment d WHERE d.generation=$1) + OR EXISTS(SELECT 1 FROM runtime_nodes n WHERE n.id=$2 AND n.removed_at IS NULL AND n.ready_generation=$1) + OR EXISTS(SELECT 1 FROM runtime_placements p WHERE p.node_id=$2 AND p.released_at IS NULL AND p.deployment_generation=$1) + OR EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.node_id=$2 AND a.state<>'released' AND a.deployment_generation=$1))::boolean AS kept +` + +type NodeGenerationKeptParams struct { + Generation int64 `json:"generation"` + NodeID pgtype.UUID `json:"node_id"` +} + +func (q *Queries) NodeGenerationKept(ctx context.Context, arg NodeGenerationKeptParams) (bool, error) { + row := q.db.QueryRow(ctx, nodeGenerationKept, arg.Generation, arg.NodeID) + var kept bool + err := row.Scan(&kept) + return kept, err +} + +const nodeGenerationReady = `-- name: NodeGenerationReady :one +SELECT EXISTS(SELECT 1 FROM runtime_node_generation_status g JOIN runtime_nodes n ON n.id=g.node_id CROSS JOIN runtime_deployment d + WHERE n.id=$1 AND g.generation=$2 AND n.removed_at IS NULL + AND n.connection_id=g.connection_id AND n.connected_epoch=g.owner_epoch AND g.owner_epoch=d.owner_epoch + AND n.last_seen_at>clock_timestamp()-interval '45 seconds' AND g.state='ready')::boolean AS ready +` + +type NodeGenerationReadyParams struct { + NodeID pgtype.UUID `json:"node_id"` + Generation int64 `json:"generation"` +} + +func (q *Queries) NodeGenerationReady(ctx context.Context, arg NodeGenerationReadyParams) (bool, error) { + row := q.db.QueryRow(ctx, nodeGenerationReady, arg.NodeID, arg.Generation) + var ready bool + err := row.Scan(&ready) + return ready, err +} + +const promoteNodeServingGeneration = `-- name: PromoteNodeServingGeneration :exec +UPDATE runtime_nodes SET ready_generation=$2 WHERE id=$1 AND removed_at IS NULL +AND $2=(SELECT generation FROM runtime_deployment) AND (ready_generation IS NULL OR ready_generation<$2) +` + +type PromoteNodeServingGenerationParams struct { + ID pgtype.UUID `json:"id"` + ReadyGeneration pgtype.Int8 `json:"ready_generation"` +} + +func (q *Queries) PromoteNodeServingGeneration(ctx context.Context, arg PromoteNodeServingGenerationParams) error { + _, err := q.db.Exec(ctx, promoteNodeServingGeneration, arg.ID, arg.ReadyGeneration) + return err +} + +const refreshNodeServingReadiness = `-- name: RefreshNodeServingReadiness :exec +UPDATE runtime_nodes n SET protocol_version=$2, provider_ready=EXISTS( + SELECT 1 FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=n.ready_generation + AND g.connection_id=n.connection_id AND g.owner_epoch=n.connected_epoch AND g.state='ready') +WHERE n.id=$1 AND n.removed_at IS NULL +` + +type RefreshNodeServingReadinessParams struct { + ID pgtype.UUID `json:"id"` + ProtocolVersion int32 `json:"protocol_version"` +} + +func (q *Queries) RefreshNodeServingReadiness(ctx context.Context, arg RefreshNodeServingReadinessParams) error { + _, err := q.db.Exec(ctx, refreshNodeServingReadiness, arg.ID, arg.ProtocolVersion) + return err +} + +const upsertNodeGenerationStatus = `-- name: UpsertNodeGenerationStatus :exec +INSERT INTO runtime_node_generation_status(node_id,generation,specification_digest,connection_id,owner_epoch,state,diagnostic) +VALUES($1,$2,$3,$4,$5,$6,$7) +ON CONFLICT(node_id,generation) DO UPDATE SET specification_digest=EXCLUDED.specification_digest,connection_id=EXCLUDED.connection_id, + owner_epoch=EXCLUDED.owner_epoch,state=EXCLUDED.state,diagnostic=EXCLUDED.diagnostic,observed_at=clock_timestamp() +` + +type UpsertNodeGenerationStatusParams struct { + NodeID pgtype.UUID `json:"node_id"` + Generation int64 `json:"generation"` + SpecificationDigest string `json:"specification_digest"` + ConnectionID pgtype.UUID `json:"connection_id"` + OwnerEpoch int64 `json:"owner_epoch"` + State string `json:"state"` + Diagnostic string `json:"diagnostic"` +} + +func (q *Queries) UpsertNodeGenerationStatus(ctx context.Context, arg UpsertNodeGenerationStatusParams) error { + _, err := q.db.Exec(ctx, upsertNodeGenerationStatus, + arg.NodeID, + arg.Generation, + arg.SpecificationDigest, + arg.ConnectionID, + arg.OwnerEpoch, + arg.State, + arg.Diagnostic, + ) + return err +} diff --git a/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go b/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go index 4a577dd6e..bba7d1a89 100644 --- a/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go +++ b/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go @@ -133,16 +133,17 @@ func (q *Queries) CreateRuntimePlacement(ctx context.Context, arg CreateRuntimeP const createSessionRuntimePlacement = `-- name: CreateSessionRuntimePlacement :exec INSERT INTO runtime_placements(environment_id,node_id,deployment_generation) -SELECT id,$2,(SELECT generation FROM runtime_deployment) FROM environments WHERE session_id=$1 +SELECT id,$2,$3::bigint FROM environments WHERE session_id=$1 ` type CreateSessionRuntimePlacementParams struct { - SessionID pgtype.UUID `json:"session_id"` - NodeID pgtype.UUID `json:"node_id"` + SessionID pgtype.UUID `json:"session_id"` + NodeID pgtype.UUID `json:"node_id"` + Generation int64 `json:"generation"` } func (q *Queries) CreateSessionRuntimePlacement(ctx context.Context, arg CreateSessionRuntimePlacementParams) error { - _, err := q.db.Exec(ctx, createSessionRuntimePlacement, arg.SessionID, arg.NodeID) + _, err := q.db.Exec(ctx, createSessionRuntimePlacement, arg.SessionID, arg.NodeID, arg.Generation) return err } @@ -219,7 +220,7 @@ func (q *Queries) GetRuntimeEnrollment(ctx context.Context, tokenSha256 string) } const getRuntimeNode = `-- name: GetRuntimeNode :one -SELECT id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, provider_ready, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url, enrollment_id, ready_generation FROM runtime_nodes WHERE id=$1 AND removed_at IS NULL +SELECT id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, provider_ready, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url, enrollment_id, ready_generation, protocol_version FROM runtime_nodes WHERE id=$1 AND removed_at IS NULL ` func (q *Queries) GetRuntimeNode(ctx context.Context, id pgtype.UUID) (RuntimeNode, error) { @@ -245,12 +246,13 @@ func (q *Queries) GetRuntimeNode(ctx context.Context, id pgtype.UUID) (RuntimeNo &i.CoreUrl, &i.EnrollmentID, &i.ReadyGeneration, + &i.ProtocolVersion, ) return i, err } const getRuntimePlacement = `-- name: GetRuntimePlacement :one -SELECT p.environment_id, p.node_id, p.reserved_at, p.released_at, p.deployment_generation, n.name, (n.provider_ready AND n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND n.last_seen_at>clock_timestamp()-interval '45 seconds' AND n.removed_at IS NULL)::boolean AS available, +SELECT p.environment_id, p.node_id, p.reserved_at, p.released_at, p.deployment_generation, n.name, (EXISTS(SELECT 1 FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=p.deployment_generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch AND g.state='ready') AND n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND n.last_seen_at>clock_timestamp()-interval '45 seconds' AND n.removed_at IS NULL)::boolean AS available, COALESCE(a.observation_error,'')::text AS observation_error, COALESCE(a.state,'reserved')::text AS state, COALESCE(a.compute_phase,'disabled')::text AS compute_phase FROM runtime_placements p JOIN runtime_nodes n ON n.id=p.node_id CROSS JOIN runtime_deployment d LEFT JOIN runtime_allocations a ON a.environment_id=p.environment_id WHERE p.environment_id=$1 @@ -316,7 +318,7 @@ func (q *Queries) HeartbeatRuntimeNode(ctx context.Context, arg HeartbeatRuntime const insertRuntimeNode = `-- name: InsertRuntimeNode :one INSERT INTO runtime_nodes(id,installation_id,name,backend_fingerprint,credential_sha256,max_active,max_retained,specification_digest,deployment_generation,core_url,enrollment_id,ready_generation) -VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$9) RETURNING id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, provider_ready, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url, enrollment_id, ready_generation +VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$9) RETURNING id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, provider_ready, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url, enrollment_id, ready_generation, protocol_version ` type InsertRuntimeNodeParams struct { @@ -368,6 +370,7 @@ func (q *Queries) InsertRuntimeNode(ctx context.Context, arg InsertRuntimeNodePa &i.CoreUrl, &i.EnrollmentID, &i.ReadyGeneration, + &i.ProtocolVersion, ) return i, err } @@ -484,8 +487,11 @@ func (q *Queries) ListNodeRuntimeAllocations(ctx context.Context, nodeID pgtype. } const listRuntimeNodes = `-- name: ListRuntimeNodes :many -SELECT n.id, n.installation_id, n.name, n.backend_fingerprint, n.credential_sha256, n.max_active, n.max_retained, n.connection_id, n.provider_ready, n.health, n.connected_epoch, n.last_seen_at, n.created_at, n.removed_at, n.specification_digest, n.deployment_generation, n.core_url, n.enrollment_id, n.ready_generation, (n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND n.last_seen_at > clock_timestamp()-interval '45 seconds')::boolean AS online, +SELECT n.id, n.installation_id, n.name, n.backend_fingerprint, n.credential_sha256, n.max_active, n.max_retained, n.connection_id, n.provider_ready, n.health, n.connected_epoch, n.last_seen_at, n.created_at, n.removed_at, n.specification_digest, n.deployment_generation, n.core_url, n.enrollment_id, n.ready_generation, n.protocol_version, (n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND n.last_seen_at > clock_timestamp()-interval '45 seconds')::boolean AS online, d.provider_kind, d.generation AS target_generation, + EXISTS(SELECT 1 FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=n.ready_generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch AND g.state='ready')::boolean AS serving_ready, + COALESCE((SELECT g.state FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=d.generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch),'')::text AS target_state, + COALESCE((SELECT g.diagnostic FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=d.generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch),'')::text AS target_diagnostic, (SELECT count(*) FROM runtime_placements p LEFT JOIN runtime_allocations a ON a.environment_id=p.environment_id WHERE p.node_id=n.id AND p.released_at IS NULL AND (a.id IS NULL OR a.compute_phase <> 'suspended'))::bigint AS active, (SELECT count(*) FROM runtime_placements p WHERE p.node_id=n.id AND p.released_at IS NULL)::bigint AS retained, (SELECT count(*) FROM runtime_placements p WHERE p.node_id=n.id AND p.released_at IS NULL AND NOT EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.environment_id=p.environment_id))::bigint AS reserved, @@ -517,9 +523,13 @@ type ListRuntimeNodesRow struct { CoreUrl string `json:"core_url"` EnrollmentID pgtype.UUID `json:"enrollment_id"` ReadyGeneration pgtype.Int8 `json:"ready_generation"` + ProtocolVersion int32 `json:"protocol_version"` Online bool `json:"online"` ProviderKind string `json:"provider_kind"` TargetGeneration int64 `json:"target_generation"` + ServingReady bool `json:"serving_ready"` + TargetState string `json:"target_state"` + TargetDiagnostic string `json:"target_diagnostic"` Active int64 `json:"active"` Retained int64 `json:"retained"` Reserved int64 `json:"reserved"` @@ -557,9 +567,13 @@ func (q *Queries) ListRuntimeNodes(ctx context.Context, nodeID pgtype.UUID) ([]L &i.CoreUrl, &i.EnrollmentID, &i.ReadyGeneration, + &i.ProtocolVersion, &i.Online, &i.ProviderKind, &i.TargetGeneration, + &i.ServingReady, + &i.TargetState, + &i.TargetDiagnostic, &i.Active, &i.Retained, &i.Reserved, @@ -652,7 +666,7 @@ func (q *Queries) SetRuntimeObservation(ctx context.Context, arg SetRuntimeObser } const updateRuntimeNode = `-- name: UpdateRuntimeNode :one -UPDATE runtime_nodes SET name=$2,max_active=$3,max_retained=$4 WHERE id=$1 AND removed_at IS NULL RETURNING id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, provider_ready, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url, enrollment_id, ready_generation +UPDATE runtime_nodes SET name=$2,max_active=$3,max_retained=$4 WHERE id=$1 AND removed_at IS NULL RETURNING id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, provider_ready, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url, enrollment_id, ready_generation, protocol_version ` type UpdateRuntimeNodeParams struct { @@ -690,6 +704,7 @@ func (q *Queries) UpdateRuntimeNode(ctx context.Context, arg UpdateRuntimeNodePa &i.CoreUrl, &i.EnrollmentID, &i.ReadyGeneration, + &i.ProtocolVersion, ) return i, err } diff --git a/services/agents-api/internal/db/sqlc/sandbox_reset.sql.go b/services/agents-api/internal/db/sqlc/sandbox_reset.sql.go index 976ebcf2a..8f3bd6c85 100644 --- a/services/agents-api/internal/db/sqlc/sandbox_reset.sql.go +++ b/services/agents-api/internal/db/sqlc/sandbox_reset.sql.go @@ -80,11 +80,10 @@ held AS ( ), rollout_nodes AS ( SELECT CASE WHEN n.connection_id IS NULL OR n.connected_epoch <> d.owner_epoch OR n.last_seen_at IS NULL OR n.last_seen_at <= observed.as_of - interval '45 seconds' THEN 'unknown' - WHEN n.deployment_generation <> d.generation THEN 'update_required' - WHEN n.provider_ready AND n.ready_generation = d.generation THEN 'ready' - WHEN COALESCE(n.health->>'diagnostic','') <> '' THEN 'failed' - ELSE 'unknown' END AS state + WHEN n.protocol_version=1 AND n.deployment_generation <> d.generation THEN 'update_required' + ELSE COALESCE(g.state,'unknown') END AS state FROM runtime_nodes n CROSS JOIN runtime_deployment d CROSS JOIN observed + LEFT JOIN runtime_node_generation_status g ON g.node_id=n.id AND g.generation=d.generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id ), offline AS ( SELECT node_id, name, count(*)::bigint AS resources FROM classified WHERE offline GROUP BY node_id, name @@ -99,11 +98,11 @@ SELECT d.singleton, d.installation_id, d.backend_fingerprint, d.admission_paused 'on_offline_nodes', (SELECT count(*) FROM classified WHERE offline), 'offline_nodes', COALESCE((SELECT jsonb_agg(jsonb_build_object('node_id', node_id, 'name', name, 'resources', resources) ORDER BY node_id) FROM offline), '[]'::jsonb) )::jsonb AS remaining, - jsonb_build_object('state','settled', + jsonb_build_object('state',CASE WHEN EXISTS(SELECT 1 FROM rollout_nodes WHERE state='preparing') THEN 'preparing' ELSE 'settled' END, 'previous_generation_sandboxes',(SELECT count(*) FROM held h WHERE h.deployment_generation <> d.generation), 'nodes',CASE WHEN d.mode='nodes' THEN jsonb_build_object( 'ready',(SELECT count(*) FROM rollout_nodes WHERE state='ready'), - 'preparing',0, + 'preparing',(SELECT count(*) FROM rollout_nodes WHERE state='preparing'), 'failed',(SELECT count(*) FROM rollout_nodes WHERE state='failed'), 'update_required',(SELECT count(*) FROM rollout_nodes WHERE state='update_required'), 'unknown',(SELECT count(*) FROM rollout_nodes WHERE state='unknown')) ELSE NULL END diff --git a/services/agents-api/internal/sandbox/generation.go b/services/agents-api/internal/sandbox/generation.go new file mode 100644 index 000000000..f191d2859 --- /dev/null +++ b/services/agents-api/internal/sandbox/generation.go @@ -0,0 +1,49 @@ +package sandbox + +import ( + "bytes" + "encoding/json" + "errors" + "io" +) + +// GenerationStatus is one sparse observation, not a complete provider inventory. +// A serving pin is owned by Core and is independent of target preparation. +type GenerationStatus struct { + Generation uint64 `json:"generation"` + SpecificationDigest string `json:"specification_digest"` + State string `json:"state"` + Diagnostic string `json:"diagnostic,omitempty"` +} + +type GenerationReference struct { + Generation uint64 `json:"generation"` + SpecificationDigest string `json:"specification_digest"` +} + +type GenerationRetention struct { + GenerationReference + Keep bool `json:"keep"` +} + +type NodeDeployment struct { + SpecificationDigest string `json:"specification_digest"` + Generation uint64 `json:"generation"` + ServingGeneration *uint64 `json:"serving_generation"` +} + +// Omission or null cannot become an implicit destructive false grant. +func (g *GenerationRetention) UnmarshalJSON(raw []byte) error { + var value struct { + Generation uint64 `json:"generation"` + SpecificationDigest string `json:"specification_digest"` + Keep *bool `json:"keep"` + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if decoder.Decode(&value) != nil || decoder.Decode(new(any)) != io.EOF || value.Keep == nil { + return errors.New("invalid generation retention grant") + } + *g = GenerationRetention{GenerationReference: GenerationReference{Generation: value.Generation, SpecificationDigest: value.SpecificationDigest}, Keep: *value.Keep} + return nil +} diff --git a/services/agents-api/internal/sandbox/microsandbox/process.go b/services/agents-api/internal/sandbox/microsandbox/process.go index 54ccba087..be1eb1740 100644 --- a/services/agents-api/internal/sandbox/microsandbox/process.go +++ b/services/agents-api/internal/sandbox/microsandbox/process.go @@ -9,14 +9,17 @@ import ( "os" "os/exec" "strings" + "sync/atomic" ) // ProcessCaller never kills a mutating helper on a Core response timeout. // The helper retains its allocation flock until the SDK mutation settles, // including after Core exits. Output is still drained and bounded by this waiter. -type ProcessCaller struct{} +type ProcessCaller struct{ active atomic.Int64 } -func (*ProcessCaller) Call(ctx context.Context, q Request) (Response, error) { +func (p *ProcessCaller) Quiescent() bool { return p.active.Load() == 0 } + +func (p *ProcessCaller) Call(ctx context.Context, q Request) (Response, error) { data, e := json.Marshal(q) if e != nil || len(data) > MaxRequestBytes { return Response{}, errors.New("invalid helper request") @@ -27,11 +30,13 @@ func (*ProcessCaller) Call(ctx context.Context, q Request) (Response, error) { stdout := &limitBuffer{limit: MaxResponseBytes} stderr := &limitBuffer{limit: MaxOutputBytes} cmd.Stdout, cmd.Stderr = stdout, stderr + p.active.Add(1) if e := cmd.Start(); e != nil { + p.active.Add(-1) return Response{}, errors.New("helper unavailable") } done := make(chan error, 1) - go func() { done <- cmd.Wait() }() + go func() { err := cmd.Wait(); p.active.Add(-1); done <- err }() select { case <-ctx.Done(): return Response{}, ctx.Err() diff --git a/services/agents-api/internal/sandbox/microsandbox/provider.go b/services/agents-api/internal/sandbox/microsandbox/provider.go index c156bf4fa..25439cad7 100644 --- a/services/agents-api/internal/sandbox/microsandbox/provider.go +++ b/services/agents-api/internal/sandbox/microsandbox/provider.go @@ -214,3 +214,9 @@ func (p *Provider) NewCompute(ctx context.Context, r sandbox.Reference, generati } return c, nil } + +// Quiescent includes a helper that outlived the caller's canceled context. +func (p *Provider) Quiescent() bool { + v, ok := p.caller.(interface{ Quiescent() bool }) + return ok && v.Quiescent() +} diff --git a/services/agents-api/internal/sandbox/node/agent.go b/services/agents-api/internal/sandbox/node/agent.go index a14f5ba87..02c25ed84 100644 --- a/services/agents-api/internal/sandbox/node/agent.go +++ b/services/agents-api/internal/sandbox/node/agent.go @@ -17,6 +17,7 @@ import ( ) type AgentConfig struct { + Generations *GenerationManager CoreURL string StateDirectory string Identity Identity @@ -38,12 +39,17 @@ type agent struct { healthSeen atomic.Bool } type agentConnection struct { - conn *websocket.Conn - id string - epoch uint64 - send sync.Mutex - done chan struct{} - once sync.Once + controlBusy bool + version int + controls sync.Mutex + pending *generationControl + controlSequence, controlCursor uint64 + conn *websocket.Conn + id string + epoch uint64 + send sync.Mutex + done chan struct{} + once sync.Once } func (c *agentConnection) close() { c.once.Do(func() { close(c.done); _ = c.conn.Close() }) } @@ -54,6 +60,7 @@ func (c *agentConnection) write(f frame) error { case <-c.done: return ErrUnavailable default: + f.Version = c.version return writeFrame(c.conn, f) } } @@ -61,16 +68,19 @@ func (c *agentConnection) write(f frame) error { type work struct { request request connection *agentConnection + provider sandbox.Provider + release func() + ready bool } // Run owns one persistent node identity and reconnects its transport only. It // never resends a Provider request. The bounded worker outlives each connection. func Run(ctx context.Context, config AgentConfig) error { - if config.Provider == nil || config.Probe == nil { + if config.Generations == nil && (config.Provider == nil || config.Probe == nil) { return sandbox.ErrInvalid } _, checkpoint := config.Provider.(sandbox.CheckpointProvider) - if checkpoint != (config.Identity.Provider == "microsandbox") { + if config.Generations == nil && checkpoint != (config.Identity.Provider == "microsandbox") { return sandbox.ErrInvalid } release, err := lockDirectory(config.StateDirectory) @@ -140,6 +150,11 @@ func Run(ctx context.Context, config AgentConfig) error { } } func (a *agent) health(ctx context.Context, host *hostHealthSampler) (Health, error) { + if a.config.Generations != nil { + health := Health{Generations: a.config.Generations.Statuses(), ObservedAt: time.Now().UTC(), ActiveOperations: int(a.active.Load())} + host.fill(&health, a.config.StateDirectory) + return health, nil + } probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() h, e := a.config.Probe(probeCtx) @@ -193,14 +208,18 @@ func (a *agent) connect(ctx context.Context) error { host := new(hostHealthSampler) health, _ := a.health(ctx, host) _ = conn.SetReadDeadline(time.Now().Add(15 * time.Second)) - if err = writeFrame(conn, frame{Type: "hello", Identity: &a.config.Identity, Health: &health}); err != nil { + version := ProtocolVersion + if a.config.Generations != nil { + version = GenerationProtocolVersion + } + if err = writeFrame(conn, frame{Version: version, Type: "hello", Identity: &a.config.Identity, Health: &health}); err != nil { return err } welcome, err := readFrame(conn) if err != nil { return err } - if welcome.Type != "welcome" || !validID(welcome.ConnectionID) || welcome.OwnerEpoch == 0 { + if welcome.Version != version || welcome.Type != "welcome" || !validID(welcome.ConnectionID) || welcome.OwnerEpoch == 0 { return sandbox.ErrInvalid } a.mu.Lock() @@ -215,7 +234,7 @@ func (a *agent) connect(ctx context.Context) error { return err } } - current := &agentConnection{conn: conn, id: welcome.ConnectionID, epoch: welcome.OwnerEpoch, done: make(chan struct{})} + current := &agentConnection{version: version, conn: conn, id: welcome.ConnectionID, epoch: welcome.OwnerEpoch, done: make(chan struct{})} a.current = current a.mu.Unlock() defer current.close() @@ -224,14 +243,37 @@ func (a *agent) connect(ctx context.Context) error { detach := context.AfterFunc(connectionCtx, current.close) defer detach() go a.heartbeats(connectionCtx, current, host) + var controlDone chan struct{} + controlWork := make(chan []sandbox.GenerationRetention, 1) + if a.config.Generations != nil { + a.config.Generations.Deployment(*welcome.Deployment) + controlDone = make(chan struct{}) + go func() { defer close(controlDone); a.garbageCollection(connectionCtx, current, controlWork) }() + defer func() { cancel(); <-controlDone }() + } sequence := uint64(0) for { _ = conn.SetReadDeadline(time.Now().Add(35 * time.Second)) f, e := readFrame(conn) - if e != nil { - return e + if e != nil || f.Version != version { + if e != nil { + return e + } + return sandbox.ErrInvalid } if f.Type == "heartbeat_ack" && f.ConnectionID == current.id { + if version == GenerationProtocolVersion { + if f.OwnerEpoch != current.epoch { + return sandbox.ErrOwnership + } + a.config.Generations.Deployment(*f.Deployment) + } + continue + } + if f.Type == "retention_ack" && version == GenerationProtocolVersion { + if err := a.acceptRetention(current, f, controlWork); err != nil { + return err + } continue } if f.Type != "request" || f.Request == nil { @@ -246,10 +288,20 @@ func (a *agent) connect(ctx context.Context) error { _ = current.write(frame{Type: "response", Response: &response{ID: q.ID, ConnectionID: current.id, ErrorCode: "invalid"}}) continue } + task := work{request: q, connection: current, provider: a.config.Provider, ready: a.ready.Load()} + if a.config.Generations != nil { + task.provider, task.ready, task.release, err = a.config.Generations.Acquire(q.DeploymentGeneration) + if err != nil { + _ = current.write(frame{Type: "response", Response: &response{ID: q.ID, ConnectionID: current.id, ErrorCode: "unconfirmed"}}) + continue + } + } select { - case a.queue <- work{q, current}: + case a.queue <- task: default: - // Closing makes every pending result unknown; queued calls are never replayed. + if task.release != nil { + task.release() + } return ErrUnavailable } } @@ -265,7 +317,11 @@ func (a *agent) heartbeats(ctx context.Context, c *agentConnection, host *hostHe return case <-ticker.C: h, _ := a.health(ctx, host) - if c.write(frame{Type: "heartbeat", ConnectionID: c.id, Health: &h}) != nil { + if c.write(frame{Type: "heartbeat", ConnectionID: c.id, OwnerEpoch: c.epoch, Health: &h}) != nil { + c.close() + return + } + if a.config.Generations != nil && a.requestRetention(c) != nil { c.close() return } @@ -273,38 +329,55 @@ func (a *agent) heartbeats(ctx context.Context, c *agentConnection, host *hostHe } } func (a *agent) worker(ctx context.Context) { + defer func() { + for { + select { + case task := <-a.queue: + if task.release != nil { + task.release() + } + default: + return + } + } + }() for { select { case <-ctx.Done(): return case task := <-a.queue: - a.mu.Lock() - current := a.current - a.mu.Unlock() - if current != task.connection || ctx.Err() != nil { - continue - } - select { - case <-task.connection.done: - continue - default: - } - if !time.Now().Before(task.request.deadline) { - _ = task.connection.write(frame{Type: "response", Response: &response{ID: task.request.ID, ConnectionID: task.connection.id, ErrorCode: "unconfirmed"}}) - continue - } - if requiresReady(task.request) && !a.ready.Load() { - _ = task.connection.write(frame{Type: "response", Response: &response{ID: task.request.ID, ConnectionID: task.connection.id, ErrorCode: "unconfirmed"}}) - continue - } - // All calls serialize across reconnects. The Provider may retain its own - // allocation flock beyond this deadline when a native mutation is uncertain. - operationCtx, cancel := context.WithDeadline(context.Background(), task.request.deadline) - a.active.Add(1) - result := execute(operationCtx, a.config.Provider, task.request) - a.active.Add(-1) - cancel() - _ = task.connection.write(frame{Type: "response", Response: &result}) + a.executeWork(ctx, task) } } } +func (a *agent) executeWork(ctx context.Context, task work) { + if task.release != nil { + defer task.release() + } + a.mu.Lock() + current := a.current + a.mu.Unlock() + if current != task.connection || ctx.Err() != nil { + return + } + select { + case <-task.connection.done: + return + default: + } + ready := a.ready.Load() + if a.config.Generations != nil { + ready = a.config.Generations.Ready(task.request.DeploymentGeneration) + } + if !time.Now().Before(task.request.deadline) || requiresReady(task.request) && !ready { + _ = task.connection.write(frame{Type: "response", Response: &response{ID: task.request.ID, ConnectionID: task.connection.id, ErrorCode: "unconfirmed"}}) + return + } + // Detached native completion retains its local provider reference across cancellation. + operationCtx, cancel := context.WithDeadline(context.Background(), task.request.deadline) + a.active.Add(1) + result := execute(operationCtx, task.provider, task.request) + a.active.Add(-1) + cancel() + _ = task.connection.write(frame{Type: "response", Response: &result}) +} diff --git a/services/agents-api/internal/sandbox/node/generation_agent.go b/services/agents-api/internal/sandbox/node/generation_agent.go new file mode 100644 index 000000000..e5918b578 --- /dev/null +++ b/services/agents-api/internal/sandbox/node/generation_agent.go @@ -0,0 +1,76 @@ +package node + +import ( + "context" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/google/uuid" +) + +func (a *agent) requestRetention(c *agentConnection) error { + c.controls.Lock() + defer c.controls.Unlock() + if c.pending != nil || c.controlBusy { + return nil + } + refs := a.config.Generations.Retained(c.controlCursor) + if len(refs) == 0 { + return nil + } + c.controlSequence++ + control := &generationControl{ID: uuid.NewString(), Sequence: c.controlSequence, ConnectionID: c.id, OwnerEpoch: c.epoch, References: refs} + c.pending = control + return c.write(frame{Type: "retention", Control: control}) +} + +func (a *agent) acceptRetention(c *agentConnection, f frame, work chan<- []sandbox.GenerationRetention) error { + c.controls.Lock() + defer c.controls.Unlock() + pending, reply := c.pending, f.Control + if pending == nil || reply == nil || reply.ID != pending.ID || reply.Sequence != pending.Sequence || reply.ConnectionID != c.id || reply.OwnerEpoch != c.epoch || len(reply.Retentions) != len(pending.References) { + return sandbox.ErrOwnership + } + // Validate the whole reply before authorizing even its first deletion. + for i, g := range reply.Retentions { + if g.GenerationReference != pending.References[i] { + return sandbox.ErrOwnership + } + } + a.config.Generations.Deployment(*f.Deployment) + select { + case work <- reply.Retentions: + c.pending = nil + c.controlBusy = true + return nil + default: + return sandbox.ErrInvalid + } +} + +func (a *agent) garbageCollection(ctx context.Context, c *agentConnection, work <-chan []sandbox.GenerationRetention) { + for { + select { + case <-ctx.Done(): + return + case grants := <-work: + for _, grant := range grants { + if ctx.Err() != nil { + return + } + select { + case <-c.done: + return + default: + } + // A busy provider retains its files and requests another grant later. + _ = a.config.Generations.Drop(ctx, grant) + } + c.controls.Lock() + if len(grants) > 0 { + c.controlCursor = grants[len(grants)-1].Generation + } + c.controlBusy = false + c.controls.Unlock() + } + } +} diff --git a/services/agents-api/internal/sandbox/node/generation_wire.go b/services/agents-api/internal/sandbox/node/generation_wire.go new file mode 100644 index 000000000..bedf2eddf --- /dev/null +++ b/services/agents-api/internal/sandbox/node/generation_wire.go @@ -0,0 +1,109 @@ +package node + +import ( + "encoding/hex" + "math" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" +) + +// A control reply grants only its correlated sparse entries. There is no +// complete-inventory or omission-based deletion interpretation. +type generationControl struct { + ID string `json:"id"` + Sequence uint64 `json:"sequence"` + ConnectionID string `json:"connection_id"` + OwnerEpoch uint64 `json:"owner_epoch"` + References []sandbox.GenerationReference `json:"references,omitempty"` + Retentions []sandbox.GenerationRetention `json:"retentions,omitempty"` +} + +func validGeneration(g uint64) bool { return g > 0 && g <= math.MaxInt64 } +func validSpecificationDigest(v string) bool { + b, err := hex.DecodeString(v) + return err == nil && len(b) == 32 && hex.EncodeToString(b) == v +} + +func validateVersionFrame(f frame, size int) error { + if f.Version == ProtocolVersion { + if f.Deployment != nil || f.Control != nil || f.Health != nil && f.Health.Generations != nil || f.Request != nil && f.Request.DeploymentGeneration != 0 { + return sandbox.ErrInvalid + } + return nil + } + if f.Version != GenerationProtocolVersion { + return sandbox.ErrInvalid + } + if f.Type != "request" && f.Type != "response" && size > MaxControlFrameBytes { + return sandbox.ErrInvalid + } + if f.Deployment != nil && (!validGeneration(f.Deployment.Generation) || !validSpecificationDigest(f.Deployment.SpecificationDigest) || f.Deployment.ServingGeneration != nil && !validGeneration(*f.Deployment.ServingGeneration)) { + return sandbox.ErrInvalid + } + if f.Health != nil { + if len(f.Health.Generations) > 8 { + return sandbox.ErrInvalid + } + seen := map[uint64]bool{} + for _, g := range f.Health.Generations { + if !validGeneration(g.Generation) || !validSpecificationDigest(g.SpecificationDigest) || seen[g.Generation] || (g.State != "ready" && g.State != "preparing" && g.State != "failed") || g.State == "ready" && g.Diagnostic != "" || len(g.Diagnostic) > 64 || sandbox.NormalizeNodeDiagnostic(g.Diagnostic) != g.Diagnostic { + return sandbox.ErrInvalid + } + seen[g.Generation] = true + } + } + if f.Request != nil && !validGeneration(f.Request.DeploymentGeneration) { + return sandbox.ErrInvalid + } + if c := f.Control; c != nil { + if !validID(c.ID) || !validID(c.ConnectionID) || !validGeneration(c.OwnerEpoch) || !validGeneration(c.Sequence) || len(c.References) > 8 || len(c.Retentions) > 8 || c.References != nil && c.Retentions != nil { + return sandbox.ErrInvalid + } + seen := map[uint64]bool{} + for _, g := range c.References { + if !validGeneration(g.Generation) || !validSpecificationDigest(g.SpecificationDigest) || seen[g.Generation] { + return sandbox.ErrInvalid + } + seen[g.Generation] = true + } + for _, g := range c.Retentions { + if !validGeneration(g.Generation) || !validSpecificationDigest(g.SpecificationDigest) || seen[g.Generation] { + return sandbox.ErrInvalid + } + seen[g.Generation] = true + } + } + switch f.Type { + case "hello": + if f.Identity == nil || f.Health == nil || f.Request != nil || f.Response != nil || f.Control != nil || f.Deployment != nil { + return sandbox.ErrInvalid + } + case "welcome", "heartbeat_ack": + if f.Deployment == nil || !validID(f.ConnectionID) || !validGeneration(f.OwnerEpoch) || f.Health != nil || f.Identity != nil || f.Request != nil || f.Response != nil || f.Control != nil { + return sandbox.ErrInvalid + } + case "heartbeat": + if f.Health == nil || !validID(f.ConnectionID) || !validGeneration(f.OwnerEpoch) || f.Identity != nil || f.Request != nil || f.Response != nil || f.Control != nil || f.Deployment != nil { + return sandbox.ErrInvalid + } + case "retention": + if f.Control == nil || f.Control.References == nil || f.Control.Retentions != nil || f.Deployment != nil || f.Health != nil || f.Identity != nil || f.Request != nil || f.Response != nil { + return sandbox.ErrInvalid + } + case "retention_ack": + if f.Control == nil || f.Control.References != nil || f.Control.Retentions == nil || f.Deployment == nil || f.Health != nil || f.Identity != nil || f.Request != nil || f.Response != nil { + return sandbox.ErrInvalid + } + case "request": + if f.Request == nil || f.Response != nil || f.Deployment != nil || f.Control != nil || f.Health != nil || f.Identity != nil { + return sandbox.ErrInvalid + } + case "response": + if f.Response == nil || f.Request != nil || f.Deployment != nil || f.Control != nil || f.Health != nil || f.Identity != nil { + return sandbox.ErrInvalid + } + default: + return sandbox.ErrInvalid + } + return nil +} diff --git a/services/agents-api/internal/sandbox/node/generations.go b/services/agents-api/internal/sandbox/node/generations.go new file mode 100644 index 000000000..1d69ea162 --- /dev/null +++ b/services/agents-api/internal/sandbox/node/generations.go @@ -0,0 +1,306 @@ +package node + +import ( + "context" + "errors" + "sort" + "sync" + "time" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" +) + +type GenerationProvider struct { + Generation uint64 + SpecificationDigest string + Provider sandbox.Provider + Probe func(context.Context) error + Close func() +} + +type GenerationManagerOptions struct { + Initial []GenerationProvider + Prepare func(context.Context, uint64, string) (GenerationProvider, error) + Remove func(context.Context, GenerationProvider) error +} + +type localGeneration struct { + value GenerationProvider + state, diagnostic string + refs int + retryAt time.Time + failures int + removing bool +} + +// GenerationManager owns local providers, preparation and all queued/in-flight +// references. Wire metadata is sparse; only correlated Core grants call Drop. +type GenerationManager struct { + mu sync.Mutex + values map[uint64]*localGeneration + target sandbox.NodeDeployment + statusCursor, probeCursor uint64 + options GenerationManagerOptions + ctx context.Context + cancel context.CancelFunc + preparingCancel context.CancelFunc + wake chan struct{} + workers sync.WaitGroup +} + +func NewGenerationManager(ctx context.Context, options GenerationManagerOptions) (*GenerationManager, error) { + if options.Prepare == nil || options.Remove == nil { + return nil, sandbox.ErrInvalid + } + owned, cancel := context.WithCancel(ctx) + m := &GenerationManager{values: map[uint64]*localGeneration{}, options: options, ctx: owned, cancel: cancel, wake: make(chan struct{}, 1)} + for _, v := range options.Initial { + if !validGeneration(v.Generation) || !validSpecificationDigest(v.SpecificationDigest) || v.Provider == nil || v.Probe == nil || m.values[v.Generation] != nil { + cancel() + return nil, sandbox.ErrInvalid + } + m.values[v.Generation] = &localGeneration{value: v, state: "preparing"} + } + m.workers.Add(2) + go func() { defer m.workers.Done(); m.prepareLoop() }() + go func() { defer m.workers.Done(); m.probeLoop() }() + return m, nil +} + +func (m *GenerationManager) Close() { + m.cancel() + m.workers.Wait() + m.mu.Lock() + defer m.mu.Unlock() + for _, g := range m.values { + if g.value.Close != nil { + g.value.Close() + } + } +} + +func (m *GenerationManager) Deployment(value sandbox.NodeDeployment) { + m.mu.Lock() + defer m.mu.Unlock() + if value.Generation < m.target.Generation { + return + } + if value.Generation != m.target.Generation && m.preparingCancel != nil { + m.preparingCancel() + } + m.target = value + if g := m.values[value.Generation]; g == nil { + m.values[value.Generation] = &localGeneration{value: GenerationProvider{Generation: value.Generation, SpecificationDigest: value.SpecificationDigest}, state: "preparing"} + } + select { + case m.wake <- struct{}{}: + default: + } +} + +func (m *GenerationManager) orderedLocked(after uint64) []uint64 { + keys := make([]uint64, 0, len(m.values)) + for g, v := range m.values { + if !v.removing { + keys = append(keys, g) + } + } + sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) + pivot := sort.Search(len(keys), func(i int) bool { return keys[i] > after }) + return append(keys[pivot:], keys[:pivot]...) +} + +func (m *GenerationManager) Statuses() []sandbox.GenerationStatus { + m.mu.Lock() + defer m.mu.Unlock() + keys := []uint64{m.target.Generation} + if m.target.ServingGeneration != nil { + keys = append(keys, *m.target.ServingGeneration) + } + keys = append(keys, m.orderedLocked(m.statusCursor)...) + result := make([]sandbox.GenerationStatus, 0, 8) + seen := map[uint64]bool{} + for _, key := range keys { + g := m.values[key] + if g == nil || g.removing || seen[key] { + continue + } + seen[key] = true + result = append(result, sandbox.GenerationStatus{Generation: key, SpecificationDigest: g.value.SpecificationDigest, State: g.state, Diagnostic: g.diagnostic}) + if key != m.target.Generation && (m.target.ServingGeneration == nil || key != *m.target.ServingGeneration) { + m.statusCursor = key + } + if len(result) == 8 { + break + } + } + return result +} + +func (m *GenerationManager) Retained(after uint64) []sandbox.GenerationReference { + m.mu.Lock() + defer m.mu.Unlock() + keys := m.orderedLocked(after) + result := make([]sandbox.GenerationReference, 0, 8) + for _, key := range keys { + g := m.values[key] + result = append(result, sandbox.GenerationReference{Generation: key, SpecificationDigest: g.value.SpecificationDigest}) + if len(result) == 8 { + break + } + } + return result +} + +// Acquire is called before queue admission, so queued work also prevents GC. +func (m *GenerationManager) Acquire(generation uint64) (sandbox.Provider, bool, func(), error) { + m.mu.Lock() + defer m.mu.Unlock() + g := m.values[generation] + if g == nil || g.removing || g.value.Provider == nil { + return nil, false, nil, ErrUnavailable + } + g.refs++ + var once sync.Once + release := func() { once.Do(func() { m.mu.Lock(); g.refs--; m.mu.Unlock() }) } + return g.value.Provider, g.state == "ready", release, nil +} + +func (m *GenerationManager) Drop(ctx context.Context, grant sandbox.GenerationRetention) error { + if grant.Keep { + return nil + } + m.mu.Lock() + g := m.values[grant.Generation] + if g == nil { + m.mu.Unlock() + return nil + } + if g.value.SpecificationDigest != grant.SpecificationDigest { + m.mu.Unlock() + return sandbox.ErrOwnership + } + quiet := true + if provider, ok := g.value.Provider.(interface{ Quiescent() bool }); ok { + quiet = provider.Quiescent() + } + if !quiet || g.refs != 0 || g.removing || m.target.Generation == grant.Generation || m.target.ServingGeneration != nil && *m.target.ServingGeneration == grant.Generation { + m.mu.Unlock() + return ErrUnavailable + } + g.removing = true + m.mu.Unlock() + err := m.options.Remove(ctx, g.value) + m.mu.Lock() + defer m.mu.Unlock() + if err != nil { + g.removing = false + return err + } + if g.value.Close != nil { + g.value.Close() + } + delete(m.values, grant.Generation) + return nil +} + +func (m *GenerationManager) prepareLoop() { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-m.ctx.Done(): + return + case <-m.wake: + case <-ticker.C: + } + m.mu.Lock() + g := m.values[m.target.Generation] + if g == nil || g.removing || g.value.Provider != nil || time.Now().Before(g.retryAt) { + m.mu.Unlock() + continue + } + ctx, cancel := context.WithTimeout(m.ctx, 30*time.Minute) + m.preparingCancel = cancel + g.refs++ + g.state = "preparing" + g.diagnostic = "" + generation, digest := g.value.Generation, g.value.SpecificationDigest + m.mu.Unlock() + value, err := m.options.Prepare(ctx, generation, digest) + cancel() + m.mu.Lock() + m.preparingCancel = nil + g.refs-- + if err == nil && (value.Generation != generation || value.SpecificationDigest != digest || value.Provider == nil || value.Probe == nil) { + err = sandbox.ErrOwnership + } + if err == nil { + g.value = value + g.state = "preparing" + } else { + if value.Close != nil { + value.Close() + } + g.state = "failed" + g.diagnostic = sandbox.NodeRuntimeDownloadFailed + g.failures++ + delays := []time.Duration{time.Minute, 2 * time.Minute, 5 * time.Minute, 10 * time.Minute, 30 * time.Minute} + index := g.failures - 1 + if index >= len(delays) { + index = len(delays) - 1 + } + g.retryAt = time.Now().Add(delays[index]) + } + m.mu.Unlock() + } +} + +func (m *GenerationManager) probeLoop() { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-m.ctx.Done(): + return + case <-ticker.C: + } + m.mu.Lock() + keys := m.orderedLocked(m.probeCursor) + var g *localGeneration + for _, key := range keys { + value := m.values[key] + if value.value.Provider != nil && !value.removing { + g = value + m.probeCursor = key + g.refs++ + break + } + } + m.mu.Unlock() + if g == nil { + continue + } + ctx, cancel := context.WithTimeout(m.ctx, 5*time.Second) + err := g.value.Probe(ctx) + cancel() + m.mu.Lock() + g.refs-- + if !errors.Is(err, context.Canceled) { + g.state = "ready" + g.diagnostic = "" + if err != nil { + g.state = "failed" + g.diagnostic = sandbox.NodeDiagnostic(err) + } + } + m.mu.Unlock() + } +} + +func (m *GenerationManager) Ready(generation uint64) bool { + m.mu.Lock() + defer m.mu.Unlock() + g := m.values[generation] + return g != nil && !g.removing && g.value.Provider != nil && g.state == "ready" +} diff --git a/services/agents-api/internal/sandbox/node/hub.go b/services/agents-api/internal/sandbox/node/hub.go index 4c9837241..9e755c69f 100644 --- a/services/agents-api/internal/sandbox/node/hub.go +++ b/services/agents-api/internal/sandbox/node/hub.go @@ -14,6 +14,8 @@ import ( ) type HubOptions struct { + Generations func(context.Context, Identity, string, uint64, Health) error + Retention func(context.Context, Identity, string, uint64, []sandbox.GenerationReference) (sandbox.NodeDeployment, []sandbox.GenerationRetention, error) Authenticate func(context.Context, string, string) (Identity, error) OwnerEpoch func(context.Context) (uint64, error) Connected func(context.Context, Identity, string, uint64) error @@ -33,18 +35,21 @@ type Hub struct { } type peer struct { - identity Identity - id string - epoch uint64 - conn *websocket.Conn - send chan struct{} - mu sync.Mutex - sequence uint64 - ready bool - pending map[string]chan response - done chan struct{} - once sync.Once - cancel context.CancelFunc + version int + generations map[uint64]sandbox.GenerationStatus + controlSequence uint64 + identity Identity + id string + epoch uint64 + conn *websocket.Conn + send chan struct{} + mu sync.Mutex + sequence uint64 + ready bool + pending map[string]chan response + done chan struct{} + once sync.Once + cancel context.CancelFunc } func NewHub(options HubOptions) *Hub { @@ -153,7 +158,7 @@ func (h *Hub) ServeHTTP(w http.ResponseWriter, r *http.Request) { _ = conn.Close() return } - p := &peer{identity: identity, id: uuid.NewString(), epoch: epoch, conn: conn, cancel: cancel, send: make(chan struct{}, 1), pending: map[string]chan response{}, ready: hello.Health.ProviderReady, done: make(chan struct{})} + p := &peer{version: hello.Version, generations: map[uint64]sandbox.GenerationStatus{}, identity: identity, id: uuid.NewString(), epoch: epoch, conn: conn, cancel: cancel, send: make(chan struct{}, 1), pending: map[string]chan response{}, ready: hello.Health.ProviderReady, done: make(chan struct{})} stopPeerClose := context.AfterFunc(ctx, p.close) defer stopPeerClose() presenceAttempted := false @@ -182,16 +187,18 @@ func (h *Hub) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } } - if h.options.Heartbeat != nil { - presenceAttempted = true - err = callback(ctx, func(ctx context.Context) error { - return h.options.Heartbeat(ctx, identity, p.id, p.epoch, *hello.Health) - }) + if err = h.recordHealth(ctx, p, *hello.Health); err != nil { + return + } + welcome := frame{Version: p.version, Type: "welcome", ConnectionID: p.id, OwnerEpoch: epoch} + if p.version == GenerationProtocolVersion { + deployment, _, err := h.retention(ctx, p, nil) if err != nil { return } + welcome.Deployment = &deployment } - if ctx.Err() != nil || writeFrame(conn, frame{Type: "welcome", ConnectionID: p.id, OwnerEpoch: epoch}) != nil { + if ctx.Err() != nil || writeFrame(conn, welcome) != nil { return } h.mu.Lock() @@ -205,7 +212,7 @@ func (h *Hub) ServeHTTP(w http.ResponseWriter, r *http.Request) { for { _ = conn.SetReadDeadline(time.Now().Add(35 * time.Second)) f, e := readFrame(conn) - if e != nil { + if e != nil || f.Version != p.version { return } switch f.Type { @@ -234,20 +241,58 @@ func (h *Hub) ServeHTTP(w http.ResponseWriter, r *http.Request) { if e != nil || current != epoch { return } - if h.options.Heartbeat != nil && callback(ctx, func(ctx context.Context) error { return h.options.Heartbeat(ctx, identity, p.id, p.epoch, *f.Health) }) != nil { + if p.version == GenerationProtocolVersion && f.OwnerEpoch != p.epoch { + return + } + if h.recordHealth(ctx, p, *f.Health) != nil { return } - p.mu.Lock() - p.ready = f.Health.ProviderReady - p.mu.Unlock() if e = p.lockSend(ctx); e != nil { return } - e = writeFrame(conn, frame{Type: "heartbeat_ack", ConnectionID: p.id}) + ack := frame{Version: p.version, Type: "heartbeat_ack", ConnectionID: p.id, OwnerEpoch: p.epoch} + if p.version == GenerationProtocolVersion { + deployment, _, readErr := h.retention(ctx, p, nil) + if readErr != nil { + p.unlockSend() + return + } + ack.Deployment = &deployment + } + e = writeFrame(conn, ack) p.unlockSend() if e != nil { return } + case "retention": + if p.version != GenerationProtocolVersion || f.Control == nil { + return + } + c := f.Control + if c.ConnectionID != p.id || c.OwnerEpoch != p.epoch || c.Sequence != p.controlSequence+1 { + return + } + p.controlSequence = c.Sequence + deployment, grants, readErr := h.retention(ctx, p, c.References) + if readErr != nil { + return + } + ack := frame{Version: p.version, Type: "retention_ack", Deployment: &deployment, Control: &generationControl{ID: c.ID, Sequence: c.Sequence, ConnectionID: p.id, OwnerEpoch: p.epoch, Retentions: grants}} + if err := p.lockSend(ctx); err != nil { + return + } + err := writeFrame(conn, ack) + p.unlockSend() + if err != nil { + return + } + p.mu.Lock() + for _, g := range grants { + if !g.Keep { + delete(p.generations, g.Generation) + } + } + p.mu.Unlock() default: return } @@ -284,10 +329,20 @@ func (h *Hub) call(ctx context.Context, id string, q request) (response, error) } p.mu.Lock() ready := p.ready + if p.version == GenerationProtocolVersion { + status, ok := p.generations[q.DeploymentGeneration] + ready = ok && status.State == "ready" + } else if q.DeploymentGeneration != 0 && q.DeploymentGeneration != p.identity.DeploymentGeneration { + p.mu.Unlock() + return response{}, uncertain(q.Operation, ErrUnavailable) + } p.mu.Unlock() if requiresReady(q) && !ready { return response{}, uncertain(q.Operation, ErrUnavailable) } + if p.version == ProtocolVersion { + q.DeploymentGeneration = 0 + } deadline, _ := ctx.Deadline() q.ID, q.ConnectionID, q.OwnerEpoch = uuid.NewString(), p.id, p.epoch if err = q.setTimeout(deadline); err != nil { @@ -314,7 +369,7 @@ func (h *Hub) call(ctx context.Context, id string, q request) (response, error) } else if err = q.setTimeout(deadline); err == nil { p.sequence++ q.Sequence = p.sequence - err = writeFrame(p.conn, frame{Type: "request", Request: &q}) + err = writeFrame(p.conn, frame{Version: p.version, Type: "request", Request: &q}) } } p.unlockSend() @@ -337,3 +392,55 @@ func (h *Hub) call(ctx context.Context, id string, q request) (response, error) } type Resolver func(context.Context, sandbox.Reference) (string, error) + +func (h *Hub) recordHealth(ctx context.Context, p *peer, health Health) error { + if p.version == GenerationProtocolVersion { + if h.options.Generations == nil { + return sandbox.ErrInvalid + } + if err := callback(ctx, func(ctx context.Context) error { return h.options.Generations(ctx, p.identity, p.id, p.epoch, health) }); err != nil { + return err + } + refs := make([]sandbox.GenerationReference, 0, len(health.Generations)) + for _, g := range health.Generations { + refs = append(refs, sandbox.GenerationReference{Generation: g.Generation, SpecificationDigest: g.SpecificationDigest}) + } + _, grants, err := h.retention(ctx, p, refs) + if err != nil { + return err + } + p.mu.Lock() + for i, g := range health.Generations { + if grants[i].Keep { + p.generations[g.Generation] = g + } else { + delete(p.generations, g.Generation) + } + } + p.mu.Unlock() + return nil + } + if h.options.Heartbeat != nil { + if err := callback(ctx, func(ctx context.Context) error { return h.options.Heartbeat(ctx, p.identity, p.id, p.epoch, health) }); err != nil { + return err + } + } + p.mu.Lock() + p.ready = health.ProviderReady + p.mu.Unlock() + return nil +} +func (h *Hub) retention(ctx context.Context, p *peer, refs []sandbox.GenerationReference) (sandbox.NodeDeployment, []sandbox.GenerationRetention, error) { + if h.options.Retention == nil { + return sandbox.NodeDeployment{}, nil, sandbox.ErrInvalid + } + type result struct { + deployment sandbox.NodeDeployment + grants []sandbox.GenerationRetention + } + value, err := callbackValue(ctx, func(ctx context.Context) (result, error) { + d, g, err := h.options.Retention(ctx, p.identity, p.id, p.epoch, refs) + return result{d, g}, err + }) + return value.deployment, value.grants, err +} diff --git a/services/agents-api/internal/sandbox/node/proxy.go b/services/agents-api/internal/sandbox/node/proxy.go index 955b03d48..ba969a5d4 100644 --- a/services/agents-api/internal/sandbox/node/proxy.go +++ b/services/agents-api/internal/sandbox/node/proxy.go @@ -6,9 +6,10 @@ import ( ) type provider struct { - hub *Hub - resolve Resolver - kind string + hub *Hub + resolve Resolver + resolveGeneration func(context.Context, sandbox.Reference) (string, uint64, error) + kind string } type checkpointProvider struct{ *provider } @@ -27,7 +28,13 @@ func (h *Hub) Proxy(id, kind string) sandbox.Provider { return h.Provider(kind, func(context.Context, sandbox.Reference) (string, error) { return id, nil }) } func (p *provider) call(ctx context.Context, q request) (response, error) { - id, err := p.resolve(ctx, q.Reference) + var id string + var err error + if p.resolveGeneration != nil { + id, q.DeploymentGeneration, err = p.resolveGeneration(ctx, q.Reference) + } else { + id, err = p.resolve(ctx, q.Reference) + } if err != nil { return response{}, err } @@ -140,3 +147,13 @@ func (p *checkpointProvider) RunCommandCompute(ctx context.Context, r sandbox.Re func (p *checkpointProvider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { return p.state(ctx, request{Operation: "resume_compute", Reference: r, Compute: &c}) } + +// GenerationProvider routes every operation with allocation-owned generation, +// distinct from the request's compute generation. +func (h *Hub) GenerationProvider(kind string, resolve func(context.Context, sandbox.Reference) (string, uint64, error)) sandbox.Provider { + p := &provider{hub: h, kind: kind, resolveGeneration: resolve} + if kind == "microsandbox" { + return &checkpointProvider{p} + } + return p +} diff --git a/services/agents-api/internal/sandbox/node/wire.go b/services/agents-api/internal/sandbox/node/wire.go index bda6b2451..3947e6472 100644 --- a/services/agents-api/internal/sandbox/node/wire.go +++ b/services/agents-api/internal/sandbox/node/wire.go @@ -17,6 +17,8 @@ import ( ) const ProtocolVersion = 1 +const GenerationProtocolVersion = 2 +const MaxControlFrameBytes = 32 * 1024 const MaxFrameBytes = 72 * 1024 * 1024 const maxPending = 32 const maxRequestTimeoutMillis int64 = 120000 @@ -36,16 +38,17 @@ type Identity struct { } type Health struct { - Diagnostic string `json:"diagnostic,omitempty"` - ProviderReady bool `json:"provider_ready"` - ObservedAt time.Time `json:"observed_at"` - ActiveOperations int `json:"active_operations"` - CPUCount *int64 `json:"cpu_count,omitempty"` - CPUUtilization *float64 `json:"cpu_utilization"` - TotalMemoryBytes *int64 `json:"total_memory_bytes"` - EffectiveCPUCores *float64 `json:"effective_cpu_cores"` - AvailableMemoryBytes *int64 `json:"available_memory_bytes,omitempty"` - AvailableDiskBytes *int64 `json:"available_disk_bytes,omitempty"` + Generations []sandbox.GenerationStatus `json:"generations,omitempty"` + Diagnostic string `json:"diagnostic,omitempty"` + ProviderReady bool `json:"provider_ready"` + ObservedAt time.Time `json:"observed_at"` + ActiveOperations int `json:"active_operations"` + CPUCount *int64 `json:"cpu_count,omitempty"` + CPUUtilization *float64 `json:"cpu_utilization"` + TotalMemoryBytes *int64 `json:"total_memory_bytes"` + EffectiveCPUCores *float64 `json:"effective_cpu_cores"` + AvailableMemoryBytes *int64 `json:"available_memory_bytes,omitempty"` + AvailableDiskBytes *int64 `json:"available_disk_bytes,omitempty"` } type EnrollmentRequest struct { @@ -73,12 +76,13 @@ type EnrollmentResponse struct { } type request struct { - ID string `json:"id"` - Sequence uint64 `json:"sequence"` - ConnectionID string `json:"connection_id"` - OwnerEpoch uint64 `json:"owner_epoch"` - Operation string `json:"operation"` - TimeoutMillis int64 `json:"timeout_ms"` + DeploymentGeneration uint64 `json:"deployment_generation,omitempty"` + ID string `json:"id"` + Sequence uint64 `json:"sequence"` + ConnectionID string `json:"connection_id"` + OwnerEpoch uint64 `json:"owner_epoch"` + Operation string `json:"operation"` + TimeoutMillis int64 `json:"timeout_ms"` // deadline is anchored to the receiving host and never crosses the wire. deadline time.Time Reference sandbox.Reference `json:"reference"` @@ -104,14 +108,16 @@ type response struct { } type frame struct { - Version int `json:"version"` - Type string `json:"type"` - Identity *Identity `json:"identity,omitempty"` - Health *Health `json:"health,omitempty"` - ConnectionID string `json:"connection_id,omitempty"` - OwnerEpoch uint64 `json:"owner_epoch,omitempty"` - Request *request `json:"request,omitempty"` - Response *response `json:"response,omitempty"` + Deployment *sandbox.NodeDeployment `json:"deployment,omitempty"` + Control *generationControl `json:"control,omitempty"` + Version int `json:"version"` + Type string `json:"type"` + Identity *Identity `json:"identity,omitempty"` + Health *Health `json:"health,omitempty"` + ConnectionID string `json:"connection_id,omitempty"` + OwnerEpoch uint64 `json:"owner_epoch,omitempty"` + Request *request `json:"request,omitempty"` + Response *response `json:"response,omitempty"` } func validID(s string) bool { @@ -132,18 +138,23 @@ func readFrame(conn *websocket.Conn) (frame, error) { } d := json.NewDecoder(bytes.NewReader(data)) d.DisallowUnknownFields() - if d.Decode(&f) != nil || d.Decode(new(any)) != io.EOF || f.Version != ProtocolVersion { + if d.Decode(&f) != nil || d.Decode(new(any)) != io.EOF || (f.Version != ProtocolVersion && f.Version != GenerationProtocolVersion) { return frame{}, sandbox.ErrInvalid } + if err := validateVersionFrame(f, len(data)); err != nil { + return frame{}, err + } return f, nil } func writeFrame(conn *websocket.Conn, f frame) error { - f.Version = ProtocolVersion + if f.Version == 0 { + f.Version = ProtocolVersion + } if err := conn.SetWriteDeadline(time.Now().Add(10 * time.Second)); err != nil { return err } data, err := json.Marshal(f) - if err != nil || len(data) > MaxFrameBytes { + if err != nil || len(data) > MaxFrameBytes || validateVersionFrame(f, len(data)) != nil { return sandbox.ErrInvalid } return conn.WriteMessage(websocket.TextMessage, data) diff --git a/services/agents-api/internal/sandbox/node_diagnostic.go b/services/agents-api/internal/sandbox/node_diagnostic.go index 640a04f96..00b1ed898 100644 --- a/services/agents-api/internal/sandbox/node_diagnostic.go +++ b/services/agents-api/internal/sandbox/node_diagnostic.go @@ -6,6 +6,7 @@ import "errors" // first failed check. Only the fixed code crosses the node transport; the error // text and any wrapped detail, such as host paths or daemon messages, stay local. var ( + ErrRuntimeDownloadFailed = errors.New("Runtime preparation failed") ErrDockerUnavailable = errors.New("Docker daemon is unavailable") ErrDockerLimitsUnsupported = errors.New("Docker host does not enforce CPU and memory limits") ErrRuntimeImageUnavailable = errors.New("pinned Runtime image is unavailable") @@ -16,11 +17,13 @@ var ( // NodeProviderUnavailable reports every readiness failure without a fixed cause. const NodeProviderUnavailable = "provider_unavailable" +const NodeRuntimeDownloadFailed = "runtime_download_failed" var nodeDiagnostics = []struct { err error code string }{ + {ErrRuntimeDownloadFailed, NodeRuntimeDownloadFailed}, {ErrDockerUnavailable, "docker_unavailable"}, {ErrDockerLimitsUnsupported, "docker_limits_unsupported"}, {ErrRuntimeImageUnavailable, "runtime_image_unavailable"}, diff --git a/services/agents-api/internal/store/runtime_node_configuration.go b/services/agents-api/internal/store/runtime_node_configuration.go index d864a17f6..fe7821c58 100644 --- a/services/agents-api/internal/store/runtime_node_configuration.go +++ b/services/agents-api/internal/store/runtime_node_configuration.go @@ -24,6 +24,12 @@ type RuntimeNodeConfiguration struct { // The deployment lock keeps authentication and the returned generation consistent. // The credential is authenticated before any deployment state is reported. func (s *Store) RuntimeNodeConfiguration(ctx context.Context, nodeID, token string) (RuntimeNodeConfiguration, error) { + return s.RuntimeNodeGenerationConfiguration(ctx, nodeID, token, 0) +} + +// Exact generation recovery is restricted to this node's current target, serving +// pin and unreleased ownership. It is never a general history read. +func (s *Store) RuntimeNodeGenerationConfiguration(ctx context.Context, nodeID, token string, generation uint64) (RuntimeNodeConfiguration, error) { var result RuntimeNodeConfiguration err := s.runtimeDeploymentTransaction(ctx, func(q *sqlc.Queries, d sqlc.RuntimeDeployment) error { var node *sqlc.RuntimeNode @@ -60,17 +66,28 @@ func (s *Store) RuntimeNodeConfiguration(ctx context.Context, nodeID, token stri if d.Mode != "nodes" { return ErrSandboxDeploymentConflict } - spec, err := deploymentSpecification(d) + selected := uint64(d.Generation) + if generation != 0 { + if node == nil { + return ErrRuntimeNodeCredential + } + selected = generation + kept, err := q.NodeGenerationKept(ctx, sqlc.NodeGenerationKeptParams{NodeID: node.ID, Generation: int64(generation)}) + if err != nil { + return err + } + if !kept { + return ErrRuntimeSpecificationMismatch + } + } + spec, err := nodeGenerationSpec(ctx, q, d, selected) if err != nil { return err } if node == nil && d.AdmissionPaused { return ErrSandboxDeploymentConflict } - if node != nil && (node.DeploymentGeneration != d.Generation || node.SpecificationDigest != spec.Digest(d.ProviderKind)) { - return ErrRuntimeSpecificationMismatch - } - result = RuntimeNodeConfiguration{MaxActive: int(active), MaxRetained: retainedLimit(d.ProviderKind, int(active), int(retained)), InstallationID: runtimeUUID(d.InstallationID), Provider: d.ProviderKind, CoreURL: s.publicURL, Generation: uint64(d.Generation), Specification: spec, SpecificationDigest: spec.Digest(d.ProviderKind)} + result = RuntimeNodeConfiguration{MaxActive: int(active), MaxRetained: retainedLimit(d.ProviderKind, int(active), int(retained)), InstallationID: runtimeUUID(d.InstallationID), Provider: d.ProviderKind, CoreURL: s.publicURL, Generation: selected, Specification: spec, SpecificationDigest: spec.Digest(d.ProviderKind)} return nil }) return result, err diff --git a/services/agents-api/internal/store/runtime_node_generations.go b/services/agents-api/internal/store/runtime_node_generations.go new file mode 100644 index 000000000..b8546f76e --- /dev/null +++ b/services/agents-api/internal/store/runtime_node_generations.go @@ -0,0 +1,138 @@ +package store + +import ( + "context" + "encoding/json" + "math" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/db/sqlc" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/jackc/pgx/v5/pgtype" +) + +func nodeGenerationSpec(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment, generation uint64) (sandbox.DeploymentSpec, error) { + var spec sandbox.DeploymentSpec + if generation == 0 || generation > math.MaxInt64 { + return spec, ErrInvalidInput + } + row, err := q.GetNodeGenerationSpecification(ctx, int64(generation)) + if err != nil { + return spec, err + } + if row.ProviderKind != d.ProviderKind || json.Unmarshal(row.Specification, &spec) != nil || spec.Validate(d.ProviderKind) != nil { + return spec, ErrRuntimeSpecificationMismatch + } + return spec, nil +} + +func nodeConnection(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment, nodeID, connectionID string, epoch uint64) (sqlc.RuntimeNode, error) { + id, err := parseConnectionGeneration(nodeID) + if err != nil { + return sqlc.RuntimeNode{}, ErrRuntimeNodeCredential + } + n, err := q.GetRuntimeNode(ctx, id) + if err != nil { + return n, err + } + if runtimeUUID(n.ConnectionID) != connectionID || n.ConnectedEpoch != int64(epoch) || epoch == 0 || epoch > math.MaxInt64 || d.OwnerEpoch != int64(epoch) || n.InstallationID != d.InstallationID || d.Mode != "nodes" { + return n, ErrRuntimeNodeCredential + } + return n, nil +} + +// RuntimeNodeRetention is a bounded read under the same serialization as pin +// promotion and placement. A drop cannot subsequently gain fresh old ownership. +func (s *Store) RuntimeNodeRetention(ctx context.Context, nodeID, connectionID string, epoch uint64, refs []sandbox.GenerationReference) (sandbox.NodeDeployment, []sandbox.GenerationRetention, error) { + var deployment sandbox.NodeDeployment + grants := make([]sandbox.GenerationRetention, 0, len(refs)) + if len(refs) > 8 { + return deployment, nil, ErrInvalidInput + } + ctx, cancel := context.WithTimeout(ctx, executionTransactionTimeout) + defer cancel() + err := s.runtimeDeploymentTransaction(ctx, func(q *sqlc.Queries, d sqlc.RuntimeDeployment) error { + n, err := nodeConnection(ctx, q, d, nodeID, connectionID, epoch) + if err != nil { + return err + } + deployment.Generation = uint64(d.Generation) + spec, err := nodeGenerationSpec(ctx, q, d, uint64(d.Generation)) + if err != nil { + return err + } + deployment.SpecificationDigest = spec.Digest(d.ProviderKind) + if n.ReadyGeneration.Valid { + pin := uint64(n.ReadyGeneration.Int64) + deployment.ServingGeneration = &pin + } + seen := map[uint64]bool{} + for _, ref := range refs { + if ref.Generation == 0 || ref.Generation > math.MaxInt64 || !validRuntimeDigest(ref.SpecificationDigest) || seen[ref.Generation] { + return ErrInvalidInput + } + seen[ref.Generation] = true + kept, err := q.NodeGenerationKept(ctx, sqlc.NodeGenerationKeptParams{NodeID: n.ID, Generation: int64(ref.Generation)}) + if err != nil { + return err + } + if kept { + spec, err := nodeGenerationSpec(ctx, q, d, ref.Generation) + if err != nil { + return err + } + if spec.Digest(d.ProviderKind) != ref.SpecificationDigest { + return ErrRuntimeSpecificationMismatch + } + } + grants = append(grants, sandbox.GenerationRetention{GenerationReference: ref, Keep: kept}) + } + return nil + }) + if err != nil { + return sandbox.NodeDeployment{}, nil, err + } + return deployment, grants, nil +} + +func (s *Store) HeartbeatRuntimeNodeGenerations(ctx context.Context, nodeID, connectionID string, epoch uint64, health RuntimeNodeHealth, statuses []sandbox.GenerationStatus) error { + if len(statuses) > 8 { + return ErrInvalidInput + } + return s.heartbeatRuntimeNode(ctx, nodeID, connectionID, epoch, health, statuses, 2) +} + +func recordNodeGenerations(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment, n sqlc.RuntimeNode, statuses []sandbox.GenerationStatus, protocol int32) error { + seen := map[uint64]bool{} + for _, status := range statuses { + if status.Generation == 0 || status.Generation > math.MaxInt64 || !validRuntimeDigest(status.SpecificationDigest) || seen[status.Generation] || (status.State != "ready" && status.State != "preparing" && status.State != "failed") || status.State == "ready" && status.Diagnostic != "" { + return ErrInvalidInput + } + seen[status.Generation] = true + kept, err := q.NodeGenerationKept(ctx, sqlc.NodeGenerationKeptParams{NodeID: n.ID, Generation: int64(status.Generation)}) + if err != nil { + return err + } + // A sparse report may race collection of a skipped target. It creates no + // readiness or pin; only the later correlated retention reply can drop it. + if !kept { + continue + } + spec, err := nodeGenerationSpec(ctx, q, d, status.Generation) + if err != nil { + return err + } + if spec.Digest(d.ProviderKind) != status.SpecificationDigest { + return ErrRuntimeSpecificationMismatch + } + diagnostic := sandbox.NormalizeNodeDiagnostic(status.Diagnostic) + if err := q.UpsertNodeGenerationStatus(ctx, sqlc.UpsertNodeGenerationStatusParams{NodeID: n.ID, Generation: int64(status.Generation), SpecificationDigest: status.SpecificationDigest, ConnectionID: n.ConnectionID, OwnerEpoch: d.OwnerEpoch, State: status.State, Diagnostic: diagnostic}); err != nil { + return err + } + if status.State == "ready" && status.Generation == uint64(d.Generation) { + if err := q.PromoteNodeServingGeneration(ctx, sqlc.PromoteNodeServingGenerationParams{ID: n.ID, ReadyGeneration: pgtype.Int8{Int64: int64(status.Generation), Valid: true}}); err != nil { + return err + } + } + } + return q.RefreshNodeServingReadiness(ctx, sqlc.RefreshNodeServingReadinessParams{ID: n.ID, ProtocolVersion: protocol}) +} diff --git a/services/agents-api/internal/store/runtime_node_presence.go b/services/agents-api/internal/store/runtime_node_presence.go index b66443d28..5f8e7636c 100644 --- a/services/agents-api/internal/store/runtime_node_presence.go +++ b/services/agents-api/internal/store/runtime_node_presence.go @@ -33,6 +33,9 @@ func (s *Store) ConnectRuntimeNode(ctx context.Context, nodeID, connectionID str }) } func (s *Store) HeartbeatRuntimeNode(ctx context.Context, nodeID, connectionID string, epoch uint64, health RuntimeNodeHealth) error { + return s.heartbeatRuntimeNode(ctx, nodeID, connectionID, epoch, health, nil, 1) +} +func (s *Store) heartbeatRuntimeNode(ctx context.Context, nodeID, connectionID string, epoch uint64, health RuntimeNodeHealth, statuses []sandbox.GenerationStatus, protocol int32) error { id, err := parseConnectionGeneration(nodeID) if err != nil { return err @@ -59,11 +62,31 @@ func (s *Store) HeartbeatRuntimeNode(ctx context.Context, nodeID, connectionID s if err != nil { return err } - changed, err := s.queries.HeartbeatRuntimeNode(ctx, sqlc.HeartbeatRuntimeNodeParams{ID: id, ConnectionID: connection, OwnerEpoch: int64(epoch), ProviderReady: health.ProviderReady, Health: raw}) - if err == nil && changed != 1 { - return ErrRuntimeNodeCredential - } - return err + return s.runtimeDeploymentTransaction(ctx, func(q *sqlc.Queries, d sqlc.RuntimeDeployment) error { + n, err := nodeConnection(ctx, q, d, nodeID, connectionID, epoch) + if err != nil { + return err + } + changed, err := q.HeartbeatRuntimeNode(ctx, sqlc.HeartbeatRuntimeNodeParams{ID: id, ConnectionID: connection, OwnerEpoch: int64(epoch), ProviderReady: health.ProviderReady, Health: raw}) + if err != nil { + return err + } + if changed != 1 { + return ErrRuntimeNodeCredential + } + if protocol == 1 { + // Unspecified legacy nodes keep cleanup connectivity without inventing a generation. + if n.DeploymentGeneration == 0 { + return nil + } + state := "failed" + if health.ProviderReady { + state = "ready" + } + statuses = []sandbox.GenerationStatus{{Generation: uint64(n.DeploymentGeneration), SpecificationDigest: n.SpecificationDigest, State: state, Diagnostic: health.Diagnostic}} + } + return recordNodeGenerations(ctx, q, d, n, statuses, protocol) + }) } func (s *Store) DisconnectRuntimeNode(ctx context.Context, nodeID, connectionID string, epoch uint64) error { id, err := parseConnectionGeneration(nodeID) diff --git a/services/agents-api/internal/store/runtime_nodes.go b/services/agents-api/internal/store/runtime_nodes.go index 8df1179d9..ff5eb64b7 100644 --- a/services/agents-api/internal/store/runtime_nodes.go +++ b/services/agents-api/internal/store/runtime_nodes.go @@ -131,7 +131,7 @@ func runtimeNodeViews(rows []sqlc.ListRuntimeNodesRow) ([]RuntimeNode, error) { if err := json.Unmarshal(n.Health, &health); err != nil { return nil, err } - health.ProviderReady = n.ProviderReady + health.ProviderReady = n.Online && n.ServingReady out = append(out, RuntimeNode{Rollout: nodeRollout(n), RuntimeNodeHealth: health, Running: n.Running, Snapshots: n.Snapshots, ID: runtimeUUID(n.ID), Name: n.Name, CoreURL: n.CoreUrl, EnrollmentID: optionalUUID(n.EnrollmentID), Provider: n.ProviderKind, Online: n.Online, LastSeenAt: seen, MaxActive: int(n.MaxActive), MaxRetained: retainedLimit(n.ProviderKind, int(n.MaxActive), int(n.MaxRetained)), Active: n.Active, Reserved: n.Reserved, Retained: n.Retained, CleanupPending: n.CleanupPending, CreatedAt: n.CreatedAt.Time}) } return out, nil @@ -264,8 +264,10 @@ func (s *Store) AuthenticateRuntimeNode(ctx context.Context, nodeID, credential } return nodeIdentity(n, d.ProviderKind), nil } - spec, err := deploymentSpecification(d) - if err != nil || n.SpecificationDigest != spec.Digest(d.ProviderKind) || n.DeploymentGeneration != d.Generation { + // Reset retires nodes before another backend lineage can be selected. + // Enrollment generation and digest remain immutable identity history; the + // current target and per-generation readiness do not replace that history. + if n.DeploymentGeneration <= 0 || !validRuntimeDigest(n.SpecificationDigest) { return RuntimeNodeIdentity{}, ErrRuntimeSpecificationMismatch } return nodeIdentity(n, d.ProviderKind), nil @@ -358,18 +360,16 @@ func nodeRollout(n sqlc.ListRuntimeNodesRow) SandboxNodeRollout { if !n.Online { return out } - if n.DeploymentGeneration != n.TargetGeneration { + if n.ProtocolVersion == 1 && n.DeploymentGeneration != n.TargetGeneration { out.State = "update_required" return out } - if n.ProviderReady && n.ReadyGeneration.Valid && n.ReadyGeneration.Int64 == n.TargetGeneration { - out.State = "ready" - return out + switch n.TargetState { + case "ready", "preparing", "failed": + out.State = n.TargetState } - var health RuntimeNodeHealth - if json.Unmarshal(n.Health, &health) == nil && health.Diagnostic != "" { - out.State = "failed" - out.Diagnostic = sandbox.NormalizeNodeDiagnostic(health.Diagnostic) + if out.State == "failed" && n.TargetDiagnostic != "" { + out.Diagnostic = sandbox.NormalizeNodeDiagnostic(n.TargetDiagnostic) } return out } diff --git a/services/agents-api/internal/store/runtime_placements.go b/services/agents-api/internal/store/runtime_placements.go index fa52ffe46..44af7d279 100644 --- a/services/agents-api/internal/store/runtime_placements.go +++ b/services/agents-api/internal/store/runtime_placements.go @@ -2,6 +2,7 @@ package store import ( "context" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/db/sqlc" "github.com/jackc/pgx/v5/pgtype" @@ -46,17 +47,17 @@ func reserveRuntimePlacement(ctx context.Context, q *sqlc.Queries, session pgtyp var chosen *sqlc.ListRuntimeNodesRow for i := range rows { n := &rows[i] - if !n.Online || !n.ProviderReady || n.Active >= int64(n.MaxActive) || n.Retained >= int64(n.MaxRetained) || (d.WebManaged && n.CoreUrl != publicURL) { + if !n.Online || !n.ServingReady || !n.ReadyGeneration.Valid || n.Active >= int64(n.MaxActive) || n.Retained >= int64(n.MaxRetained) || (d.WebManaged && n.CoreUrl != publicURL) { continue } - if chosen == nil || n.Active < chosen.Active { + if chosen == nil || n.ReadyGeneration.Int64 > chosen.ReadyGeneration.Int64 || n.ReadyGeneration.Int64 == chosen.ReadyGeneration.Int64 && n.Active < chosen.Active { chosen = n } } if chosen == nil { return ErrRuntimeNodeUnavailable } - return q.CreateSessionRuntimePlacement(ctx, sqlc.CreateSessionRuntimePlacementParams{SessionID: session, NodeID: chosen.ID}) + return q.CreateSessionRuntimePlacement(ctx, sqlc.CreateSessionRuntimePlacementParams{SessionID: session, NodeID: chosen.ID, Generation: chosen.ReadyGeneration.Int64}) } // ResolveRuntimeNode includes deleted Sessions so owned cleanup remains routable. @@ -70,7 +71,7 @@ func (s *Store) ResolveRuntimeNode(ctx context.Context, tenant, environment stri } return allocation.NodeID, nil } -func reserveRuntimeRestore(ctx context.Context, q *sqlc.Queries, node pgtype.UUID) error { +func reserveRuntimeRestore(ctx context.Context, q *sqlc.Queries, node pgtype.UUID, generation pgtype.Int8) error { if !node.Valid { return nil } @@ -83,7 +84,11 @@ func reserveRuntimeRestore(ctx context.Context, q *sqlc.Queries, node pgtype.UUI } for _, n := range nodes { if n.ID == node { - if !n.Online || !n.ProviderReady || n.Active >= int64(n.MaxActive) { + ready, err := q.NodeGenerationReady(ctx, sqlc.NodeGenerationReadyParams{NodeID: node, Generation: generation.Int64}) + if err != nil { + return err + } + if !generation.Valid || !n.Online || !ready || n.Active >= int64(n.MaxActive) { return ErrRuntimeNodeUnavailable } return nil @@ -91,3 +96,15 @@ func reserveRuntimeRestore(ctx context.Context, q *sqlc.Queries, node pgtype.UUI } return ErrRuntimeNodeUnavailable } + +// ResolveRuntimeGeneration includes deleted Sessions and never substitutes the target. +func (s *Store) ResolveRuntimeGeneration(ctx context.Context, ref sandbox.Reference) (string, uint64, error) { + a, err := s.GetRuntimeAllocation(ctx, ref.TenantID, ref.EnvironmentID) + if err != nil { + return "", 0, err + } + if a.ID != ref.AllocationID || a.NodeID == "" || a.DeploymentGeneration == 0 { + return "", 0, ErrRuntimeNodeUnavailable + } + return a.NodeID, a.DeploymentGeneration, nil +} diff --git a/services/agents-api/internal/store/runtime_suspension.go b/services/agents-api/internal/store/runtime_suspension.go index 03b58b06a..67b922d89 100644 --- a/services/agents-api/internal/store/runtime_suspension.go +++ b/services/agents-api/internal/store/runtime_suspension.go @@ -57,7 +57,7 @@ func (s *Store) SetRuntimeCompute(ctx context.Context, owner RuntimeAllocation, } } if row.ComputePhase == "suspended" && phase == "restoring" { - if err := reserveRuntimeRestore(ctx, q, row.NodeID); err != nil { + if err := reserveRuntimeRestore(ctx, q, row.NodeID, row.DeploymentGeneration); err != nil { return sqlc.RuntimeAllocation{}, err } } diff --git a/services/agents-api/migrations/000081_node_generations.sql b/services/agents-api/migrations/000081_node_generations.sql new file mode 100644 index 000000000..b367fb86d --- /dev/null +++ b/services/agents-api/migrations/000081_node_generations.sql @@ -0,0 +1,29 @@ +-- +goose Up +ALTER TABLE runtime_nodes ADD COLUMN protocol_version integer NOT NULL DEFAULT 1 CHECK (protocol_version IN (1,2)); +CREATE TABLE runtime_node_generation_status ( + node_id uuid NOT NULL REFERENCES runtime_nodes(id), + generation bigint NOT NULL CHECK (generation > 0), + specification_digest text NOT NULL CHECK (specification_digest ~ '^[0-9a-f]{64}$'), + connection_id uuid NOT NULL, + owner_epoch bigint NOT NULL CHECK (owner_epoch > 0), + state text NOT NULL CHECK (state IN ('ready','preparing','failed')), + diagnostic text NOT NULL DEFAULT '', + observed_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY(node_id,generation) +); +INSERT INTO runtime_node_generation_status(node_id,generation,specification_digest,connection_id,owner_epoch,state) +SELECT id,deployment_generation,specification_digest,connection_id,connected_epoch, + CASE WHEN provider_ready THEN 'ready' ELSE 'failed' END FROM runtime_nodes +WHERE removed_at IS NULL AND connection_id IS NOT NULL AND connected_epoch>0 +AND deployment_generation>0 AND specification_digest ~ '^[0-9a-f]{64}$'; + +-- +goose Down +-- +goose StatementBegin +DO $$ BEGIN + IF EXISTS(SELECT 1 FROM runtime_nodes WHERE removed_at IS NULL AND protocol_version=2) THEN + RAISE EXCEPTION 'Cannot downgrade while protocol v2 nodes remain; remove them after releasing their resources first'; + END IF; +END $$; +-- +goose StatementEnd +DROP TABLE runtime_node_generation_status; +ALTER TABLE runtime_nodes DROP COLUMN protocol_version; diff --git a/services/core-console/node_installation.go b/services/core-console/node_installation.go index cbfeffb32..89f704f09 100644 --- a/services/core-console/node_installation.go +++ b/services/core-console/node_installation.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "os" + "regexp" "strings" ) @@ -26,38 +27,81 @@ var optionalPayloadFiles = map[string]bool{ "images/runtime.tar.gz": true, "runtime/seccomp.json": true, } -func (h *console) allowedNodePayload(name string) bool { - if nodePayloadFiles[name] { - return true +var payloadRevision = regexp.MustCompile(`^[0-9a-f]{40}$`) + +// activePayloadPrefix reads one atomic pointer per request. Legacy flat payloads +// remain readable until the installer publishes its first versioned release. +func activePayloadPrefix(root *os.Root) (string, error) { + raw, err := root.ReadFile("active.json") + if errors.Is(err, os.ErrNotExist) { + return "", nil + } + if err != nil || len(raw) > 256 { + return "", errors.New("invalid active node payload") + } + var pointer struct { + SourceCommit string `json:"source_commit"` + } + if json.Unmarshal(raw, &pointer) != nil || !payloadRevision.MatchString(pointer.SourceCommit) { + return "", errors.New("invalid active node payload") + } + return "releases/" + pointer.SourceCommit + "/", nil +} + +func (h *console) resolveNodePayload(name string) (string, bool) { + prefix := "" + if strings.HasPrefix(name, "releases/") { + parts := strings.SplitN(name, "/", 3) + if len(parts) != 3 || !payloadRevision.MatchString(parts[1]) { + return "", false + } + prefix, name = "releases/"+parts[1]+"/", parts[2] + } else { + var err error + prefix, err = activePayloadPrefix(h.nodePayload) + if err != nil { + return "", false + } } - if !strings.HasPrefix(name, "artifacts/") || strings.Contains(strings.TrimPrefix(name, "artifacts/"), "/") { - return false + if !nodePayloadFiles[name] && (!strings.HasPrefix(name, "artifacts/") || strings.Contains(strings.TrimPrefix(name, "artifacts/"), "/")) { + return "", false } - f, err := h.nodePayload.Open("manifest.json") + f, err := h.nodePayload.Open(prefix + "manifest.json") if err != nil { - return false + return "", false } defer f.Close() var manifest struct { - Artifacts map[string]struct { + SourceCommit string `json:"source_commit"` + Artifacts map[string]struct { Filename string `json:"filename"` } `json:"artifacts"` } if json.NewDecoder(io.LimitReader(f, 1024*1024)).Decode(&manifest) != nil { - return false + return "", false + } + if prefix != "" && prefix != "releases/"+manifest.SourceCommit+"/" { + return "", false + } + if nodePayloadFiles[name] { + return prefix + name, true } for logical, entry := range manifest.Artifacts { if optionalPayloadFiles[logical] && entry.Filename != "" && "artifacts/"+entry.Filename == name { - return true + return prefix + name, true } } - return false + return "", false } // installerDigest returns the SHA-256 that Web's install commands verify // before running the named installer from the payload. func installerDigest(root *os.Root, name string) (string, error) { - f, err := root.Open(name) + prefix, err := activePayloadPrefix(root) + if err != nil { + return "", err + } + f, err := root.Open(prefix + name) if err != nil { return "", errors.New("installer " + name + " is missing from the node installation payload") } @@ -80,11 +124,12 @@ func (h *console) serveNodePayload(w http.ResponseWriter, r *http.Request) { return } name := strings.TrimPrefix(r.URL.Path, "/node-install/") - if !h.allowedNodePayload(name) { + resolved, allowed := h.resolveNodePayload(name) + if !allowed { http.NotFound(w, r) return } - f, err := h.nodePayload.Open(name) + f, err := h.nodePayload.Open(resolved) if err != nil { http.NotFound(w, r) return @@ -116,7 +161,11 @@ func (h *console) nodeArtifacts() []string { if h.nodePayload == nil { return available } - f, err := h.nodePayload.Open("manifest.json") + prefix, err := activePayloadPrefix(h.nodePayload) + if err != nil { + return available + } + f, err := h.nodePayload.Open(prefix + "manifest.json") if err != nil { return available } @@ -134,7 +183,7 @@ func (h *console) nodeArtifacts() []string { complete := true for _, logical := range providerArtifacts[provider] { entry, ok := manifest.Artifacts[logical] - info, err := h.nodePayload.Stat("artifacts/" + entry.Filename) + info, err := h.nodePayload.Stat(prefix + "artifacts/" + entry.Filename) if !ok || entry.Filename == "" || strings.Contains(entry.Filename, "/") || err != nil || !info.Mode().IsRegular() || info.Size() != entry.Size { complete = false break From 226c8fb94c414a01bc3169c4903afa73072951d9 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 21:56:42 +0000 Subject: [PATCH 02/16] Checkpoint retained payload HTTP and bounded node control regressions --- deploy/install/install.py | 2 +- deploy/install/test_install.py | 38 +++- scripts/core-distribution-manifest.test.py | 4 +- .../internal/sandbox/node/generations_test.go | 192 ++++++++++++++++++ services/core-console/distribution_test.go | 56 +++++ services/core-console/node_installation.go | 3 + 6 files changed, 290 insertions(+), 5 deletions(-) create mode 100644 services/agents-api/internal/sandbox/node/generations_test.go diff --git a/deploy/install/install.py b/deploy/install/install.py index 2460aa65c..c3995df23 100644 --- a/deploy/install/install.py +++ b/deploy/install/install.py @@ -342,7 +342,7 @@ def publish(source): raise InstallError("Offline artifact verification failed: " + logical) names.append(name) target = destination / "releases" / revision - if target.is_symlink(): + if target.is_symlink() or target.parent.is_symlink(): raise InstallError("Installed node payload differs; preserve it and inspect the distribution") target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) if target.exists(): diff --git a/deploy/install/test_install.py b/deploy/install/test_install.py index 313e3b72d..b7c59f8fc 100644 --- a/deploy/install/test_install.py +++ b/deploy/install/test_install.py @@ -66,7 +66,7 @@ def test_fresh_install_writes_config_json_and_the_layout(self): self.assertEqual((state["mode"], state["native_core"], state["source_commit"]), ("all", False, "a" * 40)) self.assertEqual(set(state["images"]), {"core", "database", "web"}) self.assertEqual(set(state["secrets_sha256"]), {"credential.key", "database.password"}) - names = {str(path.relative_to(self.root)) for path in self.root.rglob("*") if path.parent.name != "node-payload" + names = {str(path.relative_to(self.root)) for path in self.root.rglob("*") if "node-payload" not in path.relative_to(self.root).parts[:-1] and "runtime" not in path.parts} self.assertEqual(names, { ".oac.lock", "config.json", "state.json", "oac", "node-payload", "secrets", "secrets/core.key", @@ -319,7 +319,7 @@ def test_a_refused_selection_leaves_the_services_running(self): def test_node_payload_exports_only_matched_distribution_files(self): self.install() - payload = self.root / "node-payload" + payload = self.root / "node-payload/releases" / ("a" * 40) exported = {str(path.relative_to(payload)) for path in payload.rglob("*") if path.is_file()} self.assertEqual(exported, {"node-install.pyz", "self-hosted-install.pyz", "manifest.json", "SHA256SUMS", "runtime/seccomp.json"}) @@ -327,6 +327,40 @@ def test_node_payload_exports_only_matched_distribution_files(self): with self.assertRaisesRegex(install.InstallError, "node payload differs"): install.prepare_node_payload(self.root, self.document("state.json"), self.bundle) + def test_node_payload_upgrade_retains_immutable_old_release(self): + self.install() + payload = self.root / "node-payload" + old = payload / "releases" / ("a" * 40) + snapshot = {str(p.relative_to(old)): p.read_bytes() for p in old.rglob("*") if p.is_file()} + manifest = dict(MANIFEST, source_commit="b" * 40) + bundle, _ = make_bundle(self.work / "bundle-b", manifest) + install.prepare_node_payload(self.root, self.document("state.json"), bundle, replace=True) + self.assertEqual(json.loads((payload / "active.json").read_text()), {"source_commit": "b" * 40}) + self.assertEqual(snapshot, {str(p.relative_to(old)): p.read_bytes() for p in old.rglob("*") if p.is_file()}) + self.assertTrue((payload / "releases" / ("b" * 40) / "node-install.pyz").is_file()) + # A changed release cannot overwrite its immutable name or publication. + (bundle / "node-install.pyz").write_text("changed release") + with self.assertRaisesRegex(install.InstallError, "node payload differs"): + install.prepare_node_payload(self.root, self.document("state.json"), bundle, replace=True) + self.assertEqual(json.loads((payload / "active.json").read_text()), {"source_commit": "b" * 40}) + + def test_legacy_flat_payload_is_preserved_before_publication(self): + import shutil + self.install() + payload = self.root / "node-payload" + old = payload / "releases" / ("a" * 40) + for path in old.rglob("*"): + if path.is_file(): + target = payload / path.relative_to(old) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(path, target) + shutil.rmtree(payload / "releases") + (payload / "active.json").unlink() + bundle, _ = make_bundle(self.work / "bundle-b", dict(MANIFEST, source_commit="b" * 40)) + install.prepare_node_payload(self.root, self.document("state.json"), bundle, replace=True) + self.assertEqual((old / "node-install.pyz").read_bytes(), (self.bundle / "node-install.pyz").read_bytes()) + self.assertEqual(json.loads((payload / "active.json").read_text()), {"source_commit": "b" * 40}) + def test_bundle_verifies_transferred_bytes_and_checksum_list(self): self.assertEqual(install.verify_bundle(self.bundle)["source_commit"], "a" * 40) for name in ("images/core.tar", "manifest.json", "oac.pyz", "config.schema.json"): diff --git a/scripts/core-distribution-manifest.test.py b/scripts/core-distribution-manifest.test.py index cf2d2d448..797e94c5d 100644 --- a/scripts/core-distribution-manifest.test.py +++ b/scripts/core-distribution-manifest.test.py @@ -232,7 +232,7 @@ def test_release_location_and_explicit_offline_requirements(self): self.assertEqual(json.loads((self.bundle / "manifest.json").read_text())["artifact_base_url"], "") def test_bootstraps_include_shared_downloader_and_are_reproducible(self): - for name in ("node_install.py", "self_hosted_install.py", *distribution.OAC_CLI_MODULES): + for name in ("node_install.py", "node_generations.py", "self_hosted_install.py", *distribution.OAC_CLI_MODULES): (self.bundle / name).write_text("# " + name + "\n") distribution.bootstraps(self.bundle, "1700000000") first = [(self.bundle / name).read_bytes() for name in ("node-install.pyz", "oac.pyz")] @@ -244,7 +244,7 @@ def test_bootstraps_include_shared_downloader_and_are_reproducible(self): for script, filename in (("node_install.py", "node-install.pyz"), ("self_hosted_install.py", "self-hosted-install.pyz")): with zipfile.ZipFile(self.bundle / filename) as contents: - self.assertEqual(set(contents.namelist()), {"__main__.py", "distribution.py"} | ({"node_spec.py"} if script == "node_install.py" else set())) + self.assertEqual(set(contents.namelist()), {"__main__.py", "distribution.py"} | ({"node_spec.py", "node_generations.py"} if script == "node_install.py" else set())) self.assertEqual(contents.read("__main__.py"), (self.bundle / script).read_bytes()) diff --git a/services/agents-api/internal/sandbox/node/generations_test.go b/services/agents-api/internal/sandbox/node/generations_test.go new file mode 100644 index 000000000..f75d90054 --- /dev/null +++ b/services/agents-api/internal/sandbox/node/generations_test.go @@ -0,0 +1,192 @@ +package node + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/microsandbox" + "github.com/google/uuid" +) + +func generationFixture(count int) *GenerationManager { + m := &GenerationManager{values: map[uint64]*localGeneration{}, wake: make(chan struct{}, 1)} + for i := 1; i <= count; i++ { + g := uint64(i) + m.values[g] = &localGeneration{value: GenerationProvider{Generation: g, SpecificationDigest: strings.Repeat("a", 64), Provider: &fakeProvider{}}, state: "ready"} + } + pin := uint64(1) + m.target = sandbox.NodeDeployment{Generation: uint64(count), SpecificationDigest: strings.Repeat("a", 64), ServingGeneration: &pin} + return m +} + +func TestSparseGenerationControlHasNoLifetimeLimit(t *testing.T) { + for _, count := range []int{9, 17, 257} { + t.Run(fmt.Sprint(count), func(t *testing.T) { + m := generationFixture(count) + statuses, retained := map[uint64]bool{}, map[uint64]bool{} + cursor := uint64(0) + for i := 0; i < count; i++ { + batch := m.Statuses() + if len(batch) > 8 || len(batch) < 2 || batch[0].Generation != uint64(count) || batch[1].Generation != 1 { + t.Fatal("unbounded batch or missing priorities", batch) + } + for _, item := range batch { + statuses[item.Generation] = true + } + refs := m.Retained(cursor) + if len(refs) > 8 { + t.Fatal("unbounded retention batch") + } + for _, ref := range refs { + retained[ref.Generation] = true + } + cursor = refs[len(refs)-1].Generation + } + if len(statuses) != count || len(retained) != count || len(m.values) != count { + t.Fatal("sparse rotation lost generations", len(statuses), len(retained), len(m.values)) + } + }) + } +} + +func TestRetentionReplyMustMatchWholePendingExchange(t *testing.T) { + for _, mutation := range []string{"id", "sequence", "connection", "epoch", "partial", "reordered", "digest"} { + t.Run(mutation, func(t *testing.T) { + m := generationFixture(3) + a := &agent{config: AgentConfig{Generations: m}} + c := &agentConnection{id: uuid.NewString(), epoch: 7} + refs := m.Retained(0) + c.pending = &generationControl{ID: uuid.NewString(), Sequence: 1, ConnectionID: c.id, OwnerEpoch: c.epoch, References: refs} + reply := &generationControl{ID: c.pending.ID, Sequence: 1, ConnectionID: c.id, OwnerEpoch: c.epoch} + for _, ref := range refs { + reply.Retentions = append(reply.Retentions, sandbox.GenerationRetention{GenerationReference: ref, Keep: false}) + } + switch mutation { + case "id": + reply.ID = uuid.NewString() + case "sequence": + reply.Sequence++ + case "connection": + reply.ConnectionID = uuid.NewString() + case "epoch": + reply.OwnerEpoch++ + case "partial": + reply.Retentions = reply.Retentions[:1] + case "reordered": + reply.Retentions[0], reply.Retentions[1] = reply.Retentions[1], reply.Retentions[0] + case "digest": + reply.Retentions[0].SpecificationDigest = strings.Repeat("b", 64) + } + work := make(chan []sandbox.GenerationRetention, 1) + f := frame{Control: reply, Deployment: &m.target} + if err := a.acceptRetention(c, f, work); err == nil { + t.Fatal("invalid grant accepted") + } + if len(work) != 0 || len(m.values) != 3 || c.pending == nil { + t.Fatal("invalid exchange authorized partial collection") + } + }) + } + m := generationFixture(3) + a := &agent{config: AgentConfig{Generations: m}} + c := &agentConnection{id: uuid.NewString(), epoch: 7} + refs := m.Retained(0) + c.pending = &generationControl{ID: uuid.NewString(), Sequence: 1, ConnectionID: c.id, OwnerEpoch: c.epoch, References: refs} + reply := &generationControl{ID: c.pending.ID, Sequence: 1, ConnectionID: c.id, OwnerEpoch: c.epoch} + for _, ref := range refs { + reply.Retentions = append(reply.Retentions, sandbox.GenerationRetention{GenerationReference: ref}) + } + work := make(chan []sandbox.GenerationRetention, 1) + f := frame{Control: reply, Deployment: &m.target} + if err := a.acceptRetention(c, f, work); err != nil { + t.Fatal(err) + } + if err := a.acceptRetention(c, f, work); err == nil { + t.Fatal("replayed grant accepted") + } + if len(work) != 1 { + t.Fatal("grant queued more than once") + } +} + +type helperOwnedProvider struct { + *fakeProvider + caller *microsandbox.ProcessCaller +} + +func (p *helperOwnedProvider) Quiescent() bool { return p.caller.Quiescent() } + +func TestGrantedDropWaitsForReferencesAndActualHelperExit(t *testing.T) { + root := t.TempDir() + helper := filepath.Join(root, "helper") + started := filepath.Join(root, "started") + release := filepath.Join(root, "release") + artifact := filepath.Join(root, "runtime") + // A local test helper uses files only as deterministic process barriers. Its + // response waiter returns on cancellation while the actual process stays alive. + script := "#!/bin/sh\ncat >/dev/null\nprintf started > '" + started + "'\nwhile [ ! -f '" + release + "' ]; do sleep 0.01; done\nprintf '{}\\n'\n" + if err := os.WriteFile(helper, []byte(script), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(artifact, []byte("retained bytes"), 0600); err != nil { + t.Fatal(err) + } + caller := µsandbox.ProcessCaller{} + defer func() { _ = os.WriteFile(release, nil, 0600); wait(t, caller.Quiescent) }() + m := generationFixture(3) + m.target.ServingGeneration = nil + m.values[1].value.Provider = &helperOwnedProvider{fakeProvider: &fakeProvider{}, caller: caller} + m.options.Remove = func(context.Context, GenerationProvider) error { return os.Remove(artifact) } + _, _, unref, err := m.Acquire(1) + if err != nil { + t.Fatal(err) + } + grant := sandbox.GenerationRetention{GenerationReference: sandbox.GenerationReference{Generation: 1, SpecificationDigest: strings.Repeat("a", 64)}} + if err := m.Drop(t.Context(), grant); !errors.Is(err, ErrUnavailable) { + t.Fatal("queued reference did not retain bytes", err) + } + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { + _, err := caller.Call(ctx, microsandbox.Request{Config: microsandbox.Config{HelperPath: helper}}) + done <- err + }() + wait(t, func() bool { _, err := os.Stat(started); return err == nil }) + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + unref() + if caller.Quiescent() { + t.Fatal("caller cancellation pretended helper exited") + } + if err := m.Drop(t.Context(), grant); !errors.Is(err, ErrUnavailable) { + t.Fatal("live helper did not retain generation", err) + } + if _, err := os.Stat(artifact); err != nil { + t.Fatal("granted drop removed live helper bytes", err) + } + if err := os.WriteFile(release, nil, 0600); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(5 * time.Second) + for !caller.Quiescent() && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if !caller.Quiescent() { + t.Fatal("actual helper Wait did not settle") + } + if err := m.Drop(t.Context(), grant); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(artifact); !os.IsNotExist(err) { + t.Fatal("settled unreferenced generation was not collected", err) + } +} diff --git a/services/core-console/distribution_test.go b/services/core-console/distribution_test.go index 543b57deb..0372022c5 100644 --- a/services/core-console/distribution_test.go +++ b/services/core-console/distribution_test.go @@ -108,3 +108,59 @@ func TestConsoleReportsServableNodeProviders(t *testing.T) { t.Fatal("a console without a node payload did not report an empty list:", body) } } + +func TestRetainedPayloadsRemainHTTPReachableAfterPublication(t *testing.T) { + root := t.TempDir() + write := func(name, body string) { + t.Helper() + path := filepath.Join(root, name) + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0600); err != nil { + t.Fatal(err) + } + } + a, b := strings.Repeat("a", 40), strings.Repeat("b", 40) + for _, revision := range []string{a, b} { + prefix := "releases/" + revision + "/" + write(prefix+"manifest.json", `{"source_commit":"`+revision+`","artifacts":{"native/bin/oac-node":{"filename":"node"},"private/key":{"filename":"key"}}}`) + write(prefix+"node-install.pyz", "installer-"+revision) + write(prefix+"artifacts/node", "binary-"+revision) + write(prefix+"artifacts/key", "secret-must-not-be-served") + } + payload, err := os.OpenRoot(root) + if err != nil { + t.Fatal(err) + } + defer payload.Close() + h := &console{nodePayload: payload} + request := func(path string) (int, string) { + t.Helper() + w := httptest.NewRecorder() + h.serveNodePayload(w, httptest.NewRequest("GET", "/node-install/"+path, nil)) + return w.Code, w.Body.String() + } + for _, revision := range []string{a, b} { + write("active.json", `{"source_commit":"`+revision+`"}`) + if code, body := request("node-install.pyz"); code != 200 || body != "installer-"+revision { + t.Fatal("active release not served", code, body) + } + if code, body := request("releases/" + a + "/artifacts/node"); code != 200 || body != "binary-"+a { + t.Fatal("retained release unreachable", code, body) + } + for _, name := range []string{"active.json", "releases/" + a + "/artifacts/key", "releases/" + a + "/secrets/core.key", "releases/../../node-install.pyz"} { + if code, _ := request(name); code != 404 { + t.Fatal("non-public file exposed", name, code) + } + } + } + write("releases/"+a+"/manifest.json", `{"source_commit":"`+b+`"}`) + if code, _ := request("releases/" + a + "/node-install.pyz"); code != 404 { + t.Fatal("mismatched release identity was served") + } + write("active.json", `{"source_commit":"../../"}`) + if code, _ := request("node-install.pyz"); code != 404 { + t.Fatal("invalid pointer did not fail closed") + } +} diff --git a/services/core-console/node_installation.go b/services/core-console/node_installation.go index 89f704f09..464d7bbda 100644 --- a/services/core-console/node_installation.go +++ b/services/core-console/node_installation.go @@ -63,6 +63,9 @@ func (h *console) resolveNodePayload(name string) (string, bool) { return "", false } } + if prefix == "" && nodePayloadFiles[name] { + return name, true + } if !nodePayloadFiles[name] && (!strings.HasPrefix(name, "artifacts/") || strings.Contains(strings.TrimPrefix(name, "artifacts/"), "/")) { return "", false } From b605d770d6c3434082dd1660d25a964c6f44eef1 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 22:31:41 +0000 Subject: [PATCH 03/16] Preserve helper leases and node identity across program updates --- .../agents-api/node-generation-protocol.md | 110 +++++++++ deploy/install/node_generations.py | 35 ++- deploy/install/node_install.py | 7 + deploy/install/node_update.py | 229 ++++++++++++++++++ deploy/install/test_node_generations.py | 84 +++++++ deploy/install/test_node_update.py | 148 +++++++++++ docs/api/README.md | 4 +- scripts/build-core-distribution.sh | 2 +- scripts/core-distribution-manifest.py | 2 +- scripts/core-distribution-manifest.test.py | 4 +- .../cmd/sandbox-node/generations.go | 17 +- services/agents-api/cmd/sandbox-node/main.go | 4 + .../agents-api/cmd/sandbox-node/main_test.go | 10 + .../internal/sandbox/config/microsandbox.go | 5 +- .../internal/sandbox/microsandbox/process.go | 58 ++++- .../sandbox/microsandbox/process_test.go | 139 +++++++++++ .../internal/sandbox/node/generation_json.go | 141 +++++++++++ .../sandbox/node/generation_json_test.go | 63 +++++ .../internal/sandbox/node/generation_wire.go | 30 ++- .../internal/sandbox/node/generations.go | 3 + .../internal/sandbox/node/generations_test.go | 16 ++ .../agents-api/internal/sandbox/node/wire.go | 8 + .../tools/microsandbox-provider/main.go | 6 + 23 files changed, 1107 insertions(+), 18 deletions(-) create mode 100644 contracts/agents-api/node-generation-protocol.md create mode 100644 deploy/install/node_update.py create mode 100644 deploy/install/test_node_generations.py create mode 100644 deploy/install/test_node_update.py create mode 100644 services/agents-api/internal/sandbox/node/generation_json.go create mode 100644 services/agents-api/internal/sandbox/node/generation_json_test.go diff --git a/contracts/agents-api/node-generation-protocol.md b/contracts/agents-api/node-generation-protocol.md new file mode 100644 index 000000000..a3caeb02f --- /dev/null +++ b/contracts/agents-api/node-generation-protocol.md @@ -0,0 +1,110 @@ +# Node generation protocol + +This is the internal, authenticated Core-to-node provider protocol. It does not +change the pinned public Agent API. Version 1 retains its existing envelope; +version 2 carries exact deployment-generation routing and sparse local provider +observations. A version 1 connection never receives version 2 fields. + +## Bounded control + +A version 2 hello or heartbeat contains at most eight generation observations. +Each names a positive signed-64-bit generation, its lowercase SHA-256 specification +digest, a `ready`, `preparing` or `failed` state, and an optional fixed diagnostic. +The target and serving generation take priority; other records rotate fairly. +Eight bounds one message, not the number of generations a node may retain. +Omitted observations never authorize deletion or imply absence. + +Welcome and heartbeat acknowledgements carry the Core-owned target generation, +its specification digest, and an explicitly nullable durable serving generation. +Target preparation is independent of the readiness of a retained serving provider. +A provider request carries its exact immutable `deployment_generation` separately +from the allocation's compute generation. + +Retention exchanges use an independent bounded control path. A request identifies +at most eight local `(generation, specification_digest)` references, a UUID request +ID, a monotonically increasing sequence, the current connection UUID and owner +epoch. Its acknowledgement must match the complete pending request, including +entry order and identity, and must explicitly supply a boolean `keep` for every +entry. Only one pending exchange exists per connection. A disconnect discards it; +an unsolicited, replayed, stale, partial or mixed acknowledgement deletes nothing. + +Control envelopes are at most 32 KiB. Their member names are exact and unique; +unknown members, case aliases, duplicate members and unexpected nulls are rejected. +The nullable serving pin and host measurements preserve unknown values. Provider +request/response frames retain the existing global size bound; control traffic +does not increase that bound or consume the provider-operation queue. + +## Local retention and helper lifetime + +A Core drop grant is necessary but insufficient for collection. Queued and running +provider calls, preparation, the local target and serving pin retain references. +Collection rechecks those references and refuses an already canceled connection. +A canceled provider caller does not establish that its native helper has stopped: +the parent counts that helper through its actual `Wait` completion. + +Every generation also owns a permanent private lease file at +`state/node/generations/.lease`. Before starting a native helper, the +node acquires its shared flock and checks the generation's dropped tombstone under +that lock. The helper inherits the descriptor. The node closes its own descriptor +only after `Wait`; it never explicitly unlocks the shared open-file description. +Thus caller cancellation or node exit does not release a live helper's reference. +New native helpers set this descriptor close-on-exec before calling the SDK so VM +and daemon descendants do not inherit a helper reference. + +Collection acquires the exclusive nonblocking flock before inspecting references, +removing shared images or release files, or publishing the dropped tombstone. It +keeps the lock through those changes. Lock files belong to stable node state and +are never removed or atomically replaced during collection. Symlinks, multiply +linked files, foreign ownership, unsafe permissions and replaced lock paths are +refused. A dropped generation cannot be prepared or used again; a future rollback +would require a new generation and a separate policy. + +An immutable older native helper may pass its inherited descriptor to descendants. +That conservatively retains its generation's bytes until those descriptors close; +collection must not kill historical VMs or a host daemon merely to reclaim disk. +A helper's exit is local file-lifetime evidence, not proof that a remote mutation +or an uncertain provider receipt has been released. Core's durable allocation and +placement retention requirements remain independent. + +## Identity-preserving program updates + +Run the checksum-verified `node-install.pyz --update` as the original installation +owner, with the same installation ID and Core URL and the selected release's +`--source-url` or `--bundle`. Do not supply a new enrollment token. The installer +first authenticates the retained node through a read-only configuration request. +It verifies and stages the new node executable and matching private preparer, +checks the executable's `protocol-version`, and persists a resume journal before +stopping the fixed service unit. This capability check is not Runtime readiness. + +The journal pins the staged file hashes. Rerunning an interrupted update resumes +those files even if the console has since published another release. Atomic file +replacement changes the node program and preparer; the enrolled provider, +credential, Runtime identity, resource files and retained generation state remain. +The previous executable is preserved separately from Runtime releases. Core must +confirm connection and provider readiness before the journal becomes complete. +An unavailable Core or failed restart leaves recoverable files and journal; rerun +the same command after resolving the reported error. + +The existing units use `KillMode=process`: update stops and restarts their main +node process without terminating native VMs or helper processes. System-mode +updates reuse the existing recorded service account; all user-owned files and +programs are handled after dropping privileges. Root controls only the fixed +root-owned service unit and host installation lock. + +A v1-to-v2 update cannot retrospectively add a file lock to an already-running v1 +helper. Before stopping v1, it durably marks precisely the original enrolled +generation `legacy-unfenced`. Automatic collection retains that generation's local +payload and store across interruptions and node restarts. Unknown original +generation identity refuses the update. Neither an empty allocation list nor PID +absence clears this marker. It is a single-generation disk-retention limitation, +not a Core resource, serving-readiness or admission claim. Fresh v2 installations +and subsequent generations use normal flock-based collection. No historical VM +or daemon is killed to clear the marker. + +## Qualification boundary + +Protocol and process tests do not qualify Runtime readiness, VM coexistence, +native image/store locking, or identity-preserving updates. Those require the +separate exact-artifact KVM and Docker acceptance matrix. In particular, immutable +legacy helper descendant FD behavior must be recorded against its actual artifact; +absence of local test coverage is not evidence of successful garbage collection. diff --git a/deploy/install/node_generations.py b/deploy/install/node_generations.py index 4ddd06e12..7d5a14088 100644 --- a/deploy/install/node_generations.py +++ b/deploy/install/node_generations.py @@ -1,5 +1,8 @@ """Private node generation preparation and collection; Core owns all authorization.""" import copy +import contextlib +import fcntl +import stat import hashlib import json import os @@ -9,7 +12,6 @@ import subprocess import sys import tempfile -from types import SimpleNamespace import zipapp @@ -51,7 +53,7 @@ def install_helper(root, args, installer): package = Path(directory) / "package" package.mkdir(mode=0o700) source_dir = Path(installer.__file__).parent - for name in ("node_install.py", "node_spec.py", "distribution.py", "node_generations.py"): + for name in ("node_install.py", "node_spec.py", "distribution.py", "node_generations.py", "node_update.py"): shutil.copyfile(source_dir / name, package / ("__main__.py" if name == "node_install.py" else name)) zipapp.create_archive(package, staged, compressed=True) os.chmod(staged, 0o600) @@ -191,15 +193,42 @@ def prepare(args, installer): print(json.dumps(value)) +@contextlib.contextmanager +def collection_lease(root, generation, installer): + directory = root / "state/node/generations" + installer.safe_directory(directory) + path = directory / (str(generation) + ".lease") + descriptor = os.open(path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600) + try: + info = os.fstat(descriptor) + if (not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600 + or info.st_uid != os.geteuid() or info.st_nlink != 1): + raise installer.InstallError("Invalid generation helper lease") + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as error: + raise installer.InstallError("Generation helper is still active") from error + named = path.lstat() + if (named.st_dev, named.st_ino) != (info.st_dev, info.st_ino): + raise installer.InstallError("Generation helper lease was replaced") + yield + finally: + # Never unlink a lease: a replacement inode could bypass an old helper. + os.close(descriptor) + + def collect(args, installer): root, _ = owned_root(args, installer) - with installer.install_lock(root): + with installer.install_lock(root), collection_lease(root, args.generation, installer): configurations = retained_configs(root, installer) value = configurations.get(args.generation) if value is None: return if installer.node_spec.digest(value["provider"], value["specification"]) != args.specification_digest: raise installer.InstallError("Collection grant does not match the local generation") + marker = root / "state/node/generations" / (str(args.generation) + ".legacy-unfenced") + if marker.exists() or marker.is_symlink(): + raise installer.InstallError("The original v1 generation retains unfenced legacy helpers; its local payload is kept") others = [item for generation, item in configurations.items() if generation != args.generation] if value["provider"] == "microsandbox": micro = value["microsandbox"] diff --git a/deploy/install/node_install.py b/deploy/install/node_install.py index a1a8d5532..f886bb6fc 100644 --- a/deploy/install/node_install.py +++ b/deploy/install/node_install.py @@ -35,6 +35,7 @@ import distribution import node_spec import node_generations +import node_update class InstallError(Exception): @@ -1430,6 +1431,7 @@ def main(argv=None): parser.add_argument("--generation-action", choices=("prepare", "collect"), help=argparse.SUPPRESS) parser.add_argument("--generation", type=int, help=argparse.SUPPRESS) parser.add_argument("--specification-digest", help=argparse.SUPPRESS) + parser.add_argument("--update", action="store_true", help="Update this node program while preserving its identity and Runtime state") parser.add_argument("--uninstall", action="store_true", help="Remove this host's node after it was removed on the Nodes page") parser.add_argument("--force", action="store_true", help="With --uninstall: skip the Core check, for a Core that no longer exists") args = parser.parse_args(argv) @@ -1438,6 +1440,8 @@ def main(argv=None): "--enrollment-token-stdin.\n") if str(uuid.UUID(args.installation_id)) != args.installation_id: raise InstallError("Installation ID must be a canonical UUID") + if args.update and (args.uninstall or args.force or args.enrollment_token_stdin or args.generation_action): + parser.error("--update cannot enroll, uninstall, force removal, or execute a generation action") if args.generation_action: if args.generation is None or not 1 <= args.generation <= 9223372036854775807 or not re.fullmatch(r"[0-9a-f]{64}", args.specification_digest or ""): parser.error("Invalid generation authorization") @@ -1454,6 +1458,9 @@ def main(argv=None): parser.error("--source-url (or --bundle) and --core-url are required") if args.bundle is not None and (not args.bundle.is_absolute() or args.bundle.resolve() != args.bundle): raise InstallError("Local bundle must be an absolute directory without symlinks") + if args.update: + node_update.update(args, sys.modules[__name__]) + return token = read_token(args) if len(token) > 4096 or any(c.isspace() for c in token): raise InstallError("A valid one-time enrollment credential is required") diff --git a/deploy/install/node_update.py b/deploy/install/node_update.py new file mode 100644 index 000000000..06aef9af5 --- /dev/null +++ b/deploy/install/node_update.py @@ -0,0 +1,229 @@ +"""Resume a node program update without re-enrolling or replacing its Runtime.""" +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile + +import node_generations + + +def retained(root, args, installer): + installer.no_links(root) + if not root.is_dir() or not installer.existing_file(root / "registered.json"): + raise installer.InstallError("--update requires this user's registered node; use the original installation owner") + installer.existing_file(root / "state/node/identity.json") + installer.existing_file(root / "provider.json") + identity = installer.private_json(root / "state/node/identity.json") + provider = installer.private_json(root / "provider.json") + if not identity or not provider: + raise installer.InstallError("Retained node identity or provider is unreadable; preserve state") + value = identity["identity"] + generation = value.get("deployment_generation") + if (type(generation) is not int or not 1 <= generation <= 9223372036854775807 + or identity.get("core_url") != args.core_url or value.get("installation_id") != args.installation_id + or provider.get("installation_id") != args.installation_id or provider.get("generation") != generation + or provider.get("provider") != value.get("provider") + or installer.node_spec.digest(provider["provider"], provider["specification"]) != value.get("specification_digest") + or args.provider not in (None, value["provider"])): + raise installer.InstallError("Cannot identify the original enrolled generation; preserve this node") + args.provider = value["provider"] + # Refresh only authentication and backend lineage; the target may have moved. + installer.node_spec.fetch(args, "", identity, installer.open_request, allow_selection_change=True) + return identity, provider + + +def protocol(binary, installer): + result = subprocess.run([str(binary), "protocol-version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=10, check=False) + if result.returncode == 0 and result.stdout == b"2\n" and not result.stderr: + return 2 + if (result.returncode == 1 and not result.stdout + and result.stderr.strip() == b"usage: oac-node register|run --config PATH --state-dir PATH"): + return 1 + raise installer.InstallError("Cannot establish the node executable protocol; preserve state and inspect its release") + + +def atomic_copy(source, target, mode): + descriptor, temporary = tempfile.mkstemp(prefix=".node-update-", dir=target.parent) + try: + with os.fdopen(descriptor, "wb") as stream, source.open("rb") as original: + shutil.copyfileobj(original, stream) + os.fchmod(stream.fileno(), mode) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, target) + descriptor = os.open(target.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + +def journal(root, installer): + value = installer.private_json(root / "program-update.json") + if value is None: + if (root / "program-update.json").exists(): + raise installer.InstallError("Node update journal is unreadable; preserve it") + return None + if (value.get("format") != 1 or value.get("phase") not in ("prepared", "installed", "complete") + or any(not re.fullmatch(r"[0-9a-f]{64}", value.get(key, "")) + for key in ("node_sha256", "preparer_sha256", "settings_sha256"))): + raise installer.InstallError("Node update journal differs; preserve it") + return value + + +def verify_stage(root, value, installer): + stage = root / "node-programs" / value["node_sha256"] + installer.no_links(stage) + for name, key in ((installer.COMMON[0], "node_sha256"), ("generation-preparer.pyz", "preparer_sha256"), + ("preparation.json", "settings_sha256")): + path = stage / name + installer.no_links(path) + if not installer.existing_file(path) or installer.file_digest(path) != value[key]: + raise installer.InstallError("Retained update payload differs; preserve the journal and retry with its release") + return stage + + +def prepare(root, args, installer): + with installer.install_lock(root): + identity, provider = retained(root, args, installer) + value = journal(root, installer) + node_id = identity["identity"]["node_id"] + generation = provider["generation"] + if value and value["phase"] != "complete": + if value.get("node_id") != node_id or value.get("enrolled_generation") != generation: + raise installer.InstallError("Pending update belongs to another retained identity") + verify_stage(root, value, installer) + retain_legacy(root, value, provider, installer) + return + binary = root / installer.COMMON[0] + installer.existing_file(binary) + old_protocol = protocol(binary, installer) + manifest, sums = installer.metadata(args.source_url, args.bundle) + checksum = installer.distribution.artifact(manifest, installer.COMMON[0])["sha256"] + programs = root / "node-programs" + installer.safe_directory(programs) + stage = programs / checksum + installer.safe_directory(stage) + installer.safe_directory((stage / installer.COMMON[0]).parent) + installer.distribution.obtain_artifact(manifest, installer.COMMON[0], stage / installer.COMMON[0], args.bundle) + os.chmod(stage / installer.COMMON[0], 0o700) + if protocol(stage / installer.COMMON[0], installer) != 2: + raise installer.InstallError("This release does not support node generation updates") + node_generations.install_helper(stage, args, installer) + if installer.file_digest(stage / "generation-preparer.pyz") != sums.get("node-install.pyz"): + raise installer.InstallError("Use the checksum-matched node-install.pyz from the selected release") + # Preserve the previous executable independently of every Runtime release. + old_sha = installer.file_digest(binary) + previous = programs / old_sha + installer.safe_directory(previous) + saved = previous / "previous-node" + if not installer.existing_file(saved): + atomic_copy(binary, saved, 0o700) + elif installer.file_digest(saved) != old_sha: + raise installer.InstallError("Preserved node executable differs") + value = {"format": 1, "phase": "prepared", "node_id": node_id, "enrolled_generation": generation, + "legacy_unfenced_generation": generation if old_protocol == 1 else None, + "node_sha256": checksum, "preparer_sha256": installer.file_digest(stage / "generation-preparer.pyz"), + "settings_sha256": installer.file_digest(stage / "preparation.json")} + verify_stage(root, value, installer) + # The durable journal must never refer to unwritten staged bytes or + # directory entries after a power interruption. + paths = [stage / installer.COMMON[0], stage / "generation-preparer.pyz", stage / "preparation.json", saved] + directories = {root, programs, stage, previous, (stage / installer.COMMON[0]).parent, (stage / installer.COMMON[0]).parent.parent} + for path in paths + sorted(directories, key=lambda entry: len(entry.parts), reverse=True): + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + node_generations.atomic_json(root / "program-update.json", value) + retain_legacy(root, value, provider, installer) + + +def retain_legacy(root, value, provider, installer): + generation = value.get("legacy_unfenced_generation") + if generation is None: + return + if type(generation) is not int or generation != provider["generation"]: + raise installer.InstallError("Legacy helper retention differs from the enrolled generation") + directory = root / "state/node/generations" + installer.safe_directory(directory) + path = directory / (str(generation) + ".legacy-unfenced") + expected = {"specification_digest": installer.node_spec.digest(provider["provider"], provider["specification"])} + if path.exists() and installer.private_json(path) != expected: + raise installer.InstallError("Legacy helper retention marker differs") + node_generations.atomic_json(path, expected) + descriptor = os.open(directory.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def apply(root, args, installer): + with installer.install_lock(root): + identity, provider = retained(root, args, installer) + value = journal(root, installer) + if not value or value.get("node_id") != identity["identity"]["node_id"] or value.get("enrolled_generation") != provider["generation"]: + raise installer.InstallError("No matching prepared node update") + stage = verify_stage(root, value, installer) + retain_legacy(root, value, provider, installer) + for name, mode in ((installer.COMMON[0], 0o700), ("generation-preparer.pyz", 0o600), ("preparation.json", 0o600)): + installer.existing_file(root / name) + atomic_copy(stage / name, root / name, mode) + node_generations.atomic_json(root / "program-update.json", dict(value, phase="installed")) + + +def finish(root, args, installer): + installer.wait_ready(root, args) + with installer.install_lock(root): + value = journal(root, installer) + if not value or value["phase"] != "installed": + raise installer.InstallError("Node update journal changed before readiness confirmation") + node_generations.atomic_json(root / "program-update.json", dict(value, phase="complete")) + + +def update(args, installer): + if os.geteuid() == 0: + os.environ["PATH"] = installer.SAFE_PATH + installer.host_checks() + record = installer.node_record(args.installation_id) + account, account_record = installer.account_plan() + if not record or not account or not account_record or record["core_url"] != args.core_url: + raise installer.InstallError("--update requires the existing registered system node and service account") + if args.provider not in (None, record["provider"]): + raise installer.InstallError("The asserted provider differs from the retained node") + args.provider, args.system = record["provider"], True + root = installer.SERVICE_HOME / ".oac/nodes" / args.installation_id + unit = installer.SYSTEM_UNITS / installer.unit_name(args.installation_id) + if unit.is_symlink() or unit.read_text() != installer.system_unit(root, args.provider): + raise installer.InstallError("The existing system node unit differs; preserve it") + with installer.host_lock(): + current = installer.service_account() + if installer.node_record(args.installation_id) != record or current != account: + raise installer.InstallError("The system node changed meanwhile; rerun --update") + if unit.is_symlink() or unit.read_text() != installer.system_unit(root, args.provider): + raise installer.InstallError("The system node unit changed meanwhile") + installer.run_as(account, prepare, root, args, installer) + installer.checked(["systemctl", "stop", unit.name], "Cannot stop the node main process") + installer.run_as(account, apply, root, args, installer) + installer.checked(["systemctl", "start", unit.name], "Cannot restart the retained node; rerun --update") + installer.run_as(account, finish, root, args, installer) + else: + root = Path.home() / ".oac/nodes" / args.installation_id + unit = root / installer.unit_name(args.installation_id) + if not installer.existing_file(unit) or unit.read_text() != installer.service_unit(root): + raise installer.InstallError("The existing user node unit differs; preserve it") + with installer.host_lock(): + prepare(root, args, installer) + installer.checked(["systemctl", "--user", "stop", unit.name], "Cannot stop the node main process") + apply(root, args, installer) + installer.checked(["systemctl", "--user", "start", unit.name], "Cannot restart the retained node; rerun --update") + finish(root, args, installer) + print("Node program updated with retained identity and Runtime state. Inspect the Nodes page for generation readiness.") diff --git a/deploy/install/test_node_generations.py b/deploy/install/test_node_generations.py new file mode 100644 index 000000000..682f7e8f0 --- /dev/null +++ b/deploy/install/test_node_generations.py @@ -0,0 +1,84 @@ +"""Collection retains bytes until local helper ownership settles.""" +import fcntl +import os +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest import mock + +import node_generations +import node_install as installer +import node_spec + + +class CollectionTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.directory = self.root / "state/node/generations" + self.directory.mkdir(parents=True, mode=0o700) + self.value = {"generation": 1, "provider": "docker", "docker": {"image": "sha256:" + "a" * 64}, + "specification": {"resources": {"cpus": 1, "memory_mib": 1024}, "runtime": {"source_commit": "b" * 40, "image_id": "sha256:" + "a" * 64, "image_manifest_digest": "sha256:" + "c" * 64, "microsandbox_ref": "oac-runtime@sha256:" + "d" * 64, "runtime_sha256": "e" * 64, "firmware_sha256": "f" * 64}}} + self.args = SimpleNamespace(generation=1, specification_digest=node_spec.digest("docker", self.value["specification"])) + self.release = self.root / "releases" / ("b" * 40) + self.release.mkdir(parents=True) + (self.release / "artifact").write_bytes(b"immutable bytes") + node_generations.atomic_json(self.root / "provider.json", self.value) + for patch in (mock.patch.object(node_generations, "owned_root", return_value=(self.root, {})), + mock.patch.object(installer, "checked", return_value="")): + patch.start() + self.addCleanup(patch.stop) + + def test_busy_helper_refuses_all_mutations_then_same_inode_collects(self): + lease = self.directory / "1.lease" + descriptor = os.open(lease, os.O_CREAT | os.O_RDWR, 0o600) + self.addCleanup(os.close, descriptor) + original = os.fstat(descriptor) + fcntl.flock(descriptor, fcntl.LOCK_SH) + with self.assertRaisesRegex(installer.InstallError, "helper is still active"): + node_generations.collect(self.args, installer) + installer.checked.assert_not_called() + self.assertEqual((self.release / "artifact").read_bytes(), b"immutable bytes") + self.assertFalse((self.directory / "1.dropped").exists()) + fcntl.flock(descriptor, fcntl.LOCK_UN) + node_generations.collect(self.args, installer) + self.assertFalse(self.release.exists()) + self.assertTrue((self.directory / "1.dropped").exists()) + node_generations.collect(self.args, installer) + installer.checked.assert_called_once() + final = lease.stat() + self.assertEqual((original.st_dev, original.st_ino), (final.st_dev, final.st_ino)) + + def test_collection_holds_exclusive_lease_during_deletion(self): + lease = self.directory / "1.lease" + def remove(*_args, **_kwargs): + descriptor = os.open(lease, os.O_RDWR) + try: + with self.assertRaises(BlockingIOError): + fcntl.flock(descriptor, fcntl.LOCK_SH | fcntl.LOCK_NB) + finally: + os.close(descriptor) + return "" + installer.checked.side_effect = remove + node_generations.collect(self.args, installer) + + def test_symlink_and_shared_inode_are_not_collection_authority(self): + lease = self.directory / "1.lease" + foreign = self.root / "foreign" + foreign.write_bytes(b"") + foreign.chmod(0o600) + for link in (os.symlink, os.link): + link(foreign, lease) + try: + with self.assertRaises((OSError, installer.InstallError)): + node_generations.collect(self.args, installer) + installer.checked.assert_not_called() + self.assertTrue((self.release / "artifact").exists()) + finally: + lease.unlink() + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/install/test_node_update.py b/deploy/install/test_node_update.py new file mode 100644 index 000000000..f96501940 --- /dev/null +++ b/deploy/install/test_node_update.py @@ -0,0 +1,148 @@ +"""Program updates preserve enrolled state and resume their pinned payload.""" +import contextlib +import hashlib +import json +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest import mock +import node_generations +import node_install as installer +import node_spec +import node_update + + +class UpdateTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.home = Path(temporary.name) + self.args = SimpleNamespace(installation_id="94be54a1-138c-4f30-bc87-b13686272dbe", core_url="https://core.example", source_url="https://console.example", bundle=None, provider=None) + self.root = self.home / ".oac/nodes" / self.args.installation_id + (self.root / "state/node").mkdir(parents=True, mode=0o700) + (self.root / installer.COMMON[0]).parent.mkdir(parents=True, mode=0o700) + self.old = b'#!/bin/sh\necho "usage: oac-node register|run --config PATH --state-dir PATH" >&2\nexit 1\n' + self.new = b'#!/bin/sh\necho 2\n' + binary = self.root / installer.COMMON[0] + binary.write_bytes(self.old) + binary.chmod(0o700) + runtime = {"source_commit": "b" * 40, "image_id": "sha256:" + "a" * 64, "image_manifest_digest": "sha256:" + "c" * 64, "microsandbox_ref": "oac-runtime@sha256:" + "d" * 64, "runtime_sha256": "e" * 64, "firmware_sha256": "f" * 64} + spec = {"runtime": runtime, "resources": {"cpus": 1, "memory_mib": 1024}} + self.provider = {"generation": 1, "provider": "docker", "installation_id": self.args.installation_id, "specification": spec, "docker": {"image": runtime["image_id"]}} + self.identity = {"core_url": self.args.core_url, "credential": "a" * 64, "identity": {"installation_id": self.args.installation_id, "provider": "docker", "node_id": "7073346d-3c83-4361-9354-2709a9cf17ed", "deployment_generation": 1, "specification_digest": node_spec.digest("docker", spec)}} + for name, value in (("provider.json", self.provider), ("state/node/identity.json", self.identity), ("registered.json", {})): + node_generations.atomic_json(self.root / name, value) + unit = self.root / installer.unit_name(self.args.installation_id) + unit.write_text(installer.service_unit(self.root)) + unit.chmod(0o600) + self.preparer = b"verified fixture preparer" + checksum = hashlib.sha256(self.new).hexdigest() + self.manifest = {"source_commit": "c" * 40, "artifacts": {installer.COMMON[0]: {"filename": "node-" + "c" * 40, "size": len(self.new), "sha256": checksum}}} + self.sums = {"node-install.pyz": hashlib.sha256(self.preparer).hexdigest()} + self.original = {name: (self.root / name).read_bytes() for name in ("provider.json", "registered.json", "state/node/identity.json")} + def obtain(_manifest, _name, target, _bundle): + target.write_bytes(self.new) + def helper(stage, args, _installer): + (stage / "generation-preparer.pyz").write_bytes(self.preparer) + (stage / "generation-preparer.pyz").chmod(0o600) + node_generations.atomic_json(stage / "preparation.json", {"source_url": args.source_url}) + for patch in (mock.patch.object(Path, "home", return_value=self.home), mock.patch.object(installer.node_spec, "fetch", return_value={"generation": 9}), mock.patch.object(installer, "metadata", return_value=(self.manifest, self.sums)), mock.patch.object(installer.distribution, "obtain_artifact", side_effect=obtain), mock.patch.object(node_generations, "install_helper", side_effect=helper), mock.patch.object(installer, "wait_ready"), mock.patch.object(installer, "checked", return_value="")): + patch.start() + self.addCleanup(patch.stop) + + def assert_original_state(self): + for name, raw in self.original.items(): + self.assertEqual((self.root / name).read_bytes(), raw, name) + + def test_interruption_after_stop_keeps_pinned_payload_and_legacy_generation(self): + marker = self.root / "state/node/generations/1.legacy-unfenced" + def interrupted(arguments, *_args, **_kwargs): + if "stop" in arguments: + self.assertEqual(json.loads(marker.read_text())["specification_digest"], self.identity["identity"]["specification_digest"]) + self.assertEqual(node_update.journal(self.root, installer)["phase"], "prepared") + raise installer.InstallError("interrupted after stop") + installer.checked.side_effect = interrupted + with self.assertRaisesRegex(installer.InstallError, "interrupted"): + node_update.update(self.args, installer) + self.assert_original_state() + self.assertEqual((self.root / installer.COMMON[0]).read_bytes(), self.old) + installer.metadata.side_effect = AssertionError("resume re-fetched current release") + installer.checked.side_effect = None + node_update.update(self.args, installer) + self.assertEqual((self.root / installer.COMMON[0]).read_bytes(), self.new) + self.assertEqual(node_update.journal(self.root, installer)["phase"], "complete") + self.assert_original_state() + old_sha = hashlib.sha256(self.old).hexdigest() + self.assertEqual((self.root / "node-programs" / old_sha / "previous-node").read_bytes(), self.old) + installer.node_spec.fetch.assert_called_with(self.args, "", self.identity, installer.open_request, allow_selection_change=True) + with mock.patch.object(node_generations, "owned_root", return_value=(self.root, self.identity)): + installer.checked.reset_mock() + grant = SimpleNamespace(generation=1, specification_digest=self.identity["identity"]["specification_digest"]) + with self.assertRaisesRegex(installer.InstallError, "unfenced legacy helpers"): + node_generations.collect(grant, installer) + installer.checked.assert_not_called() + later = json.loads(json.dumps(self.provider)) + later["generation"] = 2 + later["docker"]["image"] = "sha256:" + "9" * 64 + node_generations.atomic_json(self.root / "state/node/generations/2.json", later) + grant.generation = 2 + node_generations.collect(grant, installer) + self.assertTrue((self.root / "state/node/generations/2.dropped").exists()) + self.assertTrue(marker.exists()) + + def test_v2_update_does_not_invent_legacy_retention(self): + (self.root / installer.COMMON[0]).write_bytes(self.new) + node_update.update(self.args, installer) + self.assertFalse((self.root / "state/node/generations/1.legacy-unfenced").exists()) + self.assert_original_state() + + def test_unknown_original_generation_refuses_before_staging_or_stop(self): + self.identity["identity"]["deployment_generation"] = 0 + node_generations.atomic_json(self.root / "state/node/identity.json", self.identity) + with self.assertRaisesRegex(installer.InstallError, "original enrolled generation"): + node_update.update(self.args, installer) + installer.metadata.assert_not_called() + installer.checked.assert_not_called() + self.assertFalse((self.root / "program-update.json").exists()) + + def test_corrupt_preparer_never_stops_old_service(self): + self.sums["node-install.pyz"] = "0" * 64 + with self.assertRaisesRegex(installer.InstallError, "checksum-matched"): + node_update.update(self.args, installer) + installer.checked.assert_not_called() + self.assert_original_state() + + def test_partial_atomic_install_reuses_verified_stage(self): + node_update.prepare(self.root, self.args, installer) + original = node_update.atomic_copy + def interrupted(source, target, mode): + if target.name == "generation-preparer.pyz": + raise installer.InstallError("interrupted between files") + original(source, target, mode) + with mock.patch.object(node_update, "atomic_copy", side_effect=interrupted): + with self.assertRaisesRegex(installer.InstallError, "between files"): + node_update.apply(self.root, self.args, installer) + installer.metadata.side_effect = AssertionError("resume re-fetched release") + node_update.prepare(self.root, self.args, installer) + node_update.apply(self.root, self.args, installer) + self.assertEqual((self.root / "generation-preparer.pyz").read_bytes(), self.preparer) + self.assert_original_state() + + def test_root_runs_only_fixed_service_operations_and_drops_privileges_for_files(self): + account = SimpleNamespace(pw_uid=12345, pw_gid=12345, pw_dir=str(self.home)) + record = {"provider": "docker", "core_url": self.args.core_url} + units = self.home / "system-units" + units.mkdir() + unit = units / installer.unit_name(self.args.installation_id) + unit.write_text(installer.system_unit(self.root, "docker")) + with mock.patch.object(node_update.os, "geteuid", return_value=0), mock.patch.object(installer, "host_checks"), mock.patch.object(installer, "node_record", return_value=record), mock.patch.object(installer, "account_plan", return_value=(account, {"uid":12345})), mock.patch.object(installer, "service_account", return_value=account), mock.patch.object(installer, "SERVICE_HOME", self.home), mock.patch.object(installer, "SYSTEM_UNITS", units), mock.patch.object(installer, "host_lock", side_effect=contextlib.nullcontext), mock.patch.object(installer, "run_as") as run_as, mock.patch.object(installer, "prepare_account", side_effect=AssertionError("update recreated account")): + node_update.update(self.args, installer) + self.assertEqual([call.args[:2] for call in run_as.call_args_list], [(account, node_update.prepare), (account, node_update.apply), (account, node_update.finish)]) + self.assertEqual([call.args[0] for call in installer.checked.call_args_list], [["systemctl", "stop", unit.name], ["systemctl", "start", unit.name]]) + installer.metadata.assert_not_called() + self.assert_original_state() + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/api/README.md b/docs/api/README.md index e9e128db9..0059c859a 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -81,8 +81,8 @@ the Core key or a Project API key. - [Core OpenAPI](../../contracts/agents-api/core.openapi.yaml): generated `/core/v1` routes, all authenticated by the Core key. - [Machine connection OpenAPI](../../contracts/agents-api/runtime.openapi.yaml): - generated `/api/v1` sandbox node routes. The node WebSocket and the private - daemon transport are described in the node and Runtime credential guides. + generated `/api/v1` sandbox node routes. The node WebSocket is described in the [generation protocol](../../contracts/agents-api/node-generation-protocol.md); the private + daemon transport is described in the Runtime credential guide. - [Administrator contract](../../contracts/agents-api/admin-api.md): Project/key lifecycle, resources, explicit hosted Session archive, summary, errors/deletion preconditions, audit and historical copy provenance. diff --git a/scripts/build-core-distribution.sh b/scripts/build-core-distribution.sh index 5cb6c0a0f..ae8907cdf 100755 --- a/scripts/build-core-distribution.sh +++ b/scripts/build-core-distribution.sh @@ -99,7 +99,7 @@ if [[ "$(go env GOVERSION)" != "$required_go" ]]; then exit 1 fi for file in install.sh install.py configuration.py config_model.py config.schema.json oac_cli.py convert.py rename.py \ - native_service.py node_install.py node_spec.py node_generations.py sandbox_setup.py distribution.py self_hosted_install.py \ + native_service.py node_install.py node_spec.py node_generations.py node_update.py sandbox_setup.py distribution.py self_hosted_install.py \ model_provider_sessions.py; do cp "deploy/install/$file" "$bundle/$file" done diff --git a/scripts/core-distribution-manifest.py b/scripts/core-distribution-manifest.py index 560f5fa33..7c37d8a6e 100644 --- a/scripts/core-distribution-manifest.py +++ b/scripts/core-distribution-manifest.py @@ -253,7 +253,7 @@ def bootstraps(bundle, epoch): shutil.copyfile(bundle / original, target) os.utime(target, (int(epoch), int(epoch))) if source == "node_install.py": - for name in ("node_spec.py", "node_generations.py"): + for name in ("node_spec.py", "node_generations.py", "node_update.py"): target = pathlib.Path(directory) / name shutil.copyfile(bundle / name, target) os.utime(target, (int(epoch), int(epoch))) diff --git a/scripts/core-distribution-manifest.test.py b/scripts/core-distribution-manifest.test.py index 797e94c5d..961c9f652 100644 --- a/scripts/core-distribution-manifest.test.py +++ b/scripts/core-distribution-manifest.test.py @@ -232,7 +232,7 @@ def test_release_location_and_explicit_offline_requirements(self): self.assertEqual(json.loads((self.bundle / "manifest.json").read_text())["artifact_base_url"], "") def test_bootstraps_include_shared_downloader_and_are_reproducible(self): - for name in ("node_install.py", "node_generations.py", "self_hosted_install.py", *distribution.OAC_CLI_MODULES): + for name in ("node_install.py", "node_generations.py", "node_update.py", "self_hosted_install.py", *distribution.OAC_CLI_MODULES): (self.bundle / name).write_text("# " + name + "\n") distribution.bootstraps(self.bundle, "1700000000") first = [(self.bundle / name).read_bytes() for name in ("node-install.pyz", "oac.pyz")] @@ -244,7 +244,7 @@ def test_bootstraps_include_shared_downloader_and_are_reproducible(self): for script, filename in (("node_install.py", "node-install.pyz"), ("self_hosted_install.py", "self-hosted-install.pyz")): with zipfile.ZipFile(self.bundle / filename) as contents: - self.assertEqual(set(contents.namelist()), {"__main__.py", "distribution.py"} | ({"node_spec.py", "node_generations.py"} if script == "node_install.py" else set())) + self.assertEqual(set(contents.namelist()), {"__main__.py", "distribution.py"} | ({"node_spec.py", "node_generations.py", "node_update.py"} if script == "node_install.py" else set())) self.assertEqual(contents.read("__main__.py"), (self.bundle / script).read_bytes()) diff --git a/services/agents-api/cmd/sandbox-node/generations.go b/services/agents-api/cmd/sandbox-node/generations.go index 9111fd0ea..1a1f373c8 100644 --- a/services/agents-api/cmd/sandbox-node/generations.go +++ b/services/agents-api/cmd/sandbox-node/generations.go @@ -71,7 +71,7 @@ func runGenerations(ctx context.Context, configFile, stateDir string) error { if _, ok := values[config.Generation]; ok { continue } - value, err := buildGeneration(config) + value, err := buildGeneration(config, stateDir) if err != nil { closeValues() return err @@ -108,7 +108,7 @@ func runGenerations(ctx context.Context, configFile, stateDir string) error { if config.InstallationID != base.InstallationID || config.Provider != base.Provider || config.Generation != generation || config.Specification.Digest(config.Provider) != digest { return node.GenerationProvider{}, sandbox.ErrOwnership } - value, err := buildGeneration(config) + value, err := buildGeneration(config, stateDir) if err != nil { return value, err } @@ -130,7 +130,18 @@ func runGenerations(ctx context.Context, configFile, stateDir string) error { return node.Run(ctx, node.AgentConfig{CoreURL: stored.CoreURL, StateDirectory: stateDir, Identity: stored.Identity, Credential: stored.Credential, Generations: manager}) } -func buildGeneration(config providerconfig.Config) (node.GenerationProvider, error) { +func buildGeneration(config providerconfig.Config, stateDir string) (node.GenerationProvider, error) { + if config.Microsandbox != nil { + directory := filepath.Join(stateDir, "generations") + if err := os.MkdirAll(directory, 0700); err != nil { + return node.GenerationProvider{}, err + } + info, err := os.Lstat(directory) + if err != nil || !info.IsDir() || info.Mode().Perm() != 0700 { + return node.GenerationProvider{}, sandbox.ErrOwnership + } + config.Microsandbox.HelperLeasePath = filepath.Join(directory, strconv.FormatUint(config.Generation, 10)+".lease") + } built, closeProvider, err := providerconfig.Build(config) if err != nil { return node.GenerationProvider{}, err diff --git a/services/agents-api/cmd/sandbox-node/main.go b/services/agents-api/cmd/sandbox-node/main.go index 1cb80b83e..d40842849 100644 --- a/services/agents-api/cmd/sandbox-node/main.go +++ b/services/agents-api/cmd/sandbox-node/main.go @@ -47,6 +47,10 @@ func run(ctx context.Context, args []string) error { return errors.New("OpenAgentCore renamed these settings; set the new names and remove the old ones: " + strings.Join(renamed, ", ")) } + if len(args) == 1 && args[0] == "protocol-version" { + fmt.Println(node.GenerationProtocolVersion) + return nil + } if len(args) == 0 || (args[0] != "register" && args[0] != "run") { return errors.New("usage: oac-node register|run --config PATH --state-dir PATH") } diff --git a/services/agents-api/cmd/sandbox-node/main_test.go b/services/agents-api/cmd/sandbox-node/main_test.go index b308f6a09..5b93b471f 100644 --- a/services/agents-api/cmd/sandbox-node/main_test.go +++ b/services/agents-api/cmd/sandbox-node/main_test.go @@ -67,3 +67,13 @@ func TestNodeRejectsRetiredLoggingSettingsBeforeStartup(t *testing.T) { t.Fatalf("retired logging settings were ignored or exposed: %v", err) } } + +func TestProtocolVersionRequiresNoProviderOrIdentity(t *testing.T) { + // This is a binary capability check, not a provider readiness probe. + if err := run(t.Context(), []string{"protocol-version"}); err != nil { + t.Fatal(err) + } + if err := run(t.Context(), []string{"protocol-version", "--config", "/missing"}); err == nil { + t.Fatal("extra protocol capability arguments accepted") + } +} diff --git a/services/agents-api/internal/sandbox/config/microsandbox.go b/services/agents-api/internal/sandbox/config/microsandbox.go index ac5e5e333..dac0cb2c8 100644 --- a/services/agents-api/internal/sandbox/config/microsandbox.go +++ b/services/agents-api/internal/sandbox/config/microsandbox.go @@ -11,6 +11,7 @@ import ( // Single-host providers pin the helper, runtime, firmware, image and resource // limits explicitly. The helper owns local paths; no ambient backend is selected. type Microsandbox struct { + HelperLeasePath string `json:"-"` HelperPath string `json:"helper_path"` RuntimeHome string `json:"runtime_home"` RuntimePath string `json:"runtime_path"` @@ -47,11 +48,11 @@ func configureMicrosandbox(entry Microsandbox, resources sandbox.Resources, resu for _, rule := range entry.Network.Rules { network.Rules = append(network.Rules, sandboxmicro.NetworkRule{Action: rule.Action, Direction: rule.Direction, Destination: rule.Destination, Protocol: rule.Protocol, Port: rule.Port}) } - provider, err := sandboxmicro.New(sandboxmicro.Config{ + provider, err := sandboxmicro.NewWithCaller(sandboxmicro.Config{ InstallationID: result.InstallationID, HelperPath: entry.HelperPath, RuntimeHome: entry.RuntimeHome, RuntimePath: entry.RuntimePath, FirmwarePath: entry.FirmwarePath, RuntimeSHA256: entry.RuntimeSHA256, FirmwareSHA256: entry.FirmwareSHA256, Image: entry.Image, MemoryMiB: entry.MemoryMiB, CPUs: entry.CPUs, RootDiskMiB: entry.RootDiskMiB, EnvironmentDiskMiB: entry.EnvironmentDiskMiB, Network: network, - }) + }, &sandboxmicro.ProcessCaller{LeasePath: entry.HelperLeasePath}) if err != nil { return errors.New("invalid managed microsandbox provider configuration") } diff --git a/services/agents-api/internal/sandbox/microsandbox/process.go b/services/agents-api/internal/sandbox/microsandbox/process.go index be1eb1740..4f4558b12 100644 --- a/services/agents-api/internal/sandbox/microsandbox/process.go +++ b/services/agents-api/internal/sandbox/microsandbox/process.go @@ -10,12 +10,18 @@ import ( "os/exec" "strings" "sync/atomic" + "syscall" ) // ProcessCaller never kills a mutating helper on a Core response timeout. // The helper retains its allocation flock until the SDK mutation settles, // including after Core exits. Output is still drained and bounded by this waiter. -type ProcessCaller struct{ active atomic.Int64 } +type ProcessCaller struct { + active atomic.Int64 + // LeasePath belongs to the permanent node state namespace, not a release. + // It is never unlinked, including after the generation is collected. + LeasePath string +} func (p *ProcessCaller) Quiescent() bool { return p.active.Load() == 0 } @@ -24,19 +30,33 @@ func (p *ProcessCaller) Call(ctx context.Context, q Request) (Response, error) { if e != nil || len(data) > MaxRequestBytes { return Response{}, errors.New("invalid helper request") } + lease, err := p.acquireLease() + if err != nil { + return Response{}, errors.New("generation helper lease unavailable") + } + releaseLease := func() { + if lease != nil { + _ = lease.Close() + } + } cmd := exec.Command(q.Config.HelperPath) cmd.Stdin = bytes.NewReader(data) cmd.Env = HelperEnvironment(os.Environ(), q.Config) + if lease != nil { + cmd.ExtraFiles = []*os.File{lease} + cmd.Env = append(cmd.Env, "OAC_NODE_GENERATION_LEASE_FD=3") + } stdout := &limitBuffer{limit: MaxResponseBytes} stderr := &limitBuffer{limit: MaxOutputBytes} cmd.Stdout, cmd.Stderr = stdout, stderr p.active.Add(1) if e := cmd.Start(); e != nil { p.active.Add(-1) + releaseLease() return Response{}, errors.New("helper unavailable") } done := make(chan error, 1) - go func() { err := cmd.Wait(); p.active.Add(-1); done <- err }() + go func() { err := cmd.Wait(); releaseLease(); p.active.Add(-1); done <- err }() select { case <-ctx.Done(): return Response{}, ctx.Err() @@ -87,3 +107,37 @@ func (b *limitBuffer) Write(v []byte) (int, error) { _, _ = b.Buffer.Write(v) return n, nil // Drain excess output without unbounded retention or pipe deadlock. } + +// A shared open-file-description flock survives node exit through the inherited +// helper descriptor. Close only our descriptor; LOCK_UN would also unlock the +// helper's copy. Collection holds the exclusive lock before touching any bytes. +func (p *ProcessCaller) acquireLease() (*os.File, error) { + if p.LeasePath == "" { + return nil, nil + } + fd, err := syscall.Open(p.LeasePath, syscall.O_RDWR|syscall.O_CREAT|syscall.O_NOFOLLOW|syscall.O_CLOEXEC, 0600) + if err != nil { + return nil, err + } + file := os.NewFile(uintptr(fd), p.LeasePath) + fail := func(err error) (*os.File, error) { _ = file.Close(); return nil, err } + info, err := file.Stat() + if err != nil { + return fail(err) + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || !info.Mode().IsRegular() || info.Mode().Perm() != 0600 || stat.Uid != uint32(os.Geteuid()) || stat.Nlink != 1 { + return fail(errors.New("invalid generation lease")) + } + if err := syscall.Flock(fd, syscall.LOCK_SH|syscall.LOCK_NB); err != nil { + return fail(err) + } + named, err := os.Lstat(p.LeasePath) + if err != nil || !os.SameFile(info, named) { + return fail(errors.New("generation lease replaced")) + } + if _, err := os.Lstat(strings.TrimSuffix(p.LeasePath, ".lease") + ".dropped"); !os.IsNotExist(err) { + return fail(errors.New("generation collected or unavailable")) + } + return file, nil +} diff --git a/services/agents-api/internal/sandbox/microsandbox/process_test.go b/services/agents-api/internal/sandbox/microsandbox/process_test.go index 4da81b2ac..81c7dd43e 100644 --- a/services/agents-api/internal/sandbox/microsandbox/process_test.go +++ b/services/agents-api/internal/sandbox/microsandbox/process_test.go @@ -6,19 +6,49 @@ import ( "errors" "fmt" "os" + "os/exec" "path/filepath" "strings" + "syscall" "testing" "time" ) func TestMain(m *testing.M) { + if filepath.Base(os.Args[0]) == "microsandbox-test-agent" { + var q Request + if json.NewDecoder(os.Stdin).Decode(&q) != nil { + os.Exit(2) + } + _, _ = (&ProcessCaller{LeasePath: q.Config.RuntimeHome}).Call(context.Background(), q) + os.Exit(0) + } if filepath.Base(os.Args[0]) == "microsandbox-test-helper" { var q Request if json.NewDecoder(os.Stdin).Decode(&q) != nil { os.Exit(2) } switch q.Operation { + case "lease-held": + // Model the new native helper's entrypoint before any SDK subprocess. + if os.Getenv("OAC_NODE_GENERATION_LEASE_FD") != "3" { + os.Exit(4) + } + syscall.CloseOnExec(3) + if os.WriteFile(q.Config.RuntimePath, []byte("started"), 0600) != nil { + os.Exit(3) + } + until := time.Now().Add(15 * time.Second) + for { + if _, err := os.Stat(q.Config.RuntimePath + ".release"); err == nil { + break + } + if time.Now().After(until) { + os.Exit(5) + } + time.Sleep(time.Millisecond) + } + case "delayed": time.Sleep(150 * time.Millisecond) if os.WriteFile(q.Config.RuntimePath, []byte("settled"), 0600) != nil { @@ -80,3 +110,112 @@ func TestInvalidOrUnboundedHelperOutputIsUnconfirmed(t *testing.T) { }) } } + +func TestGenerationLeaseSurvivesNodeProcessExit(t *testing.T) { + c := processConfig(t) + c.RuntimeHome = filepath.Join(filepath.Dir(c.RuntimePath), "1.lease") + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + agent := filepath.Join(filepath.Dir(c.RuntimePath), "microsandbox-test-agent") + if err := os.Symlink(executable, agent); err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(Request{Operation: "lease-held", Config: c}) + command := exec.Command(agent) + command.Stdin = strings.NewReader(string(raw)) + if err := command.Start(); err != nil { + t.Fatal(err) + } + defer func() { _ = command.Process.Kill(); _ = command.Wait() }() + defer func() { + _ = os.WriteFile(c.RuntimePath+".release", nil, 0600) + waitProcessCondition(t, func() bool { return exclusiveLease(c.RuntimeHome) == nil }) + }() + waitProcessCondition(t, func() bool { _, err := os.Stat(c.RuntimePath); return err == nil }) + before, err := os.Stat(c.RuntimeHome) + if err != nil { + t.Fatal(err) + } + if err := command.Process.Kill(); err != nil { + t.Fatal(err) + } + _ = command.Wait() + // A new node has no in-memory references, but cannot collect the old helper. + restarted := &ProcessCaller{LeasePath: c.RuntimeHome} + if !restarted.Quiescent() { + t.Fatal("fresh process unexpectedly has references") + } + if err := exclusiveLease(c.RuntimeHome); !errors.Is(err, syscall.EWOULDBLOCK) { + t.Fatalf("live orphan helper lost lease: %v", err) + } + if err := os.WriteFile(c.RuntimePath+".release", nil, 0600); err != nil { + t.Fatal(err) + } + waitProcessCondition(t, func() bool { return exclusiveLease(c.RuntimeHome) == nil }) + after, err := os.Stat(c.RuntimeHome) + if err != nil || !os.SameFile(before, after) { + t.Fatal("lease inode changed", err) + } +} + +func exclusiveLease(path string) error { + file, err := os.OpenFile(path, os.O_RDWR, 0600) + if err != nil { + return err + } + defer file.Close() + return syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) +} + +func waitProcessCondition(t *testing.T, condition func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if condition() { + return + } + time.Sleep(time.Millisecond) + } + t.Fatal("process condition did not settle") +} + +func TestGenerationLeaseExcludesCollectionAndDroppedGeneration(t *testing.T) { + c := processConfig(t) + lease := filepath.Join(filepath.Dir(c.RuntimePath), "1.lease") + file, err := os.OpenFile(lease, os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + t.Fatal(err) + } + defer file.Close() + if err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + t.Fatal(err) + } + caller := &ProcessCaller{LeasePath: lease} + if _, err := caller.Call(t.Context(), Request{Operation: "delayed", Config: c}); err == nil { + t.Fatal("helper started during exclusive collection") + } + if err := os.WriteFile(strings.TrimSuffix(lease, ".lease")+".dropped", []byte("{}"), 0600); err != nil { + t.Fatal(err) + } + if err := syscall.Flock(int(file.Fd()), syscall.LOCK_UN); err != nil { + t.Fatal(err) + } + if _, err := caller.Call(t.Context(), Request{Operation: "delayed", Config: c}); err == nil { + t.Fatal("helper started after payload collection") + } + if _, err := os.Stat(c.RuntimePath); !os.IsNotExist(err) { + t.Fatal("refused helper mutated files", err) + } + // Never accept an alternate inode through symlinks or multiply linked files. + if err := os.Remove(lease); err != nil { + t.Fatal(err) + } + if err := os.Symlink(c.RuntimePath, lease); err != nil { + t.Fatal(err) + } + if _, err := caller.acquireLease(); err == nil { + t.Fatal("symlink lease accepted") + } +} diff --git a/services/agents-api/internal/sandbox/node/generation_json.go b/services/agents-api/internal/sandbox/node/generation_json.go new file mode 100644 index 000000000..ad98b470e --- /dev/null +++ b/services/agents-api/internal/sandbox/node/generation_json.go @@ -0,0 +1,141 @@ +package node + +import ( + "bytes" + "encoding/json" + "io" + "strings" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" +) + +// Decode control objects before trusting Go's struct decoder: that decoder +// accepts repeated and case-insensitive keys, and silently maps null to zero. +// Each list is the actual wire surface, not a second provider payload schema. +func generationObject(raw []byte, required, optional, nullable string) (map[string]json.RawMessage, error) { + allowed := map[string]bool{} + nulls := map[string]bool{} + for _, key := range strings.Fields(required + " " + optional) { + allowed[key] = true + } + for _, key := range strings.Fields(nullable) { + nulls[key] = true + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + token, err := decoder.Token() + if err != nil || token != json.Delim('{') { + return nil, sandbox.ErrInvalid + } + fields := map[string]json.RawMessage{} + for decoder.More() { + token, err := decoder.Token() + key, ok := token.(string) + if err != nil || !ok || !allowed[key] || fields[key] != nil { + return nil, sandbox.ErrInvalid + } + var value json.RawMessage + if decoder.Decode(&value) != nil || bytes.Equal(value, []byte("null")) && !nulls[key] { + return nil, sandbox.ErrInvalid + } + fields[key] = value + } + if token, err = decoder.Token(); err != nil || token != json.Delim('}') { + return nil, sandbox.ErrInvalid + } + if decoder.Decode(new(any)) != io.EOF { + return nil, sandbox.ErrInvalid + } + for _, key := range strings.Fields(required) { + if fields[key] == nil { + return nil, sandbox.ErrInvalid + } + } + return fields, nil +} + +func generationRecords(raw []byte, required, optional string) error { + if raw == nil { + return nil + } + var entries []json.RawMessage + if json.Unmarshal(raw, &entries) != nil || entries == nil || len(entries) > 8 { + return sandbox.ErrInvalid + } + for _, entry := range entries { + if _, err := generationObject(entry, required, optional, ""); err != nil { + return err + } + } + return nil +} + +func validateGenerationJSON(raw []byte, kind string) error { + var fields string + switch kind { + case "hello": + fields = "identity health" + case "welcome", "heartbeat_ack": + fields = "deployment connection_id owner_epoch" + case "heartbeat": + fields = "health connection_id owner_epoch" + case "retention": + fields = "control" + case "retention_ack": + fields = "control deployment" + case "request": + fields = "request" + case "response": + fields = "response" + default: + return sandbox.ErrInvalid + } + values, err := generationObject(raw, "version type "+fields, "", "") + if err != nil { + return err + } + if value := values["deployment"]; value != nil { + if _, err := generationObject(value, "generation specification_digest serving_generation", "", "serving_generation"); err != nil { + return err + } + } + if value := values["identity"]; value != nil { + if _, err := generationObject(value, "specification_digest deployment_generation node_id installation_id provider backend_fingerprint max_active max_retained", "", ""); err != nil { + return err + } + } + if value := values["health"]; value != nil { + health, err := generationObject(value, "provider_ready observed_at active_operations cpu_utilization total_memory_bytes effective_cpu_cores", "generations diagnostic cpu_count available_memory_bytes available_disk_bytes", "cpu_utilization total_memory_bytes effective_cpu_cores cpu_count available_memory_bytes available_disk_bytes") + if err != nil { + return err + } + if err := generationRecords(health["generations"], "generation specification_digest state", "diagnostic"); err != nil { + return err + } + } + if value := values["control"]; value != nil { + records := "references" + required := "generation specification_digest" + if kind == "retention_ack" { + records = "retentions" + required += " keep" + } + control, err := generationObject(value, "id sequence connection_id owner_epoch "+records, "", "") + if err != nil { + return err + } + if err := generationRecords(control[records], required, ""); err != nil { + return err + } + } + if value := values["request"]; value != nil { + if _, err := generationObject(value, "deployment_generation id sequence connection_id owner_epoch operation timeout_ms reference", "bootstrap compute generation command suspend resume snapshot observation", ""); err != nil { + return err + } + } + if value := values["response"]; value != nil { + if _, err := generationObject(value, "id connection_id", "error_code info compute state command sample", ""); err != nil { + return err + } + } + return nil +} diff --git a/services/agents-api/internal/sandbox/node/generation_json_test.go b/services/agents-api/internal/sandbox/node/generation_json_test.go new file mode 100644 index 000000000..9c3b7242f --- /dev/null +++ b/services/agents-api/internal/sandbox/node/generation_json_test.go @@ -0,0 +1,63 @@ +package node + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/google/uuid" +) + +func TestGenerationGrantJSONRefusesAmbiguousDeletionAuthority(t *testing.T) { + f := frame{Version: 2, Type: "retention_ack", Deployment: &sandbox.NodeDeployment{Generation: 2, SpecificationDigest: strings.Repeat("a", 64)}, Control: &generationControl{ID: uuid.NewString(), ConnectionID: uuid.NewString(), Sequence: 1, OwnerEpoch: 1, Retentions: []sandbox.GenerationRetention{{GenerationReference: sandbox.GenerationReference{Generation: 1, SpecificationDigest: strings.Repeat("b", 64)}, Keep: true}}}} + raw, err := json.Marshal(f) + if err != nil { + t.Fatal(err) + } + original := string(raw) + if _, err := decodeFrame(raw); err != nil { + t.Fatal("valid grant rejected", err) + } + for name, replacement := range map[string]string{ + "duplicate": strings.Replace(original, `"keep":true`, `"keep":true,"keep":false`, 1), + "case alias": strings.Replace(original, `"keep":true`, `"Keep":false`, 1), + "null grant": strings.Replace(original, `"keep":true`, `"keep":null`, 1), + "missing grant": strings.Replace(original, `,"keep":true`, ``, 1), + "foreign null": strings.TrimSuffix(original, "}") + `,"request":null}`, + "mixed null control": strings.Replace(original, `"retentions":`, `"references":null,"retentions":`, 1), + "null serving omission": strings.Replace(original, `,"serving_generation":null`, ``, 1), + "duplicate envelope": strings.TrimSuffix(original, "}") + `,"version":2}`, + "sequence overflow": strings.Replace(original, `"sequence":1`, `"sequence":9223372036854775808`, 1), + } { + t.Run(name, func(t *testing.T) { + if replacement == original { + t.Fatal("fixture did not change") + } + if _, err := decodeFrame([]byte(replacement)); err == nil { + t.Fatal("ambiguous grant accepted") + } + }) + } +} + +func TestGenerationHealthRejectsUnboundedOrAmbiguousNumbers(t *testing.T) { + f := frame{Version: 2, Type: "heartbeat", ConnectionID: uuid.NewString(), OwnerEpoch: 1, Health: &Health{ObservedAt: time.Now().UTC()}} + good, _ := json.Marshal(f) + if _, err := decodeFrame(good); err != nil { + t.Fatal(err) + } + for _, replacement := range []struct{ from, to string }{ + {`"active_operations":0`, `"active_operations":-1`}, + {`"active_operations":0`, `"active_operations":33`}, + {`"cpu_utilization":null`, `"cpu_utilization":1.1`}, + {`"total_memory_bytes":null`, `"total_memory_bytes":9007199254740992`}, + {`"effective_cpu_cores":null`, `"effective_cpu_cores":0`}, + {`"provider_ready":false`, `"Provider_Ready":false`}, + } { + if _, err := decodeFrame([]byte(strings.Replace(string(good), replacement.from, replacement.to, 1))); err == nil { + t.Fatal("invalid health accepted", replacement.to) + } + } +} diff --git a/services/agents-api/internal/sandbox/node/generation_wire.go b/services/agents-api/internal/sandbox/node/generation_wire.go index bedf2eddf..7aab4d4cd 100644 --- a/services/agents-api/internal/sandbox/node/generation_wire.go +++ b/services/agents-api/internal/sandbox/node/generation_wire.go @@ -41,7 +41,7 @@ func validateVersionFrame(f frame, size int) error { return sandbox.ErrInvalid } if f.Health != nil { - if len(f.Health.Generations) > 8 { + if len(f.Health.Generations) > 8 || !validGenerationHealth(*f.Health) { return sandbox.ErrInvalid } seen := map[uint64]bool{} @@ -52,7 +52,7 @@ func validateVersionFrame(f frame, size int) error { seen[g.Generation] = true } } - if f.Request != nil && !validGeneration(f.Request.DeploymentGeneration) { + if f.Request != nil && (!validGeneration(f.Request.DeploymentGeneration) || !validGeneration(f.Request.Sequence) || !validGeneration(f.Request.OwnerEpoch) || !validID(f.Request.ConnectionID)) { return sandbox.ErrInvalid } if c := f.Control; c != nil { @@ -107,3 +107,29 @@ func validateVersionFrame(f frame, size int) error { } return nil } + +func validGenerationHealth(h Health) bool { + if h.ObservedAt.IsZero() || h.ObservedAt.Year() < 1970 || h.ObservedAt.Year() > 9999 || h.ActiveOperations < 0 || h.ActiveOperations > maxPending { + return false + } + if len(h.Diagnostic) > 64 || sandbox.NormalizeNodeDiagnostic(h.Diagnostic) != h.Diagnostic { + return false + } + for _, value := range []*int64{h.CPUCount, h.TotalMemoryBytes, h.AvailableMemoryBytes, h.AvailableDiskBytes} { + if value != nil && (*value < 0 || *value > 1<<53-1) { + return false + } + } + for _, value := range []*float64{h.CPUUtilization, h.EffectiveCPUCores} { + if value != nil && (math.IsNaN(*value) || math.IsInf(*value, 0) || *value < 0) { + return false + } + } + if h.CPUUtilization != nil && *h.CPUUtilization > 1 || h.EffectiveCPUCores != nil && *h.EffectiveCPUCores == 0 { + return false + } + if h.TotalMemoryBytes != nil && (*h.TotalMemoryBytes == 0 || h.AvailableMemoryBytes != nil && *h.AvailableMemoryBytes > *h.TotalMemoryBytes) { + return false + } + return true +} diff --git a/services/agents-api/internal/sandbox/node/generations.go b/services/agents-api/internal/sandbox/node/generations.go index 1d69ea162..5ef2f5baf 100644 --- a/services/agents-api/internal/sandbox/node/generations.go +++ b/services/agents-api/internal/sandbox/node/generations.go @@ -167,6 +167,9 @@ func (m *GenerationManager) Acquire(generation uint64) (sandbox.Provider, bool, } func (m *GenerationManager) Drop(ctx context.Context, grant sandbox.GenerationRetention) error { + if err := ctx.Err(); err != nil { + return err + } if grant.Keep { return nil } diff --git a/services/agents-api/internal/sandbox/node/generations_test.go b/services/agents-api/internal/sandbox/node/generations_test.go index f75d90054..fe663eddd 100644 --- a/services/agents-api/internal/sandbox/node/generations_test.go +++ b/services/agents-api/internal/sandbox/node/generations_test.go @@ -190,3 +190,19 @@ func TestGrantedDropWaitsForReferencesAndActualHelperExit(t *testing.T) { t.Fatal("settled unreferenced generation was not collected", err) } } + +func TestCanceledConnectionCannotBeginGenerationCollection(t *testing.T) { + m := generationFixture(3) + m.target.ServingGeneration = nil + removed := false + m.options.Remove = func(context.Context, GenerationProvider) error { removed = true; return nil } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + grant := sandbox.GenerationRetention{GenerationReference: sandbox.GenerationReference{Generation: 1, SpecificationDigest: strings.Repeat("a", 64)}} + if err := m.Drop(ctx, grant); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + if removed || m.values[1] == nil { + t.Fatal("canceled connection began collection") + } +} diff --git a/services/agents-api/internal/sandbox/node/wire.go b/services/agents-api/internal/sandbox/node/wire.go index 3947e6472..8fc7b2c39 100644 --- a/services/agents-api/internal/sandbox/node/wire.go +++ b/services/agents-api/internal/sandbox/node/wire.go @@ -136,11 +136,19 @@ func readFrame(conn *websocket.Conn) (frame, error) { if kind != websocket.TextMessage || len(data) > MaxFrameBytes { return f, sandbox.ErrInvalid } + return decodeFrame(data) +} + +func decodeFrame(data []byte) (frame, error) { + var f frame d := json.NewDecoder(bytes.NewReader(data)) d.DisallowUnknownFields() if d.Decode(&f) != nil || d.Decode(new(any)) != io.EOF || (f.Version != ProtocolVersion && f.Version != GenerationProtocolVersion) { return frame{}, sandbox.ErrInvalid } + if f.Version == GenerationProtocolVersion && validateGenerationJSON(data, f.Type) != nil { + return frame{}, sandbox.ErrInvalid + } if err := validateVersionFrame(f, len(data)); err != nil { return frame{}, err } diff --git a/services/agents-api/tools/microsandbox-provider/main.go b/services/agents-api/tools/microsandbox-provider/main.go index 036e5accb..67cfc19e3 100644 --- a/services/agents-api/tools/microsandbox-provider/main.go +++ b/services/agents-api/tools/microsandbox-provider/main.go @@ -26,6 +26,12 @@ import ( ) func main() { + // The inherited node generation lease covers this helper's actual lifetime. + // Set CLOEXEC before SDK initialization or any possible subprocess spawn so + // long-lived VMs and host daemons cannot inherit a local helper reference. + if os.Getenv("OAC_NODE_GENERATION_LEASE_FD") == "3" { + syscall.CloseOnExec(3) + } response := serve(os.Stdin) // Errors and upstream diagnostics can contain secrets: expose only stable codes. _ = json.NewEncoder(os.Stdout).Encode(response) From 87bf5071b80b682162a65d18bfa92adaef9b0a44 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 22:35:33 +0000 Subject: [PATCH 04/16] Pair current node programs with Core-selected retained Runtime releases --- .../agents-api/node-generation-protocol.md | 18 +++++ deploy/install/node_install.py | 19 +++-- deploy/install/test_node_install.py | 75 ++++++++++++++++++- 3 files changed, 104 insertions(+), 8 deletions(-) diff --git a/contracts/agents-api/node-generation-protocol.md b/contracts/agents-api/node-generation-protocol.md index a3caeb02f..782fcb793 100644 --- a/contracts/agents-api/node-generation-protocol.md +++ b/contracts/agents-api/node-generation-protocol.md @@ -66,6 +66,24 @@ A helper's exit is local file-lifetime evidence, not proof that a remote mutatio or an uncertain provider receipt has been released. Core's durable allocation and placement retention requirements remain independent. +## Matched fresh installation + +The host program release and Core's selected Runtime release are independent. +A fresh node gets its executable and private preparer from the current console +release. It reads the exact Runtime source, image identities and native runtime / +firmware digests from the authenticated Core configuration. If that Runtime is +older, the console must still serve its immutable `releases//` manifest, +checksums and allowlisted artifacts. Runtime helper, firmware, seccomp and image +bytes come from that selected release; the enrolled specification records it. +Artifact URLs are pinned to their verified manifest source even if the console's +current release changes during download. + +A missing retained release refuses installation rather than substituting the +current Runtime. A local bundle that contains only a different Runtime also +refuses with guidance to use the console origin retaining the selected release. +These refusals occur before writing the installation identity, importing the +Runtime, registering the node or starting its service. + ## Identity-preserving program updates Run the checksum-verified `node-install.pyz --update` as the original installation diff --git a/deploy/install/node_install.py b/deploy/install/node_install.py index f886bb6fc..1c5c257d1 100644 --- a/deploy/install/node_install.py +++ b/deploy/install/node_install.py @@ -230,7 +230,7 @@ def read(name): for name in (COMMON[0], "images/runtime.tar.gz") + MICRO: distribution.artifact(manifest, name) # Nodes download only from their console, never from a release URL the build recorded. - manifest["artifact_base_url"] = source + "/node-install/" + prefix + "artifacts" if source else "" + manifest["artifact_base_url"] = source + "/node-install/releases/" + manifest["source_commit"] + "/artifacts" if source else "" return manifest, sums @@ -273,7 +273,7 @@ def file_digest(path): return result.hexdigest() -def download(source, name, root, expected): +def download(source, name, root, expected, prefix=""): target = root / name safe_directory(target.parent) if existing_file(target): @@ -282,7 +282,7 @@ def download(source, name, root, expected): return descriptor, temporary = tempfile.mkstemp(prefix=".download-", dir=target.parent) try: - with os.fdopen(descriptor, "wb") as output, fetch(source, name) as response: + with os.fdopen(descriptor, "wb") as output, fetch(source, prefix + name) as response: for block in iter(lambda: response.read(1024 * 1024), b""): output.write(block) if file_digest(Path(temporary)) != expected: @@ -415,8 +415,15 @@ def install_lock(root): def register_node(root, args, token): """Download and verify the payload, prepare the Runtime and register; not the service.""" print("Downloading and verifying node files...", flush=True) - manifest, sums = metadata(args.source_url, getattr(args, "bundle", None)) + program_manifest, program_sums = metadata(args.source_url, getattr(args, "bundle", None)) + selected = args.configuration["specification"]["runtime"] + manifest, sums = program_manifest, program_sums + if selected["source_commit"] != program_manifest["source_commit"]: + if getattr(args, "bundle", None) is not None: + raise InstallError("This bundle does not contain Core's selected Runtime; install through the console --source-url that retains its release") + manifest, sums = metadata(args.source_url, prefix="releases/" + selected["source_commit"] + "/") node_spec.verify_release(args.configuration, manifest) + runtime_prefix = "releases/" + manifest["source_commit"] + "/" names = COMMON + (MICRO if args.provider == "microsandbox" else ()) if "runtime/seccomp.json" not in sums: raise InstallError("The distribution is missing required node checksums") @@ -427,7 +434,7 @@ def register_node(root, args, token): for name in names: if name == "runtime/seccomp.json": if getattr(args, "bundle", None) is None: - download(args.source_url, name, root, sums[name]) + download(args.source_url, name, root, sums[name], prefix=runtime_prefix) else: source = args.bundle / name if source.is_symlink() or file_digest(source) != sums[name]: @@ -438,7 +445,7 @@ def register_node(root, args, token): target = root / name safe_directory(target.parent) existing_file(target) - distribution.obtain_artifact(manifest, name, target, getattr(args, "bundle", None)) + distribution.obtain_artifact(program_manifest if name == COMMON[0] else manifest, name, target, getattr(args, "bundle", None)) os.chmod(target, 0o700) node_generations.install_helper(root, args, sys.modules[__name__]) safe_directory(root / "state/node") diff --git a/deploy/install/test_node_install.py b/deploy/install/test_node_install.py index eef0db8c5..6f92a4da2 100644 --- a/deploy/install/test_node_install.py +++ b/deploy/install/test_node_install.py @@ -1,5 +1,6 @@ """Exercise node installation without running providers or changing user services.""" import argparse +import copy import fcntl import hashlib import gzip @@ -72,7 +73,7 @@ def setUp(self): mock.patch.object(installer, "open_request", side_effect=self.configuration_response), mock.patch.object(installer.distribution.urllib.request, "build_opener", return_value=mock.Mock(open=self.artifact_response)), mock.patch.object(installer, "micro_home", return_value=self.home / "m"), - mock.patch.object(installer, "fetch", side_effect=lambda source, name: io.BytesIO(self.payloads[name])), + mock.patch.object(installer, "fetch", side_effect=lambda source, name: io.BytesIO(self.payloads[name.split("/", 2)[2] if name.startswith("releases/") else name])), mock.patch.object(installer, "checked", side_effect=self.checked), mock.patch.object(installer.distribution, "docker_command", side_effect=self.docker_command), mock.patch.object(installer.subprocess, "run", return_value=subprocess.CompletedProcess([], 0, b"statically linked", b""))): @@ -92,7 +93,7 @@ def configuration_response(self, request, **kwargs): def artifact_response(self, request, **kwargs): url = getattr(request, "full_url", request) # The fixture manifest records a release URL; nodes still download from their console. - self.assertTrue(url.startswith(self.args.source_url + "/node-install/artifacts/"), url) + self.assertTrue(url.startswith(self.args.source_url + "/node-install/releases/" + self.manifest["source_commit"] + "/artifacts/"), url) for name, item in self.manifest["artifacts"].items(): if url.endswith("/" + item["filename"]): return Response(self.payloads[name]) @@ -161,6 +162,76 @@ def docker_command(self, arguments, **kwargs): def install(self): installer.install(self.args, "synthetic-once-token") + def install_current_program_with_retained_runtime(self, provider): + self.args.provider = provider + old_source, new_source = self.manifest["source_commit"], "f" * 40 + program = copy.deepcopy(self.manifest) + program["source_commit"] = new_source + payloads = dict(self.payloads) + payloads[installer.COMMON[0]] = b"new protocol program from current release" + for name, artifact in program["artifacts"].items(): + artifact["filename"] = artifact["filename"].replace(old_source, new_source) + artifact["size"] = len(payloads[name]) + artifact["sha256"] = hashlib.sha256(payloads[name]).hexdigest() + payloads["manifest.json"] = json.dumps(program).encode() + payloads["SHA256SUMS"] = "".join(hashlib.sha256(raw).hexdigest() + " " + name + "\n" + for name, raw in payloads.items() if name != "SHA256SUMS").encode() + fetched = [] + def fetch(_source, name): + fetched.append(name) + if name.startswith("releases/" + old_source + "/"): + return io.BytesIO(self.payloads[name.split("/", 2)[2]]) + self.assertIn(name, ("manifest.json", "SHA256SUMS")) + return io.BytesIO(payloads[name]) + def artifact_response(request, **_kwargs): + url = getattr(request, "full_url", request) + for manifest, files in ((program, payloads), (self.manifest, self.payloads)): + prefix = self.args.source_url + "/node-install/releases/" + manifest["source_commit"] + "/artifacts/" + for name, item in manifest["artifacts"].items(): + if url == prefix + item["filename"]: + # Only the node executable comes from the host release. + self.assertEqual(manifest["source_commit"], new_source if name == installer.COMMON[0] else old_source) + return Response(files[name]) + raise AssertionError("Unexpected immutable artifact URL " + url) + with mock.patch.object(installer, "fetch", side_effect=fetch), mock.patch.object(installer.distribution.urllib.request, "build_opener", return_value=mock.Mock(open=artifact_response)): + self.install() + self.assertEqual((self.root / installer.COMMON[0]).read_bytes(), payloads[installer.COMMON[0]]) + config = json.loads((self.root / "provider.json").read_text()) + self.assertEqual(config["specification"]["runtime"], node_spec.release(self.manifest)) + self.assertEqual(json.loads((self.root / "registered.json").read_text())["source_commit"], old_source) + self.assertIn("releases/" + old_source + "/runtime/seccomp.json", fetched) + if provider == "microsandbox": + for name in installer.MICRO: + self.assertEqual((self.root / name).read_bytes(), self.payloads[name]) + + def test_current_program_pairs_with_core_selected_retained_docker_runtime(self): + self.install_current_program_with_retained_runtime("docker") + + def test_current_program_pairs_with_core_selected_retained_micro_runtime(self): + self.install_current_program_with_retained_runtime("microsandbox") + + def test_bundle_without_selected_runtime_refuses_before_payload_or_registration(self): + program = copy.deepcopy(self.manifest) + program["source_commit"] = "f" * 40 + self.args.bundle = self.home / "bundle" + self.args.bundle.mkdir() + with mock.patch.object(installer, "metadata", return_value=(program, {})): + with self.assertRaisesRegex(installer.InstallError, "does not contain Core's selected Runtime"): + self.install() + self.assertFalse((self.root / "installation.json").exists()) + self.assertFalse((self.root / installer.COMMON[0]).exists()) + self.assertFalse(any("register" in command or "load" in command or "enable" in command for command, _ in self.calls)) + + def test_missing_retained_runtime_never_falls_back_to_current_runtime(self): + program = copy.deepcopy(self.manifest) + program["source_commit"] = "f" * 40 + with mock.patch.object(installer, "metadata", side_effect=[(program, {}), installer.InstallError("Retained release unavailable")]) as metadata: + with self.assertRaisesRegex(installer.InstallError, "Retained release unavailable"): + self.install() + self.assertEqual(metadata.call_args_list[1].kwargs, {"prefix": "releases/" + self.manifest["source_commit"] + "/"}) + self.assertFalse((self.root / "installation.json").exists()) + self.assertFalse(any("register" in command or "load" in command or "enable" in command for command, _ in self.calls)) + def test_docker_installs_matched_payload_registers_and_starts_persistent_service(self): self.install() config = json.loads((self.root / "provider.json").read_text()) From ff8f1e88f13e831670b4bbdebdda10b975853544 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 22:42:36 +0000 Subject: [PATCH 05/16] Recover missing retained Runtime artifacts without changing generation identity --- .../agents-api/node-generation-protocol.md | 24 ++++ deploy/install/node_generations.py | 120 +++++++++++------- deploy/install/test_node_install.py | 51 ++++++++ .../cmd/sandbox-node/generations.go | 11 +- .../internal/sandbox/config/config.go | 8 +- .../internal/sandbox/node/generations.go | 79 +++++++++--- .../internal/sandbox/node/generations_test.go | 72 +++++++++++ 7 files changed, 296 insertions(+), 69 deletions(-) diff --git a/contracts/agents-api/node-generation-protocol.md b/contracts/agents-api/node-generation-protocol.md index 782fcb793..30415334a 100644 --- a/contracts/agents-api/node-generation-protocol.md +++ b/contracts/agents-api/node-generation-protocol.md @@ -84,6 +84,30 @@ refuses with guidance to use the console origin retaining the selected release. These refusals occur before writing the installation identity, importing the Runtime, registering the node or starting its service. +## Restart recovery + +Missing retained Runtime bytes do not switch a pinned placement to the current +Runtime. The node retains the original generation and specification digest as +unready state, and asks Core's authenticated configuration endpoint for that exact +kept generation before recovery. Missing seccomp bytes may leave an unready +provider placeholder; missing image or native artifacts discovered by a provider +probe queue repair without advertising readiness. + +Preparation and repair are serialized. Target and serving generations take +priority, with bounded progress over other retained generations. Each attempt has +a 30-minute deadline; failures back off for 1, 2, 5, 10 and then at most 30 minutes. +No connection-established deployment facts means no preparation starts. Repair +preserves existing configurations and paths, verifies the selected release and all +existing sibling checksums, and downloads only absent immutable files. Conflicting +bytes or a different retained specification refuse repair. A missing complete +provider configuration remains a refusal rather than a guessed reconstruction. + +Repair takes the same exclusive generation lease and installation lock used by +collection. A live helper or concurrent collector therefore retains ownership; +repair retries later without replacing in-use files. Once bytes are restored, the +node still runs the actual provider readiness probe. File presence and executable +capability alone never establish serving readiness. + ## Identity-preserving program updates Run the checksum-verified `node-install.pyz --update` as the original installation diff --git a/deploy/install/node_generations.py b/deploy/install/node_generations.py index 7d5a14088..6e272b736 100644 --- a/deploy/install/node_generations.py +++ b/deploy/install/node_generations.py @@ -107,9 +107,18 @@ def generation_home(root, configuration, base, installer): def image_available(value, installer): try: if value["provider"] == "docker": + seccomp = Path(value["docker"]["seccomp_file"]) + installer.existing_file(seccomp) + json.loads(seccomp.read_text()) raw = installer.checked(list(installer.DOCKER) + ["image", "inspect", value["docker"]["image"], "--format", "{{.Id}} {{.Os}}/{{.Architecture}}"], "Cannot inspect pinned image") return raw.strip() == value["docker"]["image"] + " linux/amd64" micro = value["microsandbox"] + for key in ("helper_path", "runtime_path", "firmware_path"): + if not installer.existing_file(Path(micro[key])): + return False + if (installer.file_digest(Path(micro["runtime_path"])) != micro["runtime_sha256"] + or installer.file_digest(Path(micro["firmware_path"])) != micro["firmware_sha256"]): + return False env = dict(os.environ, MSB_BACKEND="local", MSB_HOME=micro["runtime_home"], MSB_PATH=micro["runtime_path"], MSB_LIBKRUNFW_PATH=micro["firmware_path"]) image = json.loads(installer.checked([micro["runtime_path"], "image", "inspect", micro["image"], "--format", "json"], "Cannot inspect pinned image", env=env)) return image.get("digest") == micro["image"].split("@", 1)[1] and image.get("architecture") == "amd64" and image.get("os") == "linux" @@ -117,11 +126,54 @@ def image_available(value, installer): return False +def runtime_files(root, value, args, manifest, sums, installer): + """Restore only absent immutable bytes; existing conflicts are never replaced.""" + source = args.configuration["specification"]["runtime"]["source_commit"] + release = root / "releases" / source + if value is not None: + if args.provider == "microsandbox": + release = Path(value["microsandbox"]["helper_path"]).parents[2] + if any(Path(value["microsandbox"][key]) != release / name for key, name in zip( + ("helper_path", "runtime_path", "firmware_path"), installer.MICRO)): + raise installer.InstallError("Retained Runtime artifact paths differ") + else: + release = Path(value["docker"]["seccomp_file"]).parents[1] + if Path(value["docker"]["seccomp_file"]) != release / "runtime/seccomp.json": + raise installer.InstallError("Retained Runtime seccomp path differs") + if release not in (root, root / "releases" / source): + raise installer.InstallError("Retained Runtime artifacts are outside this installation") + installer.no_links(release) + installer.safe_directory(release) + names = ("runtime/seccomp.json",) + (installer.MICRO if args.provider == "microsandbox" else ()) + saved = installer.private_json(release / "manifest.json") + if (release / "manifest.json").exists(): + if saved is None: + raise installer.InstallError("Retained Runtime manifest is unreadable") + installer.node_spec.verify_release(args.configuration, saved) + # Inspect every existing byte before starting any repair, so a missing file + # cannot hide a conflicting sibling or redirect a later download. + for name in names: + path = release / name + installer.no_links(path) + if installer.existing_file(path): + expected = sums[name] if name == "runtime/seccomp.json" else installer.distribution.artifact(manifest, name)["sha256"] + if installer.file_digest(path) != expected: + raise installer.InstallError("Retained Runtime artifact checksum differs") + for name in names: + installer.safe_directory((release / name).parent) + if name == "runtime/seccomp.json": + installer.download(args.source_url, name, release, sums[name], prefix="releases/" + source + "/") + else: + installer.distribution.obtain_artifact(manifest, name, release / name) + os.chmod(release / name, 0o700) + atomic_json(release / "manifest.json", manifest) + return release + + def prepare(args, installer): root, identity = owned_root(args, installer) - with installer.install_lock(root): + with installer.install_lock(root), collection_lease(root, args.generation, installer): directory = root / "state/node/generations" - installer.safe_directory(directory) if (directory / (str(args.generation) + ".dropped")).exists(): raise installer.InstallError("A dropped generation cannot be adopted again") args.core_url = identity["core_url"] @@ -133,64 +185,42 @@ def prepare(args, installer): configurations = retained_configs(root, installer) base = installer.private_json(root / "provider.json") runtime = args.configuration["specification"]["runtime"] - value = None - for candidate in configurations.values(): - if candidate["specification"]["runtime"] == runtime and image_available(candidate, installer): - value = copy.deepcopy(candidate) - value["generation"] = args.generation - value["specification"] = args.configuration["specification"] - if args.provider == "microsandbox": - value["microsandbox"].update(value["specification"]["resources"]) - break - if value is None: + value = configurations.get(args.generation) + retained = value is not None + if retained: + installer.node_spec.verify_provider(value, args.configuration, value.get("docker", {}).get("image")) + else: + for candidate in configurations.values(): + if candidate["specification"]["runtime"] == runtime and image_available(candidate, installer): + value = copy.deepcopy(candidate) + value["generation"] = args.generation + value["specification"] = args.configuration["specification"] + if args.provider == "microsandbox": + value["microsandbox"].update(value["specification"]["resources"]) + break + if value is None or not image_available(value, installer): settings = installer.private_json(root / "preparation.json") args.source_url = installer.origin(settings["source_url"]) args.bundle = None - prefix = "releases/" + runtime["source_commit"] + "/" - manifest, sums = installer.metadata(args.source_url, prefix=prefix) + manifest, sums = installer.metadata(args.source_url, prefix="releases/" + runtime["source_commit"] + "/") installer.node_spec.verify_release(args.configuration, manifest) - release = root / "releases" / runtime["source_commit"] - installer.safe_directory(release.parent) - names = ("runtime/seccomp.json",) + (installer.MICRO if args.provider == "microsandbox" else ()) - if not release.exists(): - with tempfile.TemporaryDirectory(prefix=".release-", dir=release.parent) as temporary: - stage = Path(temporary) - for name in names: - installer.safe_directory((stage / name).parent) - if name == "runtime/seccomp.json": - with installer.fetch(args.source_url, prefix + name) as stream: - raw = stream.read(1024 * 1024 + 1) - if hashlib.sha256(raw).hexdigest() != sums.get(name): - raise installer.InstallError("Runtime seccomp checksum differs") - (stage / name).write_bytes(raw) - os.chmod(stage / name, 0o600) - else: - installer.distribution.obtain_artifact(manifest, name, stage / name) - os.chmod(stage / name, 0o700) - atomic_json(stage / "manifest.json", manifest) - os.rename(stage, release) - saved = installer.private_json(release / "manifest.json") - # The origin URL is transport metadata, not immutable release identity. - installer.node_spec.verify_release(args.configuration, saved) - for name in names: - installer.existing_file(release / name) - expected = sums[name] if name == "runtime/seccomp.json" else installer.distribution.artifact(manifest, name)["sha256"] - if installer.file_digest(release / name) != expected: - raise installer.InstallError("Retained Runtime artifact checksum differs") + release = runtime_files(root, value if retained else None, args, manifest, sums, installer) if args.provider == "microsandbox": - args.runtime_home = generation_home(root, args.configuration, base, installer) + args.runtime_home = Path(value["microsandbox"]["runtime_home"]) if retained else generation_home(root, args.configuration, base, installer) installer.safe_directory(args.runtime_home) owner = args.runtime_home / "oac-installation.json" if not owner.exists() and any(args.runtime_home.iterdir()): raise installer.InstallError("Versioned microsandbox store contains unowned state") installer.write_once(owner, installer.json_text({"installation_id": args.installation_id})) runtime_image = installer.prepare_runtime(release, args, manifest) - value = installer.provider_config(release, args, manifest, runtime_image) + if retained: + installer.node_spec.verify_provider(value, args.configuration, runtime_image) + else: + value = installer.provider_config(release, args, manifest, runtime_image) target = directory / (str(args.generation) + ".json") if installer.existing_file(target) and installer.private_json(target) != value: raise installer.InstallError("Immutable generation configuration differs") atomic_json(target, value) - print(json.dumps(value)) @contextlib.contextmanager diff --git a/deploy/install/test_node_install.py b/deploy/install/test_node_install.py index 6f92a4da2..e962fede6 100644 --- a/deploy/install/test_node_install.py +++ b/deploy/install/test_node_install.py @@ -232,6 +232,57 @@ def test_missing_retained_runtime_never_falls_back_to_current_runtime(self): self.assertFalse((self.root / "installation.json").exists()) self.assertFalse(any("register" in command or "load" in command or "enable" in command for command, _ in self.calls)) + def recovery_args(self): + self.args.generation = 1 + self.args.specification_digest = json.loads((self.root / "state/node/identity.json").read_text())["identity"]["specification_digest"] + return self.args + + def test_restart_repairs_only_missing_micro_bytes_at_original_paths(self): + self.args.provider = "microsandbox" + self.install() + before = (self.root / "provider.json").read_bytes() + helper = self.root / installer.MICRO[0] + runtime = self.root / installer.MICRO[1] + old_inode = runtime.stat().st_ino + helper.unlink() + installer.node_generations.prepare(self.recovery_args(), installer) + self.assertEqual(helper.read_bytes(), self.payloads[installer.MICRO[0]]) + self.assertEqual(runtime.stat().st_ino, old_inode) + self.assertEqual((self.root / "provider.json").read_bytes(), before) + self.assertEqual(json.loads((self.root / "state/node/generations/1.json").read_text()), json.loads(before)) + + def test_restart_refuses_conflicting_sibling_before_repairing_missing_file(self): + self.args.provider = "microsandbox" + self.install() + helper = self.root / installer.MICRO[0] + runtime = self.root / installer.MICRO[1] + helper.unlink() + runtime.write_bytes(b"conflicting retained runtime") + with self.assertRaisesRegex(installer.InstallError, "artifact checksum differs"): + installer.node_generations.prepare(self.recovery_args(), installer) + self.assertFalse(helper.exists()) + self.assertEqual(runtime.read_bytes(), b"conflicting retained runtime") + + def test_live_helper_or_collection_excludes_restart_repair(self): + self.args.provider = "microsandbox" + self.install() + helper = self.root / installer.MICRO[0] + helper.unlink() + directory = self.root / "state/node/generations" + directory.mkdir(mode=0o700) + descriptor = os.open(directory / "1.lease", os.O_CREAT | os.O_RDWR, 0o600) + try: + for mode in (fcntl.LOCK_SH, fcntl.LOCK_EX): + fcntl.flock(descriptor, mode | fcntl.LOCK_NB) + with self.assertRaisesRegex(installer.InstallError, "helper is still active"): + installer.node_generations.prepare(self.recovery_args(), installer) + self.assertFalse(helper.exists()) + fcntl.flock(descriptor, fcntl.LOCK_UN) + finally: + os.close(descriptor) + installer.node_generations.prepare(self.args, installer) + self.assertEqual(helper.read_bytes(), self.payloads[installer.MICRO[0]]) + def test_docker_installs_matched_payload_registers_and_starts_persistent_service(self): self.install() config = json.loads((self.root / "provider.json").read_text()) diff --git a/services/agents-api/cmd/sandbox-node/generations.go b/services/agents-api/cmd/sandbox-node/generations.go index 1a1f373c8..f064c9321 100644 --- a/services/agents-api/cmd/sandbox-node/generations.go +++ b/services/agents-api/cmd/sandbox-node/generations.go @@ -41,6 +41,8 @@ func runGenerations(ctx context.Context, configFile, stateDir string) error { } paths = append([]string{configFile}, paths...) values := map[uint64]node.GenerationProvider{} + recovery := []sandbox.GenerationReference{} + seen := map[uint64]bool{} closeValues := func() { for _, v := range values { if v.Close != nil { @@ -68,10 +70,15 @@ func runGenerations(ctx context.Context, configFile, stateDir string) error { closeValues() return err } - if _, ok := values[config.Generation]; ok { + if seen[config.Generation] { continue } + seen[config.Generation] = true value, err := buildGeneration(config, stateDir) + if errors.Is(err, os.ErrNotExist) { + recovery = append(recovery, sandbox.GenerationReference{Generation: config.Generation, SpecificationDigest: config.Specification.Digest(config.Provider)}) + continue + } if err != nil { closeValues() return err @@ -96,7 +103,7 @@ func runGenerations(ctx context.Context, configFile, stateDir string) error { for _, v := range values { initial = append(initial, v) } - manager, err := node.NewGenerationManager(ctx, node.GenerationManagerOptions{Initial: initial, + manager, err := node.NewGenerationManager(ctx, node.GenerationManagerOptions{Initial: initial, Recover: recovery, Prepare: func(ctx context.Context, generation uint64, digest string) (node.GenerationProvider, error) { if err := runHelper(ctx, "prepare", generation, digest); err != nil { return node.GenerationProvider{}, err diff --git a/services/agents-api/internal/sandbox/config/config.go b/services/agents-api/internal/sandbox/config/config.go index 965ff439c..880208ed9 100644 --- a/services/agents-api/internal/sandbox/config/config.go +++ b/services/agents-api/internal/sandbox/config/config.go @@ -7,6 +7,7 @@ import ( "encoding/hex" "encoding/json" "errors" + "fmt" "io" "net/url" "os" @@ -95,8 +96,11 @@ func Build(config Config) (*Built, func(), error) { return nil, closeProvider, errors.New("managed Docker host must be an explicit canonical unix socket") } seccomp, err := os.ReadFile(entry.SeccompFile) - if err != nil || !json.Valid(seccomp) { - return nil, closeProvider, errors.New("cannot read managed Docker seccomp JSON") + if err != nil { + return nil, closeProvider, fmt.Errorf("cannot read managed Docker seccomp JSON: %w", err) + } + if !json.Valid(seccomp) { + return nil, closeProvider, errors.New("invalid managed Docker seccomp JSON") } c, err := client.New(client.WithHost(entry.Host)) if err != nil { diff --git a/services/agents-api/internal/sandbox/node/generations.go b/services/agents-api/internal/sandbox/node/generations.go index 5ef2f5baf..eb4bbcd48 100644 --- a/services/agents-api/internal/sandbox/node/generations.go +++ b/services/agents-api/internal/sandbox/node/generations.go @@ -20,32 +20,34 @@ type GenerationProvider struct { type GenerationManagerOptions struct { Initial []GenerationProvider + Recover []sandbox.GenerationReference Prepare func(context.Context, uint64, string) (GenerationProvider, error) Remove func(context.Context, GenerationProvider) error } type localGeneration struct { - value GenerationProvider - state, diagnostic string - refs int - retryAt time.Time - failures int - removing bool + value GenerationProvider + state, diagnostic string + refs int + retryAt time.Time + failures int + removing bool + repairing, preparing bool } // GenerationManager owns local providers, preparation and all queued/in-flight // references. Wire metadata is sparse; only correlated Core grants call Drop. type GenerationManager struct { - mu sync.Mutex - values map[uint64]*localGeneration - target sandbox.NodeDeployment - statusCursor, probeCursor uint64 - options GenerationManagerOptions - ctx context.Context - cancel context.CancelFunc - preparingCancel context.CancelFunc - wake chan struct{} - workers sync.WaitGroup + mu sync.Mutex + values map[uint64]*localGeneration + target sandbox.NodeDeployment + statusCursor, probeCursor, recoveryCursor uint64 + options GenerationManagerOptions + ctx context.Context + cancel context.CancelFunc + preparingCancel context.CancelFunc + wake chan struct{} + workers sync.WaitGroup } func NewGenerationManager(ctx context.Context, options GenerationManagerOptions) (*GenerationManager, error) { @@ -61,6 +63,13 @@ func NewGenerationManager(ctx context.Context, options GenerationManagerOptions) } m.values[v.Generation] = &localGeneration{value: v, state: "preparing"} } + for _, ref := range options.Recover { + if !validGeneration(ref.Generation) || !validSpecificationDigest(ref.SpecificationDigest) || m.values[ref.Generation] != nil { + cancel() + return nil, sandbox.ErrInvalid + } + m.values[ref.Generation] = &localGeneration{value: GenerationProvider{Generation: ref.Generation, SpecificationDigest: ref.SpecificationDigest}, state: "failed", diagnostic: sandbox.NodeRuntimeDownloadFailed, repairing: true} + } m.workers.Add(2) go func() { defer m.workers.Done(); m.prepareLoop() }() go func() { defer m.workers.Done(); m.probeLoop() }() @@ -157,7 +166,7 @@ func (m *GenerationManager) Acquire(generation uint64) (sandbox.Provider, bool, m.mu.Lock() defer m.mu.Unlock() g := m.values[generation] - if g == nil || g.removing || g.value.Provider == nil { + if g == nil || g.removing || g.preparing || g.value.Provider == nil { return nil, false, nil, ErrUnavailable } g.refs++ @@ -218,13 +227,33 @@ func (m *GenerationManager) prepareLoop() { case <-ticker.C: } m.mu.Lock() - g := m.values[m.target.Generation] - if g == nil || g.removing || g.value.Provider != nil || time.Now().Before(g.retryAt) { + var g *localGeneration + if m.target.Generation != 0 { + keys := []uint64{m.target.Generation} + if m.target.ServingGeneration != nil { + keys = append(keys, *m.target.ServingGeneration) + } + keys = append(keys, m.orderedLocked(m.recoveryCursor)...) + for _, key := range keys { + candidate := m.values[key] + if candidate == nil || candidate.removing || candidate.preparing || candidate.refs != 0 || candidate.value.Provider != nil && !candidate.repairing || time.Now().Before(candidate.retryAt) { + continue + } + if provider, ok := candidate.value.Provider.(interface{ Quiescent() bool }); ok && !provider.Quiescent() { + continue + } + g = candidate + m.recoveryCursor = key + break + } + } + if g == nil { m.mu.Unlock() continue } ctx, cancel := context.WithTimeout(m.ctx, 30*time.Minute) m.preparingCancel = cancel + g.preparing = true g.refs++ g.state = "preparing" g.diagnostic = "" @@ -234,12 +263,19 @@ func (m *GenerationManager) prepareLoop() { cancel() m.mu.Lock() m.preparingCancel = nil + g.preparing = false g.refs-- if err == nil && (value.Generation != generation || value.SpecificationDigest != digest || value.Provider == nil || value.Probe == nil) { err = sandbox.ErrOwnership } if err == nil { + if g.value.Close != nil { + g.value.Close() + } g.value = value + g.repairing = false + g.failures = 0 + g.retryAt = time.Time{} g.state = "preparing" } else { if value.Close != nil { @@ -273,7 +309,7 @@ func (m *GenerationManager) probeLoop() { var g *localGeneration for _, key := range keys { value := m.values[key] - if value.value.Provider != nil && !value.removing { + if value.value.Provider != nil && !value.removing && !value.preparing && !value.repairing { g = value m.probeCursor = key g.refs++ @@ -295,6 +331,9 @@ func (m *GenerationManager) probeLoop() { if err != nil { g.state = "failed" g.diagnostic = sandbox.NodeDiagnostic(err) + if errors.Is(err, sandbox.ErrRuntimeImageUnavailable) || errors.Is(err, sandbox.ErrMicrosandboxArtifactsUnavailable) { + g.repairing = true + } } } m.mu.Unlock() diff --git a/services/agents-api/internal/sandbox/node/generations_test.go b/services/agents-api/internal/sandbox/node/generations_test.go index fe663eddd..bfdec8642 100644 --- a/services/agents-api/internal/sandbox/node/generations_test.go +++ b/services/agents-api/internal/sandbox/node/generations_test.go @@ -206,3 +206,75 @@ func TestCanceledConnectionCannotBeginGenerationCollection(t *testing.T) { t.Fatal("canceled connection began collection") } } + +func TestRestartRecoversExactOlderGenerationBeforeAdvertisingReadiness(t *testing.T) { + digest := strings.Repeat("a", 64) + requested := make(chan uint64, 1) + download := make(chan struct{}) + probe := make(chan struct{}, 1) + qualify := make(chan struct{}) + m, err := NewGenerationManager(t.Context(), GenerationManagerOptions{ + Initial: []GenerationProvider{{Generation: 2, SpecificationDigest: digest, Provider: &fakeProvider{}, Probe: func(context.Context) error { return nil }}}, + Recover: []sandbox.GenerationReference{{Generation: 1, SpecificationDigest: digest}}, + Prepare: func(ctx context.Context, generation uint64, got string) (GenerationProvider, error) { + if got != digest { + return GenerationProvider{}, sandbox.ErrOwnership + } + requested <- generation + select { + case <-download: + case <-ctx.Done(): + return GenerationProvider{}, ctx.Err() + } + return GenerationProvider{Generation: generation, SpecificationDigest: got, Provider: &fakeProvider{}, Probe: func(ctx context.Context) error { + select { + case probe <- struct{}{}: + default: + } + select { + case <-qualify: + return nil + case <-ctx.Done(): + return ctx.Err() + } + }}, nil + }, + Remove: func(context.Context, GenerationProvider) error { return nil }, + }) + if err != nil { + t.Fatal(err) + } + defer m.Close() + pin := uint64(2) + m.Deployment(sandbox.NodeDeployment{Generation: 2, SpecificationDigest: digest, ServingGeneration: &pin}) + select { + case generation := <-requested: + if generation != 1 { + t.Fatal("replaced old placement generation", generation) + } + case <-time.After(3 * time.Second): + t.Fatal("old generation was never recovered") + } + if m.Ready(1) { + t.Fatal("missing payload advertised readiness") + } + if err := m.Drop(t.Context(), sandbox.GenerationRetention{GenerationReference: sandbox.GenerationReference{Generation: 1, SpecificationDigest: digest}}); !errors.Is(err, ErrUnavailable) { + t.Fatal("collection entered active repair", err) + } + close(download) + select { + case <-probe: + case <-time.After(4 * time.Second): + t.Fatal("restored provider never qualified") + } + if m.Ready(1) { + t.Fatal("file presence advertised provider readiness") + } + close(qualify) + wait(t, func() bool { return m.Ready(1) }) + provider, ready, release, err := m.Acquire(1) + if err != nil || !ready || provider == nil { + t.Fatal("old generation not usable after actual qualification", err) + } + release() +} From c8ac9a3a861f1aba5bc9e9456996f65b8bc4879a Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 23:04:20 +0000 Subject: [PATCH 06/16] Checkpoint node generation routing and readiness integration --- CONTRIBUTING.md | 29 ++- contracts/agents-api/core.openapi.yaml | 11 + contracts/agents-api/runtime.openapi.yaml | 4 + contracts/agents-api/sandbox-deployment.md | 62 +++-- deploy/install/node_generations.py | 13 +- .../agents-client/src/sandbox-client.test.ts | 10 +- packages/agents-client/src/sandbox-client.ts | 3 +- .../cmd/sandbox-node/generations.go | 4 +- services/agents-api/internal/api/errors.go | 2 + .../execution/sandbox_deployment_switch.go | 6 - .../agents-api/internal/sandbox/node/agent.go | 8 +- .../internal/sandbox/node/generation_agent.go | 4 +- .../node/generation_connection_test.go | 87 +++++++ .../internal/sandbox/node/generations.go | 32 ++- .../internal/sandbox/node/generations_test.go | 18 ++ .../agents-api/internal/sandbox/node/hub.go | 8 + .../internal/store/runtime_deployment_test.go | 2 +- .../store/runtime_node_configuration.go | 9 + .../store/runtime_node_generations.go | 23 ++ .../store/runtime_node_generations_test.go | 237 ++++++++++++++++++ .../internal/store/runtime_node_types.go | 5 +- .../internal/store/runtime_nodes.go | 3 + .../internal/store/runtime_placements.go | 9 +- .../store/sandbox_deployment_mutations.go | 26 +- .../store/sandbox_deployment_switch_test.go | 4 +- .../sandbox_deployment_switch_worker_test.go | 12 +- .../store/sandbox_deployment_view_test.go | 2 +- .../store/sandbox_specification_store_test.go | 26 +- ...ations.sql => 000082_node_generations.sql} | 5 +- 29 files changed, 558 insertions(+), 106 deletions(-) create mode 100644 services/agents-api/internal/sandbox/node/generation_connection_test.go create mode 100644 services/agents-api/internal/store/runtime_node_generations_test.go rename services/agents-api/migrations/{000081_node_generations.sql => 000082_node_generations.sql} (78%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f66461e8c..a2ca1f7aa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -790,10 +790,9 @@ read. All sandbox writes require the observed generation, including initial POST at zero; reject stale state before provider preparation, reset or no-op checks, and -repeat it under the committing row lock. Same-provider PUT currently requires no -reset. Node PUT keeps the zero-held-resource guard until the complete multi-generation -node protocol ships. E2B changes apply online without retiring nodes, tokens or the -owner epoch. A different backend, E2B team or unspecified old selection requires reset. +repeat it under the committing row lock. Same-provider PUT advances the target +without draining execution or retiring nodes, tokens or the owner epoch. Node +providers prepare independently and keep their old qualified serving pin. A different backend, E2B team or unspecified old selection requires reset. Preserve historical allocation ownership and placement, never migrate a Session. Persist immutable allocation and placement deployment generations, distinct from @@ -802,8 +801,20 @@ Keep superseded specification/build rows without credentials while current, owne an unreleased allocation/placement, or pinned by any nonremoved node. A durable node serving pin survives offline state and zero resources. GC uses the deployment lock and bounded pages; reset clears pins only after release. Never downgrade facts needed -for generation routing. Node readiness and no-gap placement remain PR-N work; do not -relax the node PUT guard or introduce half a protocol. +for generation routing. Bind node readiness to exact generation, current connection +and owner epoch. Promote a durable pin only for readiness of the then-current target +under deployment serialization. Late superseded readiness cannot acquire a pin. +Filter online/readiness/address/capacity before preferring the newest eligible pin; +newest-full must not mask older-free. Route Create, restore and cleanup through the +immutable allocation/placement generation. V1 keeps its qualified enrolled fallback. + +Use sparse v2 control batches of at most eight entries with no lifetime cap. Omitted +facts never authorize deletion. Correlate whole retention grants to connection, +epoch, sequence, generation and digest; recheck queued/inflight/helper references. +Permanent generation flock files survive updates and GC. Retained Runtime bytes and +the console's exact-release HTTP allowlist must agree. See +`contracts/agents-api/node-generation-protocol.md` for recovery, immutable artifacts +and the conservative v1 legacy-helper retention boundary. Use one E2B classifier. Omitted key preserves the current key; identical selection with omitted key is a no-op. Explicit key submission, including identical bytes, @@ -829,9 +840,9 @@ Project rollout and reset from the same database snapshot. Old retained resource alone never imply preparation or high-frequency polling. Rollout state is settled unless actual target preparation is active. Offline/unconfirmed nodes are unknown; only exact current connection/epoch/generation readiness is ready. A durable pin is -not connectivity. Fixed diagnostics alone may explain failed preparation. PR-G reports -no synthetic preparing state; old online v1 nodes can be update_required while their -valid pin remains available to the eventual PR-N placement contract. +not connectivity. Fixed diagnostics alone may explain failed preparation. Report +preparing only from an actual target observation; an online v1 node can be +update_required while its valid pin remains available for admission. Reset is durable execution state, advanced by the existing manager outside its counted work. Serialize start, escalation, cancel, setup, update and finalization diff --git a/contracts/agents-api/core.openapi.yaml b/contracts/agents-api/core.openapi.yaml index 00dc1bbbe..8dba36b29 100644 --- a/contracts/agents-api/core.openapi.yaml +++ b/contracts/agents-api/core.openapi.yaml @@ -976,6 +976,7 @@ definitions: - provider_unavailable - docker_unavailable - docker_limits_unsupported + - runtime_download_failed - runtime_image_unavailable - kvm_unavailable - microsandbox_artifacts_unavailable @@ -1066,6 +1067,7 @@ definitions: - provider_unavailable - docker_unavailable - docker_limits_unsupported + - runtime_download_failed - runtime_image_unavailable - kvm_unavailable - microsandbox_artifacts_unavailable @@ -1167,6 +1169,15 @@ definitions: store.SandboxNodeRollout: properties: diagnostic: + enum: + - provider_unavailable + - docker_unavailable + - docker_limits_unsupported + - runtime_download_failed + - runtime_image_unavailable + - kvm_unavailable + - microsandbox_artifacts_unavailable + - capacity_insufficient type: string ready_generation: description: Durable serving-generation pin; online and provider_ready still gate placement. diff --git a/contracts/agents-api/runtime.openapi.yaml b/contracts/agents-api/runtime.openapi.yaml index 989f0b64b..756977ace 100644 --- a/contracts/agents-api/runtime.openapi.yaml +++ b/contracts/agents-api/runtime.openapi.yaml @@ -150,6 +150,10 @@ paths: in: header name: X-OAC-Node-ID type: string + - description: Exact kept generation (registered nodes only); omitted reads the current target + in: query + name: generation + type: integer produces: - application/json responses: diff --git a/contracts/agents-api/sandbox-deployment.md b/contracts/agents-api/sandbox-deployment.md index 4cafb14fc..d7ced7fd4 100644 --- a/contracts/agents-api/sandbox-deployment.md +++ b/contracts/agents-api/sandbox-deployment.md @@ -21,7 +21,7 @@ for the operator workflow. Generated schemas cover the | --- | --- | --- | | `GET /core/v1/sandbox/deployment` | Core key | Read the safe active configuration and retained-resource counts | | `POST /core/v1/sandbox/deployment` | Core key | Select the initial provider, resources and Runtime | -| `PUT /core/v1/sandbox/deployment` | Core key | Update E2B online; node providers still require zero retained and pending resources | +| `PUT /core/v1/sandbox/deployment` | Core key | Advance the same-provider target online while retaining existing ownership | | `POST /core/v1/sandbox/deployment/reset` | Core key | Start or escalate a durable hosted clear | | `DELETE /core/v1/sandbox/deployment/reset?expected_generation=N` | Core key | Cancel the remaining clear without restoring archived work | | `GET /api/v1/sandbox-node/configuration` | Enrollment token or retained node credential | Read the active node installation configuration without consuming enrollment | @@ -208,9 +208,9 @@ PUT accepts the same provider and no active reset. Send the observed generation once; never replay an uncertain mutation automatically. E2B changes apply online: new allocations use the newly committed specification and existing allocations retain their immutable deployment generation. No node, token or owner epoch is -retired by an E2B update. Docker/microsandbox still require zero held resources, -prepare and drain before commit, and retire old nodes/tokens. Their online update -protocol is not implemented yet. +retired by a same-provider update. Docker/microsandbox advance only the target; +each node prepares it independently while continuing to serve its qualified old +pin. No execution drain or reenrollment accompanies a target change. For E2B PUT, omit `e2b.api_key` to preserve the current key. Omitted-key identical selection is a no-op. Explicit nonempty key submission, including the same key, @@ -271,26 +271,28 @@ The previous count uses the same snapshot as resource totals: old-generation unreleased allocations plus old-generation pending placements without allocations, never counting one resource twice. E2B and unconfigured deployments return null nodes. A node deployment returns counts `{ready,preparing,failed,update_required,unknown}`. -Every nonremoved node belongs to exactly one bucket. Offline or unconfirmed current -connections are `unknown` first; an online v1 node enrolled at an older target is -`update_required`; online exact-generation provider readiness is `ready`; an online -reported diagnostic is `failed`; otherwise it is `unknown`. The existing 45-second -heartbeat and owner-epoch rules define online. Pins alone never imply readiness. Target rollout is distinct from serving readiness: -unknown/preparing/failed/update_required must not erase independently confirmed old -serving readiness. provider_ready is the last provider report and requires online; -current v1 reports its enrolled provider. Future node protocol must bind serving -readiness to the exact pin and current connection/epoch. Valid PR-G v1 online ready -nodes at the current target project ready; the client also permits an old qualified -pin with unknown target preparation for the subsequent protocol. +Every nonremoved node belongs to exactly one bucket. Offline current connections +are `unknown` first; an online v1 node enrolled at an older target is +`update_required`. Otherwise the exact target observation on the current +connection and owner epoch supplies `ready`, `preparing` or `failed`; missing +observations are `unknown`. The existing 45-second heartbeat predicate defines +online. Pins alone never imply readiness. Target rollout is independent of serving +readiness: unknown/preparing/failed/update_required does not erase an independently +confirmed old serving provider. `provider_ready` requires online presence and an +exact serving-generation observation on that connection and epoch. A v1 heartbeat +qualifies only its enrolled provider. Each node adds `rollout: {state, ready_generation, diagnostic?}`. ready_generation is the nullable durable serving pin. Diagnostic is a fixed node reason; unknown values project as `provider_unavailable`. Allocation items add `deployment_generation`. `rollout.state` is the authoritative high-frequency polling signal: poll every five seconds only while it is `preparing` or reset is nonnull. Old Sessions, failed, -update-required and offline nodes alone do not keep polling active. PR-G reports -`settled`: actual node multi-generation preparation and no-gap placement remain -PR-N work, with the zero-resource node PUT guard intact. +update-required and offline nodes alone do not keep polling active. New admission +filters online, exact serving-generation readiness, address and shared capacity +before choosing the newest qualifying pin. A full newest node does not hide a free +older node. No candidate creates no provisional Session or placement. An online +node actually preparing with available capacity returns 503 `sandbox_nodes_preparing`; +a full or offline fleet returns `runtime_node_unavailable`. To change backend, explicitly start reset: @@ -381,9 +383,11 @@ belong to this installation. This read does not consume it. An active reset prev new enrollment configuration reads. An already registered node sends its durable node credential as Bearer and its -UUID in `X-OAC-Node-ID`. Its installation, saved generation and specification -digest must match the active deployment. This read remains available in -reset so the retained node can recover its exact configuration. The old +UUID in `X-OAC-Node-ID`. Its original enrollment identity and installation remain +valid across same-provider target changes. Omitted `generation` reads the current +target; `?generation=N` reads only that node's exact current, serving-pinned or +unreleased-allocation/placement generation. Unknown or unkept history is refused. +This read remains available during reset for owned recovery. The old enrollment token cannot replace a registered node's credential. The response contains `installation_id`, `provider`, `core_url` (the installation @@ -397,9 +401,10 @@ and network policy. Enrollment at `POST /api/v1/sandbox-node/enroll` includes node stores. A specification mismatch rejects with 409 `sandbox_specification_mismatch` and a `core_url` other than the installation public URL with 409 `sandbox_node_address_mismatch`, both before token consumption. A -missing `core_url` gets 400 `invalid_request_error` with `param: "core_url"`. Retained node authentication checks the same generation and -digest. A changed local resource setting, Runtime or generation must fail rather -than rewrite the retained identity or silently use a local default. +missing `core_url` gets 400 `invalid_request_error` with `param: "core_url"`. +The original enrollment identity remains immutable. New generation preparation +uses separate exact configurations, never rewrites that identity or silently +substitutes a local default. See [node generation protocol](node-generation-protocol.md). ## What each field means per sandbox provider @@ -432,8 +437,7 @@ fields from the administrator node routes; runtime fields from the Malformed selections return 400; validated configuration diagnostics use `invalid_sandbox_configuration`. Stale generation returns 409 `sandbox_generation_stale` with safe `current_generation`; -zero-resource PUT violations return 409 `sandbox_in_use` with allocation/pending -counts; a different backend returns 409 `sandbox_reset_required` with provider names. +A different backend returns 409 `sandbox_reset_required` with provider names. Unconfigured mutations return 409 `sandbox_not_configured`. Other incompatible deployments return 409 `sandbox_deployment_conflict`. A node configuration mismatch returns 409 `sandbox_specification_mismatch`; rejected @@ -450,7 +454,7 @@ evidence of cleanup. The administrator node list and node detail report an unready provider with one fixed `diagnostic` code: `docker_unavailable`, `docker_limits_unsupported`, -`runtime_image_unavailable`, `kvm_unavailable`, +`runtime_download_failed`, `runtime_image_unavailable`, `kvm_unavailable`, `microsandbox_artifacts_unavailable`, `capacity_insufficient` or `provider_unavailable`. It is absent while the provider is ready. The node classifies the first failed readiness check and sends only the code; Core stores @@ -502,3 +506,7 @@ transition is implemented. Ordinary migration retires maintenance and resumes admission; it never interprets old maintenance as authorization to clear. Downgrade refuses an active reset or an unconfigured completed reset with generation above zero; configure a provider before downgrading, never erase the generation. + +`runtime_download_failed` means the exact Runtime artifacts could not be transferred +or verified. It is distinct from provider probe and image availability failures; +the diagnostic never contains artifact URLs, credentials or transport output. diff --git a/deploy/install/node_generations.py b/deploy/install/node_generations.py index 6e272b736..70ac8098e 100644 --- a/deploy/install/node_generations.py +++ b/deploy/install/node_generations.py @@ -204,9 +204,20 @@ def prepare(args, installer): args.bundle = None manifest, sums = installer.metadata(args.source_url, prefix="releases/" + runtime["source_commit"] + "/") installer.node_spec.verify_release(args.configuration, manifest) - release = runtime_files(root, value if retained else None, args, manifest, sums, installer) if args.provider == "microsandbox": args.runtime_home = Path(value["microsandbox"]["runtime_home"]) if retained else generation_home(root, args.configuration, base, installer) + if not retained: + # Persist the exact authorized identity before any artifact/import + # mutation. Interrupted preparation must remain discoverable after + # node restart; file presence still requires a real provider probe. + value = installer.provider_config(root / "releases" / runtime["source_commit"], args, manifest, runtime["image_id"]) + target = directory / (str(args.generation) + ".json") + if installer.existing_file(target) and installer.private_json(target) != value: + raise installer.InstallError("Immutable generation configuration differs") + atomic_json(target, value) + retained = True + release = runtime_files(root, value, args, manifest, sums, installer) + if args.provider == "microsandbox": installer.safe_directory(args.runtime_home) owner = args.runtime_home / "oac-installation.json" if not owner.exists() and any(args.runtime_home.iterdir()): diff --git a/packages/agents-client/src/sandbox-client.test.ts b/packages/agents-client/src/sandbox-client.test.ts index bb36580f4..a69e877ec 100644 --- a/packages/agents-client/src/sandbox-client.test.ts +++ b/packages/agents-client/src/sandbox-client.test.ts @@ -133,7 +133,7 @@ describe("Core sandbox credential boundaries", () => { const fetch = vi.fn().mockResolvedValue(response({ data: [unready, { ...unready, diagnostic: "future_code" }, node] })); const { data } = await new SandboxAdminClient({ baseUrl: "/core/v1/sandbox", fetch }).listNodes(); expect(data.map((entry) => entry.diagnostic)).toEqual(["kvm_unavailable", "provider_unavailable", undefined]); - expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); }); it("requires each node's enrollment ID: a string, or null for nodes enrolled before Core recorded it", async () => { const fetch = vi.fn().mockResolvedValue(response({ data: [node, unready] })); @@ -365,7 +365,13 @@ it("keeps omitted-key public-URL errors actionable without reflecting a stored k }); it.each(["unknown", "preparing", "failed", "update_required"])("does not erase old serving readiness for target %s", async (state) => { - const value = { ...node, rollout: { state, ready_generation: 1, ...(state === "failed" ? { diagnostic: "runtime_image_unavailable" } : {}) } }; + const value = { ...node, rollout: { state, ready_generation: 1, ...(state === "failed" ? { diagnostic: "runtime_download_failed" } : {}) } }; const client = new SandboxAdminClient({ fetch: async () => response({ data: [value] }) }); expect((await client.listNodes()).data[0]).toEqual(value); }); + + it("preserves artifact transfer diagnostics on node and target without exposing transport details", async () => { + const value = { ...unready, online: true, diagnostic: "runtime_download_failed", rollout: { state: "failed", ready_generation: 1, diagnostic: "runtime_download_failed" } }; + const client = new SandboxAdminClient({ fetch: async () => response({ data: [value] }) }); + expect((await client.listNodes()).data[0]).toEqual(value); + }); diff --git a/packages/agents-client/src/sandbox-client.ts b/packages/agents-client/src/sandbox-client.ts index 3a8a8531f..fa1e72868 100644 --- a/packages/agents-client/src/sandbox-client.ts +++ b/packages/agents-client/src/sandbox-client.ts @@ -10,6 +10,7 @@ export type SandboxNodeDiagnostic = | "provider_unavailable" | "docker_unavailable" | "docker_limits_unsupported" + | "runtime_download_failed" | "runtime_image_unavailable" | "kvm_unavailable" | "microsandbox_artifacts_unavailable" @@ -259,7 +260,7 @@ function projectDeployment(value: unknown): SandboxDeployment { const nodeFields = ["rollout", "id", "name", "provider", "online", "provider_ready", "cpu_count", "available_memory_bytes", "available_disk_bytes", "running", "snapshots", "last_seen_at", "max_active", "max_retained", "active", "reserved", "retained", "cleanup_pending", "created_at", "core_url", "enrollment_id"]; -const nodeDiagnostics = new Set(["provider_unavailable", "docker_unavailable", "docker_limits_unsupported", "runtime_image_unavailable", "kvm_unavailable", "microsandbox_artifacts_unavailable", "capacity_insufficient"]); +const nodeDiagnostics = new Set(["provider_unavailable", "docker_unavailable", "docker_limits_unsupported", "runtime_download_failed", "runtime_image_unavailable", "kvm_unavailable", "microsandbox_artifacts_unavailable", "capacity_insufficient"]); /** Core omits an empty `diagnostic`, so a present one is a code; an unknown code reads as provider_unavailable. */ function projectNode(node: Record): SandboxNode { const { diagnostic, rollout, ...rest } = node; diff --git a/services/agents-api/cmd/sandbox-node/generations.go b/services/agents-api/cmd/sandbox-node/generations.go index f064c9321..8ab4e8cdb 100644 --- a/services/agents-api/cmd/sandbox-node/generations.go +++ b/services/agents-api/cmd/sandbox-node/generations.go @@ -119,9 +119,7 @@ func runGenerations(ctx context.Context, configFile, stateDir string) error { if err != nil { return value, err } - probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - return value, value.Probe(probeCtx) + return value, nil }, Remove: func(ctx context.Context, value node.GenerationProvider) error { ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) diff --git a/services/agents-api/internal/api/errors.go b/services/agents-api/internal/api/errors.go index d3ddfa240..bfb714ecd 100644 --- a/services/agents-api/internal/api/errors.go +++ b/services/agents-api/internal/api/errors.go @@ -146,6 +146,8 @@ func writeStoreError(w http.ResponseWriter, r *http.Request, err error, notFound writeError(w, http.StatusConflict, "runtime_node_in_use", "The sandbox node retains allocations, snapshots, reservations or pending cleanup.") case errors.Is(err, store.ErrRuntimeLocalNodeConfigured): writeError(w, http.StatusConflict, "runtime_local_node_configured", "The local sandbox node is enabled in deployment configuration. Drain it with the previous release and remove its file-managed configuration before replacing it.") + case errors.Is(err, store.ErrSandboxNodesPreparing): + writeError(w, http.StatusServiceUnavailable, "sandbox_nodes_preparing", "Sandbox nodes are preparing the requested Runtime.") case errors.Is(err, store.ErrRuntimeNodeUnavailable): writeError(w, http.StatusServiceUnavailable, "runtime_node_unavailable", "The selected sandbox node is unavailable or has no capacity.") diff --git a/services/agents-api/internal/execution/sandbox_deployment_switch.go b/services/agents-api/internal/execution/sandbox_deployment_switch.go index 6ea7dbf9b..d86727eaa 100644 --- a/services/agents-api/internal/execution/sandbox_deployment_switch.go +++ b/services/agents-api/internal/execution/sandbox_deployment_switch.go @@ -2,7 +2,6 @@ package execution import ( "context" - "errors" "time" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" @@ -176,15 +175,10 @@ func (w *Worker) UpdateSandboxDeployment(ctx context.Context, input store.Sandbo return store.RuntimeDeploymentView{}, err } } - } else if err := m.pauseDeployment(ctx); err != nil { - return store.RuntimeDeploymentView{}, errors.Join(err, m.restoreCommittedDeployment()) } input.SandboxDeploymentSetupRequest = withTemplateBuild(input.SandboxDeploymentSetupRequest, candidate) result, err := m.store.UpdateSandboxDeployment(ctx, m.setupInstallationID, input) if err != nil { - if input.Provider != "e2b" { - err = errors.Join(err, m.restoreCommittedDeployment()) - } return store.RuntimeDeploymentView{}, err } m.publishDeployment(candidate, result) diff --git a/services/agents-api/internal/sandbox/node/agent.go b/services/agents-api/internal/sandbox/node/agent.go index 02c25ed84..dd6fff58a 100644 --- a/services/agents-api/internal/sandbox/node/agent.go +++ b/services/agents-api/internal/sandbox/node/agent.go @@ -246,7 +246,9 @@ func (a *agent) connect(ctx context.Context) error { var controlDone chan struct{} controlWork := make(chan []sandbox.GenerationRetention, 1) if a.config.Generations != nil { - a.config.Generations.Deployment(*welcome.Deployment) + if err := a.config.Generations.Deployment(*welcome.Deployment); err != nil { + return err + } controlDone = make(chan struct{}) go func() { defer close(controlDone); a.garbageCollection(connectionCtx, current, controlWork) }() defer func() { cancel(); <-controlDone }() @@ -266,7 +268,9 @@ func (a *agent) connect(ctx context.Context) error { if f.OwnerEpoch != current.epoch { return sandbox.ErrOwnership } - a.config.Generations.Deployment(*f.Deployment) + if err := a.config.Generations.Deployment(*f.Deployment); err != nil { + return err + } } continue } diff --git a/services/agents-api/internal/sandbox/node/generation_agent.go b/services/agents-api/internal/sandbox/node/generation_agent.go index e5918b578..be3021445 100644 --- a/services/agents-api/internal/sandbox/node/generation_agent.go +++ b/services/agents-api/internal/sandbox/node/generation_agent.go @@ -36,7 +36,9 @@ func (a *agent) acceptRetention(c *agentConnection, f frame, work chan<- []sandb return sandbox.ErrOwnership } } - a.config.Generations.Deployment(*f.Deployment) + if err := a.config.Generations.Deployment(*f.Deployment); err != nil { + return err + } select { case work <- reply.Retentions: c.pending = nil diff --git a/services/agents-api/internal/sandbox/node/generation_connection_test.go b/services/agents-api/internal/sandbox/node/generation_connection_test.go new file mode 100644 index 000000000..0879d90d0 --- /dev/null +++ b/services/agents-api/internal/sandbox/node/generation_connection_test.go @@ -0,0 +1,87 @@ +package node + +import ( + "context" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" +) + +func TestGenerationWireRoutesOldOwnershipAndCurrentTargetSeparately(t *testing.T) { + id := identity() + id.DeploymentGeneration = 1 + id.SpecificationDigest = strings.Repeat("a", 64) + manager := generationFixture(17) + original := manager.values[1].value.Provider.(*fakeProvider) + current := manager.values[17].value.Provider.(*fakeProvider) + ninth := manager.values[9].value.Provider.(*fakeProvider) + hub := NewHub(HubOptions{ + Authenticate: func(context.Context, string, string) (Identity, error) { return id, nil }, + OwnerEpoch: func(context.Context) (uint64, error) { return 7, nil }, + Generations: func(_ context.Context, _ Identity, _ string, _ uint64, h Health) error { + if len(h.Generations) > 8 { + t.Error("unbounded wire observations") + } + return nil + }, + Retention: func(_ context.Context, _ Identity, _ string, _ uint64, refs []sandbox.GenerationReference) (sandbox.NodeDeployment, []sandbox.GenerationRetention, error) { + result := make([]sandbox.GenerationRetention, 0, len(refs)) + for _, ref := range refs { + result = append(result, sandbox.GenerationRetention{GenerationReference: ref, Keep: true}) + } + return sandbox.NodeDeployment{Generation: 17, SpecificationDigest: id.SpecificationDigest, ServingGeneration: &id.DeploymentGeneration}, result, nil + }, + }) + server := httptest.NewServer(hub) + defer server.Close() + defer hub.Close() + dir := stateDir(t) + stored, err := InitIdentity(dir, server.URL, id) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { + done <- Run(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: dir, Identity: id, Credential: stored.Credential, Generations: manager}) + }() + defer func() { + cancel() + select { + case err := <-done: + if err != nil { + t.Error(err) + } + case <-time.After(5 * time.Second): + t.Error("generation node did not stop") + } + }() + wait(t, func() bool { return hub.Online(id.NodeID) }) + for _, generation := range []uint64{1, 17, 9} { + proxy := hub.GenerationProvider("docker", func(context.Context, sandbox.Reference) (string, uint64, error) { return id.NodeID, generation, nil }) + ref := reference() + if _, err := proxy.GetInfo(ctx, ref); err != nil { + t.Fatal("retained generation info failed", generation, err) + } + if generation != 9 { + if _, err := proxy.Create(ctx, sandbox.Bootstrap{Reference: ref}); err != nil { + t.Fatal("exact ready provider Create failed", generation, err) + } + } + } + for generation, p := range map[int]*fakeProvider{1: original, 17: current, 9: ninth} { + p.mu.Lock() + reads, creates := p.reads, p.creates + p.mu.Unlock() + want := 1 + if generation == 9 { + want = 0 + } + if reads != 1 || creates != want { + t.Fatal("generation request substituted another provider", generation, reads, creates) + } + } +} diff --git a/services/agents-api/internal/sandbox/node/generations.go b/services/agents-api/internal/sandbox/node/generations.go index eb4bbcd48..36ff9763a 100644 --- a/services/agents-api/internal/sandbox/node/generations.go +++ b/services/agents-api/internal/sandbox/node/generations.go @@ -42,6 +42,7 @@ type GenerationManager struct { values map[uint64]*localGeneration target sandbox.NodeDeployment statusCursor, probeCursor, recoveryCursor uint64 + priorityProbe uint64 options GenerationManagerOptions ctx context.Context cancel context.CancelFunc @@ -88,11 +89,20 @@ func (m *GenerationManager) Close() { } } -func (m *GenerationManager) Deployment(value sandbox.NodeDeployment) { +func (m *GenerationManager) Deployment(value sandbox.NodeDeployment) error { m.mu.Lock() defer m.mu.Unlock() + if !validGeneration(value.Generation) || !validSpecificationDigest(value.SpecificationDigest) || value.ServingGeneration != nil && (!validGeneration(*value.ServingGeneration) || *value.ServingGeneration > value.Generation) { + return sandbox.ErrOwnership + } if value.Generation < m.target.Generation { - return + return nil + } + if g := m.values[value.Generation]; g != nil && g.value.SpecificationDigest != value.SpecificationDigest { + return sandbox.ErrOwnership + } + if value.Generation == m.target.Generation && value.SpecificationDigest != m.target.SpecificationDigest { + return sandbox.ErrOwnership } if value.Generation != m.target.Generation && m.preparingCancel != nil { m.preparingCancel() @@ -105,6 +115,7 @@ func (m *GenerationManager) Deployment(value sandbox.NodeDeployment) { case m.wake <- struct{}{}: default: } + return nil } func (m *GenerationManager) orderedLocked(after uint64) []uint64 { @@ -306,12 +317,25 @@ func (m *GenerationManager) probeLoop() { } m.mu.Lock() keys := m.orderedLocked(m.probeCursor) + // Alternate priority probes with the fair retained-provider cursor. A large + // history cannot postpone serving/target qualification or starve old owners. + m.priorityProbe++ + priority := uint64(0) + if m.priorityProbe%2 == 1 { + priority = m.target.Generation + if m.priorityProbe%4 == 3 && m.target.ServingGeneration != nil { + priority = *m.target.ServingGeneration + } + keys = append([]uint64{priority}, keys...) + } var g *localGeneration for _, key := range keys { value := m.values[key] - if value.value.Provider != nil && !value.removing && !value.preparing && !value.repairing { + if value != nil && value.value.Provider != nil && !value.removing && !value.preparing && !value.repairing { g = value - m.probeCursor = key + if key != priority { + m.probeCursor = key + } g.refs++ break } diff --git a/services/agents-api/internal/sandbox/node/generations_test.go b/services/agents-api/internal/sandbox/node/generations_test.go index bfdec8642..ce94f00d2 100644 --- a/services/agents-api/internal/sandbox/node/generations_test.go +++ b/services/agents-api/internal/sandbox/node/generations_test.go @@ -278,3 +278,21 @@ func TestRestartRecoversExactOlderGenerationBeforeAdvertisingReadiness(t *testin } release() } + +func TestGenerationDeploymentRejectsConflictingImmutableIdentity(t *testing.T) { + m := generationFixture(3) + bad := m.target + bad.SpecificationDigest = strings.Repeat("b", 64) + if err := m.Deployment(bad); !errors.Is(err, sandbox.ErrOwnership) { + t.Fatal("conflicting target accepted", err) + } + if m.target.SpecificationDigest != strings.Repeat("a", 64) || m.values[3].value.SpecificationDigest != m.target.SpecificationDigest { + t.Fatal("rejected metadata changed provider identity") + } + bad = m.target + future := uint64(4) + bad.ServingGeneration = &future + if err := m.Deployment(bad); !errors.Is(err, sandbox.ErrOwnership) { + t.Fatal("future serving pin accepted", err) + } +} diff --git a/services/agents-api/internal/sandbox/node/hub.go b/services/agents-api/internal/sandbox/node/hub.go index 9e755c69f..ddefcfdc5 100644 --- a/services/agents-api/internal/sandbox/node/hub.go +++ b/services/agents-api/internal/sandbox/node/hub.go @@ -409,6 +409,14 @@ func (h *Hub) recordHealth(ctx context.Context, p *peer, health Health) error { if err != nil { return err } + if len(grants) != len(refs) { + return sandbox.ErrInvalid + } + for i, grant := range grants { + if grant.GenerationReference != refs[i] { + return sandbox.ErrInvalid + } + } p.mu.Lock() for i, g := range health.Generations { if grants[i].Keep { diff --git a/services/agents-api/internal/store/runtime_deployment_test.go b/services/agents-api/internal/store/runtime_deployment_test.go index b09fe85a2..5d5392ee6 100644 --- a/services/agents-api/internal/store/runtime_deployment_test.go +++ b/services/agents-api/internal/store/runtime_deployment_test.go @@ -36,7 +36,7 @@ func legacyRuntimeSpecification(t *testing.T, w *Store, provider string) { if _, err := w.pool.Exec(t.Context(), "UPDATE runtime_deployment SET specification=$1", raw); err != nil { t.Fatal(err) } - if _, err := w.pool.Exec(t.Context(), "UPDATE runtime_nodes SET deployment_generation=1,specification_digest=$1", spec.Digest(provider)); err != nil { + if _, err := w.pool.Exec(t.Context(), "UPDATE runtime_nodes SET deployment_generation=1,ready_generation=1,specification_digest=$1", spec.Digest(provider)); err != nil { t.Fatal(err) } } diff --git a/services/agents-api/internal/store/runtime_node_configuration.go b/services/agents-api/internal/store/runtime_node_configuration.go index fe7821c58..3bdb58f90 100644 --- a/services/agents-api/internal/store/runtime_node_configuration.go +++ b/services/agents-api/internal/store/runtime_node_configuration.go @@ -2,6 +2,7 @@ package store import ( "context" + "math" "time" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/db/sqlc" @@ -31,6 +32,9 @@ func (s *Store) RuntimeNodeConfiguration(ctx context.Context, nodeID, token stri // pin and unreleased ownership. It is never a general history read. func (s *Store) RuntimeNodeGenerationConfiguration(ctx context.Context, nodeID, token string, generation uint64) (RuntimeNodeConfiguration, error) { var result RuntimeNodeConfiguration + if generation > math.MaxInt64 { + return result, ErrInvalidInput + } err := s.runtimeDeploymentTransaction(ctx, func(q *sqlc.Queries, d sqlc.RuntimeDeployment) error { var node *sqlc.RuntimeNode var installation pgtype.UUID @@ -66,6 +70,11 @@ func (s *Store) RuntimeNodeGenerationConfiguration(ctx context.Context, nodeID, if d.Mode != "nodes" { return ErrSandboxDeploymentConflict } + if node != nil { + if err := validateNodeEnrollmentIdentity(ctx, q, d, *node); err != nil { + return err + } + } selected := uint64(d.Generation) if generation != 0 { if node == nil { diff --git a/services/agents-api/internal/store/runtime_node_generations.go b/services/agents-api/internal/store/runtime_node_generations.go index b8546f76e..7f4d0061c 100644 --- a/services/agents-api/internal/store/runtime_node_generations.go +++ b/services/agents-api/internal/store/runtime_node_generations.go @@ -3,10 +3,12 @@ package store import ( "context" "encoding/json" + "errors" "math" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/db/sqlc" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" ) @@ -84,6 +86,11 @@ func (s *Store) RuntimeNodeRetention(ctx context.Context, nodeID, connectionID s return ErrRuntimeSpecificationMismatch } } + if !kept { + if err := q.DeleteNodeGenerationStatus(ctx, sqlc.DeleteNodeGenerationStatusParams{NodeID: n.ID, Generation: int64(ref.Generation)}); err != nil { + return err + } + } grants = append(grants, sandbox.GenerationRetention{GenerationReference: ref, Keep: kept}) } return nil @@ -136,3 +143,19 @@ func recordNodeGenerations(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeD } return q.RefreshNodeServingReadiness(ctx, sqlc.RefreshNodeServingReadinessParams{ID: n.ID, ProtocolVersion: protocol}) } + +// Enrollment identity survives collection of its old specification. Whenever +// that specification is still authoritative, its exact digest must agree. +func validateNodeEnrollmentIdentity(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment, n sqlc.RuntimeNode) error { + if n.DeploymentGeneration <= 0 || n.DeploymentGeneration > d.Generation || !validRuntimeDigest(n.SpecificationDigest) { + return ErrRuntimeSpecificationMismatch + } + spec, err := nodeGenerationSpec(ctx, q, d, uint64(n.DeploymentGeneration)) + if errors.Is(err, pgx.ErrNoRows) && n.DeploymentGeneration < d.Generation { + return nil + } + if err != nil || spec.Digest(d.ProviderKind) != n.SpecificationDigest { + return ErrRuntimeSpecificationMismatch + } + return nil +} diff --git a/services/agents-api/internal/store/runtime_node_generations_test.go b/services/agents-api/internal/store/runtime_node_generations_test.go new file mode 100644 index 000000000..b78ce2086 --- /dev/null +++ b/services/agents-api/internal/store/runtime_node_generations_test.go @@ -0,0 +1,237 @@ +package store + +import ( + "database/sql" + "errors" + "github.com/jackc/pgx/v5/stdlib" + "github.com/pressly/goose/v3" + "os" + "testing" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/google/uuid" +) + +func changeNodeTarget(t *testing.T, w *Store, view RuntimeDeploymentView, input SandboxDeploymentSetupRequest) (RuntimeDeploymentView, SandboxDeploymentSetupRequest) { + t.Helper() + input.Resources.CPUs++ + next, err := w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: view.Generation}) + if err != nil { + t.Fatal(err) + } + if next.Generation != view.Generation+1 || next.OwnerEpoch != view.OwnerEpoch { + t.Fatal("target change replaced execution ownership", next) + } + return next, input +} + +func generationHeartbeat(t *testing.T, s *Store, node RuntimeNodeEnrollment, connection string, view RuntimeDeploymentView, state string) { + t.Helper() + err := s.HeartbeatRuntimeNodeGenerations(t.Context(), node.NodeID, connection, view.OwnerEpoch, RuntimeNodeHealth{}, []sandbox.GenerationStatus{{Generation: view.Generation, SpecificationDigest: view.SpecificationDigest, State: state}}) + if err != nil { + t.Fatal(err) + } +} + +func placedGeneration(t *testing.T, s *Store, session Session) (string, int64) { + t.Helper() + var node string + var generation int64 + if err := s.pool.QueryRow(t.Context(), "SELECT node_id::text,deployment_generation FROM runtime_placements WHERE environment_id=$1", session.Environment.ID).Scan(&node, &generation); err != nil { + t.Fatal(err) + } + return node, generation +} + +func TestNodeGenerationsCapacityFallbackAndImmutablePending(t *testing.T) { + for _, provider := range []string{"microsandbox", "docker"} { + t.Run(provider, func(t *testing.T) { + s, w, first, input := webSpecificationFixture(t, provider) + a := specificationNode(t, s, first) + b := specificationNode(t, s, first) + ca := onlineManagerNode(t, s, a.NodeID) + cb := onlineManagerNode(t, s, b.NodeID) + tenant := uuid.NewString() + pending, err := s.CreateSession(t.Context(), tenant, managerSessionInput(uuid.NewString())) + if err != nil { + t.Fatal(err) + } + pendingNode, pendingGeneration := placedGeneration(t, s, pending) + token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 1, MaxRetained: 2})) + if err != nil { + t.Fatal(err) + } + second, input := changeNodeTarget(t, w, first, input) + if _, err = s.RuntimeNodeConfiguration(t.Context(), "", token); err != nil { + t.Fatal("update retired enrollment", err) + } + if _, err = s.AuthenticateRuntimeNode(t.Context(), a.NodeID, a.Credential); err != nil { + t.Fatal("update retired node", err) + } + generationHeartbeat(t, s, a, ca, second, "preparing") + fallback, err := s.CreateSession(t.Context(), tenant, managerSessionInput(uuid.NewString())) + if err != nil { + t.Fatal("target preparation suppressed fallback", err) + } + if _, g := placedGeneration(t, s, fallback); g != 1 { + t.Fatal(g) + } + generationHeartbeat(t, s, a, ca, second, "ready") + newest, err := s.CreateSession(t.Context(), tenant, managerSessionInput(uuid.NewString())) + if err != nil { + t.Fatal(err) + } + if n, g := placedGeneration(t, s, newest); n != a.NodeID || g != 2 { + t.Fatal(n, g) + } + // Capacity remains shared across generations. A full newest pin cannot hide B. + if _, err = s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET max_active=1 WHERE id=$1", a.NodeID); err != nil { + t.Fatal(err) + } + old, err := s.CreateSession(t.Context(), tenant, managerSessionInput(uuid.NewString())) + if err != nil { + t.Fatal(err) + } + if n, g := placedGeneration(t, s, old); n != b.NodeID || g != 1 { + t.Fatal("newest full hid older free node", n, g) + } + third, _ := changeNodeTarget(t, w, second, input) + // B finishes a superseded target late: it may describe ownership but cannot adopt it. + generationHeartbeat(t, s, b, cb, second, "ready") + nodes, err := s.ListRuntimeNodes(t.Context()) + if err != nil { + t.Fatal(err) + } + for _, n := range nodes { + want := uint64(1) + if n.ID == a.NodeID { + want = 2 + } + if n.Rollout.ReadyGeneration == nil || *n.Rollout.ReadyGeneration != want || !n.ProviderReady { + t.Fatal("late readiness moved pin or erased serving readiness", n) + } + } + owner, err := w.ReserveRuntimeAllocation(t.Context(), tenant, pending.Environment.ID, first.InstallationID, device.HashCredential("runtime")) + if err != nil { + t.Fatal(err) + } + if owner.NodeID != pendingNode || owner.DeploymentGeneration != uint64(pendingGeneration) { + t.Fatal("pending placement moved", owner) + } + n, g, err := s.ResolveRuntimeGeneration(t.Context(), sandbox.Reference{TenantID: tenant, EnvironmentID: owner.EnvironmentID, AllocationID: owner.ID}) + if err != nil || n != pendingNode || g != 1 { + t.Fatal(n, g, err) + } + for _, v := range []RuntimeDeploymentView{first, second, third} { + target := a + if v.Generation == 1 { + target = b + } + got, err := s.RuntimeNodeGenerationConfiguration(t.Context(), target.NodeID, target.Credential, v.Generation) + if err != nil || got.SpecificationDigest != v.SpecificationDigest { + t.Fatal("kept generation unrecoverable", v.Generation, err) + } + } + }) + } +} + +func TestNodeGenerationsReconnectAndV1Fallback(t *testing.T) { + s, w, first, input := webSpecificationFixture(t, "docker") + node := specificationNode(t, s, first) + old := onlineManagerNode(t, s, node.NodeID) + second, _ := changeNodeTarget(t, w, first, input) + nodes, err := s.ListRuntimeNodes(t.Context()) + if err != nil || !nodes[0].ProviderReady || nodes[0].Rollout.State != "update_required" { + t.Fatal(nodes, err) + } + if _, err = s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())); err != nil { + t.Fatal("v1 lost fallback", err) + } + connection := uuid.NewString() + if err = s.ConnectRuntimeNode(t.Context(), node.NodeID, connection, first.OwnerEpoch); err != nil { + t.Fatal(err) + } + if err = s.HeartbeatRuntimeNodeGenerations(t.Context(), node.NodeID, old, first.OwnerEpoch, RuntimeNodeHealth{}, []sandbox.GenerationStatus{{Generation: second.Generation, SpecificationDigest: second.SpecificationDigest, State: "ready"}}); !errors.Is(err, ErrRuntimeNodeCredential) { + t.Fatal("old connection qualified", err) + } + nodes, err = s.ListRuntimeNodes(t.Context()) + if err != nil || nodes[0].ProviderReady || *nodes[0].Rollout.ReadyGeneration != 1 { + t.Fatal("reconnect inherited readiness or lost pin", nodes, err) + } + if _, err = s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())); !errors.Is(err, ErrRuntimeNodeUnavailable) { + t.Fatal("unconfirmed connection admitted", err) + } + if err = s.HeartbeatRuntimeNode(t.Context(), node.NodeID, connection, first.OwnerEpoch, RuntimeNodeHealth{ProviderReady: true}); err != nil { + t.Fatal(err) + } + if _, err = s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())); err != nil { + t.Fatal("fresh v1 readiness did not restore fallback", err) + } +} + +func TestNodeGenerationDowngradePreservesServingProtocol(t *testing.T) { + for _, mode := range []string{"v2", "old_v1", "current_v1"} { + t.Run(mode, func(t *testing.T) { + s, w, first, input := webSpecificationFixture(t, "docker") + node := specificationNode(t, s, first) + switch mode { + case "v2": + generationHeartbeat(t, s, node, onlineManagerNode(t, s, node.NodeID), first, "ready") + case "old_v1": + changeNodeTarget(t, w, first, input) + } + db := sql.OpenDB(stdlib.GetConnector(*s.pool.Config().ConnConfig)) + defer db.Close() + migration, err := goose.NewProvider(goose.DialectPostgres, db, os.DirFS("../../migrations"), goose.WithTableName("agents_api_schema_version")) + if err != nil { + t.Fatal(err) + } + _, err = migration.DownTo(t.Context(), 81) + if mode == "current_v1" { + if err != nil { + t.Fatal("safe v1 downgrade refused", err) + } + } else { + if err == nil { + t.Fatal("downgrade discarded required node protocol") + } + if _, err = s.RuntimeNodeGenerationConfiguration(t.Context(), node.NodeID, node.Credential, 1); err != nil { + t.Fatal("refused downgrade damaged retained recovery", err) + } + if err = s.RemoveRuntimeNode(t.Context(), node.NodeID); err != nil { + t.Fatal(err) + } + if _, err = migration.DownTo(t.Context(), 81); err != nil { + t.Fatal("removed node blocked downgrade", err) + } + } + if _, err = migration.Up(t.Context()); err != nil { + t.Fatal("node schema could not upgrade again", err) + } + }) + } +} + +func TestNodeGenerationPreparationRefusalCreatesNoProvisionalOwnership(t *testing.T) { + s, _, first, _ := webSpecificationFixture(t, "docker") + node := specificationNode(t, s, first) + connection := uuid.NewString() + if err := s.ConnectRuntimeNode(t.Context(), node.NodeID, connection, first.OwnerEpoch); err != nil { + t.Fatal(err) + } + generationHeartbeat(t, s, node, connection, first, "preparing") + tenant := uuid.NewString() + if _, err := s.CreateSession(t.Context(), tenant, managerSessionInput(uuid.NewString())); !errors.Is(err, ErrSandboxNodesPreparing) { + t.Fatal("actual preparation was not identified", err) + } + var sessions, placements int + if err := s.pool.QueryRow(t.Context(), "SELECT (SELECT count(*) FROM sessions WHERE tenant_id=$1),(SELECT count(*) FROM runtime_placements)", tenant).Scan(&sessions, &placements); err != nil || sessions != 0 || placements != 0 { + t.Fatal("refusal left provisional ownership", sessions, placements, err) + } + generationHeartbeat(t, s, node, connection, first, "failed") + if _, err := s.CreateSession(t.Context(), tenant, managerSessionInput(uuid.NewString())); !errors.Is(err, ErrRuntimeNodeUnavailable) { + t.Fatal("failed preparation advertised active work", err) + } +} diff --git a/services/agents-api/internal/store/runtime_node_types.go b/services/agents-api/internal/store/runtime_node_types.go index 6cf19f7f3..2f90dc9fc 100644 --- a/services/agents-api/internal/store/runtime_node_types.go +++ b/services/agents-api/internal/store/runtime_node_types.go @@ -7,6 +7,7 @@ import ( ) var ( + ErrSandboxNodesPreparing = errors.New("sandbox nodes are preparing the target generation") ErrRuntimeNodeUnavailable = errors.New("sandbox node unavailable") ErrRuntimeNodeInUse = errors.New("sandbox node retains resources") ErrRuntimeNodeCredential = errors.New("invalid sandbox node credential") @@ -54,7 +55,7 @@ type RuntimeNodeEnrollment struct { type RuntimeNodeHealth struct { Host *RuntimeNodeHost `json:"-"` // Fixed reason for the last reported unreadiness; absent while the provider is ready. Clients treat an unknown value as provider_unavailable. - Diagnostic string `json:"diagnostic,omitempty" enums:"provider_unavailable,docker_unavailable,docker_limits_unsupported,runtime_image_unavailable,kvm_unavailable,microsandbox_artifacts_unavailable,capacity_insufficient"` + Diagnostic string `json:"diagnostic,omitempty" enums:"provider_unavailable,docker_unavailable,docker_limits_unsupported,runtime_download_failed,runtime_image_unavailable,kvm_unavailable,microsandbox_artifacts_unavailable,capacity_insufficient"` ProviderReady bool `json:"provider_ready"` CPUCount *int64 `json:"cpu_count"` AvailableMemoryBytes *int64 `json:"available_memory_bytes"` @@ -131,7 +132,7 @@ type SandboxNodeRollout struct { State string `json:"state" enums:"ready,preparing,failed,update_required,unknown"` // Durable serving-generation pin; online and provider_ready still gate placement. ReadyGeneration *uint64 `json:"ready_generation" extensions:"x-nullable"` - Diagnostic string `json:"diagnostic,omitempty"` + Diagnostic string `json:"diagnostic,omitempty" enums:"provider_unavailable,docker_unavailable,docker_limits_unsupported,runtime_download_failed,runtime_image_unavailable,kvm_unavailable,microsandbox_artifacts_unavailable,capacity_insufficient"` } type SandboxRolloutNodes struct { Ready int64 `json:"ready"` diff --git a/services/agents-api/internal/store/runtime_nodes.go b/services/agents-api/internal/store/runtime_nodes.go index ff5eb64b7..b18835e12 100644 --- a/services/agents-api/internal/store/runtime_nodes.go +++ b/services/agents-api/internal/store/runtime_nodes.go @@ -264,6 +264,9 @@ func (s *Store) AuthenticateRuntimeNode(ctx context.Context, nodeID, credential } return nodeIdentity(n, d.ProviderKind), nil } + if err := validateNodeEnrollmentIdentity(ctx, s.queries, d, n); err != nil { + return RuntimeNodeIdentity{}, err + } // Reset retires nodes before another backend lineage can be selected. // Enrollment generation and digest remain immutable identity history; the // current target and per-generation readiness do not replace that history. diff --git a/services/agents-api/internal/store/runtime_placements.go b/services/agents-api/internal/store/runtime_placements.go index 44af7d279..5dddee480 100644 --- a/services/agents-api/internal/store/runtime_placements.go +++ b/services/agents-api/internal/store/runtime_placements.go @@ -45,8 +45,12 @@ func reserveRuntimePlacement(ctx context.Context, q *sqlc.Queries, session pgtyp return err } var chosen *sqlc.ListRuntimeNodesRow + preparing := false for i := range rows { n := &rows[i] + if n.Online && n.TargetState == "preparing" && n.Active < int64(n.MaxActive) && n.Retained < int64(n.MaxRetained) && (!d.WebManaged || n.CoreUrl == publicURL) { + preparing = true + } if !n.Online || !n.ServingReady || !n.ReadyGeneration.Valid || n.Active >= int64(n.MaxActive) || n.Retained >= int64(n.MaxRetained) || (d.WebManaged && n.CoreUrl != publicURL) { continue } @@ -55,6 +59,9 @@ func reserveRuntimePlacement(ctx context.Context, q *sqlc.Queries, session pgtyp } } if chosen == nil { + if preparing { + return ErrSandboxNodesPreparing + } return ErrRuntimeNodeUnavailable } return q.CreateSessionRuntimePlacement(ctx, sqlc.CreateSessionRuntimePlacementParams{SessionID: session, NodeID: chosen.ID, Generation: chosen.ReadyGeneration.Int64}) @@ -103,7 +110,7 @@ func (s *Store) ResolveRuntimeGeneration(ctx context.Context, ref sandbox.Refere if err != nil { return "", 0, err } - if a.ID != ref.AllocationID || a.NodeID == "" || a.DeploymentGeneration == 0 { + if a.State == "released" || a.ID != ref.AllocationID || a.NodeID == "" || a.DeploymentGeneration == 0 { return "", 0, ErrRuntimeNodeUnavailable } return a.NodeID, a.DeploymentGeneration, nil diff --git a/services/agents-api/internal/store/sandbox_deployment_mutations.go b/services/agents-api/internal/store/sandbox_deployment_mutations.go index 6d41dc3d6..9dcd93f1f 100644 --- a/services/agents-api/internal/store/sandbox_deployment_mutations.go +++ b/services/agents-api/internal/store/sandbox_deployment_mutations.go @@ -242,16 +242,7 @@ func checkSandboxSwitch(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDepl if d.ProviderKind != input.Provider || unspecifiedNodeDeployment(d) { return &SandboxResetRequiredError{CurrentProvider: d.ProviderKind, RequestedProvider: input.Provider} } - if d.ProviderKind == "e2b" { - return nil - } - resources, err := q.CountRuntimeDeploymentResources(ctx) - if err != nil { - return err - } - if resources.Allocations != 0 || resources.Pending != 0 { - return &SandboxInUseError{SandboxDeploymentResources{Allocations: resources.Allocations, Pending: resources.Pending}} - } + return nil } @@ -285,23 +276,12 @@ func (s *Store) UpdateSandboxDeployment(ctx context.Context, installation string if err := s.saveSandboxSelection(ctx, q, d, input.SandboxDeploymentSetupRequest); err != nil { return err } - if input.Provider != "e2b" { - if err := q.RetireSandboxNodes(ctx); err != nil { - return err - } - if err := q.RetireSandboxEnrollments(ctx); err != nil { - return err - } - if err := q.AdvanceSandboxOwnerEpoch(ctx); err != nil { - return err - } - } if err := q.CollectSandboxGenerations(ctx); err != nil { return err } - if input.Provider == "e2b" { + { action := "change" - if input.E2B.ReplaceCredential { + if input.E2B != nil && input.E2B.ReplaceCredential { action = "replace_credential" } if err := recordDeploymentMutation(ctx, q, action, "sandbox_deployment", installation); err != nil { diff --git a/services/agents-api/internal/store/sandbox_deployment_switch_test.go b/services/agents-api/internal/store/sandbox_deployment_switch_test.go index fece61d4d..07efa16fc 100644 --- a/services/agents-api/internal/store/sandbox_deployment_switch_test.go +++ b/services/agents-api/internal/store/sandbox_deployment_switch_test.go @@ -77,7 +77,7 @@ func TestSandboxDirectDeploymentOwnershipAndCleanSwitch(t *testing.T) { t.Fatal(err) } update := SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: SandboxDeploymentSetupRequest{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}, ExpectedGeneration: 1} - if _, err := w.UpdateSandboxDeployment(t.Context(), id, update); !errors.Is(err, ErrSandboxResetInProgress) { + if _, err := w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), id, update); !errors.Is(err, ErrSandboxResetInProgress) { t.Fatal("reset allowed switch", err) } if _, err := w.RequestRuntimeCleanup(t.Context(), owner); err != nil { @@ -214,7 +214,7 @@ func TestSandboxResetSerializesFreshDirectSessions(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = w.UpdateSandboxDeployment(t.Context(), id, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: SandboxDeploymentSetupRequest{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}, ExpectedGeneration: 1}) + _, err = w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), id, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: SandboxDeploymentSetupRequest{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}, ExpectedGeneration: 1}) if view.Resources.Pending > 0 && !errors.Is(err, ErrSandboxResetInProgress) { t.Fatal("committed pending Session bypassed switch guard", err) } diff --git a/services/agents-api/internal/store/sandbox_deployment_switch_worker_test.go b/services/agents-api/internal/store/sandbox_deployment_switch_worker_test.go index a3558311a..6cc559427 100644 --- a/services/agents-api/internal/store/sandbox_deployment_switch_worker_test.go +++ b/services/agents-api/internal/store/sandbox_deployment_switch_worker_test.go @@ -86,7 +86,7 @@ func TestSandboxWorkerSwitchesAndRecoversFailedActivation(t *testing.T) { changed.Resources.CPUs++ input := store.SandboxDeploymentUpdateRequest{ExpectedGeneration: 1, SandboxDeploymentSetupRequest: changed} fail.Store(true) - if _, err := w.UpdateSandboxDeployment(t.Context(), input); err == nil { + if _, err := w.UpdateSandboxDeployment(store.SandboxResetTestContext(t.Context()), input); err == nil { t.Fatal("failed activation reported success") } view, err := s.GetRuntimeDeployment(t.Context()) @@ -97,12 +97,12 @@ func TestSandboxWorkerSwitchesAndRecoversFailedActivation(t *testing.T) { t.Fatal("rejected candidate retired the previous node", err) } fail.Store(false) - // Inject a real final SQL failure after successful provider preparation and - // manager drain; the surviving owner must recover before another mutation. + // Inject a real final SQL failure after successful + // candidate preparation; the surviving owner keeps its active deployment. if _, err := pool.Exec(t.Context(), `CREATE FUNCTION reject_reset_fixture_update() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.generation > OLD.generation THEN RAISE EXCEPTION 'fixture commit rejected'; END IF; RETURN NEW; END $$; CREATE TRIGGER reject_reset_fixture_update BEFORE UPDATE ON runtime_deployment FOR EACH ROW EXECUTE FUNCTION reject_reset_fixture_update()`); err != nil { t.Fatal(err) } - if _, err := w.UpdateSandboxDeployment(t.Context(), input); err == nil { + if _, err := w.UpdateSandboxDeployment(store.SandboxResetTestContext(t.Context()), input); err == nil { t.Fatal("failed final transaction accepted") } if _, err := pool.Exec(t.Context(), `DROP TRIGGER reject_reset_fixture_update ON runtime_deployment; DROP FUNCTION reject_reset_fixture_update()`); err != nil { @@ -114,7 +114,7 @@ func TestSandboxWorkerSwitchesAndRecoversFailedActivation(t *testing.T) { if view, err := s.GetRuntimeDeployment(t.Context()); err != nil || view.Generation != 1 || view.Provider != "docker" { t.Fatal("failed commit replaced provider", view, err) } - if _, err := w.UpdateSandboxDeployment(t.Context(), input); err != nil { + if _, err := w.UpdateSandboxDeployment(store.SandboxResetTestContext(t.Context()), input); err != nil { t.Fatal(err) } reset := func(generation uint64) store.RuntimeDeploymentView { @@ -151,7 +151,7 @@ func TestSandboxWorkerSwitchesAndRecoversFailedActivation(t *testing.T) { t.Fatal("direct provider not available after resume", allocation, err) } next := store.SandboxDeploymentUpdateRequest{ExpectedGeneration: 4, SandboxDeploymentSetupRequest: store.SandboxDeploymentSetupRequest{DeploymentSpec: store.SandboxDeploymentTestSpec("docker"), Provider: "docker"}} - if _, err := w.UpdateSandboxDeployment(t.Context(), next); !errors.Is(err, store.ErrSandboxDeploymentConflict) { + if _, err := w.UpdateSandboxDeployment(store.SandboxResetTestContext(t.Context()), next); !errors.Is(err, store.ErrSandboxDeploymentConflict) { t.Fatal("dirty switch accepted", err) } if err := s.DeleteSession(t.Context(), tenant, session.ID); err != nil { diff --git a/services/agents-api/internal/store/sandbox_deployment_view_test.go b/services/agents-api/internal/store/sandbox_deployment_view_test.go index ea97f9341..2e81f0fa4 100644 --- a/services/agents-api/internal/store/sandbox_deployment_view_test.go +++ b/services/agents-api/internal/store/sandbox_deployment_view_test.go @@ -68,7 +68,7 @@ func TestSandboxDeploymentViewRecordsTemplateBuildAndSuspension(t *testing.T) { t.Fatalf("identical POST did not record the build: %s %v", raw, err) } forget() - view, err = w.UpdateSandboxDeployment(t.Context(), id, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 1}) + view, err = w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), id, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 1}) raw, _ = json.Marshal(view) if err != nil || view.Generation != 1 || !bytes.Contains(raw, recorded) { t.Fatalf("identical PUT did not record the build: %s %v", raw, err) diff --git a/services/agents-api/internal/store/sandbox_specification_store_test.go b/services/agents-api/internal/store/sandbox_specification_store_test.go index 1fdcea2c5..c6c370246 100644 --- a/services/agents-api/internal/store/sandbox_specification_store_test.go +++ b/services/agents-api/internal/store/sandbox_specification_store_test.go @@ -150,7 +150,7 @@ func TestSandboxSpecificationBootstrapReadDoesNotConsumeEnrollment(t *testing.T) } } -func TestSandboxSpecificationChangesWaitForEveryRetainedResource(t *testing.T) { +func TestSandboxSpecificationChangesPreserveEveryRetainedResource(t *testing.T) { for _, state := range []string{"pending", "creating", "running", "stopped", "snapshot", "cleanup_pending"} { t.Run(state, func(t *testing.T) { s, w, view, input := webSpecificationFixture(t, "microsandbox") @@ -215,16 +215,18 @@ func TestSandboxSpecificationChangesWaitForEveryRetainedResource(t *testing.T) { changed.Runtime = &runtime } update := SandboxDeploymentUpdateRequest{ExpectedGeneration: view.Generation, SandboxDeploymentSetupRequest: changed} - if err := w.CheckSandboxDeploymentSwitch(t.Context(), view.InstallationID, update); !errors.Is(err, ErrSandboxDeploymentConflict) { - t.Fatal("precheck ignored retained resource", field, err) + if err := w.CheckSandboxDeploymentSwitch(t.Context(), view.InstallationID, update); err != nil { + t.Fatal(field, err) } - if _, err := w.UpdateSandboxDeployment(t.Context(), view.InstallationID, update); !errors.Is(err, ErrSandboxDeploymentConflict) { - t.Fatal("configuration changed with retained resource", field, err) + next, err := w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), view.InstallationID, update) + if err != nil || next.Generation != view.Generation+1 || next.OwnerEpoch != view.OwnerEpoch { + t.Fatal(field, next, err) } + view = next } after, err := s.GetRuntimeDeployment(t.Context()) - if err != nil || !reflect.DeepEqual(before, after) { - t.Fatal("blocked specification change mutated deployment", err) + if err != nil || before.Resources != after.Resources { + t.Fatal("online specification change altered ownership", err) } if owner.ID != "" { var allocationAfter []byte @@ -232,7 +234,7 @@ func TestSandboxSpecificationChangesWaitForEveryRetainedResource(t *testing.T) { t.Fatal(err) } if !bytes.Equal(allocationBefore, allocationAfter) { - t.Fatal("blocked change mutated or deleted retained ownership") + t.Fatal("online change mutated or deleted retained ownership") } owner, err = w.RequestRuntimeCleanup(t.Context(), owner) if err != nil { @@ -253,12 +255,12 @@ func TestSandboxSpecificationChangesWaitForEveryRetainedResource(t *testing.T) { runtime := *input.Runtime runtime.SourceCommit = strings.Repeat("2", 40) changed.Runtime = &runtime - committed, err := w.UpdateSandboxDeployment(t.Context(), view.InstallationID, SandboxDeploymentUpdateRequest{ExpectedGeneration: view.Generation, SandboxDeploymentSetupRequest: changed}) + committed, err := w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxDeploymentUpdateRequest{ExpectedGeneration: view.Generation, SandboxDeploymentSetupRequest: changed}) if err != nil || committed.Generation != view.Generation+1 || committed.Reset != nil || committed.Resources != (SandboxDeploymentResources{}) || committed.Specification == nil || !reflect.DeepEqual(*committed.Specification, changed.DeploymentSpec) { t.Fatal("completed cleanup did not permit the replacement", err) } - if _, err := s.AuthenticateRuntimeNode(t.Context(), node.NodeID, node.Credential); !errors.Is(err, ErrRuntimeNodeCredential) { - t.Fatal("old node survived specification replacement", err) + if _, err := s.AuthenticateRuntimeNode(t.Context(), node.NodeID, node.Credential); err != nil { + t.Fatal("online change retired serving identity", err) } }) } @@ -338,7 +340,7 @@ func TestSandboxSpecificationAllocationRaceWithMaintenance(t *testing.T) { t.Fatal("concurrent maintenance lost resource accounting", after.Resources, err) } input.Resources.CPUs++ - if _, err := w.UpdateSandboxDeployment(t.Context(), view.InstallationID, SandboxDeploymentUpdateRequest{ExpectedGeneration: view.Generation, SandboxDeploymentSetupRequest: input}); !errors.Is(err, ErrSandboxResetInProgress) { + if _, err := w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxDeploymentUpdateRequest{ExpectedGeneration: view.Generation, SandboxDeploymentSetupRequest: input}); !errors.Is(err, ErrSandboxResetInProgress) { t.Fatal("allocation race bypassed replacement guard", err) } } diff --git a/services/agents-api/migrations/000081_node_generations.sql b/services/agents-api/migrations/000082_node_generations.sql similarity index 78% rename from services/agents-api/migrations/000081_node_generations.sql rename to services/agents-api/migrations/000082_node_generations.sql index b367fb86d..1e1485518 100644 --- a/services/agents-api/migrations/000081_node_generations.sql +++ b/services/agents-api/migrations/000082_node_generations.sql @@ -20,8 +20,9 @@ AND deployment_generation>0 AND specification_digest ~ '^[0-9a-f]{64}$'; -- +goose Down -- +goose StatementBegin DO $$ BEGIN - IF EXISTS(SELECT 1 FROM runtime_nodes WHERE removed_at IS NULL AND protocol_version=2) THEN - RAISE EXCEPTION 'Cannot downgrade while protocol v2 nodes remain; remove them after releasing their resources first'; + IF EXISTS(SELECT 1 FROM runtime_nodes n CROSS JOIN runtime_deployment d WHERE n.removed_at IS NULL + AND (n.protocol_version=2 OR n.deployment_generation<>d.generation OR n.ready_generation<>d.generation)) THEN + RAISE EXCEPTION 'Cannot downgrade while protocol v2 or retained-generation nodes remain; remove them after releasing their resources first'; END IF; END $$; -- +goose StatementEnd From bd0fd3ba513387b67f670f75259fe7c985ebc0d4 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 23:45:27 +0000 Subject: [PATCH 07/16] Resume node generation collection safely after interruption --- CONTRIBUTING.md | 5 + .../agents-api/node-generation-protocol.md | 26 +++ deploy/install/node_generations.py | 153 +++++++++++++---- deploy/install/test_node_generations.py | 156 +++++++++++++++++- deploy/install/test_node_install.py | 32 ++++ deploy/install/test_node_update.py | 2 +- .../cmd/sandbox-node/generations.go | 85 +++++++++- .../cmd/sandbox-node/generations_test.go | 86 ++++++++++ .../internal/sandbox/node/generations.go | 25 ++- .../internal/sandbox/node/generations_test.go | 73 ++++++++ 10 files changed, 599 insertions(+), 44 deletions(-) create mode 100644 services/agents-api/cmd/sandbox-node/generations_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a2ca1f7aa..0b03300bb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -815,6 +815,11 @@ Permanent generation flock files survive updates and GC. Retained Runtime bytes the console's exact-release HTTP allowlist must agree. See `contracts/agents-api/node-generation-protocol.md` for recovery, immutable artifacts and the conservative v1 legacy-helper retention boundary. +Interrupted node collection keeps an exact private generation journal and immutable +configuration. Restart treats it only as a candidate for a fresh correlated Core +drop grant, never as preparation or serving readiness. Persist native cleanup +before removing its executable and durable file cleanup before the dropped marker; +CLI errors and unknown native ownership cannot establish absence. Use one E2B classifier. Omitted key preserves the current key; identical selection with omitted key is a no-op. Explicit key submission, including identical bytes, diff --git a/contracts/agents-api/node-generation-protocol.md b/contracts/agents-api/node-generation-protocol.md index 30415334a..b1a25a3d7 100644 --- a/contracts/agents-api/node-generation-protocol.md +++ b/contracts/agents-api/node-generation-protocol.md @@ -108,6 +108,32 @@ repair retries later without replacing in-use files. Once bytes are restored, th node still runs the actual provider readiness probe. File presence and executable capability alone never establish serving readiness. +## Interrupted local collection + +Before native or release deletion, the node persists a private collection journal +bound to its installation, generation and specification digest. Restart loads an +unfinished journal only as a retention-exchange candidate: it cannot prepare, +probe, acquire or advertise that generation. A fresh correlated Core drop grant +is required to resume; the journal itself never authorizes deletion. + +A successful complete native image inventory distinguishes absence from a CLI or +daemon failure. Failed queries, malformed inventories, native in-use refusals and +unknown ownership retain the local bytes. Native completion is persisted before +release-file cleanup, so a retry can finish a partly removed release without +executing an already removed helper. Shared images/releases remain until their +last local reference. The final digest-bound dropped marker follows durable file +cleanup and permanently prevents re-adoption. The small immutable configuration +and ownership journals remain as local identity records. + +New preparation records its exact identity before downloads and records import +start before invoking the native importer. An interrupted download can repair only +missing bytes at the original paths. If collection precedes any import attempt, +the preparation journal proves that this generation has no imported native image. +An older or interrupted generation whose native executable is missing and whose +import may have started remains conservatively retained; missing files do not +prove native absence. Receipt/store history and a legacy-unfenced generation are +never erased using an empty native inventory. + ## Identity-preserving program updates Run the checksum-verified `node-install.pyz --update` as the original installation diff --git a/deploy/install/node_generations.py b/deploy/install/node_generations.py index 70ac8098e..069b943cd 100644 --- a/deploy/install/node_generations.py +++ b/deploy/install/node_generations.py @@ -61,11 +61,52 @@ def install_helper(root, args, installer): atomic_json(settings, value) +def generation_marker(root, generation, suffix, digest, installation, installer): + path = root / "state/node/generations" / (str(generation) + suffix) + if not path.exists() and not path.is_symlink(): + return None + installer.no_links(path) + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK) + with os.fdopen(descriptor) as stream: + info = os.fstat(stream.fileno()) + if (not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600 + or info.st_uid != os.getuid() or info.st_nlink != 1 or info.st_size > 4096): + raise installer.InstallError("Invalid generation ownership journal") + try: + value = json.load(stream) + except ValueError as error: + raise installer.InstallError("Invalid generation ownership journal") from error + named = path.lstat() + if (named.st_dev, named.st_ino) != (info.st_dev, info.st_ino): + raise installer.InstallError("Generation ownership journal was replaced") + expected = {"installation_id": installation, "generation": generation, "specification_digest": digest} + if (not isinstance(value, dict) or type(value.get("generation")) is not int + or any(value.get(key) != item for key, item in expected.items())): + raise installer.InstallError("Invalid generation ownership journal") + keys = set(expected) + if suffix == ".collecting": + keys.add("image_removed") + if type(value.get("image_removed")) is not bool: + raise installer.InstallError("Invalid generation collection journal") + if suffix == ".preparing": + keys.add("import_started") + if type(value.get("import_started")) is not bool: + raise installer.InstallError("Invalid generation preparation journal") + if set(value) != keys: + raise installer.InstallError("Invalid generation journal fields") + return value + + +def marker_identity(args): + return {"installation_id": args.installation_id, "generation": args.generation, + "specification_digest": args.specification_digest} + + def retained_configs(root, installer): directory = root / "state/node/generations" result = {} base = installer.private_json(root / "provider.json") - if base and not (directory / (str(base["generation"]) + ".dropped")).exists(): + if base and not generation_marker(root, base["generation"], ".dropped", installer.node_spec.digest(base["provider"], base["specification"]), base["installation_id"], installer): result[base["generation"]] = base if directory.exists(): for path in sorted(directory.glob("*.json")): @@ -74,7 +115,9 @@ def retained_configs(root, installer): value = installer.private_json(path) if value is None or value.get("generation") != int(path.stem): raise installer.InstallError("Invalid retained generation configuration") - if not (directory / (path.stem + ".dropped")).exists(): + if not generation_marker(root, value["generation"], ".dropped", installer.node_spec.digest(value["provider"], value["specification"]), value["installation_id"], installer): + if value["generation"] in result and result[value["generation"]] != value: + raise installer.InstallError("Conflicting retained generation configuration") result[value["generation"]] = value return result @@ -174,8 +217,10 @@ def prepare(args, installer): root, identity = owned_root(args, installer) with installer.install_lock(root), collection_lease(root, args.generation, installer): directory = root / "state/node/generations" - if (directory / (str(args.generation) + ".dropped")).exists(): + if (generation_marker(root, args.generation, ".dropped", args.specification_digest, args.installation_id, installer) + or generation_marker(root, args.generation, ".collecting", args.specification_digest, args.installation_id, installer)): raise installer.InstallError("A dropped generation cannot be adopted again") + generation_marker(root, args.generation, ".preparing", args.specification_digest, args.installation_id, installer) args.core_url = identity["core_url"] args.configuration = installer.node_spec.fetch(args, "", identity, installer.open_request, generation=args.generation, allow_selection_change=True) @@ -214,6 +259,7 @@ def prepare(args, installer): target = directory / (str(args.generation) + ".json") if installer.existing_file(target) and installer.private_json(target) != value: raise installer.InstallError("Immutable generation configuration differs") + atomic_json(directory / (str(args.generation) + ".preparing"), dict(marker_identity(args), import_started=False)) atomic_json(target, value) retained = True release = runtime_files(root, value, args, manifest, sums, installer) @@ -223,6 +269,7 @@ def prepare(args, installer): if not owner.exists() and any(args.runtime_home.iterdir()): raise installer.InstallError("Versioned microsandbox store contains unowned state") installer.write_once(owner, installer.json_text({"installation_id": args.installation_id})) + atomic_json(directory / (str(args.generation) + ".preparing"), dict(marker_identity(args), import_started=True)) runtime_image = installer.prepare_runtime(release, args, manifest) if retained: installer.node_spec.verify_provider(value, args.configuration, runtime_image) @@ -265,39 +312,87 @@ def collect(args, installer): value = configurations.get(args.generation) if value is None: return - if installer.node_spec.digest(value["provider"], value["specification"]) != args.specification_digest: + if (value["installation_id"] != args.installation_id + or installer.node_spec.digest(value["provider"], value["specification"]) != args.specification_digest): raise installer.InstallError("Collection grant does not match the local generation") marker = root / "state/node/generations" / (str(args.generation) + ".legacy-unfenced") if marker.exists() or marker.is_symlink(): raise installer.InstallError("The original v1 generation retains unfenced legacy helpers; its local payload is kept") + source = value["specification"]["runtime"]["source_commit"] + if not re.fullmatch(r"[a-f0-9]{40}", source): + raise installer.InstallError("Invalid retained release identity") + release = root / "releases" / source + installer.no_links(release) + directory = root / "state/node/generations" + journal = generation_marker(root, args.generation, ".collecting", args.specification_digest, args.installation_id, installer) + if journal is None: + journal = dict(marker_identity(args), image_removed=False) + atomic_json(directory / (str(args.generation) + ".collecting"), journal) others = [item for generation, item in configurations.items() if generation != args.generation] - if value["provider"] == "microsandbox": - micro = value["microsandbox"] - shared = [item for item in others if item["microsandbox"]["runtime_home"] == micro["runtime_home"]] - env = dict(os.environ, MSB_BACKEND="local", MSB_HOME=micro["runtime_home"], MSB_PATH=micro["runtime_path"], MSB_LIBKRUNFW_PATH=micro["firmware_path"]) - if not any(item["microsandbox"]["image"] == micro["image"] for item in shared): - # No force: native ownership can still refuse removal after Core's grant. - installer.checked([micro["runtime_path"], "image", "remove", micro["image"], "--quiet"], "Runtime image is still in use", env=env) - if not shared: - raw = installer.checked([micro["runtime_path"], "sandbox", "list", "--format", "json"], "Cannot verify empty microsandbox store", env=env) - if json.loads(raw) != []: - raise installer.InstallError("Microsandbox store still contains native sandboxes") - home = Path(micro["runtime_home"]) - installer.safe_directory(home) - if installer.private_json(home / "oac-installation.json") != {"installation_id": args.installation_id}: - raise installer.InstallError("Microsandbox store ownership differs") - # Keep receipt/lock history conservatively; an empty VM list is not - # authority to discard unknown provider receipts or shared host state. - else: - image = value["docker"]["image"] - if not any(item["docker"]["image"] == image for item in others): - installer.checked(list(installer.DOCKER) + ["image", "rm", image], "Docker image is still in use") + if not journal["image_removed"]: + preparation = generation_marker(root, args.generation, ".preparing", args.specification_digest, args.installation_id, installer) + if preparation is None or preparation["import_started"]: + collect_image(args, value, others, installer) + # Persist native completion before deleting its executable. A fresh + # Core grant is still required after restart to finish file cleanup. + journal["image_removed"] = True + atomic_json(directory / (str(args.generation) + ".collecting"), journal) directory = root / "state/node/generations" installer.safe_directory(directory) - atomic_json(directory / (str(args.generation) + ".dropped"), {"specification_digest": args.specification_digest}) - (directory / (str(args.generation) + ".json")).unlink(missing_ok=True) - source = value["specification"]["runtime"]["source_commit"] if not any(item["specification"]["runtime"]["source_commit"] == source for item in others): release = root / "releases" / source - if release.exists() and not release.is_symlink() and release.resolve().parent == (root / "releases").resolve(): + installer.no_links(release) + if release.exists(): shutil.rmtree(release) + if release.parent.exists(): + descriptor = os.open(release.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + atomic_json(directory / (str(args.generation) + ".dropped"), marker_identity(args)) + # Keep the immutable configuration and permanent journal as restart + # identity. They contain no provider credentials and are not readiness. + + +def collect_image(args, value, others, installer): + if value["provider"] == "microsandbox": + micro = value["microsandbox"] + shared = [item for item in others if item["microsandbox"]["runtime_home"] == micro["runtime_home"]] + home = Path(micro["runtime_home"]) + installer.no_links(home) + if not home.is_dir() or installer.private_json(home / "oac-installation.json") != {"installation_id": args.installation_id}: + raise installer.InstallError("Microsandbox store ownership differs") + runtime_path = Path(micro["runtime_path"]) + installer.no_links(runtime_path) + if not installer.existing_file(runtime_path) or installer.file_digest(runtime_path) != micro["runtime_sha256"]: + raise installer.InstallError("Cannot verify retained microsandbox executable") + env = dict(os.environ, MSB_BACKEND="local", MSB_HOME=micro["runtime_home"], MSB_PATH=micro["runtime_path"], MSB_LIBKRUNFW_PATH=micro["firmware_path"]) + if not any(item["microsandbox"]["image"] == micro["image"] for item in shared): + # A failed inspect/remove is not proof of absence. A successful full + # inventory must contain only understood immutable references. + raw = installer.checked([micro["runtime_path"], "image", "list", "--quiet"], "Cannot verify microsandbox image inventory", env=env) + references = raw.splitlines() + if any(not re.fullmatch(r"[^\s@]+@sha256:[a-f0-9]{64}", item) for item in references): + raise installer.InstallError("Cannot verify microsandbox image inventory") + if any(item.split("@", 1)[1] == micro["image"].split("@", 1)[1] for item in references): + installer.checked([micro["runtime_path"], "image", "remove", micro["image"], "--quiet"], "Runtime image is still in use", env=env) + if not shared: + raw = installer.checked([micro["runtime_path"], "sandbox", "list", "--format", "json"], "Cannot verify empty microsandbox store", env=env) + if json.loads(raw) != []: + raise installer.InstallError("Microsandbox store still contains native sandboxes") + home = Path(micro["runtime_home"]) + installer.safe_directory(home) + if installer.private_json(home / "oac-installation.json") != {"installation_id": args.installation_id}: + raise installer.InstallError("Microsandbox store ownership differs") + # Native emptiness never authorizes discarding receipt/lock history. + else: + image = value["docker"]["image"] + if not any(item["docker"]["image"] == image for item in others): + raw = installer.checked(list(installer.DOCKER) + ["image", "ls", "--quiet", "--no-trunc"], "Cannot verify Docker image inventory") + images = raw.splitlines() + if any(not re.fullmatch(r"sha256:[a-f0-9]{64}", item) for item in images): + raise installer.InstallError("Cannot verify Docker image inventory") + if image in images: + installer.checked(list(installer.DOCKER) + ["image", "rm", image], "Docker image is still in use") diff --git a/deploy/install/test_node_generations.py b/deploy/install/test_node_generations.py index 682f7e8f0..182ce864d 100644 --- a/deploy/install/test_node_generations.py +++ b/deploy/install/test_node_generations.py @@ -1,6 +1,8 @@ """Collection retains bytes until local helper ownership settles.""" import fcntl import os +import json +import hashlib from pathlib import Path import tempfile from types import SimpleNamespace @@ -19,9 +21,9 @@ def setUp(self): self.root = Path(temporary.name) self.directory = self.root / "state/node/generations" self.directory.mkdir(parents=True, mode=0o700) - self.value = {"generation": 1, "provider": "docker", "docker": {"image": "sha256:" + "a" * 64}, + self.value = {"installation_id": "test-installation", "generation": 1, "provider": "docker", "docker": {"image": "sha256:" + "a" * 64}, "specification": {"resources": {"cpus": 1, "memory_mib": 1024}, "runtime": {"source_commit": "b" * 40, "image_id": "sha256:" + "a" * 64, "image_manifest_digest": "sha256:" + "c" * 64, "microsandbox_ref": "oac-runtime@sha256:" + "d" * 64, "runtime_sha256": "e" * 64, "firmware_sha256": "f" * 64}}} - self.args = SimpleNamespace(generation=1, specification_digest=node_spec.digest("docker", self.value["specification"])) + self.args = SimpleNamespace(installation_id="test-installation", generation=1, specification_digest=node_spec.digest("docker", self.value["specification"])) self.release = self.root / "releases" / ("b" * 40) self.release.mkdir(parents=True) (self.release / "artifact").write_bytes(b"immutable bytes") @@ -79,6 +81,156 @@ def test_symlink_and_shared_inode_are_not_collection_authority(self): finally: lease.unlink() + def test_restart_after_native_remove_uses_inventory_not_error_text(self): + present = [True] + removes = [] + def native(command, *_args, **_kwargs): + if "ls" in command: + return self.value["docker"]["image"] + "\n" if present[0] else "" + self.assertIn("rm", command) + present[0] = False + removes.append(command) + # Model interruption after native commit but before the response. + raise installer.InstallError("interrupted after native mutation") + installer.checked.side_effect = native + with self.assertRaises(installer.InstallError): + node_generations.collect(self.args, installer) + self.assertFalse(json.loads((self.directory / "1.collecting").read_text())["image_removed"]) + self.assertTrue(self.release.exists()) + node_generations.collect(self.args, installer) + self.assertEqual(len(removes), 1) + self.assertFalse(self.release.exists()) + self.assertTrue((self.directory / "1.dropped").exists()) + + def test_native_inventory_failure_and_unknown_output_retain_every_byte(self): + for failure in (installer.InstallError("daemon unavailable"), "not a verified image ID"): + installer.checked.side_effect = failure if isinstance(failure, Exception) else None + installer.checked.return_value = failure + with self.assertRaises(installer.InstallError): + node_generations.collect(self.args, installer) + self.assertTrue((self.release / "artifact").exists()) + self.assertFalse((self.directory / "1.dropped").exists()) + self.assertFalse(json.loads((self.directory / "1.collecting").read_text())["image_removed"]) + + def test_restart_after_partial_file_cleanup_skips_native_and_finishes(self): + def interrupted(path): + (path / "artifact").unlink() + raise OSError("interrupted file cleanup") + with mock.patch.object(node_generations.shutil, "rmtree", side_effect=interrupted): + with self.assertRaises(OSError): + node_generations.collect(self.args, installer) + self.assertTrue(json.loads((self.directory / "1.collecting").read_text())["image_removed"]) + self.assertFalse((self.directory / "1.dropped").exists()) + installer.checked.reset_mock() + node_generations.collect(self.args, installer) + installer.checked.assert_not_called() + self.assertFalse(self.release.exists()) + self.assertEqual(node_generations.retained_configs(self.root, installer), {}) + + def test_restart_after_files_removed_before_final_marker(self): + original = node_generations.atomic_json + def interrupt(path, value): + if path.suffix == ".dropped": + raise OSError("interrupted final marker") + original(path, value) + with mock.patch.object(node_generations, "atomic_json", side_effect=interrupt): + with self.assertRaises(OSError): + node_generations.collect(self.args, installer) + self.assertFalse(self.release.exists()) + installer.checked.reset_mock() + node_generations.collect(self.args, installer) + installer.checked.assert_not_called() + self.assertTrue((self.directory / "1.dropped").exists()) + + def test_invalid_journal_never_authorizes_deletion_or_hides_configuration(self): + for suffix in (".collecting", ".dropped", ".preparing"): + path = self.directory / ("1" + suffix) + value = node_generations.marker_identity(self.args) + if suffix == ".collecting": value["image_removed"] = True + if suffix == ".preparing": value["import_started"] = False + for field, invalid in (("installation_id", "foreign"), ("generation", 2), ("specification_digest", "a" * 64)): + node_generations.atomic_json(path, dict(value, **{field: invalid})) + with self.assertRaises(installer.InstallError): + node_generations.collect(self.args, installer) + self.assertTrue(self.release.exists()) + path.unlink() + installer.checked.assert_not_called() + + def test_shared_image_and_release_survive_until_last_generation(self): + other = dict(self.value, generation=2) + node_generations.atomic_json(self.directory / "2.json", other) + node_generations.collect(self.args, installer) + installer.checked.assert_not_called() + self.assertTrue((self.release / "artifact").exists()) + self.args.generation = 2 + node_generations.collect(self.args, installer) + self.assertFalse(self.release.exists()) + + def test_never_imported_generation_can_collect_missing_executable(self): + node_generations.atomic_json(self.directory / "1.preparing", dict(node_generations.marker_identity(self.args), import_started=False)) + node_generations.collect(self.args, installer) + installer.checked.assert_not_called() + self.assertFalse(self.release.exists()) + + def test_legacy_unfenced_marker_blocks_before_collection_journal(self): + (self.directory / "1.legacy-unfenced").write_text("retained") + with self.assertRaisesRegex(installer.InstallError, "legacy helpers"): + node_generations.collect(self.args, installer) + installer.checked.assert_not_called() + self.assertFalse((self.directory / "1.collecting").exists()) + self.assertTrue(self.release.exists()) + + def test_micro_interrupted_native_removal_and_missing_runtime_fail_closed(self): + self.value["provider"] = "microsandbox" + del self.value["docker"] + home = self.root / "micro-store" + home.mkdir(mode=0o700) + node_generations.atomic_json(home / "oac-installation.json", {"installation_id": self.args.installation_id}) + runtime = self.release / "msb" + runtime.write_bytes(b"verified native executable") + runtime.chmod(0o700) + image = self.value["specification"]["runtime"]["microsandbox_ref"] + self.value["microsandbox"] = {"runtime_home": str(home), "runtime_path": str(runtime), "firmware_path": str(self.release / "firmware"), "runtime_sha256": hashlib.sha256(runtime.read_bytes()).hexdigest(), "image": image} + self.args.specification_digest = node_spec.digest("microsandbox", self.value["specification"]) + node_generations.atomic_json(self.root / "provider.json", self.value) + present = [True] + def native(command, *_args, **_kwargs): + if command[1:3] == ["image", "list"]: return image + "\n" if present[0] else "" + if command[1:3] == ["image", "remove"]: + present[0] = False + raise installer.InstallError("interrupted native removal") + if command[1:3] == ["sandbox", "list"]: return "[]" + raise AssertionError(command) + installer.checked.side_effect = native + with self.assertRaises(installer.InstallError): node_generations.collect(self.args, installer) + raw = runtime.read_bytes() + runtime.unlink() + installer.checked.reset_mock() + with self.assertRaisesRegex(installer.InstallError, "executable"): + node_generations.collect(self.args, installer) + installer.checked.assert_not_called() + self.assertTrue(self.release.exists()) + runtime.write_bytes(raw) + runtime.chmod(0o700) + node_generations.collect(self.args, installer) + self.assertFalse(self.release.exists()) + self.assertTrue((home / "oac-installation.json").exists()) + + def test_fifo_journal_is_refused_without_blocking(self): + path = self.directory / "1.collecting" + os.mkfifo(path, 0o600) + with self.assertRaises(installer.InstallError): node_generations.collect(self.args, installer) + installer.checked.assert_not_called() + self.assertTrue(self.release.exists()) + + def test_release_path_conflict_is_refused_before_native_mutation(self): + foreign = self.root / "foreign" + self.release.rename(foreign) + self.release.symlink_to(foreign, target_is_directory=True) + with self.assertRaises(installer.InstallError): node_generations.collect(self.args, installer) + installer.checked.assert_not_called() + self.assertTrue((foreign / "artifact").exists()) + if __name__ == "__main__": unittest.main() diff --git a/deploy/install/test_node_install.py b/deploy/install/test_node_install.py index e962fede6..9a180284f 100644 --- a/deploy/install/test_node_install.py +++ b/deploy/install/test_node_install.py @@ -237,6 +237,38 @@ def recovery_args(self): self.args.specification_digest = json.loads((self.root / "state/node/identity.json").read_text())["identity"]["specification_digest"] return self.args + def test_interrupted_new_generation_recovers_partial_download_at_same_identity(self): + self.args.provider = "microsandbox" + self.install() + original = (self.root / "provider.json").read_bytes() + self.manifest["source_commit"] = "f" * 40 + self.refresh_manifest() + config = json.loads(self.configuration_response(None).read()) + config["generation"] = 2 + self.args.generation = 2 + self.args.specification_digest = config["specification_digest"] + real_obtain = installer.distribution.obtain_artifact + def interrupted(manifest, name, path): + if name == installer.MICRO[1]: + raise OSError("interrupted partial download") + return real_obtain(manifest, name, path) + with mock.patch.object(installer.node_spec, "fetch", return_value=config): + with mock.patch.object(installer.distribution, "obtain_artifact", side_effect=interrupted): + with self.assertRaises(OSError): + installer.node_generations.prepare(self.args, installer) + directory = self.root / "state/node/generations" + saved = json.loads((directory / "2.json").read_text()) + self.assertEqual(saved["specification"], config["specification"]) + self.assertFalse(json.loads((directory / "2.preparing").read_text())["import_started"]) + helper = Path(saved["microsandbox"]["helper_path"]) + inode = helper.stat().st_ino + self.assertFalse(Path(saved["microsandbox"]["runtime_path"]).exists()) + installer.node_generations.prepare(self.args, installer) + self.assertEqual(helper.stat().st_ino, inode) + self.assertEqual(json.loads((directory / "2.json").read_text()), saved) + self.assertTrue(json.loads((directory / "2.preparing").read_text())["import_started"]) + self.assertEqual((self.root / "provider.json").read_bytes(), original) + def test_restart_repairs_only_missing_micro_bytes_at_original_paths(self): self.args.provider = "microsandbox" self.install() diff --git a/deploy/install/test_node_update.py b/deploy/install/test_node_update.py index f96501940..c760aa5a5 100644 --- a/deploy/install/test_node_update.py +++ b/deploy/install/test_node_update.py @@ -78,7 +78,7 @@ def interrupted(arguments, *_args, **_kwargs): installer.node_spec.fetch.assert_called_with(self.args, "", self.identity, installer.open_request, allow_selection_change=True) with mock.patch.object(node_generations, "owned_root", return_value=(self.root, self.identity)): installer.checked.reset_mock() - grant = SimpleNamespace(generation=1, specification_digest=self.identity["identity"]["specification_digest"]) + grant = SimpleNamespace(installation_id=self.args.installation_id, generation=1, specification_digest=self.identity["identity"]["specification_digest"]) with self.assertRaisesRegex(installer.InstallError, "unfenced legacy helpers"): node_generations.collect(grant, installer) installer.checked.assert_not_called() diff --git a/services/agents-api/cmd/sandbox-node/generations.go b/services/agents-api/cmd/sandbox-node/generations.go index 8ab4e8cdb..5856cf2e0 100644 --- a/services/agents-api/cmd/sandbox-node/generations.go +++ b/services/agents-api/cmd/sandbox-node/generations.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "strconv" "strings" "syscall" @@ -42,7 +43,8 @@ func runGenerations(ctx context.Context, configFile, stateDir string) error { paths = append([]string{configFile}, paths...) values := map[uint64]node.GenerationProvider{} recovery := []sandbox.GenerationReference{} - seen := map[uint64]bool{} + collection := []sandbox.GenerationReference{} + seen := map[uint64]providerconfig.Config{} closeValues := func() { for _, v := range values { if v.Close != nil { @@ -64,16 +66,26 @@ func runGenerations(ctx context.Context, configFile, stateDir string) error { closeValues() return sandbox.ErrOwnership } - if _, err := os.Lstat(filepath.Join(stateDir, "generations", strconv.FormatUint(config.Generation, 10)+".dropped")); err == nil { - continue - } else if !os.IsNotExist(err) { + state, err := generationLocalState(config, stateDir) + if err != nil { closeValues() return err } - if seen[config.Generation] { + if state == "dropped" { + continue + } + if previous, ok := seen[config.Generation]; ok { + if !reflect.DeepEqual(previous, config) { + closeValues() + return sandbox.ErrOwnership + } + continue + } + seen[config.Generation] = config + if state == "collecting" { + collection = append(collection, sandbox.GenerationReference{Generation: config.Generation, SpecificationDigest: config.Specification.Digest(config.Provider)}) continue } - seen[config.Generation] = true value, err := buildGeneration(config, stateDir) if errors.Is(err, os.ErrNotExist) { recovery = append(recovery, sandbox.GenerationReference{Generation: config.Generation, SpecificationDigest: config.Specification.Digest(config.Provider)}) @@ -103,7 +115,7 @@ func runGenerations(ctx context.Context, configFile, stateDir string) error { for _, v := range values { initial = append(initial, v) } - manager, err := node.NewGenerationManager(ctx, node.GenerationManagerOptions{Initial: initial, Recover: recovery, + manager, err := node.NewGenerationManager(ctx, node.GenerationManagerOptions{Initial: initial, Recover: recovery, Collect: collection, Prepare: func(ctx context.Context, generation uint64, digest string) (node.GenerationProvider, error) { if err := runHelper(ctx, "prepare", generation, digest); err != nil { return node.GenerationProvider{}, err @@ -180,3 +192,62 @@ func buildGeneration(config providerconfig.Config, stateDir string) (node.Genera } return node.GenerationProvider{Generation: config.Generation, SpecificationDigest: built.SpecificationDigest, Provider: built.Provider, Probe: probe, Close: closeProvider}, nil } + +// A persisted local journal never substitutes for a fresh Core drop grant. It +// only prevents restart from preparing or serving a partly collected generation. +func generationLocalState(config providerconfig.Config, stateDir string) (string, error) { + directory := filepath.Join(stateDir, "generations") + info, err := os.Lstat(directory) + if os.IsNotExist(err) { + return "", nil + } + if err != nil { + return "", err + } + owner, ok := info.Sys().(*syscall.Stat_t) + resolved, err := filepath.EvalSymlinks(directory) + if err != nil || resolved != directory || !info.IsDir() || info.Mode().Perm() != 0700 || !ok || owner.Uid != uint32(os.Getuid()) { + return "", sandbox.ErrOwnership + } + state := "" + for _, suffix := range []string{".collecting", ".dropped"} { + path := filepath.Join(stateDir, "generations", strconv.FormatUint(config.Generation, 10)+suffix) + fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_CLOEXEC|syscall.O_NONBLOCK, 0) + if errors.Is(err, syscall.ENOENT) { + continue + } + if err != nil { + return "", err + } + file := os.NewFile(uintptr(fd), path) + var st syscall.Stat_t + err = syscall.Fstat(fd, &st) + resolved, pathErr := filepath.EvalSymlinks(path) + if err != nil || pathErr != nil || resolved != path || st.Mode&syscall.S_IFMT != syscall.S_IFREG || st.Mode&0777 != 0600 || st.Uid != uint32(os.Getuid()) || st.Nlink != 1 || st.Size > 4096 { + file.Close() + return "", sandbox.ErrOwnership + } + var journal struct { + InstallationID string `json:"installation_id"` + Generation uint64 `json:"generation"` + SpecificationDigest string `json:"specification_digest"` + ImageRemoved *bool `json:"image_removed,omitempty"` + } + decoder := json.NewDecoder(io.LimitReader(file, 4097)) + decoder.DisallowUnknownFields() + err = decoder.Decode(&journal) + var trailing any + end := decoder.Decode(&trailing) + opened, statErr := file.Stat() + named, namedErr := os.Lstat(path) + file.Close() + if statErr != nil || namedErr != nil || !os.SameFile(opened, named) { + return "", sandbox.ErrOwnership + } + if err != nil || end != io.EOF || journal.InstallationID != config.InstallationID || journal.Generation != config.Generation || journal.SpecificationDigest != config.Specification.Digest(config.Provider) || (suffix == ".collecting") != (journal.ImageRemoved != nil) { + return "", sandbox.ErrOwnership + } + state = strings.TrimPrefix(suffix, ".") + } + return state, nil +} diff --git a/services/agents-api/cmd/sandbox-node/generations_test.go b/services/agents-api/cmd/sandbox-node/generations_test.go new file mode 100644 index 000000000..c1f381410 --- /dev/null +++ b/services/agents-api/cmd/sandbox-node/generations_test.go @@ -0,0 +1,86 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "syscall" + "testing" + + providerconfig "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/config" +) + +func TestGenerationJournalRestartIdentity(t *testing.T) { + config := providerconfig.Config{InstallationID: "installation", Generation: 9, Provider: "docker"} + stateDir := t.TempDir() + directory := filepath.Join(stateDir, "generations") + if err := os.Mkdir(directory, 0700); err != nil { + t.Fatal(err) + } + journal := map[string]any{"installation_id": config.InstallationID, "generation": config.Generation, "specification_digest": config.Specification.Digest(config.Provider)} + for _, suffix := range []string{".collecting", ".dropped"} { + t.Run(suffix, func(t *testing.T) { + path := filepath.Join(directory, "9"+suffix) + if suffix == ".collecting" { + journal["image_removed"] = true + } else { + delete(journal, "image_removed") + } + write := func(value map[string]any) { + raw, _ := json.Marshal(value) + if err := os.WriteFile(path, raw, 0600); err != nil { + t.Fatal(err) + } + } + write(journal) + state, err := generationLocalState(config, stateDir) + if err != nil || state != suffix[1:] { + t.Fatal(state, err) + } + for _, field := range []string{"installation_id", "generation", "specification_digest"} { + previous := journal[field] + journal[field] = "foreign" + write(journal) + if _, err = generationLocalState(config, stateDir); err == nil { + t.Fatal("accepted mismatched", field) + } + journal[field] = previous + } + write(journal) + if err = os.Chmod(path, 0644); err != nil { + t.Fatal(err) + } + if _, err = generationLocalState(config, stateDir); err == nil { + t.Fatal("accepted non-private marker") + } + if err = os.Remove(path); err != nil { + t.Fatal(err) + } + foreign := filepath.Join(directory, "foreign") + raw, _ := json.Marshal(journal) + if err = os.WriteFile(foreign, raw, 0600); err != nil { + t.Fatal(err) + } + for _, link := range []func(string, string) error{os.Symlink, os.Link} { + if err = link(foreign, path); err != nil { + t.Fatal(err) + } + if _, err = generationLocalState(config, stateDir); err == nil { + t.Fatal("accepted linked journal") + } + if err = os.Remove(path); err != nil { + t.Fatal(err) + } + } + if err = syscall.Mkfifo(path, 0600); err != nil { + t.Fatal(err) + } + if _, err = generationLocalState(config, stateDir); err == nil { + t.Fatal("accepted FIFO journal") + } + if err = os.Remove(path); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/services/agents-api/internal/sandbox/node/generations.go b/services/agents-api/internal/sandbox/node/generations.go index 36ff9763a..6c7ff148b 100644 --- a/services/agents-api/internal/sandbox/node/generations.go +++ b/services/agents-api/internal/sandbox/node/generations.go @@ -21,6 +21,7 @@ type GenerationProvider struct { type GenerationManagerOptions struct { Initial []GenerationProvider Recover []sandbox.GenerationReference + Collect []sandbox.GenerationReference Prepare func(context.Context, uint64, string) (GenerationProvider, error) Remove func(context.Context, GenerationProvider) error } @@ -33,6 +34,7 @@ type localGeneration struct { failures int removing bool repairing, preparing bool + collecting bool } // GenerationManager owns local providers, preparation and all queued/in-flight @@ -71,6 +73,13 @@ func NewGenerationManager(ctx context.Context, options GenerationManagerOptions) } m.values[ref.Generation] = &localGeneration{value: GenerationProvider{Generation: ref.Generation, SpecificationDigest: ref.SpecificationDigest}, state: "failed", diagnostic: sandbox.NodeRuntimeDownloadFailed, repairing: true} } + for _, ref := range options.Collect { + if !validGeneration(ref.Generation) || !validSpecificationDigest(ref.SpecificationDigest) || m.values[ref.Generation] != nil { + cancel() + return nil, sandbox.ErrInvalid + } + m.values[ref.Generation] = &localGeneration{value: GenerationProvider{Generation: ref.Generation, SpecificationDigest: ref.SpecificationDigest}, collecting: true} + } m.workers.Add(2) go func() { defer m.workers.Done(); m.prepareLoop() }() go func() { defer m.workers.Done(); m.probeLoop() }() @@ -98,9 +107,14 @@ func (m *GenerationManager) Deployment(value sandbox.NodeDeployment) error { if value.Generation < m.target.Generation { return nil } - if g := m.values[value.Generation]; g != nil && g.value.SpecificationDigest != value.SpecificationDigest { + if g := m.values[value.Generation]; g != nil && (g.value.SpecificationDigest != value.SpecificationDigest || g.collecting) { return sandbox.ErrOwnership } + if value.ServingGeneration != nil { + if g := m.values[*value.ServingGeneration]; g != nil && g.collecting { + return sandbox.ErrOwnership + } + } if value.Generation == m.target.Generation && value.SpecificationDigest != m.target.SpecificationDigest { return sandbox.ErrOwnership } @@ -142,7 +156,7 @@ func (m *GenerationManager) Statuses() []sandbox.GenerationStatus { seen := map[uint64]bool{} for _, key := range keys { g := m.values[key] - if g == nil || g.removing || seen[key] { + if g == nil || g.removing || g.collecting || seen[key] { continue } seen[key] = true @@ -177,7 +191,7 @@ func (m *GenerationManager) Acquire(generation uint64) (sandbox.Provider, bool, m.mu.Lock() defer m.mu.Unlock() g := m.values[generation] - if g == nil || g.removing || g.preparing || g.value.Provider == nil { + if g == nil || g.removing || g.collecting || g.preparing || g.value.Provider == nil { return nil, false, nil, ErrUnavailable } g.refs++ @@ -211,6 +225,7 @@ func (m *GenerationManager) Drop(ctx context.Context, grant sandbox.GenerationRe m.mu.Unlock() return ErrUnavailable } + g.collecting = true g.removing = true m.mu.Unlock() err := m.options.Remove(ctx, g.value) @@ -247,7 +262,7 @@ func (m *GenerationManager) prepareLoop() { keys = append(keys, m.orderedLocked(m.recoveryCursor)...) for _, key := range keys { candidate := m.values[key] - if candidate == nil || candidate.removing || candidate.preparing || candidate.refs != 0 || candidate.value.Provider != nil && !candidate.repairing || time.Now().Before(candidate.retryAt) { + if candidate == nil || candidate.removing || candidate.collecting || candidate.preparing || candidate.refs != 0 || candidate.value.Provider != nil && !candidate.repairing || time.Now().Before(candidate.retryAt) { continue } if provider, ok := candidate.value.Provider.(interface{ Quiescent() bool }); ok && !provider.Quiescent() { @@ -331,7 +346,7 @@ func (m *GenerationManager) probeLoop() { var g *localGeneration for _, key := range keys { value := m.values[key] - if value != nil && value.value.Provider != nil && !value.removing && !value.preparing && !value.repairing { + if value != nil && value.value.Provider != nil && !value.removing && !value.collecting && !value.preparing && !value.repairing { g = value if key != priority { m.probeCursor = key diff --git a/services/agents-api/internal/sandbox/node/generations_test.go b/services/agents-api/internal/sandbox/node/generations_test.go index ce94f00d2..fca4ebf53 100644 --- a/services/agents-api/internal/sandbox/node/generations_test.go +++ b/services/agents-api/internal/sandbox/node/generations_test.go @@ -296,3 +296,76 @@ func TestGenerationDeploymentRejectsConflictingImmutableIdentity(t *testing.T) { t.Fatal("future serving pin accepted", err) } } + +func TestInterruptedCollectionNeverPreparesOrServesAfterRestart(t *testing.T) { + digest := strings.Repeat("a", 64) + ref := sandbox.GenerationReference{Generation: 1, SpecificationDigest: digest} + probed := make(chan struct{}, 1) + removes := 0 + manager, err := NewGenerationManager(t.Context(), GenerationManagerOptions{ + Initial: []GenerationProvider{{Generation: 2, SpecificationDigest: digest, Provider: &fakeProvider{}, Probe: func(context.Context) error { + select { + case probed <- struct{}{}: + default: + } + return nil + }}}, + Collect: []sandbox.GenerationReference{ref}, + Prepare: func(context.Context, uint64, string) (GenerationProvider, error) { + t.Error("collection entered preparation") + return GenerationProvider{}, sandbox.ErrInvalid + }, + Remove: func(_ context.Context, value GenerationProvider) error { + removes++ + if value.Generation != 1 { + t.Error("wrong generation") + } + if removes == 1 { + return errors.New("interrupted cleanup") + } + return nil + }, + }) + if err != nil { + t.Fatal(err) + } + defer manager.Close() + pin := uint64(2) + if err = manager.Deployment(sandbox.NodeDeployment{Generation: 2, SpecificationDigest: digest, ServingGeneration: &pin}); err != nil { + t.Fatal(err) + } + select { + case <-probed: + case <-time.After(3 * time.Second): + t.Fatal("provider loop did not run") + } + if _, _, _, err = manager.Acquire(1); err == nil { + t.Fatal("unfinished collection admitted an operation") + } + for _, status := range manager.Statuses() { + if status.Generation == 1 { + t.Fatal("unfinished collection advertised readiness", status) + } + } + if len(manager.Retained(0)) != 2 || removes != 0 { + t.Fatal("restart consumed old drop authorization") + } + cancelled, cancel := context.WithCancel(t.Context()) + cancel() + grant := sandbox.GenerationRetention{GenerationReference: ref} + if err = manager.Drop(cancelled, grant); err == nil || removes != 0 { + t.Fatal("cancelled connection collected") + } + if err = manager.Drop(t.Context(), grant); err == nil { + t.Fatal("expected interrupted cleanup") + } + if _, _, _, err = manager.Acquire(1); err == nil { + t.Fatal("failed cleanup resumed serving") + } + if err = manager.Drop(t.Context(), grant); err != nil { + t.Fatal(err) + } + if len(manager.Retained(0)) != 1 || removes != 2 { + t.Fatal("cleanup did not settle") + } +} From 84b7bcec92568126836c06b9eb20da6643b5d9ab Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 23:28:54 +0000 Subject: [PATCH 08/16] Bound generation snapshot singleton planning within reset deadline (cherry picked from commit 619d8e55de23b6649c428cc7177f062b20a20294) --- CONTRIBUTING.md | 8 +- .../internal/db/queries/sandbox_reset.sql | 13 +- .../internal/db/sqlc/sandbox_reset.sql.go | 11 +- .../internal/execution/sandbox_reset_test.go | 8 ++ .../execution/sandbox_snapshot_budget_test.go | 114 ++++++++++++++++++ .../store/sandbox_generations_test.go | 34 ++++++ .../internal/store/sandbox_reset_test.go | 2 + .../store/sandbox_snapshot_plan_test.go | 106 ++++++++++++++++ 8 files changed, 285 insertions(+), 11 deletions(-) create mode 100644 services/agents-api/internal/execution/sandbox_snapshot_budget_test.go create mode 100644 services/agents-api/internal/store/sandbox_snapshot_plan_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0b03300bb..a1ec428a5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3024,8 +3024,12 @@ replaced; do not carry obsolete compatibility code forward to satisfy this secti every Session transaction: binding, claim/reconciliation, journal/Items/Usage, function callbacks/application receipts and terminal/native continuity. Serialize these short transactions and lease pings; execution transactions have a five-second - deadline including gate and Session-lock waits. Never hold a transaction across - daemon/model work, reconnect the writer or fall back to the pool after lease loss. + deadline including gate and Session-lock waits. Sandbox reset snapshots bound the + deployment relation explicitly to its legal singleton row before resource joins. + Keep this cardinality visible even on fresh databases without statistics: inflated + join estimates can trigger expensive JIT compilation inside the lease deadline. + Never hold a transaction across daemon/model work, reconnect the writer or fall + back to the pool after lease loss. The original Store handles public admission and device/auth maintenance on pooled connections. Execution reads may also use the pool; a read grants no write authority. At startup, reconcile previously claimed work as failed, preserve queued inputs diff --git a/services/agents-api/internal/db/queries/sandbox_reset.sql b/services/agents-api/internal/db/queries/sandbox_reset.sql index 27e1cc52a..01afae0f9 100644 --- a/services/agents-api/internal/db/queries/sandbox_reset.sql +++ b/services/agents-api/internal/db/queries/sandbox_reset.sql @@ -55,7 +55,8 @@ ORDER BY s.id LIMIT 32; SELECT id FROM projects WHERE tenant_id = $1; -- name: GetSandboxDeploymentSnapshot :one -WITH observed AS MATERIALIZED (SELECT clock_timestamp() AS as_of), +WITH deployment AS MATERIALIZED (SELECT * FROM runtime_deployment WHERE singleton = true LIMIT 1), +observed AS MATERIALIZED (SELECT clock_timestamp() AS as_of), held AS ( SELECT a.deployment_generation, a.node_id, s.id AS session_id, e.id AS environment_id, false AS pending, (a.state = 'cleanup_pending' OR s.deleted_at IS NOT NULL OR e.status IN ('failed', 'expired') @@ -63,7 +64,7 @@ held AS ( THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= observed.as_of ELSE a.node_id IS NULL AND d.mode <> 'direct' AND a.kept_at <= observed.as_of - interval '1 hour' END) AS cleanup FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id - JOIN sessions s ON s.id = e.session_id CROSS JOIN runtime_deployment d CROSS JOIN observed + JOIN sessions s ON s.id = e.session_id CROSS JOIN deployment d CROSS JOIN observed WHERE a.state <> 'released' UNION ALL SELECT p.deployment_generation, p.node_id, s.id, e.id, true, false @@ -80,13 +81,13 @@ held AS ( THEN 'busy' ELSE 'idle' END AS category, n.name, (h.node_id IS NOT NULL AND NOT COALESCE(n.removed_at IS NULL AND n.connection_id IS NOT NULL AND n.connected_epoch = d.owner_epoch AND n.last_seen_at > observed.as_of - interval '45 seconds', false)) AS offline - FROM held h LEFT JOIN runtime_nodes n ON n.id = h.node_id CROSS JOIN runtime_deployment d CROSS JOIN observed + FROM held h LEFT JOIN runtime_nodes n ON n.id = h.node_id CROSS JOIN deployment d CROSS JOIN observed ), rollout_nodes AS ( SELECT CASE WHEN n.connection_id IS NULL OR n.connected_epoch <> d.owner_epoch OR n.last_seen_at IS NULL OR n.last_seen_at <= observed.as_of - interval '45 seconds' THEN 'unknown' WHEN n.protocol_version=1 AND n.deployment_generation <> d.generation THEN 'update_required' ELSE COALESCE(g.state,'unknown') END AS state - FROM runtime_nodes n CROSS JOIN runtime_deployment d CROSS JOIN observed + FROM runtime_nodes n CROSS JOIN deployment d CROSS JOIN observed LEFT JOIN runtime_node_generation_status g ON g.node_id=n.id AND g.generation=d.generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id ), offline AS ( @@ -111,4 +112,6 @@ SELECT sqlc.embed(d), 'update_required',(SELECT count(*) FROM rollout_nodes WHERE state='update_required'), 'unknown',(SELECT count(*) FROM rollout_nodes WHERE state='unknown')) ELSE NULL END )::jsonb AS rollout -FROM runtime_deployment d; +FROM runtime_deployment d +WHERE d.singleton = true +LIMIT 1; diff --git a/services/agents-api/internal/db/sqlc/sandbox_reset.sql.go b/services/agents-api/internal/db/sqlc/sandbox_reset.sql.go index 8f3bd6c85..823e5ef91 100644 --- a/services/agents-api/internal/db/sqlc/sandbox_reset.sql.go +++ b/services/agents-api/internal/db/sqlc/sandbox_reset.sql.go @@ -51,7 +51,8 @@ func (q *Queries) ForceSandboxReset(ctx context.Context) error { } const getSandboxDeploymentSnapshot = `-- name: GetSandboxDeploymentSnapshot :one -WITH observed AS MATERIALIZED (SELECT clock_timestamp() AS as_of), +WITH deployment AS MATERIALIZED (SELECT singleton, installation_id, backend_fingerprint, admission_paused, updated_at, provider_kind, local_node_id, owner_epoch, web_managed, idle_seconds, retention_seconds, generation, mode, e2b_template, e2b_credential, specification, e2b_template_build_status, e2b_template_cpus, e2b_template_memory_mib, e2b_template_root_disk_mib, reset_clear, reset_requested_at, reset_deadline_at, reset_forced_at, reset_audit FROM runtime_deployment WHERE singleton = true LIMIT 1), +observed AS MATERIALIZED (SELECT clock_timestamp() AS as_of), held AS ( SELECT a.deployment_generation, a.node_id, s.id AS session_id, e.id AS environment_id, false AS pending, (a.state = 'cleanup_pending' OR s.deleted_at IS NOT NULL OR e.status IN ('failed', 'expired') @@ -59,7 +60,7 @@ held AS ( THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= observed.as_of ELSE a.node_id IS NULL AND d.mode <> 'direct' AND a.kept_at <= observed.as_of - interval '1 hour' END) AS cleanup FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id - JOIN sessions s ON s.id = e.session_id CROSS JOIN runtime_deployment d CROSS JOIN observed + JOIN sessions s ON s.id = e.session_id CROSS JOIN deployment d CROSS JOIN observed WHERE a.state <> 'released' UNION ALL SELECT p.deployment_generation, p.node_id, s.id, e.id, true, false @@ -76,13 +77,13 @@ held AS ( THEN 'busy' ELSE 'idle' END AS category, n.name, (h.node_id IS NOT NULL AND NOT COALESCE(n.removed_at IS NULL AND n.connection_id IS NOT NULL AND n.connected_epoch = d.owner_epoch AND n.last_seen_at > observed.as_of - interval '45 seconds', false)) AS offline - FROM held h LEFT JOIN runtime_nodes n ON n.id = h.node_id CROSS JOIN runtime_deployment d CROSS JOIN observed + FROM held h LEFT JOIN runtime_nodes n ON n.id = h.node_id CROSS JOIN deployment d CROSS JOIN observed ), rollout_nodes AS ( SELECT CASE WHEN n.connection_id IS NULL OR n.connected_epoch <> d.owner_epoch OR n.last_seen_at IS NULL OR n.last_seen_at <= observed.as_of - interval '45 seconds' THEN 'unknown' WHEN n.protocol_version=1 AND n.deployment_generation <> d.generation THEN 'update_required' ELSE COALESCE(g.state,'unknown') END AS state - FROM runtime_nodes n CROSS JOIN runtime_deployment d CROSS JOIN observed + FROM runtime_nodes n CROSS JOIN deployment d CROSS JOIN observed LEFT JOIN runtime_node_generation_status g ON g.node_id=n.id AND g.generation=d.generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id ), offline AS ( @@ -108,6 +109,8 @@ SELECT d.singleton, d.installation_id, d.backend_fingerprint, d.admission_paused 'unknown',(SELECT count(*) FROM rollout_nodes WHERE state='unknown')) ELSE NULL END )::jsonb AS rollout FROM runtime_deployment d +WHERE d.singleton = true +LIMIT 1 ` type GetSandboxDeploymentSnapshotRow struct { diff --git a/services/agents-api/internal/execution/sandbox_reset_test.go b/services/agents-api/internal/execution/sandbox_reset_test.go index c90038f4e..f2fcca3b5 100644 --- a/services/agents-api/internal/execution/sandbox_reset_test.go +++ b/services/agents-api/internal/execution/sandbox_reset_test.go @@ -22,6 +22,11 @@ import ( // The execution lease is database-scoped, so this manager test owns a database. func resetManagerStore(t *testing.T) (*store.Store, *store.ExecutionLease) { + t.Helper() + return resetManagerStoreConfig(t, nil) +} + +func resetManagerStoreConfig(t *testing.T, configure func(*pgxpool.Config)) (*store.Store, *store.ExecutionLease) { t.Helper() url := os.Getenv("OAC_TEST_DATABASE_URL") if url == "" { @@ -56,6 +61,9 @@ func resetManagerStore(t *testing.T) (*store.Store, *store.ExecutionLease) { if err != nil { t.Fatal(err) } + if configure != nil { + configure(cfg) + } pool, err := pgxpool.NewWithConfig(t.Context(), cfg) if err != nil { t.Fatal(err) diff --git a/services/agents-api/internal/execution/sandbox_snapshot_budget_test.go b/services/agents-api/internal/execution/sandbox_snapshot_budget_test.go new file mode 100644 index 000000000..339f18f5c --- /dev/null +++ b/services/agents-api/internal/execution/sandbox_snapshot_budget_test.go @@ -0,0 +1,114 @@ +package execution + +import ( + "context" + "runtime/debug" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/adminaudit" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/node" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +type snapshotBudgetKey struct{} +type snapshotBudgetQuery struct { + started time.Time + ordinal int64 +} +type snapshotBudget struct { + t *testing.T + armed atomic.Bool + reads atomic.Int64 +} + +func (d *snapshotBudget) TraceQueryStart(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryStartData) context.Context { + q := snapshotBudgetQuery{started: time.Now()} + if d.armed.Load() && strings.Contains(data.SQL, "GetSandboxDeploymentSnapshot") && strings.Contains(string(debug.Stack()), "runtimeManager).resetPage") { + q.ordinal = d.reads.Add(1) + } + return context.WithValue(ctx, snapshotBudgetKey{}, q) +} +func (d *snapshotBudget) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) { + q, _ := ctx.Value(snapshotBudgetKey{}).(snapshotBudgetQuery) + if q.ordinal == 0 { + return + } + d.t.Logf("snapshot=%d query_elapsed=%s err=%v ctx=%v", q.ordinal, time.Since(q.started), data.Err, ctx.Err()) + // Model scheduling pressure on the first two reads without changing the real + // five-second page deadline. The final snapshot uses the lease connection. + if q.ordinal <= 2 && data.Err == nil { + delay := 2200*time.Millisecond - time.Since(q.started) + if delay > 0 { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-timer.C: + case <-ctx.Done(): + } + } + } +} + +func TestSandboxResetSnapshotFitsPageBudget(t *testing.T) { + budget := &snapshotBudget{t: t} + s, lease := resetManagerStoreConfig(t, func(cfg *pgxpool.Config) { + cfg.ConnConfig.RuntimeParams["jit"] = "on" + cfg.ConnConfig.Tracer = budget + }) + w := lease.Store() + id := uuid.NewString() + if err := w.ClaimWebSandboxDeployment(t.Context(), id); err != nil { + t.Fatal(err) + } + selection := store.SandboxDeploymentSetupRequest{Provider: "e2b", E2B: &store.SandboxE2BConfiguration{APIKey: "fixture-key", Template: "runtime:" + uuid.NewString()}} + selection.Resources.CPUs = 2 + selection.Resources.MemoryMiB = 2048 + if _, err := w.InitializeSandboxDeployment(t.Context(), id, selection); err != nil { + t.Fatal(err) + } + hub := node.NewHub(node.HubOptions{}) + defer hub.Close() + configuration := NewDeferredRuntimeProvider(id, func(ctx context.Context) (*RuntimeProvider, error) { + setup, err := s.GetSandboxSetup(ctx) + if err != nil || setup.Provider == "" { + return nil, err + } + return &RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: hub.Proxy(uuid.NewString(), "docker")}, nil + }) + m, err := newRuntimeManager(w, gateway.NewRegistry(), configuration) + if err != nil { + t.Fatal(err) + } + defer func() { m.stop(); m.drain() }() + if _, err = m.ensureDeployment(t.Context()); err != nil { + t.Fatal(err) + } + if _, err = w.StartSandboxReset(adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "reset-test", ActorLabel: "operator", RequestID: "request", TraceID: "trace"}), id, store.SandboxResetRequest{ExpectedGeneration: 1, Clear: "force"}); err != nil { + t.Fatal(err) + } + budget.armed.Store(true) + started := time.Now() + err = m.resetStep(t.Context()) + ping := lease.Ping(t.Context()) + t.Logf("reset_elapsed=%s reset_error=%v lease_ping=%v", time.Since(started), err, ping) + if err != nil || ping != nil { + t.Fatal("bounded snapshot lost execution ownership", err, ping) + } + if budget.reads.Load() < 3 { + t.Fatal("did not exercise final leased snapshot") + } + if err := w.CollectSandboxGenerations(t.Context()); err != nil { + t.Fatal("next generation collection lost ownership", err) + } + view, err := s.GetRuntimeDeployment(t.Context()) + if err != nil || view.Generation != 2 || view.Provider != "" || view.Reset != nil { + t.Fatal("reset did not commit", view, err) + } +} diff --git a/services/agents-api/internal/store/sandbox_generations_test.go b/services/agents-api/internal/store/sandbox_generations_test.go index 05e7b6221..714b3ebf2 100644 --- a/services/agents-api/internal/store/sandbox_generations_test.go +++ b/services/agents-api/internal/store/sandbox_generations_test.go @@ -27,6 +27,7 @@ func TestE2BGenerationsRetainOwnershipAndUseCurrentCredential(t *testing.T) { if err != nil || changed.Generation != 2 || changed.OwnerEpoch != view.OwnerEpoch || changed.Rollout.PreviousGenerationSandboxes != 1 || changed.Rollout.State != "settled" { t.Fatal(changed, err) } + assertSandboxSnapshotEquivalent(t, s.pool) ref := sandbox.Reference{TenantID: tenant, EnvironmentID: owner.EnvironmentID, AllocationID: owner.ID} retained, err := s.GetSandboxAllocationSetup(t.Context(), ref) if err != nil || retained.Generation != 1 || retained.E2B.Template != oldTemplate || retained.Specification.Resources.CPUs == input.Resources.CPUs { @@ -173,6 +174,7 @@ func TestPendingPlacementGenerationSurvivesRepeatedUpdates(t *testing.T) { t.Fatal(err) } } + assertSandboxSnapshotEquivalent(t, s.pool) owner := archiveAllocation(t, w, tenant, session, view.InstallationID) if owner.DeploymentGeneration != 1 || owner.NodeID != node.NodeID { t.Fatal("pending allocation rebound to newest generation", owner) @@ -332,3 +334,35 @@ func TestNodeRolloutSeparatesOfflinePinAndTargetReadiness(t *testing.T) { }) } } + +func TestSandboxSnapshotRolloutEquivalence(t *testing.T) { + s, _, view, _ := webSpecificationFixture(t, "docker") + node := specificationNode(t, s, view) + for _, state := range []string{"ready", "failed", "unconfirmed", "offline", "update_required"} { + t.Run(state, func(t *testing.T) { + runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET provider_ready=true,ready_generation=1,health='{}',last_seen_at=clock_timestamp(),connected_epoch=(SELECT owner_epoch FROM runtime_deployment) WHERE id=$1`, node.NodeID) + want := SandboxRolloutNodes{} + switch state { + case "ready": + want.Ready = 1 + case "failed": + runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET provider_ready=false,health='{"diagnostic":"provider_unavailable"}' WHERE id=$1`, node.NodeID) + want.Failed = 1 + case "unconfirmed": + runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET provider_ready=false WHERE id=$1`, node.NodeID) + want.Unknown = 1 + case "offline": + runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET last_seen_at=clock_timestamp()-interval '46 seconds',health='{"diagnostic":"provider_unavailable"}' WHERE id=$1`, node.NodeID) + want.Unknown = 1 + case "update_required": + runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_deployment SET generation=2`) + want.UpdateRequired = 1 + } + assertSandboxSnapshotEquivalent(t, s.pool) + got, err := s.GetRuntimeDeployment(t.Context()) + if err != nil || got.Rollout.Nodes == nil || *got.Rollout.Nodes != want || got.Rollout.State != "settled" { + t.Fatal("rollout classification changed", got.Rollout, err) + } + }) + } +} diff --git a/services/agents-api/internal/store/sandbox_reset_test.go b/services/agents-api/internal/store/sandbox_reset_test.go index 317d19be5..83e903403 100644 --- a/services/agents-api/internal/store/sandbox_reset_test.go +++ b/services/agents-api/internal/store/sandbox_reset_test.go @@ -78,6 +78,7 @@ func TestSandboxResetAutoUsesStartedWorkAndLockedRecheck(t *testing.T) { t.Fatal(err) } assertResetPartition(t, reset) + assertSandboxSnapshotEquivalent(t, s.pool) if (reset.Reset.Remaining.Busy == 1) != busy { t.Fatalf("wrong busy classification: %+v", reset.Reset.Remaining) } @@ -268,6 +269,7 @@ func TestSandboxResetSnapshotCountsOfflineOwnershipOnce(t *testing.T) { t.Fatal(err) } assertResetPartition(t, view) + assertSandboxSnapshotEquivalent(t, s.pool) want := int64(3) if state == "preparing" { want = 0 diff --git a/services/agents-api/internal/store/sandbox_snapshot_plan_test.go b/services/agents-api/internal/store/sandbox_snapshot_plan_test.go new file mode 100644 index 000000000..feb3e86df --- /dev/null +++ b/services/agents-api/internal/store/sandbox_snapshot_plan_test.go @@ -0,0 +1,106 @@ +package store + +import ( + "encoding/json" + "os" + "strings" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +func sandboxSnapshotSQL(t *testing.T) string { + t.Helper() + raw, err := os.ReadFile("../db/queries/sandbox_reset.sql") + if err != nil { + t.Fatal(err) + } + query := strings.Split(string(raw), "-- name: GetSandboxDeploymentSnapshot :one\n")[1] + return strings.TrimSuffix(strings.TrimSpace(strings.ReplaceAll(strings.Split(query, "\n-- name:")[0], "sqlc.embed(d)", "d.*")), ";") +} + +func TestSandboxSnapshotFreshPlan(t *testing.T) { + _, pool := newManagedTestStore(t) + tx, err := pool.Begin(t.Context()) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(t.Context()) + if _, err = tx.Exec(t.Context(), "SET LOCAL jit=on"); err != nil { + t.Fatal(err) + } + var stats int + if err = tx.QueryRow(t.Context(), "SELECT count(*) FROM pg_stats WHERE tablename='runtime_deployment'").Scan(&stats); err != nil || stats != 0 { + t.Fatal("fixture must exercise fresh cardinality", stats, err) + } + query := sandboxSnapshotSQL(t) + var raw []byte + if err = tx.QueryRow(t.Context(), "EXPLAIN (ANALYZE, FORMAT JSON) "+query).Scan(&raw); err != nil { + t.Fatal(err) + } + var plan []map[string]any + if err = json.Unmarshal(raw, &plan); err != nil { + t.Fatal(err) + } + if _, jit := plan[0]["JIT"]; jit { + t.Fatal("fresh singleton snapshot unexpectedly requires JIT", string(raw)) + } + bounded := false + var visit func(any) + visit = func(v any) { + switch v := v.(type) { + case map[string]any: + if v["Subplan Name"] == "CTE deployment" { + bounded = v["Plan Rows"] == float64(1) + } + for _, child := range v { + visit(child) + } + case []any: + for _, child := range v { + visit(child) + } + } + } + visit(plan[0]) + if !bounded { + t.Fatal("planner lost singleton bound", string(raw)) + } + t.Logf("fresh snapshot execution_ms=%v planning_ms=%v; singleton rows=1, no JIT", plan[0]["Execution Time"], plan[0]["Planning Time"]) + // A legal empty singleton table must not fabricate a deployment. + if _, err = tx.Exec(t.Context(), "DELETE FROM runtime_deployment"); err != nil { + t.Fatal(err) + } + var empty any + if err = tx.QueryRow(t.Context(), query).Scan(&empty); err != pgx.ErrNoRows { + t.Fatal("empty deployment produced a row", err) + } +} + +// Compare both complete projections in one MVCC statement and at one as_of, +// including every resource/rollout field. Only the cardinality annotation differs. +func assertSandboxSnapshotEquivalent(t *testing.T, pool *pgxpool.Pool) { + t.Helper() + query := sandboxSnapshotSQL(t) + legacy := strings.Replace(query, "WITH deployment AS MATERIALIZED (SELECT * FROM runtime_deployment WHERE singleton = true LIMIT 1),\nobserved AS", "WITH observed AS", 1) + legacy = strings.ReplaceAll(legacy, "CROSS JOIN deployment d", "CROSS JOIN runtime_deployment d") + legacy = strings.TrimSuffix(legacy, "\nWHERE d.singleton = true\nLIMIT 1") + query = strings.Replace(query, "clock_timestamp() AS as_of", "(SELECT as_of FROM snapshot_time) AS as_of", 1) + legacy = strings.Replace(legacy, "clock_timestamp() AS as_of", "(SELECT as_of FROM snapshot_time) AS as_of", 1) + tx, err := pool.Begin(t.Context()) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(t.Context()) + // Avoid paying the legacy planner's JIT cost in this result-equivalence check. + // The separate fresh-plan test keeps JIT enabled. + if _, err = tx.Exec(t.Context(), "SET LOCAL jit=off"); err != nil { + t.Fatal(err) + } + var same bool + err = tx.QueryRow(t.Context(), "WITH snapshot_time AS MATERIALIZED (SELECT clock_timestamp() AS as_of) SELECT (SELECT to_jsonb(s) FROM ("+query+") s) IS NOT DISTINCT FROM (SELECT to_jsonb(s) FROM ("+legacy+") s)").Scan(&same) + if err != nil || !same { + t.Fatal("singleton annotation changed snapshot", same, err) + } +} From dcaf3b823af1096f7b9ceac4a9afe9b059c5e883 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 23:15:10 +0000 Subject: [PATCH 09/16] test(node): await peer publication before responsiveness checks (cherry picked from commit f620ceac89a6d4e5fa0002e8a9c733301aa12e1b) --- services/agents-api/internal/sandbox/node/hub_lifetime_test.go | 1 + services/agents-api/internal/sandbox/node/hub_postgres_test.go | 1 + 2 files changed, 2 insertions(+) diff --git a/services/agents-api/internal/sandbox/node/hub_lifetime_test.go b/services/agents-api/internal/sandbox/node/hub_lifetime_test.go index 283614a93..d72021c21 100644 --- a/services/agents-api/internal/sandbox/node/hub_lifetime_test.go +++ b/services/agents-api/internal/sandbox/node/hub_lifetime_test.go @@ -194,6 +194,7 @@ func TestSlowOpeningAndClosingLeaveOtherNodesResponsive(t *testing.T) { live := connectRawNode(t, server.URL, fast) defer live.Close() go serveRawInfo(live) + wait(t, func() bool { return hub.Online(fast.NodeID) }) beginRawNode(t, server.URL, slow) select { case <-opening: diff --git a/services/agents-api/internal/sandbox/node/hub_postgres_test.go b/services/agents-api/internal/sandbox/node/hub_postgres_test.go index ac29125e1..38a751663 100644 --- a/services/agents-api/internal/sandbox/node/hub_postgres_test.go +++ b/services/agents-api/internal/sandbox/node/hub_postgres_test.go @@ -113,6 +113,7 @@ func TestHubPostgresBlockedOpeningIsBounded(t *testing.T) { live := connectRawNode(t, server.URL, fast) defer live.Close() go serveRawInfo(live) + wait(t, func() bool { return hub.Online(fast.NodeID) }) start := time.Now() beginRawNode(t, server.URL, slow) var pid uint32 From 36a306b42900bd2d9bdeca23f383b5cfdb67155f Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 23:50:49 +0000 Subject: [PATCH 10/16] Exercise snapshot equivalence through node generation readiness --- .../store/sandbox_generations_test.go | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/services/agents-api/internal/store/sandbox_generations_test.go b/services/agents-api/internal/store/sandbox_generations_test.go index 714b3ebf2..c50c1ed03 100644 --- a/services/agents-api/internal/store/sandbox_generations_test.go +++ b/services/agents-api/internal/store/sandbox_generations_test.go @@ -336,31 +336,42 @@ func TestNodeRolloutSeparatesOfflinePinAndTargetReadiness(t *testing.T) { } func TestSandboxSnapshotRolloutEquivalence(t *testing.T) { - s, _, view, _ := webSpecificationFixture(t, "docker") + s, w, view, input := webSpecificationFixture(t, "docker") node := specificationNode(t, s, view) - for _, state := range []string{"ready", "failed", "unconfirmed", "offline", "update_required"} { + for _, state := range []string{"ready", "preparing", "failed", "unconfirmed", "offline", "update_required"} { t.Run(state, func(t *testing.T) { - runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET provider_ready=true,ready_generation=1,health='{}',last_seen_at=clock_timestamp(),connected_epoch=(SELECT owner_epoch FROM runtime_deployment) WHERE id=$1`, node.NodeID) + connection := onlineManagerNode(t, s, node.NodeID) want := SandboxRolloutNodes{} + wantState := "settled" switch state { case "ready": want.Ready = 1 + case "preparing": + generationHeartbeat(t, s, node, connection, view, "preparing") + want.Preparing = 1 + wantState = "preparing" case "failed": - runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET provider_ready=false,health='{"diagnostic":"provider_unavailable"}' WHERE id=$1`, node.NodeID) + generationHeartbeat(t, s, node, connection, view, "failed") want.Failed = 1 case "unconfirmed": - runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET provider_ready=false WHERE id=$1`, node.NodeID) + connection = uuid.NewString() + if err := s.ConnectRuntimeNode(t.Context(), node.NodeID, connection, view.OwnerEpoch); err != nil { + t.Fatal(err) + } + if err := s.HeartbeatRuntimeNodeGenerations(t.Context(), node.NodeID, connection, view.OwnerEpoch, RuntimeNodeHealth{}, nil); err != nil { + t.Fatal(err) + } want.Unknown = 1 case "offline": runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET last_seen_at=clock_timestamp()-interval '46 seconds',health='{"diagnostic":"provider_unavailable"}' WHERE id=$1`, node.NodeID) want.Unknown = 1 case "update_required": - runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_deployment SET generation=2`) + changeNodeTarget(t, w, view, input) want.UpdateRequired = 1 } assertSandboxSnapshotEquivalent(t, s.pool) got, err := s.GetRuntimeDeployment(t.Context()) - if err != nil || got.Rollout.Nodes == nil || *got.Rollout.Nodes != want || got.Rollout.State != "settled" { + if err != nil || got.Rollout.Nodes == nil || *got.Rollout.Nodes != want || got.Rollout.State != wantState { t.Fatal("rollout classification changed", got.Rollout, err) } }) From d9ac7ad21de8cefbc55476b8f51377925f89e680 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 27 Sep 2026 23:50:49 +0000 Subject: [PATCH 11/16] Preserve shared Docker images during installation-local collection --- CONTRIBUTING.md | 5 +- .../agents-api/node-generation-protocol.md | 12 ++-- deploy/install/node_generations.py | 22 +++--- deploy/install/test_node_generations.py | 68 +++++++++++-------- .../cmd/sandbox-node/generations.go | 10 +-- .../cmd/sandbox-node/generations_test.go | 14 +++- 6 files changed, 76 insertions(+), 55 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a1ec428a5..7e956e9ae 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -819,7 +819,10 @@ Interrupted node collection keeps an exact private generation journal and immuta configuration. Restart treats it only as a candidate for a fresh correlated Core drop grant, never as preparation or serving readiness. Persist native cleanup before removing its executable and durable file cleanup before the dropped marker; -CLI errors and unknown native ownership cannot establish absence. +CLI errors and unknown native ownership cannot establish absence. Docker daemon +images are shared host content, so automatic node GC retains them; only the host +administrator can establish whole-host authority to remove them. Microsandbox +image GC remains scoped to the installation-private store. Use one E2B classifier. Omitted key preserves the current key; identical selection with omitted key is a no-op. Explicit key submission, including identical bytes, diff --git a/contracts/agents-api/node-generation-protocol.md b/contracts/agents-api/node-generation-protocol.md index b1a25a3d7..a36ff4f5b 100644 --- a/contracts/agents-api/node-generation-protocol.md +++ b/contracts/agents-api/node-generation-protocol.md @@ -116,12 +116,16 @@ unfinished journal only as a retention-exchange candidate: it cannot prepare, probe, acquire or advertise that generation. A fresh correlated Core drop grant is required to resume; the journal itself never authorizes deletion. -A successful complete native image inventory distinguishes absence from a CLI or -daemon failure. Failed queries, malformed inventories, native in-use refusals and +For an installation-private microsandbox store, a successful complete native image +inventory distinguishes absence from a CLI failure. Failed queries, malformed inventories, native in-use refusals and unknown ownership retain the local bytes. Native completion is persisted before release-file cleanup, so a retry can finish a partly removed release without -executing an already removed helper. Shared images/releases remain until their -last local reference. The final digest-bound dropped marker follows durable file +executing an already removed helper. Shared microsandbox images and private releases remain until their last local +reference. Docker imported images belong to the shared host daemon and are retained, +including when another installation has only an idle serving pin. Automatic node +GC never runs Docker image removal or pruning. The host administrator may remove +those images only after confirming that no installation on the host needs them; +local generation collection does not claim physical Docker image GC. The final digest-bound dropped marker follows durable file cleanup and permanently prevents re-adoption. The small immutable configuration and ownership journals remain as local identity records. diff --git a/deploy/install/node_generations.py b/deploy/install/node_generations.py index 069b943cd..f311b2fbd 100644 --- a/deploy/install/node_generations.py +++ b/deploy/install/node_generations.py @@ -85,8 +85,8 @@ def generation_marker(root, generation, suffix, digest, installation, installer) raise installer.InstallError("Invalid generation ownership journal") keys = set(expected) if suffix == ".collecting": - keys.add("image_removed") - if type(value.get("image_removed")) is not bool: + keys.add("native_complete") + if type(value.get("native_complete")) is not bool: raise installer.InstallError("Invalid generation collection journal") if suffix == ".preparing": keys.add("import_started") @@ -326,16 +326,16 @@ def collect(args, installer): directory = root / "state/node/generations" journal = generation_marker(root, args.generation, ".collecting", args.specification_digest, args.installation_id, installer) if journal is None: - journal = dict(marker_identity(args), image_removed=False) + journal = dict(marker_identity(args), native_complete=False) atomic_json(directory / (str(args.generation) + ".collecting"), journal) others = [item for generation, item in configurations.items() if generation != args.generation] - if not journal["image_removed"]: + if not journal["native_complete"]: preparation = generation_marker(root, args.generation, ".preparing", args.specification_digest, args.installation_id, installer) if preparation is None or preparation["import_started"]: collect_image(args, value, others, installer) # Persist native completion before deleting its executable. A fresh # Core grant is still required after restart to finish file cleanup. - journal["image_removed"] = True + journal["native_complete"] = True atomic_json(directory / (str(args.generation) + ".collecting"), journal) directory = root / "state/node/generations" installer.safe_directory(directory) @@ -388,11 +388,7 @@ def collect_image(args, value, others, installer): raise installer.InstallError("Microsandbox store ownership differs") # Native emptiness never authorizes discarding receipt/lock history. else: - image = value["docker"]["image"] - if not any(item["docker"]["image"] == image for item in others): - raw = installer.checked(list(installer.DOCKER) + ["image", "ls", "--quiet", "--no-trunc"], "Cannot verify Docker image inventory") - images = raw.splitlines() - if any(not re.fullmatch(r"sha256:[a-f0-9]{64}", item) for item in images): - raise installer.InstallError("Cannot verify Docker image inventory") - if image in images: - installer.checked(list(installer.DOCKER) + ["image", "rm", image], "Docker image is still in use") + # Docker imported content belongs to the host daemon, including idle + # serving pins in other installations. This installation has no authority + # to remove it; its private generation/release files can still be collected. + return diff --git a/deploy/install/test_node_generations.py b/deploy/install/test_node_generations.py index 182ce864d..31a6b9ae8 100644 --- a/deploy/install/test_node_generations.py +++ b/deploy/install/test_node_generations.py @@ -49,22 +49,23 @@ def test_busy_helper_refuses_all_mutations_then_same_inode_collects(self): self.assertFalse(self.release.exists()) self.assertTrue((self.directory / "1.dropped").exists()) node_generations.collect(self.args, installer) - installer.checked.assert_called_once() + installer.checked.assert_not_called() final = lease.stat() self.assertEqual((original.st_dev, original.st_ino), (final.st_dev, final.st_ino)) def test_collection_holds_exclusive_lease_during_deletion(self): lease = self.directory / "1.lease" - def remove(*_args, **_kwargs): + original = node_generations.shutil.rmtree + def remove(path): descriptor = os.open(lease, os.O_RDWR) try: with self.assertRaises(BlockingIOError): fcntl.flock(descriptor, fcntl.LOCK_SH | fcntl.LOCK_NB) finally: os.close(descriptor) - return "" - installer.checked.side_effect = remove - node_generations.collect(self.args, installer) + return original(path) + with mock.patch.object(node_generations.shutil, "rmtree", side_effect=remove): + node_generations.collect(self.args, installer) def test_symlink_and_shared_inode_are_not_collection_authority(self): lease = self.directory / "1.lease" @@ -81,28 +82,8 @@ def test_symlink_and_shared_inode_are_not_collection_authority(self): finally: lease.unlink() - def test_restart_after_native_remove_uses_inventory_not_error_text(self): - present = [True] - removes = [] - def native(command, *_args, **_kwargs): - if "ls" in command: - return self.value["docker"]["image"] + "\n" if present[0] else "" - self.assertIn("rm", command) - present[0] = False - removes.append(command) - # Model interruption after native commit but before the response. - raise installer.InstallError("interrupted after native mutation") - installer.checked.side_effect = native - with self.assertRaises(installer.InstallError): - node_generations.collect(self.args, installer) - self.assertFalse(json.loads((self.directory / "1.collecting").read_text())["image_removed"]) - self.assertTrue(self.release.exists()) - node_generations.collect(self.args, installer) - self.assertEqual(len(removes), 1) - self.assertFalse(self.release.exists()) - self.assertTrue((self.directory / "1.dropped").exists()) - def test_native_inventory_failure_and_unknown_output_retain_every_byte(self): + self.micro_fixture() for failure in (installer.InstallError("daemon unavailable"), "not a verified image ID"): installer.checked.side_effect = failure if isinstance(failure, Exception) else None installer.checked.return_value = failure @@ -110,7 +91,7 @@ def test_native_inventory_failure_and_unknown_output_retain_every_byte(self): node_generations.collect(self.args, installer) self.assertTrue((self.release / "artifact").exists()) self.assertFalse((self.directory / "1.dropped").exists()) - self.assertFalse(json.loads((self.directory / "1.collecting").read_text())["image_removed"]) + self.assertFalse(json.loads((self.directory / "1.collecting").read_text())["native_complete"]) def test_restart_after_partial_file_cleanup_skips_native_and_finishes(self): def interrupted(path): @@ -119,7 +100,7 @@ def interrupted(path): with mock.patch.object(node_generations.shutil, "rmtree", side_effect=interrupted): with self.assertRaises(OSError): node_generations.collect(self.args, installer) - self.assertTrue(json.loads((self.directory / "1.collecting").read_text())["image_removed"]) + self.assertTrue(json.loads((self.directory / "1.collecting").read_text())["native_complete"]) self.assertFalse((self.directory / "1.dropped").exists()) installer.checked.reset_mock() node_generations.collect(self.args, installer) @@ -146,7 +127,7 @@ def test_invalid_journal_never_authorizes_deletion_or_hides_configuration(self): for suffix in (".collecting", ".dropped", ".preparing"): path = self.directory / ("1" + suffix) value = node_generations.marker_identity(self.args) - if suffix == ".collecting": value["image_removed"] = True + if suffix == ".collecting": value["native_complete"] = True if suffix == ".preparing": value["import_started"] = False for field, invalid in (("installation_id", "foreign"), ("generation", 2), ("specification_digest", "a" * 64)): node_generations.atomic_json(path, dict(value, **{field: invalid})) @@ -167,6 +148,8 @@ def test_shared_image_and_release_survive_until_last_generation(self): self.assertFalse(self.release.exists()) def test_never_imported_generation_can_collect_missing_executable(self): + runtime, _, _ = self.micro_fixture() + runtime.unlink() node_generations.atomic_json(self.directory / "1.preparing", dict(node_generations.marker_identity(self.args), import_started=False)) node_generations.collect(self.args, installer) installer.checked.assert_not_called() @@ -180,7 +163,7 @@ def test_legacy_unfenced_marker_blocks_before_collection_journal(self): self.assertFalse((self.directory / "1.collecting").exists()) self.assertTrue(self.release.exists()) - def test_micro_interrupted_native_removal_and_missing_runtime_fail_closed(self): + def micro_fixture(self): self.value["provider"] = "microsandbox" del self.value["docker"] home = self.root / "micro-store" @@ -193,6 +176,10 @@ def test_micro_interrupted_native_removal_and_missing_runtime_fail_closed(self): self.value["microsandbox"] = {"runtime_home": str(home), "runtime_path": str(runtime), "firmware_path": str(self.release / "firmware"), "runtime_sha256": hashlib.sha256(runtime.read_bytes()).hexdigest(), "image": image} self.args.specification_digest = node_spec.digest("microsandbox", self.value["specification"]) node_generations.atomic_json(self.root / "provider.json", self.value) + return runtime, image, home + + def test_micro_interrupted_native_removal_and_missing_runtime_fail_closed(self): + runtime, image, home = self.micro_fixture() present = [True] def native(command, *_args, **_kwargs): if command[1:3] == ["image", "list"]: return image + "\n" if present[0] else "" @@ -216,6 +203,27 @@ def native(command, *_args, **_kwargs): self.assertFalse(self.release.exists()) self.assertTrue((home / "oac-installation.json").exists()) + def test_docker_collection_preserves_another_installations_idle_serving_image(self): + second = self.root.parent / (self.root.name + "-second") + second.mkdir(mode=0o700) + self.addCleanup(lambda: node_generations.shutil.rmtree(second)) + # The second installation's enrolled generation still backs an idle + # serving pin. No container is needed to protect its daemon-level image. + other = dict(self.value, installation_id="second-installation") + node_generations.atomic_json(second / "provider.json", other) + before = (second / "provider.json").read_bytes() + host_images = {self.value["docker"]["image"]} + def docker(command, *_args, **_kwargs): + if "rm" in command or "prune" in command: host_images.clear() + raise AssertionError("generation GC must not manage shared Docker images") + installer.checked.side_effect = docker + node_generations.collect(self.args, installer) + installer.checked.assert_not_called() + self.assertEqual(host_images, {other["docker"]["image"]}) + self.assertEqual((second / "provider.json").read_bytes(), before) + self.assertFalse(self.release.exists()) + self.assertEqual(node_generations.retained_configs(self.root, installer), {}) + def test_fifo_journal_is_refused_without_blocking(self): path = self.directory / "1.collecting" os.mkfifo(path, 0o600) diff --git a/services/agents-api/cmd/sandbox-node/generations.go b/services/agents-api/cmd/sandbox-node/generations.go index 5856cf2e0..5b98cb632 100644 --- a/services/agents-api/cmd/sandbox-node/generations.go +++ b/services/agents-api/cmd/sandbox-node/generations.go @@ -228,10 +228,10 @@ func generationLocalState(config providerconfig.Config, stateDir string) (string return "", sandbox.ErrOwnership } var journal struct { - InstallationID string `json:"installation_id"` - Generation uint64 `json:"generation"` - SpecificationDigest string `json:"specification_digest"` - ImageRemoved *bool `json:"image_removed,omitempty"` + InstallationID string `json:"installation_id"` + Generation uint64 `json:"generation"` + SpecificationDigest string `json:"specification_digest"` + NativeComplete json.RawMessage `json:"native_complete,omitempty"` } decoder := json.NewDecoder(io.LimitReader(file, 4097)) decoder.DisallowUnknownFields() @@ -244,7 +244,7 @@ func generationLocalState(config providerconfig.Config, stateDir string) (string if statErr != nil || namedErr != nil || !os.SameFile(opened, named) { return "", sandbox.ErrOwnership } - if err != nil || end != io.EOF || journal.InstallationID != config.InstallationID || journal.Generation != config.Generation || journal.SpecificationDigest != config.Specification.Digest(config.Provider) || (suffix == ".collecting") != (journal.ImageRemoved != nil) { + if err != nil || end != io.EOF || journal.InstallationID != config.InstallationID || journal.Generation != config.Generation || journal.SpecificationDigest != config.Specification.Digest(config.Provider) || (suffix == ".collecting" && string(journal.NativeComplete) != "true" && string(journal.NativeComplete) != "false") || (suffix == ".dropped" && len(journal.NativeComplete) != 0) { return "", sandbox.ErrOwnership } state = strings.TrimPrefix(suffix, ".") diff --git a/services/agents-api/cmd/sandbox-node/generations_test.go b/services/agents-api/cmd/sandbox-node/generations_test.go index c1f381410..50a9343d3 100644 --- a/services/agents-api/cmd/sandbox-node/generations_test.go +++ b/services/agents-api/cmd/sandbox-node/generations_test.go @@ -22,9 +22,9 @@ func TestGenerationJournalRestartIdentity(t *testing.T) { t.Run(suffix, func(t *testing.T) { path := filepath.Join(directory, "9"+suffix) if suffix == ".collecting" { - journal["image_removed"] = true + journal["native_complete"] = true } else { - delete(journal, "image_removed") + delete(journal, "native_complete") } write := func(value map[string]any) { raw, _ := json.Marshal(value) @@ -46,6 +46,16 @@ func TestGenerationJournalRestartIdentity(t *testing.T) { } journal[field] = previous } + journal["native_complete"] = nil + write(journal) + if _, err = generationLocalState(config, stateDir); err == nil { + t.Fatal("accepted null journal phase") + } + if suffix == ".collecting" { + journal["native_complete"] = true + } else { + delete(journal, "native_complete") + } write(journal) if err = os.Chmod(path, 0644); err != nil { t.Fatal(err) From a6cad3dc73dda9dd6636ac14e95e904b209a4679 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Mon, 28 Sep 2026 00:00:46 +0000 Subject: [PATCH 12/16] Collect verified original Runtime files without removing node identity --- CONTRIBUTING.md | 5 +- .../agents-api/node-generation-protocol.md | 12 ++ deploy/install/node_generations.py | 72 +++++++++++ deploy/install/node_install.py | 1 + deploy/install/test_node_generations.py | 3 +- deploy/install/test_node_install.py | 115 ++++++++++++++++++ deploy/install/test_node_update.py | 2 +- 7 files changed, 207 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7e956e9ae..937909fbe 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -822,7 +822,10 @@ before removing its executable and durable file cleanup before the dropped marke CLI errors and unknown native ownership cannot establish absence. Docker daemon images are shared host content, so automatic node GC retains them; only the host administrator can establish whole-host authority to remove them. Microsandbox -image GC remains scoped to the installation-private store. +image GC remains scoped to the installation-private store. Fresh nodes persist +verified original Runtime file ownership separately from the host program; collect +only exact unreferenced Runtime paths, keeping program, identity, base configuration +and manifests readable across interrupted cleanup and restart. Use one E2B classifier. Omitted key preserves the current key; identical selection with omitted key is a no-op. Explicit key submission, including identical bytes, diff --git a/contracts/agents-api/node-generation-protocol.md b/contracts/agents-api/node-generation-protocol.md index a36ff4f5b..b9f994151 100644 --- a/contracts/agents-api/node-generation-protocol.md +++ b/contracts/agents-api/node-generation-protocol.md @@ -129,6 +129,18 @@ local generation collection does not claim physical Docker image GC. The final d cleanup and permanently prevents re-adoption. The small immutable configuration and ownership journals remain as local identity records. +Fresh installations also persist the verified Runtime file checksums separately +from the host program. Collection of the original generation removes only those +exact private Runtime helper, executable, firmware, seccomp and import-cache files +that no retained configuration references. All remaining files are checked before +the first deletion; unknown hashes, changed bytes, links or missing ownership +metadata refuse cleanup. Interruption resumes under the same collection journal. +The current node executable, preparer, identity, base provider configuration and +manifests remain, so restart can read its enrolled identity and construct a newer +retained provider after the original Runtime bytes have gone. Shared native paths +are compared across all retained configurations before removal. The v1 +legacy-unfenced marker still prevents any collection of its original payload. + New preparation records its exact identity before downloads and records import start before invoking the native importer. An interrupted download can repair only missing bytes at the original paths. If collection precedes any import attempt, diff --git a/deploy/install/node_generations.py b/deploy/install/node_generations.py index f311b2fbd..da22a2233 100644 --- a/deploy/install/node_generations.py +++ b/deploy/install/node_generations.py @@ -102,6 +102,68 @@ def marker_identity(args): "specification_digest": args.specification_digest} +def record_root_runtime(root, args, manifest, sums, installer): + checksums = {"runtime/seccomp.json": sums["runtime/seccomp.json"]} + if args.provider == "microsandbox": + checksums.update({name: installer.distribution.artifact(manifest, name)["sha256"] for name in installer.MICRO}) + archive = installer.distribution.artifact(manifest, "images/runtime.tar.gz") + checksums["images/runtime.tar.gz"] = archive["sha256"] + checksums["images/runtime.tar"] = archive["unpacked_sha256"] + value = {"installation_id": args.installation_id, "source_commit": manifest["source_commit"], "sha256": checksums} + path = root / "runtime-artifacts.json" + if installer.existing_file(path): + if installer.private_json(path) != value: + raise installer.InstallError("Original Runtime artifact ownership differs") + return + atomic_json(path, value) + + +def root_runtime_files(root, value, others, installer): + """Return only verified original Runtime files that no retained config uses.""" + def paths(configuration): + if configuration["provider"] == "docker": + return [Path(configuration["docker"]["seccomp_file"])] + return [Path(configuration["microsandbox"][key]) for key in ("helper_path", "runtime_path", "firmware_path")] + names = ["runtime/seccomp.json", "images/runtime.tar.gz", "images/runtime.tar"] + if value["provider"] == "microsandbox": + names.extend(installer.MICRO) + own_paths = paths(value) + if not any(path == root / name for path in own_paths for name in names): + return [] + referenced = {Path(os.path.realpath(path)) for item in others for path in paths(item)} + auxiliary = {root / name for name in ("runtime/seccomp.json", "images/runtime.tar.gz", "images/runtime.tar")} + # An original Runtime provider also retains its import cache and policy. + if any(path in referenced for path in own_paths): + referenced.update(auxiliary) + candidates = [root / name for name in names if root / name not in referenced] + if not candidates: + return [] + record = root / "runtime-artifacts.json" + installer.no_links(record) + if not installer.existing_file(record): + raise installer.InstallError("Original Runtime artifact ownership is missing; preserve its files") + saved = installer.private_json(record) + base = installer.private_json(root / "provider.json") + if not base or base.get("installation_id") != value["installation_id"] or base.get("provider") != value["provider"]: + raise installer.InstallError("Original Runtime configuration differs") + source = base["specification"]["runtime"]["source_commit"] + if (not saved or set(saved) != {"installation_id", "source_commit", "sha256"} + or saved["installation_id"] != value["installation_id"] or saved["source_commit"] != source + or not isinstance(saved["sha256"], dict) or set(saved["sha256"]) != set(names) + or any(not isinstance(digest, str) or not re.fullmatch(r"[a-f0-9]{64}", digest) for digest in saved["sha256"].values())): + raise installer.InstallError("Original Runtime artifact ownership differs") + if value["provider"] == "microsandbox" and any( + saved["sha256"][name] != base["specification"]["runtime"][field] + for name, field in zip(installer.MICRO[1:], ("runtime_sha256", "firmware_sha256"))): + raise installer.InstallError("Original Runtime artifact hashes differ from its specification") + for path in candidates: + installer.no_links(path) + if installer.existing_file(path): + if path.stat().st_nlink != 1 or installer.file_digest(path) != saved["sha256"][str(path.relative_to(root))]: + raise installer.InstallError("Original Runtime file differs; refusing deletion") + return candidates + + def retained_configs(root, installer): directory = root / "state/node/generations" result = {} @@ -329,6 +391,7 @@ def collect(args, installer): journal = dict(marker_identity(args), native_complete=False) atomic_json(directory / (str(args.generation) + ".collecting"), journal) others = [item for generation, item in configurations.items() if generation != args.generation] + original_files = root_runtime_files(root, value, others, installer) if not journal["native_complete"]: preparation = generation_marker(root, args.generation, ".preparing", args.specification_digest, args.installation_id, installer) if preparation is None or preparation["import_started"]: @@ -351,6 +414,15 @@ def collect(args, installer): finally: os.close(descriptor) + for path in original_files: + path.unlink(missing_ok=True) + for parent in {path.parent for path in original_files}: + if parent.exists(): + descriptor = os.open(parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) atomic_json(directory / (str(args.generation) + ".dropped"), marker_identity(args)) # Keep the immutable configuration and permanent journal as restart # identity. They contain no provider credentials and are not readiness. diff --git a/deploy/install/node_install.py b/deploy/install/node_install.py index 1c5c257d1..fc5e3a11d 100644 --- a/deploy/install/node_install.py +++ b/deploy/install/node_install.py @@ -431,6 +431,7 @@ def register_node(root, args, token): "source_commit": manifest["source_commit"], "generation": args.configuration["generation"], "specification_digest": args.configuration["specification_digest"]} write_once(root / "installation.json", json_text(state)) + node_generations.record_root_runtime(root, args, manifest, sums, sys.modules[__name__]) for name in names: if name == "runtime/seccomp.json": if getattr(args, "bundle", None) is None: diff --git a/deploy/install/test_node_generations.py b/deploy/install/test_node_generations.py index 31a6b9ae8..ac0aa7cde 100644 --- a/deploy/install/test_node_generations.py +++ b/deploy/install/test_node_generations.py @@ -25,6 +25,7 @@ def setUp(self): "specification": {"resources": {"cpus": 1, "memory_mib": 1024}, "runtime": {"source_commit": "b" * 40, "image_id": "sha256:" + "a" * 64, "image_manifest_digest": "sha256:" + "c" * 64, "microsandbox_ref": "oac-runtime@sha256:" + "d" * 64, "runtime_sha256": "e" * 64, "firmware_sha256": "f" * 64}}} self.args = SimpleNamespace(installation_id="test-installation", generation=1, specification_digest=node_spec.digest("docker", self.value["specification"])) self.release = self.root / "releases" / ("b" * 40) + self.value["docker"]["seccomp_file"] = str(self.release / "runtime/seccomp.json") self.release.mkdir(parents=True) (self.release / "artifact").write_bytes(b"immutable bytes") node_generations.atomic_json(self.root / "provider.json", self.value) @@ -173,7 +174,7 @@ def micro_fixture(self): runtime.write_bytes(b"verified native executable") runtime.chmod(0o700) image = self.value["specification"]["runtime"]["microsandbox_ref"] - self.value["microsandbox"] = {"runtime_home": str(home), "runtime_path": str(runtime), "firmware_path": str(self.release / "firmware"), "runtime_sha256": hashlib.sha256(runtime.read_bytes()).hexdigest(), "image": image} + self.value["microsandbox"] = {"helper_path": str(self.release / "helper"), "runtime_home": str(home), "runtime_path": str(runtime), "firmware_path": str(self.release / "firmware"), "runtime_sha256": hashlib.sha256(runtime.read_bytes()).hexdigest(), "image": image} self.args.specification_digest = node_spec.digest("microsandbox", self.value["specification"]) node_generations.atomic_json(self.root / "provider.json", self.value) return runtime, image, home diff --git a/deploy/install/test_node_install.py b/deploy/install/test_node_install.py index 9a180284f..9e3148cc7 100644 --- a/deploy/install/test_node_install.py +++ b/deploy/install/test_node_install.py @@ -104,6 +104,8 @@ def refresh_manifest(self): self.manifest["artifacts"] = {name: {"filename": name.replace("/", "-") + "-" + self.manifest["source_commit"], "size": len(raw), "sha256": hashlib.sha256(raw).hexdigest()} for name, raw in self.payloads.items() if name.startswith("native/") or name == "images/runtime.tar.gz"} + self.manifest["microsandbox"] = {"runtime_sha256": hashlib.sha256(self.payloads[installer.MICRO[1]]).hexdigest(), + "firmware_sha256": hashlib.sha256(self.payloads[installer.MICRO[2]]).hexdigest()} self.manifest["artifacts"]["images/runtime.tar.gz"].update(unpacked_sha256=hashlib.sha256(b"runtime archive").hexdigest(), unpacked_size=len(b"runtime archive")) self.payloads["manifest.json"] = json.dumps(self.manifest).encode() self.payloads["SHA256SUMS"] = "".join(hashlib.sha256(raw).hexdigest() + " " + name + "\n" @@ -237,6 +239,119 @@ def recovery_args(self): self.args.specification_digest = json.loads((self.root / "state/node/identity.json").read_text())["identity"]["specification_digest"] return self.args + def prepare_successor_runtime(self): + self.manifest["source_commit"] = "f" * 40 + self.refresh_manifest() + config = json.loads(self.configuration_response(None).read()) + config["generation"] = 2 + self.args.generation = 2 + self.args.specification_digest = config["specification_digest"] + with mock.patch.object(installer.node_spec, "fetch", return_value=config): + installer.node_generations.prepare(self.args, installer) + return installer.private_json(self.root / "state/node/generations/2.json") + + def test_original_runtime_gc_interruption_preserves_restart_identity_and_successor(self): + self.args.provider = "microsandbox" + self.install() + successor = self.prepare_successor_runtime() + keep = (installer.COMMON[0], "generation-preparer.pyz", "provider.json", "registered.json", + "installation.json", "preparation.json", "runtime-artifacts.json", "state/node/identity.json") + before = {name: (self.root / name).read_bytes() for name in keep} + original = installer.private_json(self.root / "provider.json") + self.args.generation = 1 + self.args.specification_digest = node_spec.digest("microsandbox", original["specification"]) + real_unlink = Path.unlink + def interrupted(path, *args, **kwargs): + if path == self.root / installer.MICRO[1]: + raise OSError("interrupted original Runtime cleanup") + return real_unlink(path, *args, **kwargs) + with mock.patch.object(Path, "unlink", new=interrupted): + with self.assertRaises(OSError): installer.node_generations.collect(self.args, installer) + self.assertFalse((self.root / installer.MICRO[0]).exists()) + self.assertTrue((self.root / installer.MICRO[1]).exists()) + self.assertTrue(installer.private_json(self.root / "state/node/generations/1.collecting")["native_complete"]) + self.assertEqual(installer.private_json(self.root / "provider.json"), original) + self.assertEqual(installer.node_generations.retained_configs(self.root, installer)[2], successor) + count = len(self.calls) + installer.node_generations.collect(self.args, installer) + self.assertEqual(len(self.calls), count) + for name in ("runtime/seccomp.json", "images/runtime.tar.gz", "images/runtime.tar") + installer.MICRO: + self.assertFalse((self.root / name).exists(), name) + for name, raw in before.items(): self.assertEqual((self.root / name).read_bytes(), raw, name) + self.assertEqual(installer.node_generations.retained_configs(self.root, installer), {2: successor}) + self.assertTrue(installer.node_generations.image_available(successor, installer)) + + def test_original_runtime_gc_refuses_changed_sibling_before_deleting_any_file(self): + self.args.provider = "microsandbox" + self.install() + self.prepare_successor_runtime() + original = installer.private_json(self.root / "provider.json") + self.args.generation = 1 + self.args.specification_digest = node_spec.digest("microsandbox", original["specification"]) + firmware = self.root / installer.MICRO[2] + firmware.write_bytes(b"changed private file") + before = {name: (self.root / name).read_bytes() for name in ("runtime/seccomp.json",) + installer.MICRO} + count = len(self.calls) + with self.assertRaisesRegex(installer.InstallError, "refusing deletion"): + installer.node_generations.collect(self.args, installer) + self.assertEqual(len(self.calls), count) + for name, raw in before.items(): self.assertEqual((self.root / name).read_bytes(), raw, name) + + def test_original_runtime_gc_missing_ownership_refuses_and_preserves_unknown_files(self): + self.install() + self.prepare_successor_runtime() + (self.root / "runtime-artifacts.json").unlink() + unknown = self.root / "native/bin/operator-owned-file" + unknown.write_bytes(b"unowned") + self.args.generation = 1 + original = installer.private_json(self.root / "provider.json") + self.args.specification_digest = node_spec.digest("docker", original["specification"]) + seccomp = (self.root / "runtime/seccomp.json").read_bytes() + with self.assertRaisesRegex(installer.InstallError, "ownership is missing"): + installer.node_generations.collect(self.args, installer) + self.assertEqual((self.root / "runtime/seccomp.json").read_bytes(), seccomp) + self.assertEqual(unknown.read_bytes(), b"unowned") + + def test_shared_original_firmware_survives_until_its_last_generation(self): + self.args.provider = "microsandbox" + self.install() + successor = self.prepare_successor_runtime() + successor["microsandbox"]["firmware_path"] = str(self.root / installer.MICRO[2]) + installer.node_generations.atomic_json(self.root / "state/node/generations/2.json", successor) + original = installer.private_json(self.root / "provider.json") + self.args.generation = 1 + self.args.specification_digest = node_spec.digest("microsandbox", original["specification"]) + installer.node_generations.collect(self.args, installer) + self.assertFalse((self.root / installer.MICRO[0]).exists()) + self.assertFalse((self.root / installer.MICRO[1]).exists()) + self.assertTrue((self.root / installer.MICRO[2]).exists()) + self.assertTrue(installer.node_generations.image_available(successor, installer)) + self.args.generation = 2 + self.args.specification_digest = node_spec.digest("microsandbox", successor["specification"]) + def inventory(command, *_args, **_kwargs): + if command[1:3] == ["image", "list"]: return "" + if command[1:3] == ["sandbox", "list"]: return "[]" + raise AssertionError(command) + with mock.patch.object(installer, "checked", side_effect=inventory): + installer.node_generations.collect(self.args, installer) + self.assertFalse((self.root / installer.MICRO[2]).exists()) + self.assertTrue((self.root / "provider.json").exists()) + self.assertTrue((self.root / installer.COMMON[0]).exists()) + + def test_original_runtime_gc_preserves_exact_shared_native_file_references(self): + self.args.provider = "microsandbox" + self.install() + original = installer.private_json(self.root / "provider.json") + other = json.loads(json.dumps(original)) + other["generation"] = 2 + directory = self.root / "state/node/generations" + directory.mkdir(mode=0o700, exist_ok=True) + installer.node_generations.atomic_json(directory / "2.json", other) + before = {name: (self.root / name).read_bytes() for name in ("runtime/seccomp.json",) + installer.MICRO} + installer.node_generations.collect(self.recovery_args(), installer) + for name, raw in before.items(): self.assertEqual((self.root / name).read_bytes(), raw, name) + self.assertEqual(installer.node_generations.retained_configs(self.root, installer), {2: other}) + def test_interrupted_new_generation_recovers_partial_download_at_same_identity(self): self.args.provider = "microsandbox" self.install() diff --git a/deploy/install/test_node_update.py b/deploy/install/test_node_update.py index c760aa5a5..ed7f7aa96 100644 --- a/deploy/install/test_node_update.py +++ b/deploy/install/test_node_update.py @@ -29,7 +29,7 @@ def setUp(self): binary.chmod(0o700) runtime = {"source_commit": "b" * 40, "image_id": "sha256:" + "a" * 64, "image_manifest_digest": "sha256:" + "c" * 64, "microsandbox_ref": "oac-runtime@sha256:" + "d" * 64, "runtime_sha256": "e" * 64, "firmware_sha256": "f" * 64} spec = {"runtime": runtime, "resources": {"cpus": 1, "memory_mib": 1024}} - self.provider = {"generation": 1, "provider": "docker", "installation_id": self.args.installation_id, "specification": spec, "docker": {"image": runtime["image_id"]}} + self.provider = {"generation": 1, "provider": "docker", "installation_id": self.args.installation_id, "specification": spec, "docker": {"image": runtime["image_id"], "seccomp_file": str(self.root / "runtime/seccomp.json")}} self.identity = {"core_url": self.args.core_url, "credential": "a" * 64, "identity": {"installation_id": self.args.installation_id, "provider": "docker", "node_id": "7073346d-3c83-4361-9354-2709a9cf17ed", "deployment_generation": 1, "specification_digest": node_spec.digest("docker", spec)}} for name, value in (("provider.json", self.provider), ("state/node/identity.json", self.identity), ("registered.json", {})): node_generations.atomic_json(self.root / name, value) From db7d48d5519ec311c069445f00b111030f2d10af Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Mon, 28 Sep 2026 00:09:09 +0000 Subject: [PATCH 13/16] Regenerate documentation for node generation contracts --- apps/docs/content/docs/public-api.mdx | 4 ++-- apps/docs/content/guide-sources.json | 4 ++-- apps/docs/openapi/core-api.yaml | 11 +++++++++++ apps/docs/openapi/runtime-api.yaml | 5 +++++ apps/docs/openapi/sources.json | 8 ++++---- 5 files changed, 24 insertions(+), 8 deletions(-) diff --git a/apps/docs/content/docs/public-api.mdx b/apps/docs/content/docs/public-api.mdx index 5c55bf834..8e0ed8da4 100644 --- a/apps/docs/content/docs/public-api.mdx +++ b/apps/docs/content/docs/public-api.mdx @@ -84,8 +84,8 @@ the Core key or a Project API key. - [Core OpenAPI](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/core.openapi.yaml): generated `/core/v1` routes, all authenticated by the Core key. - [Machine connection OpenAPI](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/runtime.openapi.yaml): - generated `/api/v1` sandbox node routes. The node WebSocket and the private - daemon transport are described in the node and Runtime credential guides. + generated `/api/v1` sandbox node routes. The node WebSocket is described in the [generation protocol](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/node-generation-protocol.md); the private + daemon transport is described in the Runtime credential guide. - [Administrator contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/admin-api.md): Project/key lifecycle, resources, explicit hosted Session archive, summary, errors/deletion preconditions, audit and historical copy provenance. diff --git a/apps/docs/content/guide-sources.json b/apps/docs/content/guide-sources.json index d6c333fe1..645e517bf 100644 --- a/apps/docs/content/guide-sources.json +++ b/apps/docs/content/guide-sources.json @@ -8,7 +8,7 @@ "docs/configuration.md": "e911757a03fbce4b803a284c8bb40bdb7000ca2f70e27938706dcdcd74bc7f9a", "docs/web/core-connection.md": "46f86520e70bf41079708e2c398655aa9087ad9488c8e661d9a73035d11098c4", "docs/getting-started/quickstart.md": "5effeffcfb343f7939910f0c1758e1492d49e7bd66c409b1819cefe67af592d7", - "docs/api/README.md": "b2b4c7fdf9fbde62d2169014180698a77c671f85c1c68c2e5de0e73bbdf3b13c", + "docs/api/README.md": "3c590ca05a07b2bea5ab88ab67719741f8a73a28056ec7d943a5912eb5f9052f", "contracts/agents-api/execution-tools.md": "e0b61f6c0c236c362186c5f6d0ad69a16dd8a8d9afabe1329a1fe22e5f4a71e9", "docs/api/public-agent-api.md": "18829b366bd2e49fe5beb76dc8a709bc1aa8e35d4fbfd2484a0c52b46cf75417", "contracts/agents-api/environments.md": "89ffc903ee44d00ee87c16950202a3ca4c8345160780dcdd4ed2a71b25b1c2dc", @@ -36,7 +36,7 @@ "content/docs/bootstrap-projects-keys.zh.mdx": "509b4dcea4c865d6834bcaef78baa681180d48274931230f958a39e46011c88f", "content/docs/quickstart.mdx": "4ebc8ac16df66042e816ff51cf6672d8a85f44e119211efba584025666dce845", "content/docs/quickstart.zh.mdx": "c5f119b100b102918a45763834d0ed7fd436479fb411e16a62a408411dbcb3e1", - "content/docs/public-api.mdx": "ee1be51bf72bc48809efc0af8a460c26ae1cbf962771e132ec521447b89d5479", + "content/docs/public-api.mdx": "da6aa71753cc84081fba9ae03491717042369a12ea94037fc532f5f1f69d2319", "content/docs/public-api.zh.mdx": "fb0efb9be2fba3f4a3dc71bf053d2c7eddffbfe010f59fa5a1ae60a3f068545b", "content/docs/agents-and-tools.mdx": "b8cc50792e579cbf66bcc597685b745d8721266970b75e928d8edee2ce11b08b", "content/docs/agents-and-tools.zh.mdx": "cc8d4e0ef0ea894f068f3bbd143d5ee70c6346bf98f425b1d5fe07ec3931f128", diff --git a/apps/docs/openapi/core-api.yaml b/apps/docs/openapi/core-api.yaml index 5159dc6c2..f90dc61ac 100644 --- a/apps/docs/openapi/core-api.yaml +++ b/apps/docs/openapi/core-api.yaml @@ -5010,6 +5010,7 @@ components: - provider_unavailable - docker_unavailable - docker_limits_unsupported + - runtime_download_failed - runtime_image_unavailable - kvm_unavailable - microsandbox_artifacts_unavailable @@ -5102,6 +5103,7 @@ components: - provider_unavailable - docker_unavailable - docker_limits_unsupported + - runtime_download_failed - runtime_image_unavailable - kvm_unavailable - microsandbox_artifacts_unavailable @@ -5211,6 +5213,15 @@ components: store.SandboxNodeRollout: properties: diagnostic: + enum: + - provider_unavailable + - docker_unavailable + - docker_limits_unsupported + - runtime_download_failed + - runtime_image_unavailable + - kvm_unavailable + - microsandbox_artifacts_unavailable + - capacity_insufficient type: string ready_generation: anyOf: diff --git a/apps/docs/openapi/runtime-api.yaml b/apps/docs/openapi/runtime-api.yaml index 140773236..050619b1b 100644 --- a/apps/docs/openapi/runtime-api.yaml +++ b/apps/docs/openapi/runtime-api.yaml @@ -17,6 +17,11 @@ paths: description: Retained node UUID in: header name: X-OAC-Node-ID + - schema: + type: integer + description: Exact kept generation (registered nodes only); omitted reads the current target + in: query + name: generation responses: '200': description: OK diff --git a/apps/docs/openapi/sources.json b/apps/docs/openapi/sources.json index a7a17bfbe..a8ed47ac1 100644 --- a/apps/docs/openapi/sources.json +++ b/apps/docs/openapi/sources.json @@ -1,8 +1,8 @@ { "sources": { "contracts/agents-api/openapi.yaml": "a8896e8b67d78ed0608c7cc5636077ec33f4b4895a04bd6cfb8b038afe941e01", - "contracts/agents-api/core.openapi.yaml": "e9b159a84afef28d086f5c285d298834b9f647ae8e7c14457117572ee01cc3ab", - "contracts/agents-api/runtime.openapi.yaml": "b3fb025b20a0bcf4635d60339479b2b07f6dbed80f51e72f40a63b31e42b5f4a" + "contracts/agents-api/core.openapi.yaml": "5594e256e06a608d0be4232e395acbe0bb1d258ff4749112d2f5d85ff05e3c54", + "contracts/agents-api/runtime.openapi.yaml": "23852205d3b45892f2de9ecf3d52518c0dd0808962c0c512f219b565aa7ccef2" }, "outputs": { "content/docs/api-reference/agents.mdx": "1f7697959e9c52c9d24fbe70e116e807f49912b9a27ac638b86782e2f326af44", @@ -40,11 +40,11 @@ "content/docs/api-reference/core/vaults.mdx": "12d0aac96cd4a3472ef4eaa287f7945f78862017956e79bfc3ed0ef8e772e78a", "content/docs/api-reference/core/credentials.mdx": "27d586b9a8fd738b9947802f32c80458195370252886bc26abedccf2125ee981", "content/docs/api-reference/core/sandbox-manager.mdx": "6be7c52e057365949f0247c18c710561d680847a955f78f1bbe57825d4d5f549", - "openapi/core-api.yaml": "f6ba52edeb5c70c11f230da401e60cc442e7ac6760f63013b2293ba045786b5b", + "openapi/core-api.yaml": "d0fcaa48655905f72d7f8a0a58208e56d88570ebd882b08795625ebc8c4308d3", "content/docs/api-reference/core/meta.json": "588b163ed2de2100b0f8376688274f80698a6bf6ac037eef56d17c93fa8a5062", "content/docs/api-reference/core/index.mdx": "ddf0a8d87a6c21761beac703a32f2a40486a1c74abb341bd726f40c44a866405", "content/docs/api-reference/machine/sandbox-node.mdx": "5b367a08d43cea82686345d2ba1c75cc90627a7b2970fe42a0df9d77fb3e1e80", - "openapi/runtime-api.yaml": "bca127037ffb1e132e7f26bffada5e3da70f4b2534926a04e1e65029d885afe9", + "openapi/runtime-api.yaml": "32adc347bfa8effd501b9c0089a2a4d114cab43d5c26c7947b3a79fdc6ae1f1a", "content/docs/api-reference/machine/meta.json": "12a1a6534c1cae32918282da2110e4b64c43f17abb868486b5eff50ceb0b01c1", "content/docs/api-reference/machine/index.mdx": "792c1ce86a6653943021eb5cd03a2c6b5c604f7f012da5ef06f3ffa027b12973" } From d177474052969329e0b5c9222645be6a2d629659 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Mon, 28 Sep 2026 04:22:37 +0000 Subject: [PATCH 14/16] Fence node generation publication and durable local ownership --- CONTRIBUTING.md | 9 +- .../agents-api/node-generation-protocol.md | 64 ++++- deploy/install/distribution.py | 44 +-- deploy/install/install.py | 41 ++- deploy/install/node_generations.py | 192 ++++++++++--- deploy/install/node_install.py | 46 ++- deploy/install/node_update.py | 3 + .../test_generation_review_regressions.py | 263 ++++++++++++++++++ deploy/install/test_node_generations.py | 12 + deploy/install/test_node_install.py | 9 +- deploy/install/test_node_update.py | 2 + docs/getting-started/nodes.md | 36 ++- .../cmd/sandbox-node/generations.go | 141 +++++++--- .../cmd/sandbox-node/generations_test.go | 49 ++++ .../internal/sandbox/config/microsandbox.go | 29 +- .../internal/sandbox/microsandbox/process.go | 98 ++++++- .../sandbox/microsandbox/process_test.go | 233 +++++++++++++++- .../internal/sandbox/node/generations.go | 4 +- .../internal/sandbox/node/generations_test.go | 22 ++ .../runtime_node_lifecycle_fixture_test.go | 39 ++- .../store/runtime_node_status_test.go | 9 +- .../internal/store/runtime_nodes_test.go | 34 ++- .../store/sandbox_generations_test.go | 24 +- 23 files changed, 1227 insertions(+), 176 deletions(-) create mode 100644 deploy/install/test_generation_review_regressions.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 937909fbe..8dea4e34c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -811,7 +811,14 @@ immutable allocation/placement generation. V1 keeps its qualified enrolled fallb Use sparse v2 control batches of at most eight entries with no lifetime cap. Omitted facts never authorize deletion. Correlate whole retention grants to connection, epoch, sequence, generation and digest; recheck queued/inflight/helper references. -Permanent generation flock files survive updates and GC. Retained Runtime bytes and +Permanent generation flock files survive updates and GC. Before any helper starts, +the installer durably binds the original inode to installation/generation/specification +identity; Python and Go openers verify it after every restart and never adopt a +replacement or missing record. Preparation plans stay distinct from write-once final +provider configurations: pending-only entries may recover or collect under Core +authority, never serve. Resolve Docker's supported local image ID before publication. +Only transfer/checksum/provenance errors report runtime_download_failed; preserve +other fixed causes without parsing native error text. Retained Runtime bytes and the console's exact-release HTTP allowlist must agree. See `contracts/agents-api/node-generation-protocol.md` for recovery, immutable artifacts and the conservative v1 legacy-helper retention boundary. diff --git a/contracts/agents-api/node-generation-protocol.md b/contracts/agents-api/node-generation-protocol.md index b9f994151..17d23342b 100644 --- a/contracts/agents-api/node-generation-protocol.md +++ b/contracts/agents-api/node-generation-protocol.md @@ -44,8 +44,9 @@ the parent counts that helper through its actual `Wait` completion. Every generation also owns a permanent private lease file at `state/node/generations/.lease`. Before starting a native helper, the -node acquires its shared flock and checks the generation's dropped tombstone under -that lock. The helper inherits the descriptor. The node closes its own descriptor +node acquires its shared flock and validates the durable lease identity under +that lock. It also requires the matching published final provider configuration +and absence of preparing, collecting and dropped journals. The helper inherits the descriptor. The node closes its own descriptor only after `Wait`; it never explicitly unlocks the shared open-file description. Thus caller cancellation or node exit does not release a live helper's reference. New native helpers set this descriptor close-on-exec before calling the SDK so VM @@ -56,7 +57,19 @@ removing shared images or release files, or publishing the dropped tombstone. It keeps the lock through those changes. Lock files belong to stable node state and are never removed or atomically replaced during collection. Symlinks, multiply linked files, foreign ownership, unsafe permissions and replaced lock paths are -refused. A dropped generation cannot be prepared or used again; a future rollback +refused. Before the first helper can start, the installer exclusively creates the lease and +fsyncs it, then atomically persists and fsyncs a private `.lease-identity` record and +its parent directory. The record binds installation, generation, specification +digest, device and inode. Python exclusive collection/repair and Go shared helper +openers validate the same record on every open, including after node restart. +Neither opener adopts a missing identity, replaces its inode, or erases +it after GC. Initialization interrupted before the identity is durable refuses +re-adoption; preserve the installation for inspection. A removed identity or an +owned replacement 0600 lease still refuses, even when its current fstat/lstat agree. +The approved v1 update initializes this new fence before starting v2 but retains +its separate legacy-unfenced marker for already-running old helpers. + +A dropped generation cannot be prepared or used again; a future rollback would require a new generation and a separate policy. An immutable older native helper may pass its inherited descriptor to descendants. @@ -100,7 +113,7 @@ No connection-established deployment facts means no preparation starts. Repair preserves existing configurations and paths, verifies the selected release and all existing sibling checksums, and downloads only absent immutable files. Conflicting bytes or a different retained specification refuse repair. A missing complete -provider configuration remains a refusal rather than a guessed reconstruction. +provider configuration without an exact durable preparation plan remains a refusal rather than a guessed reconstruction. Repair takes the same exclusive generation lease and installation lock used by collection. A live helper or concurrent collector therefore retains ownership; @@ -141,14 +154,41 @@ retained provider after the original Runtime bytes have gone. Shared native path are compared across all retained configurations before removal. The v1 legacy-unfenced marker still prevents any collection of its original payload. -New preparation records its exact identity before downloads and records import -start before invoking the native importer. An interrupted download can repair only -missing bytes at the original paths. If collection precedes any import attempt, -the preparation journal proves that this generation has no imported native image. -An older or interrupted generation whose native executable is missing and whose -import may have started remains conservatively retained; missing files do not -prove native absence. Receipt/store history and a legacy-unfenced generation are -never erased using an empty native inventory. +New preparation has two distinct records. Before downloads, `.preparing` holds the +immutable installation/generation/specification identity, private provider paths +and `import_started:false`; it is a recovery/collection plan, not a published +provider. Before invoking the importer, the same plan records `import_started:true`. +Both Python retention discovery and Go restart recovery recognize pending-only +plans, but never build, probe or acquire a provider from them. Current-connection +Core authorization is still required for recovery or collection. + +Only successful preparation publishes the final `.json` provider configuration. +Docker records the actual immutable local ID returned by the resolver; either +builder-proven config or manifest identity can be valid for the same specification. +The final configuration is write-once. Publication is durable before clearing the +preparation journal. Interruption between those steps revalidates the same plan +and existing final identity; it does not permit editing a published configuration. +Plan/spec/path drift refuses. A canceled or failed import remains visible to fresh +Core retention exchange without becoming a serving generation. + +An interrupted download repairs only missing bytes at the original paths. If +collection precedes any import attempt, the preparation journal proves that this +generation has no imported native image. An older or interrupted generation whose +native executable is missing and whose import may have started remains retained; +missing files do not prove native absence. Receipt/store history and a +legacy-unfenced generation are never erased using an empty native inventory. + +Preparation diagnostics preserve fixed typed causes. Only artifact transfer, +checksum or release-provenance failures report `runtime_download_failed`. A private +preparer exit category communicates that class without parsing stderr; provider, +ownership, cancellation and unclassified failures remain their existing typed +code or `provider_unavailable`. No raw provider text crosses the node protocol. + +Published console releases keep immutable metadata and existing artifact bytes. +A rerun first validates all published metadata and every existing declared artifact, +then may atomically add only absent, checksum-matched declared artifacts. This +supports thin-release completion and retained HTTP artifact repair. Any existing +conflict prevents all additions; repair never overwrites a conflicting artifact. ## Identity-preserving program updates diff --git a/deploy/install/distribution.py b/deploy/install/distribution.py index 1a0f77fe9..843ba0013 100644 --- a/deploy/install/distribution.py +++ b/deploy/install/distribution.py @@ -21,6 +21,10 @@ class DistributionError(Exception): pass +class ArtifactError(DistributionError): + """Transfer or immutable artifact provenance failed, not provider readiness.""" + + def image_identities(manifest, name): """Both immutable IDs describe the same archive, as proven by the builder.""" identities = [] @@ -28,7 +32,7 @@ def image_identities(manifest, name): mapping = manifest.get(field) value = mapping.get(name) if isinstance(mapping, dict) else None if not isinstance(value, str) or not re.fullmatch(r'sha256:[0-9a-f]{64}', value): - raise DistributionError('Missing or invalid immutable image identity: ' + name) + raise ArtifactError('Missing or invalid immutable image identity: ' + name) identities.append(value) return tuple(identities) @@ -82,13 +86,13 @@ def artifact(manifest, name): entry = manifest.get('artifacts', {}).get(name) revision = manifest.get('source_commit', '') if not isinstance(entry, dict) or not re.fullmatch(r'[0-9a-f]{40}', revision): - raise DistributionError('Missing versioned artifact: ' + name) + raise ArtifactError('Missing versioned artifact: ' + name) filename = entry.get('filename', '') if (not isinstance(filename, str) or not re.fullmatch(r'[A-Za-z0-9._-]+', filename) or revision not in filename or filename in ('.', '..') or not re.fullmatch(r'[0-9a-f]{64}', str(entry.get('sha256', ''))) or type(entry.get('size')) is not int or entry['size'] <= 0): - raise DistributionError('Invalid artifact metadata: ' + name) + raise ArtifactError('Invalid artifact metadata: ' + name) return entry @@ -107,7 +111,7 @@ def safe_url(value): or '\\' in value or parsed.scheme != 'https' and not (parsed.scheme == 'http' and loopback)): raise ValueError() except ValueError: - raise DistributionError('Artifact downloads require HTTPS; loopback HTTP is only for local testing') from None + raise ArtifactError('Artifact downloads require HTTPS; loopback HTTP is only for local testing') from None return value @@ -115,7 +119,7 @@ class NoRedirect(urllib.request.HTTPRedirectHandler): """Downloads come only from the console, which never redirects; a redirect could hand verified names to another origin, so every one is refused.""" def redirect_request(self, request, fp, code, message, headers, newurl): - raise DistributionError('Artifact and metadata downloads do not follow redirects; check the console URL ' + raise ArtifactError('Artifact and metadata downloads do not follow redirects; check the console URL ' 'and the reverse proxy in front of it') @@ -138,7 +142,7 @@ def obtain_artifact(manifest, logical_path, destination, offline_root=None): target = checked_path(destination) if target.exists(): if not matches(target, entry): - raise DistributionError('Cached artifact differs; preserve the installation and inspect: ' + logical_path) + raise ArtifactError('Cached artifact differs; preserve the installation and inspect: ' + logical_path) return target source = None if offline_root is not None: @@ -151,7 +155,7 @@ def obtain_artifact(manifest, logical_path, destination, offline_root=None): return copy_artifact(source, target, entry, logical_path) base = manifest.get('artifact_base_url', '') if not isinstance(base, str) or not base or urlsplit(base).query: - raise DistributionError('No downloadable artifact source; use the matching offline bundle') + raise ArtifactError('No downloadable artifact source; use the matching offline bundle') url = safe_url(base.rstrip('/') + '/' + entry['filename']) # A private partial file survives interruptions and reruns; the next attempt asks # for the missing bytes only. The complete file is still verified as a whole. @@ -164,17 +168,17 @@ def obtain_artifact(manifest, logical_path, destination, offline_root=None): if error.code == 416: discard_partial(partial) elif error.code not in (408, 429, 500, 502, 503, 504) or attempt == 2: - raise DistributionError(f'Artifact download failed (HTTP {error.code}): {logical_path}. Check the console and retry.') from None + raise ArtifactError(f'Artifact download failed (HTTP {error.code}): {logical_path}. Check the console and retry.') from None except (urllib.error.URLError, socket.timeout, ConnectionError, http.client.HTTPException): if attempt == 2: - raise DistributionError('Artifact transfer interrupted: ' + logical_path + '. Check network access and rerun; ' + raise ArtifactError('Artifact transfer interrupted: ' + logical_path + '. Check network access and rerun; ' 'the download resumes where it stopped.') from None time.sleep(attempt + 1) else: - raise DistributionError('Artifact download did not complete') + raise ArtifactError('Artifact download did not complete') if digest(partial) != entry['sha256']: discard_partial(partial) - raise DistributionError('Artifact checksum mismatch: ' + logical_path) + raise ArtifactError('Artifact checksum mismatch: ' + logical_path) os.chmod(partial, 0o700 if logical_path.startswith('native/') else 0o600) os.replace(partial, target) validator_path(partial).unlink(missing_ok=True) @@ -198,10 +202,10 @@ def copy_artifact(source, target, entry, logical_path): for block in iter(lambda: stream.read(1024 * 1024), b''): count += len(block) if count > entry['size']: - raise DistributionError('Artifact exceeds published size: ' + logical_path) + raise ArtifactError('Artifact exceeds published size: ' + logical_path) output.write(block) if count != entry['size'] or digest(temporary) != entry['sha256']: - raise DistributionError('Artifact checksum mismatch: ' + logical_path) + raise ArtifactError('Artifact checksum mismatch: ' + logical_path) os.chmod(temporary, 0o700 if logical_path.startswith('native/') else 0o600) os.replace(temporary, target) return target @@ -260,12 +264,12 @@ def download_partial(url, partial, entry, logical_path): if count > size: output.close() discard_partial(partial) - raise DistributionError('Artifact exceeds published size: ' + logical_path) + raise ArtifactError('Artifact exceeds published size: ' + logical_path) output.write(block) window_count += len(block) if time.monotonic() - window >= SLOW_SECONDS: if window_count < SLOW_BYTES: - raise DistributionError(f'Artifact download stalled (under {SLOW_BYTES // 1024} KiB in {SLOW_SECONDS} s): ' + raise ArtifactError(f'Artifact download stalled (under {SLOW_BYTES // 1024} KiB in {SLOW_SECONDS} s): ' f'{logical_path}. The downloaded part is kept; check the network, then rerun ' 'the command to resume.') window, window_count = time.monotonic(), 0 @@ -286,12 +290,12 @@ def runtime_archive(manifest, cache_root, offline_root=None): expanded = {'sha256': entry.get('unpacked_sha256'), 'size': entry.get('unpacked_size')} if (not re.fullmatch(r'[0-9a-f]{64}', str(expanded['sha256'])) or type(expanded['size']) is not int or expanded['size'] <= 0): - raise DistributionError('Runtime archive is missing unpacked verification metadata') + raise ArtifactError('Runtime archive is missing unpacked verification metadata') root = Path(cache_root) target = checked_path(root / 'images/runtime.tar') if target.exists(): if not matches(target, expanded): - raise DistributionError('Cached Runtime archive differs; preserve state and inspect it') + raise ArtifactError('Cached Runtime archive differs; preserve state and inspect it') return target archive = obtain_artifact(manifest, 'images/runtime.tar.gz', root / 'images/runtime.tar.gz', offline_root) fd, temporary = tempfile.mkstemp(prefix='.runtime-', dir=target.parent) @@ -301,13 +305,13 @@ def runtime_archive(manifest, cache_root, offline_root=None): for block in iter(lambda: stream.read(1024 * 1024), b''): count += len(block) if count > expanded['size']: - raise DistributionError('Runtime archive exceeds published unpacked size') + raise ArtifactError('Runtime archive exceeds published unpacked size') output.write(block) if not matches(Path(temporary), expanded): - raise DistributionError('Unpacked Runtime checksum mismatch') + raise ArtifactError('Unpacked Runtime checksum mismatch') os.replace(temporary, target) except (gzip.BadGzipFile, EOFError): - raise DistributionError('Invalid compressed Runtime archive') from None + raise ArtifactError('Invalid compressed Runtime archive') from None finally: if os.path.exists(temporary): os.unlink(temporary) diff --git a/deploy/install/install.py b/deploy/install/install.py index c3995df23..b1dc62dbd 100644 --- a/deploy/install/install.py +++ b/deploy/install/install.py @@ -346,10 +346,47 @@ def publish(source): raise InstallError("Installed node payload differs; preserve it and inspect the distribution") target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) if target.exists(): - for name in names: + # Validate the complete published metadata and every existing declared + # artifact before filling any absence. Existing bytes are immutable. + for name in names[:5]: previous = target / name - if previous.is_symlink() or not previous.is_file() or digest(previous) != digest(source / name): + if (previous.parent.is_symlink() or previous.is_symlink() or not previous.is_file() + or digest(previous) != digest(source / name)): raise InstallError("Installed node payload differs; preserve it and inspect the distribution") + artifacts = target / "artifacts" + if artifacts.is_symlink() or artifacts.exists() and not artifacts.is_dir(): + raise InstallError("Installed node artifact directory differs") + missing = [] + for logical in manifest.get("artifacts", {}): + entry = artifact(manifest, logical) + name = "artifacts/" + entry["filename"] + previous = target / name + if previous.is_symlink() or previous.exists() and (not previous.is_file() + or previous.stat().st_size != entry["size"] or digest(previous) != entry["sha256"]): + raise InstallError("Installed node artifact differs; refusing repair") + if not previous.exists() and name in names: + missing.append((name, entry)) + if missing: + artifacts.mkdir(mode=0o700, exist_ok=True) + for name, entry in missing: + descriptor, temporary = tempfile.mkstemp(prefix=".payload-", dir=artifacts) + try: + with os.fdopen(descriptor, "wb") as outgoing, (source / name).open("rb") as incoming: + shutil.copyfileobj(incoming, outgoing) + outgoing.flush() + os.fsync(outgoing.fileno()) + if Path(temporary).stat().st_size != entry["size"] or digest(Path(temporary)) != entry["sha256"]: + raise InstallError("Node artifact changed during repair") + # Publish without replacing bytes introduced concurrently. + os.link(temporary, target / name) + finally: + os.unlink(temporary) + for directory in (artifacts, target): + descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) return revision with tempfile.TemporaryDirectory(prefix=".payload-", dir=target.parent) as temporary: stage = Path(temporary) / "release" diff --git a/deploy/install/node_generations.py b/deploy/install/node_generations.py index da22a2233..33570c589 100644 --- a/deploy/install/node_generations.py +++ b/deploy/install/node_generations.py @@ -70,7 +70,7 @@ def generation_marker(root, generation, suffix, digest, installation, installer) with os.fdopen(descriptor) as stream: info = os.fstat(stream.fileno()) if (not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600 - or info.st_uid != os.getuid() or info.st_nlink != 1 or info.st_size > 4096): + or info.st_uid != os.getuid() or info.st_nlink != 1 or info.st_size > (16384 if suffix == ".preparing" else 4096)): raise installer.InstallError("Invalid generation ownership journal") try: value = json.load(stream) @@ -79,6 +79,10 @@ def generation_marker(root, generation, suffix, digest, installation, installer) named = path.lstat() if (named.st_dev, named.st_ino) != (info.st_dev, info.st_ino): raise installer.InstallError("Generation ownership journal was replaced") + if digest is None: + digest = value.get("specification_digest") if isinstance(value, dict) else None + if not isinstance(digest, str) or not re.fullmatch(r"[a-f0-9]{64}", digest): + raise installer.InstallError("Invalid preparation specification identity") expected = {"installation_id": installation, "generation": generation, "specification_digest": digest} if (not isinstance(value, dict) or type(value.get("generation")) is not int or any(value.get(key) != item for key, item in expected.items())): @@ -90,6 +94,13 @@ def generation_marker(root, generation, suffix, digest, installation, installer) raise installer.InstallError("Invalid generation collection journal") if suffix == ".preparing": keys.add("import_started") + if "configuration" in value: + keys.add("configuration") + plan = value["configuration"] + if (not isinstance(plan, dict) or plan.get("installation_id") != installation or plan.get("generation") != generation + or plan.get("provider") not in ("docker", "microsandbox") + or installer.node_spec.digest(plan["provider"], plan.get("specification")) != digest): + raise installer.InstallError("Invalid generation preparation plan") if type(value.get("import_started")) is not bool: raise installer.InstallError("Invalid generation preparation journal") if set(value) != keys: @@ -181,9 +192,70 @@ def retained_configs(root, installer): if value["generation"] in result and result[value["generation"]] != value: raise installer.InstallError("Conflicting retained generation configuration") result[value["generation"]] = value + if directory.exists(): + for path in sorted(directory.glob("*.preparing")): + if not re.fullmatch(r"[1-9][0-9]*\.preparing", path.name): + raise installer.InstallError("Invalid preparation filename") + if not base: + raise installer.InstallError("Preparation installation identity is unavailable") + journal = generation_marker(root, int(path.stem), ".preparing", None, base["installation_id"], installer) + plan = journal.get("configuration") + if plan is None: + if int(path.stem) not in result: + raise installer.InstallError("Preparation plan is missing") + continue + validate_preparation_plan(root, plan, base, installer) + if generation_marker(root, plan["generation"], ".dropped", journal["specification_digest"], plan["installation_id"], installer): + continue + final = result.get(plan["generation"]) + if final is not None: + verify_plan_final(plan, final, installer) + else: + result[plan["generation"]] = plan return result +def validate_preparation_plan(root, plan, base, installer): + if (not base or any(plan.get(key) != base.get(key) for key in ("installation_id", "provider", "core_url")) + or type(plan.get("generation")) is not int): + raise installer.InstallError("Preparation plan installation differs") + runtime = plan["specification"]["runtime"] + source = runtime["source_commit"] + if not re.fullmatch(r"[a-f0-9]{40}", source): + raise installer.InstallError("Preparation release identity differs") + if plan["provider"] == "docker": + policy = Path(plan["docker"]["seccomp_file"]) + if policy not in (root / "runtime/seccomp.json", root / "releases" / source / "runtime/seccomp.json"): + raise installer.InstallError("Preparation policy is outside its immutable release") + if plan["docker"]["image"] not in (runtime["image_id"], runtime["image_manifest_digest"]): + raise installer.InstallError("Preparation image differs from the specification") + installer.no_links(policy) + else: + micro = plan["microsandbox"] + paths = [Path(micro[key]) for key in ("helper_path", "runtime_path", "firmware_path")] + if not any(paths == [release / name for name in installer.MICRO] for release in (root, root / "releases" / source)): + raise installer.InstallError("Preparation artifacts are outside their immutable release") + expected_home = generation_home(root, plan, base, installer) + if Path(micro["runtime_home"]) != expected_home: + raise installer.InstallError("Preparation native store differs") + for path in paths + [expected_home]: + installer.no_links(path) + configuration = dict(plan, core_url=plan["core_url"].removesuffix("/api/v1")) + installer.node_spec.verify_provider(plan, configuration, plan.get("docker", {}).get("image")) + + +def verify_plan_final(plan, final, installer): + expected = copy.deepcopy(plan) + if plan["provider"] == "docker": + image = final.get("docker", {}).get("image") + runtime = plan["specification"]["runtime"] + if image not in (runtime["image_id"], runtime["image_manifest_digest"]): + raise installer.InstallError("Final Docker image differs from the preparation specification") + expected["docker"]["image"] = image + if expected != final: + raise installer.InstallError("Final generation differs from its immutable preparation plan") + + def owned_root(args, installer): if os.getuid() == 0: raise installer.InstallError("Generation operations run only as the node service user") @@ -254,7 +326,10 @@ def runtime_files(root, value, args, manifest, sums, installer): if (release / "manifest.json").exists(): if saved is None: raise installer.InstallError("Retained Runtime manifest is unreadable") - installer.node_spec.verify_release(args.configuration, saved) + try: + installer.node_spec.verify_release(args.configuration, saved) + except installer.node_spec.SpecificationError as error: + raise installer.RuntimeDownloadError("Retained Runtime release provenance differs") from error # Inspect every existing byte before starting any repair, so a missing file # cannot hide a conflicting sibling or redirect a later download. for name in names: @@ -263,7 +338,7 @@ def runtime_files(root, value, args, manifest, sums, installer): if installer.existing_file(path): expected = sums[name] if name == "runtime/seccomp.json" else installer.distribution.artifact(manifest, name)["sha256"] if installer.file_digest(path) != expected: - raise installer.InstallError("Retained Runtime artifact checksum differs") + raise installer.RuntimeDownloadError("Retained Runtime artifact checksum differs") for name in names: installer.safe_directory((release / name).parent) if name == "runtime/seccomp.json": @@ -277,12 +352,16 @@ def runtime_files(root, value, args, manifest, sums, installer): def prepare(args, installer): root, identity = owned_root(args, installer) - with installer.install_lock(root), collection_lease(root, args.generation, installer): + with installer.install_lock(root), collection_lease( + root, args.generation, installer, marker_identity(args), + initialize=args.generation not in retained_configs(root, installer) + and not (root / "state/node/generations" / (str(args.generation) + ".json")).exists()): directory = root / "state/node/generations" + target = directory / (str(args.generation) + ".json") if (generation_marker(root, args.generation, ".dropped", args.specification_digest, args.installation_id, installer) or generation_marker(root, args.generation, ".collecting", args.specification_digest, args.installation_id, installer)): raise installer.InstallError("A dropped generation cannot be adopted again") - generation_marker(root, args.generation, ".preparing", args.specification_digest, args.installation_id, installer) + preparation = generation_marker(root, args.generation, ".preparing", args.specification_digest, args.installation_id, installer) args.core_url = identity["core_url"] args.configuration = installer.node_spec.fetch(args, "", identity, installer.open_request, generation=args.generation, allow_selection_change=True) @@ -293,11 +372,14 @@ def prepare(args, installer): base = installer.private_json(root / "provider.json") runtime = args.configuration["specification"]["runtime"] value = configurations.get(args.generation) - retained = value is not None - if retained: + finalized = target.exists() or base["generation"] == args.generation + if value is not None: installer.node_spec.verify_provider(value, args.configuration, value.get("docker", {}).get("image")) else: for candidate in configurations.values(): + # Unpublished plans must never be used as ready reuse candidates. + if any((directory / (str(candidate["generation"]) + suffix)).exists() for suffix in (".preparing", ".collecting", ".dropped")): + continue if candidate["specification"]["runtime"] == runtime and image_available(candidate, installer): value = copy.deepcopy(candidate) value["generation"] = args.generation @@ -305,25 +387,22 @@ def prepare(args, installer): if args.provider == "microsandbox": value["microsandbox"].update(value["specification"]["resources"]) break - if value is None or not image_available(value, installer): + if not finalized or value is None or not image_available(value, installer): settings = installer.private_json(root / "preparation.json") args.source_url = installer.origin(settings["source_url"]) args.bundle = None manifest, sums = installer.metadata(args.source_url, prefix="releases/" + runtime["source_commit"] + "/") - installer.node_spec.verify_release(args.configuration, manifest) + try: + installer.node_spec.verify_release(args.configuration, manifest) + except installer.node_spec.SpecificationError as error: + raise installer.RuntimeDownloadError("Runtime release provenance differs") from error if args.provider == "microsandbox": - args.runtime_home = Path(value["microsandbox"]["runtime_home"]) if retained else generation_home(root, args.configuration, base, installer) - if not retained: - # Persist the exact authorized identity before any artifact/import - # mutation. Interrupted preparation must remain discoverable after - # node restart; file presence still requires a real provider probe. + args.runtime_home = Path(value["microsandbox"]["runtime_home"]) if value else generation_home(root, args.configuration, base, installer) + if value is None: value = installer.provider_config(root / "releases" / runtime["source_commit"], args, manifest, runtime["image_id"]) - target = directory / (str(args.generation) + ".json") - if installer.existing_file(target) and installer.private_json(target) != value: - raise installer.InstallError("Immutable generation configuration differs") - atomic_json(directory / (str(args.generation) + ".preparing"), dict(marker_identity(args), import_started=False)) - atomic_json(target, value) - retained = True + if preparation is None: + preparation = dict(marker_identity(args), import_started=False, configuration=copy.deepcopy(value)) + atomic_json(directory / (str(args.generation) + ".preparing"), preparation) release = runtime_files(root, value, args, manifest, sums, installer) if args.provider == "microsandbox": installer.safe_directory(args.runtime_home) @@ -331,24 +410,57 @@ def prepare(args, installer): if not owner.exists() and any(args.runtime_home.iterdir()): raise installer.InstallError("Versioned microsandbox store contains unowned state") installer.write_once(owner, installer.json_text({"installation_id": args.installation_id})) - atomic_json(directory / (str(args.generation) + ".preparing"), dict(marker_identity(args), import_started=True)) + preparation = dict(preparation, import_started=True) + atomic_json(directory / (str(args.generation) + ".preparing"), preparation) runtime_image = installer.prepare_runtime(release, args, manifest) - if retained: + if finalized: installer.node_spec.verify_provider(value, args.configuration, runtime_image) - else: - value = installer.provider_config(release, args, manifest, runtime_image) - target = directory / (str(args.generation) + ".json") - if installer.existing_file(target) and installer.private_json(target) != value: - raise installer.InstallError("Immutable generation configuration differs") - atomic_json(target, value) + elif args.provider == "docker": + if runtime_image not in (runtime["image_id"], runtime["image_manifest_digest"]): + raise installer.InstallError("Resolved Docker image is outside the authorized specification") + value = copy.deepcopy(value) + value["docker"]["image"] = runtime_image + if preparation and preparation.get("configuration"): + verify_plan_final(preparation["configuration"], value, installer) + if installer.existing_file(target): + if installer.private_json(target) != value: + raise installer.InstallError("Immutable generation configuration differs") + else: + atomic_json(target, value) + # If interrupted after publication, restart rechecks the same plan and + # final identity. It never edits the published provider configuration. + pending = directory / (str(args.generation) + ".preparing") + pending.unlink(missing_ok=True) + descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) @contextlib.contextmanager -def collection_lease(root, generation, installer): +def collection_lease(root, generation, installer, identity=None, initialize=False): directory = root / "state/node/generations" installer.safe_directory(directory) path = directory / (str(generation) + ".lease") - descriptor = os.open(path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600) + record = directory / (str(generation) + ".lease-identity") + if identity is None: + value = retained_configs(root, installer).get(generation) + if value is None: + # Dropped configurations remain as immutable identity receipts. + value = installer.private_json(directory / (str(generation) + ".json")) or installer.private_json(root / "provider.json") + if not value or value["generation"] != generation: + raise installer.InstallError("Generation lease identity is unavailable") + identity = {"installation_id": value["installation_id"], "generation": generation, + "specification_digest": installer.node_spec.digest(value["provider"], value["specification"])} + flags = os.O_RDWR | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK + created = False + if initialize and not record.exists() and not record.is_symlink(): + # O_EXCL prevents adopting an old unrecorded inode after interruption. + descriptor = os.open(path, flags | os.O_CREAT | os.O_EXCL, 0o600) + created = True + else: + descriptor = os.open(path, flags) try: info = os.fstat(descriptor) if (not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600 @@ -361,6 +473,24 @@ def collection_lease(root, generation, installer): named = path.lstat() if (named.st_dev, named.st_ino) != (info.st_dev, info.st_ino): raise installer.InstallError("Generation helper lease was replaced") + expected = dict(identity, device=info.st_dev, inode=info.st_ino) + if created: + os.fsync(descriptor) + atomic_json(record, expected) + else: + installer.no_links(record) + fd = os.open(record, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK) + with os.fdopen(fd) as stream: + saved_info = os.fstat(stream.fileno()) + if (not stat.S_ISREG(saved_info.st_mode) or stat.S_IMODE(saved_info.st_mode) != 0o600 + or saved_info.st_uid != os.geteuid() or saved_info.st_nlink != 1 or saved_info.st_size > 4096): + raise installer.InstallError("Invalid durable generation lease identity") + saved = json.load(stream) + named_record = record.lstat() + if (named_record.st_dev, named_record.st_ino) != (saved_info.st_dev, saved_info.st_ino): + raise installer.InstallError("Durable generation lease identity was replaced") + if saved != expected or any(type(saved.get(key)) is not int for key in ("generation", "device", "inode")): + raise installer.InstallError("Generation helper lease identity differs; retain its payload") yield finally: # Never unlink a lease: a replacement inode could bypass an old helper. @@ -369,7 +499,7 @@ def collection_lease(root, generation, installer): def collect(args, installer): root, _ = owned_root(args, installer) - with installer.install_lock(root), collection_lease(root, args.generation, installer): + with installer.install_lock(root), collection_lease(root, args.generation, installer, marker_identity(args)): configurations = retained_configs(root, installer) value = configurations.get(args.generation) if value is None: diff --git a/deploy/install/node_install.py b/deploy/install/node_install.py index fc5e3a11d..83d54e081 100644 --- a/deploy/install/node_install.py +++ b/deploy/install/node_install.py @@ -42,6 +42,10 @@ class InstallError(Exception): pass +class RuntimeDownloadError(InstallError): + """A fixed private helper exit category for transfer/provenance failures.""" + + COMMON = ("native/bin/oac-node", "runtime/seccomp.json") MICRO = ("native/bin/oac-microsandbox-provider", "native/microsandbox/msb", "native/microsandbox/libkrunfw.so.5.6.1") @@ -190,12 +194,12 @@ def fetch(source, name): with open_request(source + "/node-install/" + name) as response: raw = response.read(1024 * 1024 + 1) if len(raw) > 1024 * 1024: - raise InstallError("Node bootstrap metadata is too large: " + name) + raise RuntimeDownloadError("Node bootstrap metadata is too large: " + name) return io.BytesIO(raw) except (urllib.error.URLError, TimeoutError, ConnectionError, http.client.IncompleteRead) as error: if not transient(error) or attempt == 2: status = " (HTTP " + str(error.code) + ")" if isinstance(error, urllib.error.HTTPError) else "" - raise InstallError("Cannot download node metadata " + name + status + "; check the console URL, TLS and network, then rerun") from None + raise RuntimeDownloadError("Cannot download node metadata " + name + status + "; check the console URL, TLS and network, then rerun") from None time.sleep(attempt + 1) @@ -211,21 +215,27 @@ def read(name): with read("SHA256SUMS") as response: raw = response.read(1024 * 1024 + 1) if len(raw) > 1024 * 1024: - raise InstallError("Invalid distribution checksum list") + raise RuntimeDownloadError("Invalid distribution checksum list") sums = {} - for line in raw.decode().splitlines(): - checksum, name = line.split(" ", 1) - if name in sums or not re.fullmatch(r"[0-9a-f]{64}", checksum): - raise InstallError("Invalid distribution checksum entry") - sums[name] = checksum + try: + for line in raw.decode().splitlines(): + checksum, name = line.split(" ", 1) + if name in sums or not re.fullmatch(r"[0-9a-f]{64}", checksum): + raise RuntimeDownloadError("Invalid distribution checksum entry") + sums[name] = checksum + except ValueError: + raise RuntimeDownloadError("Invalid distribution checksum list") from None with read("manifest.json") as response: raw = response.read(1024 * 1024 + 1) if len(raw) > 1024 * 1024 or hashlib.sha256(raw).hexdigest() != sums.get("manifest.json"): - raise InstallError("Distribution manifest checksum mismatch") - manifest = json.loads(raw) - if (manifest.get("platform") != "linux/amd64" or not re.fullmatch(r"[0-9a-f]{40}", manifest.get("source_commit", "")) + raise RuntimeDownloadError("Distribution manifest checksum mismatch") + try: + manifest = json.loads(raw) + except (ValueError, TypeError): + raise RuntimeDownloadError("Invalid distribution manifest") from None + if (not isinstance(manifest, dict) or manifest.get("platform") != "linux/amd64" or not re.fullmatch(r"[0-9a-f]{40}", manifest.get("source_commit", "")) or not re.fullmatch(r"sha256:[0-9a-f]{64}", manifest.get("images", {}).get("runtime", ""))): - raise InstallError("Unsupported node distribution") + raise RuntimeDownloadError("Unsupported node distribution") distribution.image_identities(manifest, "runtime") for name in (COMMON[0], "images/runtime.tar.gz") + MICRO: distribution.artifact(manifest, name) @@ -278,7 +288,7 @@ def download(source, name, root, expected, prefix=""): safe_directory(target.parent) if existing_file(target): if file_digest(target) != expected: - raise InstallError("Installed node payload differs; refusing to overwrite it") + raise RuntimeDownloadError("Installed node payload differs; refusing to overwrite it") return descriptor, temporary = tempfile.mkstemp(prefix=".download-", dir=target.parent) try: @@ -286,7 +296,7 @@ def download(source, name, root, expected, prefix=""): for block in iter(lambda: response.read(1024 * 1024), b""): output.write(block) if file_digest(Path(temporary)) != expected: - raise InstallError("Node payload checksum mismatch: " + name) + raise RuntimeDownloadError("Node payload checksum mismatch: " + name) os.replace(temporary, target) finally: if os.path.exists(temporary): @@ -450,6 +460,11 @@ def register_node(root, args, token): os.chmod(target, 0o700) node_generations.install_helper(root, args, sys.modules[__name__]) safe_directory(root / "state/node") + lease_identity = {"installation_id": args.installation_id, "generation": args.configuration["generation"], + "specification_digest": args.configuration["specification_digest"]} + with node_generations.collection_lease(root, args.configuration["generation"], sys.modules[__name__], lease_identity, + initialize=not (root / "provider.json").exists()): + pass print("Checking the sandbox runtime...", flush=True) runtime_image = prepare_runtime(root, args, manifest) # Retain the original network policy when recovering a partial installation. @@ -1481,6 +1496,9 @@ def main(argv=None): if __name__ == "__main__": try: main() + except (RuntimeDownloadError, distribution.ArtifactError): + print("Runtime artifact transfer or verification failed", file=sys.stderr) + sys.exit(65) except ChildFailed as failure: if str(failure): print(str(failure), file=sys.stderr) diff --git a/deploy/install/node_update.py b/deploy/install/node_update.py index 06aef9af5..eea3c47fc 100644 --- a/deploy/install/node_update.py +++ b/deploy/install/node_update.py @@ -159,6 +159,9 @@ def retain_legacy(root, value, provider, installer): if path.exists() and installer.private_json(path) != expected: raise installer.InstallError("Legacy helper retention marker differs") node_generations.atomic_json(path, expected) + identity = dict(expected, installation_id=provider["installation_id"], generation=generation) + with node_generations.collection_lease(root, generation, installer, identity, initialize=True): + pass descriptor = os.open(directory.parent, os.O_RDONLY | os.O_DIRECTORY) try: os.fsync(descriptor) diff --git a/deploy/install/test_generation_review_regressions.py b/deploy/install/test_generation_review_regressions.py new file mode 100644 index 000000000..dd2a8a2fd --- /dev/null +++ b/deploy/install/test_generation_review_regressions.py @@ -0,0 +1,263 @@ +"""Immutable preparation, durable lease identity and published repair regressions.""" +import fcntl +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +import distribution +import install +import node_generations +import node_install as installer +import test_node_install +import test_node_generations + + +class GenerationReviewRegressions(unittest.TestCase): + def fixture(self, kind): + case = kind() + case.setUp() + self.addCleanup(case.doCleanups) + return case + + def test_containerd_resolution_recovers_interrupted_commit(self): + case = self.fixture(test_node_install.NodeInstallTests) + case.containerd = True + case.payloads["runtime/seccomp.json"] = b"{}" + case.refresh_manifest() + case.install() + case.manifest['source_commit'] = 'f' * 40 + case.refresh_manifest() + configuration = json.loads(case.configuration_response(None).read()) + configuration['generation'] = 2 + case.args.generation = 2 + case.args.specification_digest = configuration['specification_digest'] + path = case.root / 'state/node/generations/2.json' + original = node_generations.atomic_json + def interrupted(path, value): + original(path, value) + if path.name == '2.json': + raise OSError('interrupted after resolved configuration') + with mock.patch.object(installer.node_spec, 'fetch', return_value=configuration): + with mock.patch.object(node_generations, 'atomic_json', side_effect=interrupted): + with self.assertRaises(OSError): + node_generations.prepare(case.args, installer) + self.assertEqual(installer.private_json(path)['docker']['image'], case.manifest['image_manifest_digests']['runtime']) + self.assertTrue(installer.private_json(path.with_suffix('.preparing'))['import_started']) + node_generations.prepare(case.args, installer) + saved = path.read_bytes() + node_generations.prepare(case.args, installer) + self.assertEqual(path.read_bytes(), saved) + self.assertFalse(path.with_suffix('.preparing').exists()) + self.assertTrue(node_generations.image_available(installer.private_json(path), installer)) + + def test_unresolved_import_remains_discoverable_and_collectible(self): + case = self.fixture(test_node_install.NodeInstallTests) + case.containerd = True + case.payloads["runtime/seccomp.json"] = b"{}" + case.refresh_manifest() + case.install() + case.manifest['source_commit'] = 'f' * 40 + case.refresh_manifest() + configuration = json.loads(case.configuration_response(None).read()) + configuration['generation'] = 2 + case.args.generation = 2 + case.args.specification_digest = configuration['specification_digest'] + with mock.patch.object(installer.node_spec, 'fetch', return_value=configuration), mock.patch.object(installer, 'prepare_runtime', side_effect=installer.InstallError('daemon unavailable')): + with self.assertRaises(installer.InstallError): + node_generations.prepare(case.args, installer) + self.assertIn(2, node_generations.retained_configs(case.root, installer)) + record = installer.private_json(case.root / 'state/node/generations/2.preparing') + self.assertTrue(record['import_started']) + self.assertEqual(record['configuration']['specification'], configuration['specification']) + self.assertFalse((case.root / 'state/node/generations/2.json').exists()) + node_generations.collect(case.args, installer) + node_generations.collect(case.args, installer) + self.assertNotIn(2, node_generations.retained_configs(case.root, installer)) + self.assertTrue((case.root / 'provider.json').exists()) + + def test_live_helper_replaced_inode_is_refused_after_restart(self): + case = self.fixture(test_node_generations.CollectionTests) + lease = case.directory / '1.lease' + helper = subprocess.Popen([sys.executable, '-c', 'import os,fcntl,sys; f=os.open(sys.argv[1],os.O_RDWR);fcntl.flock(f,fcntl.LOCK_SH);print("locked",flush=True);sys.stdin.read()', str(lease)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + try: + self.assertEqual(helper.stdout.readline().strip(), 'locked') + old = lease.with_suffix('.old') + lease.rename(old) + lease.touch(mode=0o600) + for initialize in (False, True): + with self.assertRaises(installer.InstallError): + with node_generations.collection_lease(case.root, 1, installer, node_generations.marker_identity(case.args), initialize=initialize): + self.fail('replacement adopted') + with self.assertRaises(installer.InstallError): + node_generations.collect(case.args, installer) + self.assertIsNone(helper.poll()) + self.assertTrue(case.release.exists()) + self.assertFalse((case.directory / '1.dropped').exists()) + with old.open('rb') as stream: + with self.assertRaises(BlockingIOError): + fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + lease.unlink() + old.rename(lease) + finally: + helper.communicate(timeout=10) + self.assertEqual(helper.returncode, 0) + node_generations.collect(case.args, installer) + self.assertFalse(case.release.exists()) + + def test_interrupted_lease_initialization_is_not_reclaimed(self): + case = self.fixture(test_node_generations.CollectionTests) + identity = dict(node_generations.marker_identity(case.args), generation=3) + with mock.patch.object(node_generations, 'atomic_json', side_effect=OSError('power loss')): + with self.assertRaises(OSError): + with node_generations.collection_lease(case.root, 3, installer, identity, initialize=True): + self.fail('initialization not durable') + with self.assertRaises(FileExistsError): + with node_generations.collection_lease(case.root, 3, installer, identity, initialize=True): + self.fail('interrupted inode adopted') + (case.directory / '1.lease-identity').unlink() + with self.assertRaises(OSError): + node_generations.collect(case.args, installer) + self.assertTrue(case.release.exists()) + + def payload_fixture(self, directory): + root = directory / 'installation'; root.mkdir() + bundle = directory / 'bundle'; bundle.mkdir() + source = 'a' * 40 + entries = {} + for name in ('node', 'helper'): + raw = (name + ' verified artifact').encode() + entries[name] = {'filename': name + '-' + source, 'size': len(raw), 'sha256': hashlib.sha256(raw).hexdigest()} + files = {'node-install.pyz': b'node', 'self-hosted-install.pyz': b'self', 'SHA256SUMS': b'fixture', 'runtime/seccomp.json': b'{}', 'manifest.json': json.dumps({'source_commit': source, 'artifacts': entries}).encode()} + for name, raw in files.items(): + path = bundle / name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(raw) + install.prepare_node_payload(root, {'mode': 'all'}, bundle) + return root, bundle, root / 'node-payload/releases' / source, entries + + def test_thin_release_adds_and_repairs_verified_missing_artifact(self): + with tempfile.TemporaryDirectory() as temp: + root, bundle, release, entries = self.payload_fixture(Path(temp)) + (bundle / 'artifacts').mkdir() + for name, entry in entries.items(): + (bundle / 'artifacts' / entry['filename']).write_bytes((name + ' verified artifact').encode()) + original = (release / 'manifest.json').read_bytes() + install.prepare_node_payload(root, {'mode': 'all'}, bundle) + artifact = release / 'artifacts' / entries['node']['filename'] + self.assertEqual(artifact.read_bytes(), b'node verified artifact') + artifact.unlink() + install.prepare_node_payload(root, {'mode': 'all'}, bundle) + self.assertEqual(artifact.read_bytes(), b'node verified artifact') + self.assertEqual((release / 'manifest.json').read_bytes(), original) + self.assertEqual(artifact.stat().st_nlink, 1) + + def test_published_conflict_refuses_every_missing_repair(self): + with tempfile.TemporaryDirectory() as temp: + root, bundle, release, entries = self.payload_fixture(Path(temp)) + (bundle / 'artifacts').mkdir(); (release / 'artifacts').mkdir() + node = entries['node']['filename']; helper = entries['helper']['filename'] + (bundle / 'artifacts' / node).write_bytes(b'node verified artifact') + (release / 'artifacts' / helper).write_bytes(b'changed') + with self.assertRaises(install.InstallError): + install.prepare_node_payload(root, {'mode': 'all'}, bundle) + self.assertFalse((release / 'artifacts' / node).exists()) + self.assertEqual((release / 'artifacts' / helper).read_bytes(), b'changed') + + def test_transfer_classification_excludes_provider_and_path_errors(self): + with self.assertRaises(distribution.ArtifactError): + distribution.image_identities({}, 'runtime') + with mock.patch.object(distribution.subprocess, 'run', side_effect=OSError('private daemon detail')): + with self.assertRaises(distribution.DistributionError) as got: + distribution.docker_command(['docker']) + self.assertNotIsInstance(got.exception, distribution.ArtifactError) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp); (root / 'link').symlink_to('/missing') + with self.assertRaises(distribution.DistributionError) as got: + distribution.checked_path(root / 'link') + self.assertNotIsInstance(got.exception, distribution.ArtifactError) + + def test_private_helper_exit_classification_never_parses_text(self): + for expression, code in (("installer.RuntimeDownloadError('secret')", 65), ("installer.distribution.ArtifactError('secret')", 65), ("installer.InstallError('runtime_download_failed secret')", 1)): + script = "import sys,runpy,node_generations; sys.argv=['node_install.py','--installation-id','94be54a1-138c-4f30-bc87-b13686272dbe','--generation-action','prepare','--generation','2','--specification-digest','" + 'a' * 64 + "'];\ndef fail(args,installer): raise " + expression + "\nnode_generations.prepare=fail; runpy.run_module('node_install',run_name='__main__',alter_sys=True)" + result = subprocess.run([sys.executable, '-c', script], capture_output=True, text=True, timeout=10, cwd=Path(__file__).parent) + self.assertEqual(result.returncode, code, result.stderr) + if code == 65: + self.assertNotIn('secret', result.stderr) + + def test_lease_record_is_fsynced_before_any_caller_can_enter(self): + case = self.fixture(test_node_generations.CollectionTests) + identity = dict(node_generations.marker_identity(case.args), generation=4) + operations = [] + original = node_generations.os.fsync + def sync(fd): + operations.append(fd) + return original(fd) + with mock.patch.object(node_generations.os, 'fsync', side_effect=sync): + with node_generations.collection_lease(case.root, 4, installer, identity, initialize=True): + self.assertGreaterEqual(len(operations), 3) # lease, identity, directory + record = installer.private_json(case.directory / '4.lease-identity') + lease = (case.directory / '4.lease').stat() + self.assertEqual((record['device'], record['inode']), (lease.st_dev, lease.st_ino)) + + def test_unpublished_plan_refuses_path_or_installation_drift(self): + case = self.fixture(test_node_install.NodeInstallTests) + case.payloads["runtime/seccomp.json"] = b"{}" + case.refresh_manifest(); case.install() + case.manifest['source_commit'] = 'f' * 40; case.refresh_manifest() + configuration = json.loads(case.configuration_response(None).read()); configuration['generation'] = 2 + case.args.generation = 2; case.args.specification_digest = configuration['specification_digest'] + with mock.patch.object(installer.node_spec, 'fetch', return_value=configuration), mock.patch.object(installer, 'prepare_runtime', side_effect=installer.InstallError('stopped')): + with self.assertRaises(installer.InstallError): node_generations.prepare(case.args, installer) + path=case.root/'state/node/generations/2.preparing' + original=installer.private_json(path) + for field,value in (("installation_id","foreign"),("core_url","http://foreign/api/v1")): + changed=json.loads(json.dumps(original)); changed['configuration'][field]=value + node_generations.atomic_json(path,changed) + with self.assertRaises(installer.InstallError): node_generations.retained_configs(case.root,installer) + changed=json.loads(json.dumps(original)); changed['configuration']['docker']['seccomp_file']=str(case.home/'foreign') + node_generations.atomic_json(path,changed) + with self.assertRaises(installer.InstallError): node_generations.collect(case.args,installer) + self.assertFalse((case.root/'state/node/generations/2.dropped').exists()) + node_generations.atomic_json(path,original) + node_generations.collect(case.args,installer) + + def test_pending_fifo_is_refused_without_opening_a_blocking_reader(self): + case = self.fixture(test_node_generations.CollectionTests) + import os + os.mkfifo(case.directory/'2.preparing',0o600) + with self.assertRaises(installer.InstallError): node_generations.retained_configs(case.root,installer) + + def test_final_identity_never_changes_on_resolver_drift(self): + case = self.fixture(test_node_install.NodeInstallTests) + case.payloads["runtime/seccomp.json"] = b"{}"; case.refresh_manifest(); case.install() + case.manifest['source_commit']='f'*40; case.refresh_manifest() + cfg=json.loads(case.configuration_response(None).read()); cfg['generation']=2 + case.args.generation=2;case.args.specification_digest=cfg['specification_digest'] + with mock.patch.object(installer.node_spec,'fetch',return_value=cfg): + node_generations.prepare(case.args,installer) + path=case.root/'state/node/generations/2.json';before=path.read_bytes() + # The same spec allows both IDs, but a finalized local provider is immutable. + case.containerd=True + with self.assertRaises(installer.node_spec.SpecificationError): node_generations.prepare(case.args,installer) + self.assertEqual(path.read_bytes(),before) + + def test_collecting_generation_is_retained_but_never_reused(self): + case=self.fixture(test_node_install.NodeInstallTests) + case.args.provider="microsandbox";case.install() + cfg=json.loads(case.configuration_response(None).read());cfg['generation']=2 + case.args.generation=2;case.args.specification_digest=cfg['specification_digest'] + original=installer.private_json(case.root/'provider.json') + marker=dict(installation_id=case.args.installation_id,generation=1,specification_digest=installer.node_spec.digest('microsandbox',original['specification']),native_complete=False) + node_generations.atomic_json(case.root/'state/node/generations/1.collecting',marker) + real=node_generations.image_available + def available(value,installer): + self.assertNotEqual(value['generation'],1,'collecting entry was considered for reuse') + return real(value,installer) + with mock.patch.object(installer.node_spec,'fetch',return_value=cfg),mock.patch.object(node_generations,'image_available',side_effect=available): + node_generations.prepare(case.args,installer) + final=installer.private_json(case.root/'state/node/generations/2.json') + self.assertNotEqual(final['microsandbox']['helper_path'],original['microsandbox']['helper_path']) + self.assertIn(1,node_generations.retained_configs(case.root,installer)) diff --git a/deploy/install/test_node_generations.py b/deploy/install/test_node_generations.py index ac0aa7cde..851628ae2 100644 --- a/deploy/install/test_node_generations.py +++ b/deploy/install/test_node_generations.py @@ -29,11 +29,17 @@ def setUp(self): self.release.mkdir(parents=True) (self.release / "artifact").write_bytes(b"immutable bytes") node_generations.atomic_json(self.root / "provider.json", self.value) + self.initialize_lease() for patch in (mock.patch.object(node_generations, "owned_root", return_value=(self.root, {})), mock.patch.object(installer, "checked", return_value="")): patch.start() self.addCleanup(patch.stop) + def initialize_lease(self): + with node_generations.collection_lease(self.root, self.args.generation, installer, + node_generations.marker_identity(self.args), initialize=True): + pass + def test_busy_helper_refuses_all_mutations_then_same_inode_collects(self): lease = self.directory / "1.lease" descriptor = os.open(lease, os.O_CREAT | os.O_RDWR, 0o600) @@ -73,6 +79,7 @@ def test_symlink_and_shared_inode_are_not_collection_authority(self): foreign = self.root / "foreign" foreign.write_bytes(b"") foreign.chmod(0o600) + lease.unlink() for link in (os.symlink, os.link): link(foreign, lease) try: @@ -145,6 +152,7 @@ def test_shared_image_and_release_survive_until_last_generation(self): installer.checked.assert_not_called() self.assertTrue((self.release / "artifact").exists()) self.args.generation = 2 + self.initialize_lease() node_generations.collect(self.args, installer) self.assertFalse(self.release.exists()) @@ -177,6 +185,10 @@ def micro_fixture(self): self.value["microsandbox"] = {"helper_path": str(self.release / "helper"), "runtime_home": str(home), "runtime_path": str(runtime), "firmware_path": str(self.release / "firmware"), "runtime_sha256": hashlib.sha256(runtime.read_bytes()).hexdigest(), "image": image} self.args.specification_digest = node_spec.digest("microsandbox", self.value["specification"]) node_generations.atomic_json(self.root / "provider.json", self.value) + # This fixture changes provider before any helper exists. + for suffix in (".lease", ".lease-identity"): + (self.directory / ("1" + suffix)).unlink() + self.initialize_lease() return runtime, image, home def test_micro_interrupted_native_removal_and_missing_runtime_fail_closed(self): diff --git a/deploy/install/test_node_install.py b/deploy/install/test_node_install.py index 9e3148cc7..483a02e3b 100644 --- a/deploy/install/test_node_install.py +++ b/deploy/install/test_node_install.py @@ -328,6 +328,8 @@ def test_shared_original_firmware_survives_until_its_last_generation(self): self.assertTrue(installer.node_generations.image_available(successor, installer)) self.args.generation = 2 self.args.specification_digest = node_spec.digest("microsandbox", successor["specification"]) + with installer.node_generations.collection_lease(self.root, 2, installer, installer.node_generations.marker_identity(self.args), initialize=True): + pass def inventory(command, *_args, **_kwargs): if command[1:3] == ["image", "list"]: return "" if command[1:3] == ["sandbox", "list"]: return "[]" @@ -372,7 +374,8 @@ def interrupted(manifest, name, path): with self.assertRaises(OSError): installer.node_generations.prepare(self.args, installer) directory = self.root / "state/node/generations" - saved = json.loads((directory / "2.json").read_text()) + saved = json.loads((directory / "2.preparing").read_text())["configuration"] + self.assertFalse((directory / "2.json").exists()) self.assertEqual(saved["specification"], config["specification"]) self.assertFalse(json.loads((directory / "2.preparing").read_text())["import_started"]) helper = Path(saved["microsandbox"]["helper_path"]) @@ -381,7 +384,7 @@ def interrupted(manifest, name, path): installer.node_generations.prepare(self.args, installer) self.assertEqual(helper.stat().st_ino, inode) self.assertEqual(json.loads((directory / "2.json").read_text()), saved) - self.assertTrue(json.loads((directory / "2.preparing").read_text())["import_started"]) + self.assertFalse((directory / "2.preparing").exists()) self.assertEqual((self.root / "provider.json").read_bytes(), original) def test_restart_repairs_only_missing_micro_bytes_at_original_paths(self): @@ -416,7 +419,7 @@ def test_live_helper_or_collection_excludes_restart_repair(self): helper = self.root / installer.MICRO[0] helper.unlink() directory = self.root / "state/node/generations" - directory.mkdir(mode=0o700) + directory.mkdir(mode=0o700, exist_ok=True) descriptor = os.open(directory / "1.lease", os.O_CREAT | os.O_RDWR, 0o600) try: for mode in (fcntl.LOCK_SH, fcntl.LOCK_EX): diff --git a/deploy/install/test_node_update.py b/deploy/install/test_node_update.py index ed7f7aa96..a2450905e 100644 --- a/deploy/install/test_node_update.py +++ b/deploy/install/test_node_update.py @@ -87,6 +87,8 @@ def interrupted(arguments, *_args, **_kwargs): later["docker"]["image"] = "sha256:" + "9" * 64 node_generations.atomic_json(self.root / "state/node/generations/2.json", later) grant.generation = 2 + with node_generations.collection_lease(self.root, 2, installer, node_generations.marker_identity(grant), initialize=True): + pass node_generations.collect(grant, installer) self.assertTrue((self.root / "state/node/generations/2.dropped").exists()) self.assertTrue(marker.exists()) diff --git a/docs/getting-started/nodes.md b/docs/getting-started/nodes.md index 4d500553d..93aca47b2 100644 --- a/docs/getting-started/nodes.md +++ b/docs/getting-started/nodes.md @@ -263,14 +263,36 @@ WebSocket doesn't pass the reverse proxy. ## Change the sandbox backend or size -Changing backend type requires an explicit reset. Docker and microsandbox size or -Runtime edits currently require zero held resources; node online updates are not -available yet. These changes retire every node. Retired nodes drop out of the list, so there is nothing to -remove in Web: on each host, run the [uninstall command](#remove-a-node) (Core no longer -accepts the node, so it needs no `--force`), then add the host again with a new command. +Changing backend type requires an explicit reset. Same-provider Docker and +microsandbox size or Runtime edits advance the target generation without retiring +nodes or moving existing Sessions. Version 2 nodes prepare the target independently; +existing allocations and suspended VMs keep their original generation. A qualified +older serving generation can still accept new Sessions when it has capacity, +including while a newer target is preparing or has failed. + +Version 1 nodes report `update_required` for a newer target while retaining their +qualified original serving generation. Run the verified current installer with +`--update`, the same installation ID, Core/source URLs and original installation +user (or sudo mode); do not provide another enrollment token. The update preserves +node identity, retained Runtime files and running VMs. Its original v1 generation +is conservatively retained locally because old helpers were not protected by the +new generation lock. This retained disk usage is not reported as completed GC. + +Fresh v2 preparation keeps a private recovery plan until the actual provider image +identity is resolved and its final configuration is published. Restart does not +serve a pending or partly collected generation. Generation lock identity is durable: +if a lease or its identity record is missing or replaced, preserve the installation +for inspection. Do not delete records, recreate lock files or re-enroll to bypass +that refusal; a process restart does not prove that older helpers have stopped. + +Automatic Docker generation GC removes only installation-owned generation/release +files. Shared daemon images remain; a host administrator may remove them only after +confirming no installation still needs them. Microsandbox image cleanup remains +scoped to the installation's private native store and verified ownership receipts. + The [operator reference](../../services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-reset) -describes today's procedure and API. Same-team E2B edits have a separate online -Core API path and do not involve node installation. +describes reset and API behavior. Same-team E2B edits have a separate online Core +API path and do not involve node installation. ## Remove a node added before the rename diff --git a/services/agents-api/cmd/sandbox-node/generations.go b/services/agents-api/cmd/sandbox-node/generations.go index 5b98cb632..14197eda4 100644 --- a/services/agents-api/cmd/sandbox-node/generations.go +++ b/services/agents-api/cmd/sandbox-node/generations.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "errors" - "fmt" "io" "os" "os/exec" @@ -40,7 +39,11 @@ func runGenerations(ctx context.Context, configFile, stateDir string) error { if err != nil { return err } - paths = append([]string{configFile}, paths...) + pending, err := filepath.Glob(filepath.Join(stateDir, "generations", "*.preparing")) + if err != nil { + return err + } + paths = append(append([]string{configFile}, paths...), pending...) values := map[uint64]node.GenerationProvider{} recovery := []sandbox.GenerationReference{} collection := []sandbox.GenerationReference{} @@ -53,7 +56,19 @@ func runGenerations(ctx context.Context, configFile, stateDir string) error { } } for _, path := range paths { - config, err := providerconfig.Load(path) + var config providerconfig.Config + if strings.HasSuffix(path, ".preparing") { + journal, readErr := readGenerationJournal(path) + err = readErr + if err == nil && journal.Configuration == nil { + err = sandbox.ErrOwnership + } + if err == nil { + config = *journal.Configuration + } + } else { + config, err = providerconfig.Load(path) + } if err != nil { closeValues() return err @@ -62,7 +77,7 @@ func runGenerations(ctx context.Context, configFile, stateDir string) error { closeValues() return sandbox.ErrOwnership } - if path != configFile && filepath.Base(path) != strconv.FormatUint(config.Generation, 10)+".json" { + if path != configFile && strings.TrimSuffix(strings.TrimSuffix(filepath.Base(path), ".json"), ".preparing") != strconv.FormatUint(config.Generation, 10) { closeValues() return sandbox.ErrOwnership } @@ -75,13 +90,17 @@ func runGenerations(ctx context.Context, configFile, stateDir string) error { continue } if previous, ok := seen[config.Generation]; ok { - if !reflect.DeepEqual(previous, config) { + if !sameGenerationPlan(previous, config) { closeValues() return sandbox.ErrOwnership } continue } seen[config.Generation] = config + if state == "preparing" { + recovery = append(recovery, sandbox.GenerationReference{Generation: config.Generation, SpecificationDigest: config.Specification.Digest(config.Provider)}) + continue + } if state == "collecting" { collection = append(collection, sandbox.GenerationReference{Generation: config.Generation, SpecificationDigest: config.Specification.Digest(config.Provider)}) continue @@ -107,7 +126,7 @@ func runGenerations(ctx context.Context, configFile, stateDir string) error { command.Stdout = os.Stderr command.Stderr = os.Stderr if err := command.Run(); err != nil { - return fmt.Errorf("%w: local generation %s did not complete", sandbox.ErrRuntimeDownloadFailed, action) + return generationHelperError(ctx, err) } return nil } @@ -157,6 +176,7 @@ func buildGeneration(config providerconfig.Config, stateDir string) (node.Genera if err != nil || !info.IsDir() || info.Mode().Perm() != 0700 { return node.GenerationProvider{}, sandbox.ErrOwnership } + config.Microsandbox.HelperLeaseGeneration = config.Generation config.Microsandbox.HelperLeasePath = filepath.Join(directory, strconv.FormatUint(config.Generation, 10)+".lease") } built, closeProvider, err := providerconfig.Build(config) @@ -193,8 +213,59 @@ func buildGeneration(config providerconfig.Config, stateDir string) (node.Genera return node.GenerationProvider{Generation: config.Generation, SpecificationDigest: built.SpecificationDigest, Provider: built.Provider, Probe: probe, Close: closeProvider}, nil } -// A persisted local journal never substitutes for a fresh Core drop grant. It -// only prevents restart from preparing or serving a partly collected generation. +type generationJournal struct { + InstallationID string `json:"installation_id"` + Generation uint64 `json:"generation"` + SpecificationDigest string `json:"specification_digest"` + NativeComplete json.RawMessage `json:"native_complete,omitempty"` + ImportStarted json.RawMessage `json:"import_started,omitempty"` + Configuration *providerconfig.Config `json:"configuration,omitempty"` +} + +func readGenerationJournal(path string) (generationJournal, error) { + var journal generationJournal + fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_CLOEXEC|syscall.O_NONBLOCK, 0) + if err != nil { + return journal, err + } + file := os.NewFile(uintptr(fd), path) + defer file.Close() + var st syscall.Stat_t + err = syscall.Fstat(fd, &st) + resolved, pathErr := filepath.EvalSymlinks(path) + if err != nil || pathErr != nil || resolved != path || st.Mode&syscall.S_IFMT != syscall.S_IFREG || st.Mode&0777 != 0600 || st.Uid != uint32(os.Getuid()) || st.Nlink != 1 || st.Size > 16384 { + return journal, sandbox.ErrOwnership + } + decoder := json.NewDecoder(io.LimitReader(file, 16385)) + decoder.DisallowUnknownFields() + err = decoder.Decode(&journal) + var trailing any + end := decoder.Decode(&trailing) + opened, statErr := file.Stat() + named, namedErr := os.Lstat(path) + if err != nil || end != io.EOF || statErr != nil || namedErr != nil || !os.SameFile(opened, named) { + return journal, sandbox.ErrOwnership + } + return journal, nil +} + +// The preparation configuration is an immutable plan, never a usable provider. +// Only Docker's two specification-proven local IDs can differ at publication. +func sameGenerationPlan(final, plan providerconfig.Config) bool { + if plan.Docker != nil && final.Docker != nil { + runtime := plan.Specification.Runtime + if final.Docker.Image != runtime.ImageID && final.Docker.Image != runtime.ImageManifestDigest { + return false + } + copyDocker := *plan.Docker + copyDocker.Image = final.Docker.Image + plan.Docker = ©Docker + } + return reflect.DeepEqual(final, plan) +} + +// Pending-only entries stay recovery/collection-only. No journal is readiness +// or authority to collect without a fresh current-connection Core grant. func generationLocalState(config providerconfig.Config, stateDir string) (string, error) { directory := filepath.Join(stateDir, "generations") info, err := os.Lstat(directory) @@ -210,44 +281,36 @@ func generationLocalState(config providerconfig.Config, stateDir string) (string return "", sandbox.ErrOwnership } state := "" - for _, suffix := range []string{".collecting", ".dropped"} { - path := filepath.Join(stateDir, "generations", strconv.FormatUint(config.Generation, 10)+suffix) - fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_CLOEXEC|syscall.O_NONBLOCK, 0) - if errors.Is(err, syscall.ENOENT) { + for _, suffix := range []string{".preparing", ".collecting", ".dropped"} { + path := filepath.Join(directory, strconv.FormatUint(config.Generation, 10)+suffix) + journal, err := readGenerationJournal(path) + if errors.Is(err, os.ErrNotExist) { continue } - if err != nil { - return "", err - } - file := os.NewFile(uintptr(fd), path) - var st syscall.Stat_t - err = syscall.Fstat(fd, &st) - resolved, pathErr := filepath.EvalSymlinks(path) - if err != nil || pathErr != nil || resolved != path || st.Mode&syscall.S_IFMT != syscall.S_IFREG || st.Mode&0777 != 0600 || st.Uid != uint32(os.Getuid()) || st.Nlink != 1 || st.Size > 4096 { - file.Close() + if err != nil || journal.InstallationID != config.InstallationID || journal.Generation != config.Generation || journal.SpecificationDigest != config.Specification.Digest(config.Provider) { return "", sandbox.ErrOwnership } - var journal struct { - InstallationID string `json:"installation_id"` - Generation uint64 `json:"generation"` - SpecificationDigest string `json:"specification_digest"` - NativeComplete json.RawMessage `json:"native_complete,omitempty"` - } - decoder := json.NewDecoder(io.LimitReader(file, 4097)) - decoder.DisallowUnknownFields() - err = decoder.Decode(&journal) - var trailing any - end := decoder.Decode(&trailing) - opened, statErr := file.Stat() - named, namedErr := os.Lstat(path) - file.Close() - if statErr != nil || namedErr != nil || !os.SameFile(opened, named) { - return "", sandbox.ErrOwnership - } - if err != nil || end != io.EOF || journal.InstallationID != config.InstallationID || journal.Generation != config.Generation || journal.SpecificationDigest != config.Specification.Digest(config.Provider) || (suffix == ".collecting" && string(journal.NativeComplete) != "true" && string(journal.NativeComplete) != "false") || (suffix == ".dropped" && len(journal.NativeComplete) != 0) { + if suffix == ".preparing" { + if len(journal.NativeComplete) != 0 || (string(journal.ImportStarted) != "true" && string(journal.ImportStarted) != "false") || journal.Configuration == nil || !sameGenerationPlan(config, *journal.Configuration) { + return "", sandbox.ErrOwnership + } + } else if len(journal.ImportStarted) != 0 || journal.Configuration != nil || suffix == ".collecting" && string(journal.NativeComplete) != "true" && string(journal.NativeComplete) != "false" || suffix == ".dropped" && len(journal.NativeComplete) != 0 { return "", sandbox.ErrOwnership } state = strings.TrimPrefix(suffix, ".") } return state, nil } + +// Exit 65 is reserved by the private preparer for artifact transfer/provenance. +// Arbitrary provider output never selects a wire diagnostic. +func generationHelperError(ctx context.Context, err error) error { + if ctx.Err() != nil { + return ctx.Err() + } + var exit *exec.ExitError + if errors.As(err, &exit) && exit.ExitCode() == 65 { + return sandbox.ErrRuntimeDownloadFailed + } + return errors.New("local generation operation did not complete") +} diff --git a/services/agents-api/cmd/sandbox-node/generations_test.go b/services/agents-api/cmd/sandbox-node/generations_test.go index 50a9343d3..b0705496e 100644 --- a/services/agents-api/cmd/sandbox-node/generations_test.go +++ b/services/agents-api/cmd/sandbox-node/generations_test.go @@ -1,8 +1,12 @@ package main import ( + "context" "encoding/json" + "errors" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" "os" + "os/exec" "path/filepath" "syscall" "testing" @@ -94,3 +98,48 @@ func TestGenerationJournalRestartIdentity(t *testing.T) { }) } } + +func TestGenerationHelperUsesOnlyTypedExit(t *testing.T) { + for _, code := range []string{"1", "65", "78"} { + err := exec.Command("sh", "-c", "exit "+code).Run() + got := generationHelperError(t.Context(), err) + if errors.Is(got, sandbox.ErrRuntimeDownloadFailed) != (code == "65") { + t.Fatal(code, got) + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if !errors.Is(generationHelperError(ctx, err), context.Canceled) { + t.Fatal("cancellation was lost") + } + } + if errors.Is(generationHelperError(t.Context(), errors.New("runtime_download_failed")), sandbox.ErrRuntimeDownloadFailed) { + t.Fatal("parsed arbitrary provider text") + } +} + +func TestUnresolvedPreparationRemainsRecoveryOnly(t *testing.T) { + config := providerconfig.Config{InstallationID: "installation", Generation: 2, Provider: "docker"} + stateDir := t.TempDir() + directory := filepath.Join(stateDir, "generations") + if err := os.Mkdir(directory, 0700); err != nil { + t.Fatal(err) + } + path := filepath.Join(directory, "2.preparing") + for _, started := range []bool{false, true} { + raw, _ := json.Marshal(map[string]any{"installation_id": config.InstallationID, "generation": config.Generation, "specification_digest": config.Specification.Digest(config.Provider), "import_started": started, "configuration": config}) + if err := os.WriteFile(path, raw, 0600); err != nil { + t.Fatal(err) + } + journal, err := readGenerationJournal(path) + if err != nil || journal.Configuration == nil { + t.Fatal("pending-only discovery", err) + } + state, err := generationLocalState(*journal.Configuration, stateDir) + if err != nil || state != "preparing" { + t.Fatal(state, err) + } + if _, err := os.Stat(filepath.Join(directory, "2.json")); !os.IsNotExist(err) { + t.Fatal("unresolved plan published") + } + } +} diff --git a/services/agents-api/internal/sandbox/config/microsandbox.go b/services/agents-api/internal/sandbox/config/microsandbox.go index dac0cb2c8..16ed01ff4 100644 --- a/services/agents-api/internal/sandbox/config/microsandbox.go +++ b/services/agents-api/internal/sandbox/config/microsandbox.go @@ -11,19 +11,20 @@ import ( // Single-host providers pin the helper, runtime, firmware, image and resource // limits explicitly. The helper owns local paths; no ambient backend is selected. type Microsandbox struct { - HelperLeasePath string `json:"-"` - HelperPath string `json:"helper_path"` - RuntimeHome string `json:"runtime_home"` - RuntimePath string `json:"runtime_path"` - FirmwarePath string `json:"firmware_path"` - RuntimeSHA256 string `json:"runtime_sha256"` - FirmwareSHA256 string `json:"firmware_sha256"` - Image string `json:"image"` - MemoryMiB uint32 `json:"memory_mib"` - CPUs uint8 `json:"cpus"` - RootDiskMiB uint32 `json:"root_disk_mib"` - EnvironmentDiskMiB uint32 `json:"environment_disk_mib"` - Network Network `json:"network"` + HelperLeaseGeneration uint64 `json:"-"` + HelperLeasePath string `json:"-"` + HelperPath string `json:"helper_path"` + RuntimeHome string `json:"runtime_home"` + RuntimePath string `json:"runtime_path"` + FirmwarePath string `json:"firmware_path"` + RuntimeSHA256 string `json:"runtime_sha256"` + FirmwareSHA256 string `json:"firmware_sha256"` + Image string `json:"image"` + MemoryMiB uint32 `json:"memory_mib"` + CPUs uint8 `json:"cpus"` + RootDiskMiB uint32 `json:"root_disk_mib"` + EnvironmentDiskMiB uint32 `json:"environment_disk_mib"` + Network Network `json:"network"` } type Network struct { @@ -52,7 +53,7 @@ func configureMicrosandbox(entry Microsandbox, resources sandbox.Resources, resu InstallationID: result.InstallationID, HelperPath: entry.HelperPath, RuntimeHome: entry.RuntimeHome, RuntimePath: entry.RuntimePath, FirmwarePath: entry.FirmwarePath, RuntimeSHA256: entry.RuntimeSHA256, FirmwareSHA256: entry.FirmwareSHA256, Image: entry.Image, MemoryMiB: entry.MemoryMiB, CPUs: entry.CPUs, RootDiskMiB: entry.RootDiskMiB, EnvironmentDiskMiB: entry.EnvironmentDiskMiB, Network: network, - }, &sandboxmicro.ProcessCaller{LeasePath: entry.HelperLeasePath}) + }, &sandboxmicro.ProcessCaller{LeasePath: entry.HelperLeasePath, LeaseIdentity: sandboxmicro.LeaseIdentity{InstallationID: result.InstallationID, Generation: entry.HelperLeaseGeneration, SpecificationDigest: result.SpecificationDigest}}) if err != nil { return errors.New("invalid managed microsandbox provider configuration") } diff --git a/services/agents-api/internal/sandbox/microsandbox/process.go b/services/agents-api/internal/sandbox/microsandbox/process.go index 4f4558b12..7600b386c 100644 --- a/services/agents-api/internal/sandbox/microsandbox/process.go +++ b/services/agents-api/internal/sandbox/microsandbox/process.go @@ -5,9 +5,11 @@ import ( "context" "encoding/json" "errors" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" "io" "os" "os/exec" + "path/filepath" "strings" "sync/atomic" "syscall" @@ -20,7 +22,8 @@ type ProcessCaller struct { active atomic.Int64 // LeasePath belongs to the permanent node state namespace, not a release. // It is never unlinked, including after the generation is collected. - LeasePath string + LeasePath string + LeaseIdentity LeaseIdentity } func (p *ProcessCaller) Quiescent() bool { return p.active.Load() == 0 } @@ -115,7 +118,7 @@ func (p *ProcessCaller) acquireLease() (*os.File, error) { if p.LeasePath == "" { return nil, nil } - fd, err := syscall.Open(p.LeasePath, syscall.O_RDWR|syscall.O_CREAT|syscall.O_NOFOLLOW|syscall.O_CLOEXEC, 0600) + fd, err := syscall.Open(p.LeasePath, syscall.O_RDWR|syscall.O_NOFOLLOW|syscall.O_CLOEXEC|syscall.O_NONBLOCK, 0600) if err != nil { return nil, err } @@ -136,8 +139,95 @@ func (p *ProcessCaller) acquireLease() (*os.File, error) { if err != nil || !os.SameFile(info, named) { return fail(errors.New("generation lease replaced")) } - if _, err := os.Lstat(strings.TrimSuffix(p.LeasePath, ".lease") + ".dropped"); !os.IsNotExist(err) { - return fail(errors.New("generation collected or unavailable")) + if err := p.verifyLeaseIdentity(stat); err != nil { + return fail(err) + } + for _, suffix := range []string{".preparing", ".collecting", ".dropped"} { + if _, err := os.Lstat(strings.TrimSuffix(p.LeasePath, ".lease") + suffix); !os.IsNotExist(err) { + return fail(errors.New("generation is not finalized and usable")) + } + } + if err := p.verifyFinalizedGeneration(); err != nil { + return fail(err) } return file, nil } + +// LeaseIdentity is recorded by the installer before a generation can spawn a +// helper. Neither opener adopts a missing record or a replacement lease inode. +type LeaseIdentity struct { + InstallationID string `json:"installation_id"` + Generation uint64 `json:"generation"` + SpecificationDigest string `json:"specification_digest"` +} + +func (p *ProcessCaller) verifyLeaseIdentity(lease *syscall.Stat_t) error { + path := strings.TrimSuffix(p.LeasePath, ".lease") + ".lease-identity" + fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_CLOEXEC|syscall.O_NONBLOCK, 0) + if err != nil { + return err + } + file := os.NewFile(uintptr(fd), path) + defer file.Close() + info, err := file.Stat() + if err != nil { + return err + } + st, ok := info.Sys().(*syscall.Stat_t) + if !ok || !info.Mode().IsRegular() || info.Mode().Perm() != 0600 || st.Uid != uint32(os.Geteuid()) || st.Nlink != 1 || st.Size > 4096 { + return errors.New("invalid durable generation lease identity") + } + var saved struct { + LeaseIdentity + Device uint64 `json:"device"` + Inode uint64 `json:"inode"` + } + decoder := json.NewDecoder(io.LimitReader(file, 4097)) + decoder.DisallowUnknownFields() + if decoder.Decode(&saved) != nil || decoder.Decode(new(any)) != io.EOF || saved.LeaseIdentity != p.LeaseIdentity || saved.Generation == 0 || saved.InstallationID == "" || len(saved.SpecificationDigest) != 64 || saved.Device != uint64(lease.Dev) || saved.Inode != lease.Ino { + return errors.New("generation lease identity differs") + } + named, err := os.Lstat(path) + if err != nil || !os.SameFile(info, named) { + return errors.New("generation lease identity replaced") + } + return nil +} + +// A durable lease alone is not a published provider. Initial enrollment stores +// its final config at provider.json; later generations use .json. +func (p *ProcessCaller) verifyFinalizedGeneration() error { + path := strings.TrimSuffix(p.LeasePath, ".lease") + ".json" + if _, err := os.Lstat(path); os.IsNotExist(err) { + path = filepath.Join(filepath.Dir(filepath.Dir(filepath.Dir(filepath.Dir(p.LeasePath)))), "provider.json") + } + fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_CLOEXEC|syscall.O_NONBLOCK, 0) + if err != nil { + return err + } + file := os.NewFile(uintptr(fd), path) + defer file.Close() + info, err := file.Stat() + if err != nil { + return err + } + st, ok := info.Sys().(*syscall.Stat_t) + if !ok || !info.Mode().IsRegular() || info.Mode().Perm() != 0600 || st.Uid != uint32(os.Geteuid()) || st.Nlink != 1 || st.Size > 16384 { + return errors.New("invalid finalized generation") + } + var value struct { + InstallationID string `json:"installation_id"` + Generation uint64 `json:"generation"` + Provider string `json:"provider"` + Specification sandbox.DeploymentSpec `json:"specification"` + } + decoder := json.NewDecoder(io.LimitReader(file, 16385)) + if decoder.Decode(&value) != nil || decoder.Decode(new(any)) != io.EOF || value.InstallationID != p.LeaseIdentity.InstallationID || value.Generation != p.LeaseIdentity.Generation || value.Provider != "microsandbox" || value.Specification.Digest(value.Provider) != p.LeaseIdentity.SpecificationDigest { + return errors.New("finalized generation identity differs") + } + named, err := os.Lstat(path) + if err != nil || !os.SameFile(info, named) { + return errors.New("finalized generation was replaced") + } + return nil +} diff --git a/services/agents-api/internal/sandbox/microsandbox/process_test.go b/services/agents-api/internal/sandbox/microsandbox/process_test.go index 81c7dd43e..6d60cda93 100644 --- a/services/agents-api/internal/sandbox/microsandbox/process_test.go +++ b/services/agents-api/internal/sandbox/microsandbox/process_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" "os" "os/exec" "path/filepath" @@ -20,7 +21,7 @@ func TestMain(m *testing.M) { if json.NewDecoder(os.Stdin).Decode(&q) != nil { os.Exit(2) } - _, _ = (&ProcessCaller{LeasePath: q.Config.RuntimeHome}).Call(context.Background(), q) + _, _ = (&ProcessCaller{LeasePath: q.Config.RuntimeHome, LeaseIdentity: testLeaseIdentity()}).Call(context.Background(), q) os.Exit(0) } if filepath.Base(os.Args[0]) == "microsandbox-test-helper" { @@ -114,6 +115,7 @@ func TestInvalidOrUnboundedHelperOutputIsUnconfirmed(t *testing.T) { func TestGenerationLeaseSurvivesNodeProcessExit(t *testing.T) { c := processConfig(t) c.RuntimeHome = filepath.Join(filepath.Dir(c.RuntimePath), "1.lease") + writeTestLease(t, c.RuntimeHome) executable, err := os.Executable() if err != nil { t.Fatal(err) @@ -143,13 +145,34 @@ func TestGenerationLeaseSurvivesNodeProcessExit(t *testing.T) { } _ = command.Wait() // A new node has no in-memory references, but cannot collect the old helper. - restarted := &ProcessCaller{LeasePath: c.RuntimeHome} + restarted := &ProcessCaller{LeasePath: c.RuntimeHome, LeaseIdentity: testLeaseIdentity()} if !restarted.Quiescent() { t.Fatal("fresh process unexpectedly has references") } if err := exclusiveLease(c.RuntimeHome); !errors.Is(err, syscall.EWOULDBLOCK) { t.Fatalf("live orphan helper lost lease: %v", err) } + // A fresh node must also refuse an owned replacement regular inode while + // the actual orphan helper still holds the original open file description. + if err := os.Rename(c.RuntimeHome, c.RuntimeHome+".old"); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(c.RuntimeHome, nil, 0600); err != nil { + t.Fatal(err) + } + if file, err := restarted.acquireLease(); err == nil { + file.Close() + t.Fatal("live helper bypassed through replacement inode") + } + if err := exclusiveLease(c.RuntimeHome + ".old"); !errors.Is(err, syscall.EWOULDBLOCK) { + t.Fatal("original helper lock lost", err) + } + if err := os.Remove(c.RuntimeHome); err != nil { + t.Fatal(err) + } + if err := os.Rename(c.RuntimeHome+".old", c.RuntimeHome); err != nil { + t.Fatal(err) + } if err := os.WriteFile(c.RuntimePath+".release", nil, 0600); err != nil { t.Fatal(err) } @@ -184,6 +207,7 @@ func waitProcessCondition(t *testing.T, condition func() bool) { func TestGenerationLeaseExcludesCollectionAndDroppedGeneration(t *testing.T) { c := processConfig(t) lease := filepath.Join(filepath.Dir(c.RuntimePath), "1.lease") + writeTestLease(t, lease) file, err := os.OpenFile(lease, os.O_CREATE|os.O_RDWR, 0600) if err != nil { t.Fatal(err) @@ -192,7 +216,7 @@ func TestGenerationLeaseExcludesCollectionAndDroppedGeneration(t *testing.T) { if err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { t.Fatal(err) } - caller := &ProcessCaller{LeasePath: lease} + caller := &ProcessCaller{LeasePath: lease, LeaseIdentity: testLeaseIdentity()} if _, err := caller.Call(t.Context(), Request{Operation: "delayed", Config: c}); err == nil { t.Fatal("helper started during exclusive collection") } @@ -219,3 +243,206 @@ func TestGenerationLeaseExcludesCollectionAndDroppedGeneration(t *testing.T) { t.Fatal("symlink lease accepted") } } + +func testLeaseIdentity() LeaseIdentity { + return LeaseIdentity{InstallationID: "test-installation", Generation: 1, SpecificationDigest: (sandbox.DeploymentSpec{}).Digest("microsandbox")} +} + +func writeTestLease(t *testing.T, path string) { + t.Helper() + if err := os.WriteFile(path, nil, 0600); err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + st := info.Sys().(*syscall.Stat_t) + value := struct { + LeaseIdentity + Device uint64 `json:"device"` + Inode uint64 `json:"inode"` + }{testLeaseIdentity(), uint64(st.Dev), st.Ino} + raw, _ := json.Marshal(value) + if err := os.WriteFile(strings.TrimSuffix(path, ".lease")+".lease-identity", raw, 0600); err != nil { + t.Fatal(err) + } + writeTestFinalGeneration(t, path) +} + +func writeTestFinalGeneration(t *testing.T, path string) { + t.Helper() + raw, _ := json.Marshal(map[string]any{"installation_id": "test-installation", "generation": 1, "provider": "microsandbox", "specification": sandbox.DeploymentSpec{}}) + if err := os.WriteFile(strings.TrimSuffix(path, ".lease")+".json", raw, 0600); err != nil { + t.Fatal(err) + } +} + +func TestGenerationLeaseReplacementRejectedAfterRestart(t *testing.T) { + path := filepath.Join(t.TempDir(), "1.lease") + writeTestLease(t, path) + caller := &ProcessCaller{LeasePath: path, LeaseIdentity: testLeaseIdentity()} + old, err := caller.acquireLease() + if err != nil { + t.Fatal(err) + } + defer old.Close() + if err := os.Rename(path, path+".old"); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, nil, 0600); err != nil { + t.Fatal(err) + } + restarted := &ProcessCaller{LeasePath: path, LeaseIdentity: testLeaseIdentity()} + if file, err := restarted.acquireLease(); err == nil { + file.Close() + t.Fatal("replacement inode was adopted") + } + if err := exclusiveLease(path + ".old"); !errors.Is(err, syscall.EWOULDBLOCK) { + t.Fatal("old helper lease lost", err) + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if err := os.Rename(path+".old", path); err != nil { + t.Fatal(err) + } + file, err := restarted.acquireLease() + if err != nil { + t.Fatal("original inode refused", err) + } + file.Close() + if err := os.Remove(strings.TrimSuffix(path, ".lease") + ".lease-identity"); err != nil { + t.Fatal(err) + } + if file, err := restarted.acquireLease(); err == nil { + file.Close() + t.Fatal("missing identity adopted") + } +} + +func TestPythonAndGoUseTheSameDurableLease(t *testing.T) { + root := t.TempDir() + installer, err := filepath.Abs("../../../../../deploy/install") + if err != nil { + t.Fatal(err) + } + script := `import sys +sys.path.insert(0,sys.argv[1]) +from pathlib import Path +import node_generations as g,node_install as i +identity={"installation_id":"test-installation","generation":1,"specification_digest":sys.argv[4]} +with g.collection_lease(Path(sys.argv[2]),1,i,identity,initialize=sys.argv[3]=="init"): pass +` + run := func(mode string) error { + return exec.Command("python3", "-c", script, installer, root, mode, testLeaseIdentity().SpecificationDigest).Run() + } + if err := run("init"); err != nil { + t.Fatal("Python initialization", err) + } + path := filepath.Join(root, "state/node/generations/1.lease") + writeTestFinalGeneration(t, path) + caller := &ProcessCaller{LeasePath: path, LeaseIdentity: testLeaseIdentity()} + file, err := caller.acquireLease() + if err != nil { + t.Fatal("Go rejected Python identity", err) + } + if err := run("collect"); err == nil { + file.Close() + t.Fatal("Python collected live Go lease") + } + file.Close() + if err := run("collect"); err != nil { + t.Fatal("settled lease not collectible", err) + } + for _, kind := range []string{"symlink", "hardlink", "fifo"} { + record := strings.TrimSuffix(path, ".lease") + ".lease-identity" + original := record + ".original" + if err := os.Rename(record, original); err != nil { + t.Fatal(err) + } + switch kind { + case "symlink": + err = os.Symlink(original, record) + case "hardlink": + err = os.Link(original, record) + case "fifo": + err = syscall.Mkfifo(record, 0600) + } + if err != nil { + t.Fatal(err) + } + if file, err := caller.acquireLease(); err == nil { + file.Close() + t.Fatal("invalid identity accepted", kind) + } + if err := run("collect"); err == nil { + t.Fatal("Python accepted invalid identity", kind) + } + if err := os.Remove(record); err != nil { + t.Fatal(err) + } + if err := os.Rename(original, record); err != nil { + t.Fatal(err) + } + } +} + +func TestHelperRequiresFinalConfigWithoutAnyUnfinishedJournal(t *testing.T) { + path := filepath.Join(t.TempDir(), "1.lease") + writeTestLease(t, path) + caller := &ProcessCaller{LeasePath: path, LeaseIdentity: testLeaseIdentity()} + for _, suffix := range []string{".preparing", ".collecting", ".dropped"} { + marker := strings.TrimSuffix(path, ".lease") + suffix + if err := os.WriteFile(marker, []byte("{}"), 0600); err != nil { + t.Fatal(err) + } + if file, err := caller.acquireLease(); err == nil { + file.Close() + t.Fatal("unfinished generation was usable", suffix) + } + if err := os.Remove(marker); err != nil { + t.Fatal(err) + } + } + if err := os.Remove(strings.TrimSuffix(path, ".lease") + ".json"); err != nil { + t.Fatal(err) + } + if file, err := caller.acquireLease(); err == nil { + file.Close() + t.Fatal("unpublished generation was usable") + } + writeTestFinalGeneration(t, path) + file, err := caller.acquireLease() + if err != nil { + t.Fatal(err) + } + file.Close() +} + +func TestOriginalFinalConfigAndWrongGeneration(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "state/node/generations") + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "1.lease") + writeTestLease(t, path) + if err := os.Rename(filepath.Join(dir, "1.json"), filepath.Join(root, "provider.json")); err != nil { + t.Fatal(err) + } + caller := &ProcessCaller{LeasePath: path, LeaseIdentity: testLeaseIdentity()} + file, err := caller.acquireLease() + if err != nil { + t.Fatal("initial published config refused", err) + } + file.Close() + raw, _ := json.Marshal(map[string]any{"installation_id": "test-installation", "generation": 2, "provider": "microsandbox", "specification": sandbox.DeploymentSpec{}}) + if err := os.WriteFile(filepath.Join(root, "provider.json"), raw, 0600); err != nil { + t.Fatal(err) + } + if file, err := caller.acquireLease(); err == nil { + file.Close() + t.Fatal("another generation final config accepted") + } +} diff --git a/services/agents-api/internal/sandbox/node/generations.go b/services/agents-api/internal/sandbox/node/generations.go index 6c7ff148b..57b45ba85 100644 --- a/services/agents-api/internal/sandbox/node/generations.go +++ b/services/agents-api/internal/sandbox/node/generations.go @@ -71,7 +71,7 @@ func NewGenerationManager(ctx context.Context, options GenerationManagerOptions) cancel() return nil, sandbox.ErrInvalid } - m.values[ref.Generation] = &localGeneration{value: GenerationProvider{Generation: ref.Generation, SpecificationDigest: ref.SpecificationDigest}, state: "failed", diagnostic: sandbox.NodeRuntimeDownloadFailed, repairing: true} + m.values[ref.Generation] = &localGeneration{value: GenerationProvider{Generation: ref.Generation, SpecificationDigest: ref.SpecificationDigest}, state: "failed", diagnostic: sandbox.NodeProviderUnavailable, repairing: true} } for _, ref := range options.Collect { if !validGeneration(ref.Generation) || !validSpecificationDigest(ref.SpecificationDigest) || m.values[ref.Generation] != nil { @@ -308,7 +308,7 @@ func (m *GenerationManager) prepareLoop() { value.Close() } g.state = "failed" - g.diagnostic = sandbox.NodeRuntimeDownloadFailed + g.diagnostic = sandbox.NodeDiagnostic(err) g.failures++ delays := []time.Duration{time.Minute, 2 * time.Minute, 5 * time.Minute, 10 * time.Minute, 30 * time.Minute} index := g.failures - 1 diff --git a/services/agents-api/internal/sandbox/node/generations_test.go b/services/agents-api/internal/sandbox/node/generations_test.go index fca4ebf53..aeb690d97 100644 --- a/services/agents-api/internal/sandbox/node/generations_test.go +++ b/services/agents-api/internal/sandbox/node/generations_test.go @@ -369,3 +369,25 @@ func TestInterruptedCollectionNeverPreparesOrServesAfterRestart(t *testing.T) { t.Fatal("cleanup did not settle") } } + +func TestPreparationDiagnosticPreservesTypedCause(t *testing.T) { + for _, cause := range []error{sandbox.ErrRuntimeDownloadFailed, sandbox.ErrDockerUnavailable, sandbox.ErrKVMUnavailable, sandbox.ErrOwnership, context.Canceled, errors.New("raw secret provider text")} { + t.Run(sandbox.NodeDiagnostic(cause)+cause.Error(), func(t *testing.T) { + m, err := NewGenerationManager(t.Context(), GenerationManagerOptions{ + Prepare: func(context.Context, uint64, string) (GenerationProvider, error) { return GenerationProvider{}, cause }, + Remove: func(context.Context, GenerationProvider) error { return nil }, + }) + if err != nil { + t.Fatal(err) + } + defer m.Close() + if err := m.Deployment(sandbox.NodeDeployment{Generation: 1, SpecificationDigest: strings.Repeat("a", 64)}); err != nil { + t.Fatal(err) + } + wait(t, func() bool { s := m.Statuses(); return len(s) == 1 && s[0].State == "failed" }) + if got := m.Statuses()[0].Diagnostic; got != sandbox.NodeDiagnostic(cause) { + t.Fatal("wrong fixed diagnostic", got) + } + }) + } +} diff --git a/services/agents-api/internal/store/runtime_node_lifecycle_fixture_test.go b/services/agents-api/internal/store/runtime_node_lifecycle_fixture_test.go index 475ec5907..638d2013d 100644 --- a/services/agents-api/internal/store/runtime_node_lifecycle_fixture_test.go +++ b/services/agents-api/internal/store/runtime_node_lifecycle_fixture_test.go @@ -170,14 +170,39 @@ func (f *nodeIsolationFixture) session(node string, initialize bool) (string, st if initialize { input.InitialFiles = []store.InitialFile{{Type: "inline", Path: "/workspace/seed", Data: []byte("retained")}} } - // Placement is automatic: only node stays provider-ready while it is created. - var others []string - if err := f.pool.QueryRow(f.t.Context(), "WITH changed AS (UPDATE runtime_nodes SET provider_ready=false WHERE provider_ready AND id<>$1 RETURNING id) SELECT coalesce(array_agg(id::text),'{}') FROM changed", node).Scan(&others); err != nil { + // Placement is automatic and generation readiness is current-connection + // authority. Do not merely change the legacy provider_ready projection. + type presence struct { + id, connection string + epoch uint64 + } + rows, err := f.pool.Query(f.t.Context(), "SELECT id::text, connection_id::text, connected_epoch FROM runtime_nodes WHERE provider_ready AND id<>$1 AND connection_id IS NOT NULL AND removed_at IS NULL", node) + if err != nil { + f.t.Fatal(err) + } + var others []presence + for rows.Next() { + var value presence + if err := rows.Scan(&value.id, &value.connection, &value.epoch); err != nil { + rows.Close() + f.t.Fatal(err) + } + others = append(others, value) + } + rows.Close() + if err := rows.Err(); err != nil { f.t.Fatal(err) } + for _, value := range others { + if err := f.store.HeartbeatRuntimeNode(f.t.Context(), value.id, value.connection, value.epoch, store.RuntimeNodeHealth{ProviderReady: false}); err != nil { + f.t.Fatal(err) + } + } session, err := f.store.CreateSession(f.t.Context(), tenant, input) - if _, restoreErr := f.pool.Exec(f.t.Context(), "UPDATE runtime_nodes SET provider_ready=true WHERE id::text=ANY($1)", others); restoreErr != nil { - f.t.Fatal(restoreErr) + for _, value := range others { + if err := f.store.HeartbeatRuntimeNode(context.WithoutCancel(f.t.Context()), value.id, value.connection, value.epoch, store.RuntimeNodeHealth{ProviderReady: true}); err != nil { + f.t.Fatal(err) + } } if err != nil { f.t.Fatal(err) @@ -186,6 +211,10 @@ func (f *nodeIsolationFixture) session(node string, initialize bool) (string, st if err != nil { f.t.Fatal(err) } + var placed string + if err := f.pool.QueryRow(f.t.Context(), "SELECT node_id::text FROM runtime_placements WHERE environment_id=$1 AND released_at IS NULL", env.ID).Scan(&placed); err != nil || placed != node { + f.t.Fatal("fixture did not place on its authenticated ready node", placed, node, err) + } return tenant, session, env } func (f *nodeIsolationFixture) provision(tenant string, env store.Environment) store.RuntimeAllocation { diff --git a/services/agents-api/internal/store/runtime_node_status_test.go b/services/agents-api/internal/store/runtime_node_status_test.go index edc2bbcd1..abbf3c372 100644 --- a/services/agents-api/internal/store/runtime_node_status_test.go +++ b/services/agents-api/internal/store/runtime_node_status_test.go @@ -24,11 +24,16 @@ func TestRuntimeNodeStatusUsesAuthenticatedFreshPresence(t *testing.T) { if _, err := s.RuntimeNodeStatus(t.Context(), d.LocalNodeID, "different-credential"); !errors.Is(err, ErrRuntimeNodeCredential) { t.Fatal("status admitted another credential", err) } - if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET provider_ready=false WHERE id=$1", d.LocalNodeID); err != nil { + connection := onlineManagerNode(t, s, d.LocalNodeID) + if err := s.HeartbeatRuntimeNode(t.Context(), d.LocalNodeID, connection, managerEpoch(t, s), RuntimeNodeHealth{ProviderReady: false}); err != nil { t.Fatal(err) } status(true, false) - if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET provider_ready=true,last_seen_at=clock_timestamp()-interval '46 seconds' WHERE id=$1", d.LocalNodeID); err != nil { + if err := s.HeartbeatRuntimeNode(t.Context(), d.LocalNodeID, connection, managerEpoch(t, s), RuntimeNodeHealth{ProviderReady: true}); err != nil { + t.Fatal(err) + } + status(true, true) + if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET last_seen_at=clock_timestamp()-interval '46 seconds' WHERE id=$1", d.LocalNodeID); err != nil { t.Fatal(err) } status(false, false) diff --git a/services/agents-api/internal/store/runtime_nodes_test.go b/services/agents-api/internal/store/runtime_nodes_test.go index bf1ef1f22..9e943435e 100644 --- a/services/agents-api/internal/store/runtime_nodes_test.go +++ b/services/agents-api/internal/store/runtime_nodes_test.go @@ -47,14 +47,40 @@ func managerSessionInput(key string) CreateSessionInput { // stays provider-ready while the Session is created. func createSessionOnNode(t *testing.T, s *Store, tenant string, input CreateSessionInput, node string) (Session, error) { t.Helper() - var others []string - if err := s.pool.QueryRow(t.Context(), "WITH changed AS (UPDATE runtime_nodes SET provider_ready=false WHERE provider_ready AND id<>$1 RETURNING id) SELECT coalesce(array_agg(id::text),'{}') FROM changed", node).Scan(&others); err != nil { + // Use the same authenticated readiness observations as the scheduler. The + // compatibility provider_ready column alone is not admission authority. + type presence struct { + id, connection string + epoch uint64 + } + rows, err := s.pool.Query(t.Context(), "SELECT id::text, connection_id::text, connected_epoch FROM runtime_nodes WHERE provider_ready AND id<>$1 AND connection_id IS NOT NULL AND removed_at IS NULL", node) + if err != nil { t.Fatal(err) } - defer func() { - if _, err := s.pool.Exec(context.WithoutCancel(t.Context()), "UPDATE runtime_nodes SET provider_ready=true WHERE id::text=ANY($1)", others); err != nil { + var others []presence + for rows.Next() { + var value presence + if err := rows.Scan(&value.id, &value.connection, &value.epoch); err != nil { + rows.Close() t.Fatal(err) } + others = append(others, value) + } + rows.Close() + if err := rows.Err(); err != nil { + t.Fatal(err) + } + for _, value := range others { + if err := s.HeartbeatRuntimeNode(t.Context(), value.id, value.connection, value.epoch, RuntimeNodeHealth{ProviderReady: false}); err != nil { + t.Fatal(err) + } + } + defer func() { + for _, value := range others { + if err := s.HeartbeatRuntimeNode(context.WithoutCancel(t.Context()), value.id, value.connection, value.epoch, RuntimeNodeHealth{ProviderReady: true}); err != nil { + t.Fatal(err) + } + } }() return s.CreateSession(t.Context(), tenant, input) } diff --git a/services/agents-api/internal/store/sandbox_generations_test.go b/services/agents-api/internal/store/sandbox_generations_test.go index c50c1ed03..8f78c8205 100644 --- a/services/agents-api/internal/store/sandbox_generations_test.go +++ b/services/agents-api/internal/store/sandbox_generations_test.go @@ -2,7 +2,6 @@ package store import ( "database/sql" - "encoding/json" "errors" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/db/sqlc" "github.com/jackc/pgx/v5/pgtype" @@ -314,21 +313,20 @@ func TestGenerationUpdateSerializesWithAllocationAdmission(t *testing.T) { func TestNodeRolloutSeparatesOfflinePinAndTargetReadiness(t *testing.T) { for _, tc := range []struct { - name string - online, ready bool - enrolled, pin, target int64 - diagnostic, state string + name string + online, ready bool + enrolled, pin, target int64 + targetState, diagnostic, state string }{ - {"offline pin", false, true, 1, 1, 2, "", "unknown"}, - {"old serving", true, true, 1, 1, 2, "", "update_required"}, - {"current serving", true, true, 2, 2, 2, "", "ready"}, - {"current failed", true, false, 2, 2, 2, "kvm_unavailable", "failed"}, - {"current unknown", true, false, 2, 2, 2, "", "unknown"}, + {"offline pin", false, true, 1, 1, 2, "ready", "", "unknown"}, + {"old serving", true, true, 1, 1, 2, "ready", "", "update_required"}, + {"current serving", true, true, 2, 2, 2, "ready", "", "ready"}, + {"current failed", true, false, 2, 2, 2, "failed", "kvm_unavailable", "failed"}, + {"current unknown", true, false, 2, 2, 2, "", "", "unknown"}, } { t.Run(tc.name, func(t *testing.T) { - raw, _ := json.Marshal(RuntimeNodeHealth{Diagnostic: tc.diagnostic}) - got := nodeRollout(sqlc.ListRuntimeNodesRow{Online: tc.online, ProviderReady: tc.ready, DeploymentGeneration: tc.enrolled, ReadyGeneration: pgtype.Int8{Int64: tc.pin, Valid: true}, TargetGeneration: tc.target, Health: raw}) - if got.State != tc.state || got.ReadyGeneration == nil || *got.ReadyGeneration != uint64(tc.pin) { + got := nodeRollout(sqlc.ListRuntimeNodesRow{Online: tc.online, ProviderReady: tc.ready, DeploymentGeneration: tc.enrolled, ReadyGeneration: pgtype.Int8{Int64: tc.pin, Valid: true}, TargetGeneration: tc.target, ProtocolVersion: 1, TargetState: tc.targetState, TargetDiagnostic: tc.diagnostic}) + if got.State != tc.state || got.Diagnostic != tc.diagnostic || got.ReadyGeneration == nil || *got.ReadyGeneration != uint64(tc.pin) { t.Fatal(got) } }) From 58e3a0057fab942fcd7e977a49f8599b235976db Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Mon, 28 Sep 2026 04:25:12 +0000 Subject: [PATCH 15/16] Keep typed artifact exits private to generation operations --- deploy/install/node_install.py | 9 +++++---- deploy/install/test_generation_review_regressions.py | 7 +++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/deploy/install/node_install.py b/deploy/install/node_install.py index 83d54e081..0e782010b 100644 --- a/deploy/install/node_install.py +++ b/deploy/install/node_install.py @@ -1468,7 +1468,11 @@ def main(argv=None): if args.generation_action: if args.generation is None or not 1 <= args.generation <= 9223372036854775807 or not re.fullmatch(r"[0-9a-f]{64}", args.specification_digest or ""): parser.error("Invalid generation authorization") - (node_generations.prepare if args.generation_action == "prepare" else node_generations.collect)(args, sys.modules[__name__]) + try: + (node_generations.prepare if args.generation_action == "prepare" else node_generations.collect)(args, sys.modules[__name__]) + except (RuntimeDownloadError, distribution.ArtifactError): + print("Runtime artifact transfer or verification failed", file=sys.stderr) + raise SystemExit(65) from None return if args.uninstall: if args.source_url or args.bundle or args.core_url or args.provider or args.enrollment_token_stdin: @@ -1496,9 +1500,6 @@ def main(argv=None): if __name__ == "__main__": try: main() - except (RuntimeDownloadError, distribution.ArtifactError): - print("Runtime artifact transfer or verification failed", file=sys.stderr) - sys.exit(65) except ChildFailed as failure: if str(failure): print(str(failure), file=sys.stderr) diff --git a/deploy/install/test_generation_review_regressions.py b/deploy/install/test_generation_review_regressions.py index dd2a8a2fd..5740e765c 100644 --- a/deploy/install/test_generation_review_regressions.py +++ b/deploy/install/test_generation_review_regressions.py @@ -261,3 +261,10 @@ def available(value,installer): final=installer.private_json(case.root/'state/node/generations/2.json') self.assertNotEqual(final['microsandbox']['helper_path'],original['microsandbox']['helper_path']) self.assertIn(1,node_generations.retained_configs(case.root,installer)) + + def test_operator_update_keeps_actionable_errors_and_original_exit(self): + for kind in ("RuntimeDownloadError", "distribution.ArtifactError"): + script = "import sys,runpy,node_update; sys.argv=['node_install.py','--installation-id','94be54a1-138c-4f30-bc87-b13686272dbe','--update','--source-url','https://core.invalid','--core-url','https://core.invalid'];\ndef fail(args,installer): raise installer." + kind + "('Check the matched release and network, then rerun')\nnode_update.update=fail; runpy.run_module('node_install',run_name='__main__',alter_sys=True)" + result = subprocess.run([sys.executable, '-c', script], capture_output=True, text=True, timeout=10, cwd=Path(__file__).parent) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn('Check the matched release and network, then rerun', result.stderr) From 6e70d1a16231f5497e8f0da7bc45e54935072b2f Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Mon, 28 Sep 2026 04:27:44 +0000 Subject: [PATCH 16/16] Regenerate node lifecycle operator guide --- apps/docs/content/docs/hosted-providers.mdx | 36 +++++++++++++++++---- apps/docs/content/guide-sources.json | 4 +-- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/apps/docs/content/docs/hosted-providers.mdx b/apps/docs/content/docs/hosted-providers.mdx index 78f24ef83..77f112757 100644 --- a/apps/docs/content/docs/hosted-providers.mdx +++ b/apps/docs/content/docs/hosted-providers.mdx @@ -266,14 +266,36 @@ WebSocket doesn't pass the reverse proxy. ## Change the sandbox backend or size -Changing backend type requires an explicit reset. Docker and microsandbox size or -Runtime edits currently require zero held resources; node online updates are not -available yet. These changes retire every node. Retired nodes drop out of the list, so there is nothing to -remove in Web: on each host, run the [uninstall command](#remove-a-node) (Core no longer -accepts the node, so it needs no `--force`), then add the host again with a new command. +Changing backend type requires an explicit reset. Same-provider Docker and +microsandbox size or Runtime edits advance the target generation without retiring +nodes or moving existing Sessions. Version 2 nodes prepare the target independently; +existing allocations and suspended VMs keep their original generation. A qualified +older serving generation can still accept new Sessions when it has capacity, +including while a newer target is preparing or has failed. + +Version 1 nodes report `update_required` for a newer target while retaining their +qualified original serving generation. Run the verified current installer with +`--update`, the same installation ID, Core/source URLs and original installation +user (or sudo mode); do not provide another enrollment token. The update preserves +node identity, retained Runtime files and running VMs. Its original v1 generation +is conservatively retained locally because old helpers were not protected by the +new generation lock. This retained disk usage is not reported as completed GC. + +Fresh v2 preparation keeps a private recovery plan until the actual provider image +identity is resolved and its final configuration is published. Restart does not +serve a pending or partly collected generation. Generation lock identity is durable: +if a lease or its identity record is missing or replaced, preserve the installation +for inspection. Do not delete records, recreate lock files or re-enroll to bypass +that refusal; a process restart does not prove that older helpers have stopped. + +Automatic Docker generation GC removes only installation-owned generation/release +files. Shared daemon images remain; a host administrator may remove them only after +confirming no installation still needs them. Microsandbox image cleanup remains +scoped to the installation's private native store and verified ownership receipts. + The [operator reference](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/services/agents-api/HOSTED-SANDBOX-MANAGER.md#removal-and-reset) -describes today's procedure and API. Same-team E2B edits have a separate online -Core API path and do not involve node installation. +describes reset and API behavior. Same-team E2B edits have a separate online Core +API path and do not involve node installation. ## Remove a node added before the rename diff --git a/apps/docs/content/guide-sources.json b/apps/docs/content/guide-sources.json index 645e517bf..3e6366f09 100644 --- a/apps/docs/content/guide-sources.json +++ b/apps/docs/content/guide-sources.json @@ -12,7 +12,7 @@ "contracts/agents-api/execution-tools.md": "e0b61f6c0c236c362186c5f6d0ad69a16dd8a8d9afabe1329a1fe22e5f4a71e9", "docs/api/public-agent-api.md": "18829b366bd2e49fe5beb76dc8a709bc1aa8e35d4fbfd2484a0c52b46cf75417", "contracts/agents-api/environments.md": "89ffc903ee44d00ee87c16950202a3ca4c8345160780dcdd4ed2a71b25b1c2dc", - "docs/getting-started/nodes.md": "71c422ba23b274b038a841dfdc960649c29a4409454f49f8916044fa88041458", + "docs/getting-started/nodes.md": "fd858830647e16554f7fef1381a0329e1b020f37fb7e826204e3ac5816e050c7", "docs/getting-started/self-hosted.md": "3f1b402cf28bb320ba414eaf36110e9df4d792b81da0456a3c42d469357d8d99", "apps/web/public/onboarding/monitor-en.webp": "29dc220cb1250c7016b7c4bf7f30e9510b07c07b814a48c3aa9303b2d76f20b2", "docs/web/README.md": "9fce295f6f6ff04acefe338679c1e54723e123dd0027a778202adfa9ab95b6c4", @@ -44,7 +44,7 @@ "content/docs/sessions.zh.mdx": "b1b1a7f8aacd9a902320f9692173ede24036124f85761cf02f23ef3ef2bcc4c1", "content/docs/environments-and-files.mdx": "0fce7c8806d532d64eff6319bba7d0a8e44da46f888d224644b3ab5c23428c41", "content/docs/environments-and-files.zh.mdx": "ee64577220a798f023b330c65302c14438f25a34a78c4e36144bfdb4304a3ae5", - "content/docs/hosted-providers.mdx": "02a39c9fd07c7adfc065ca9510136362503c791334d8b084b7b9dc0958eec81b", + "content/docs/hosted-providers.mdx": "3a0560420c7564acb273c4accf278cee466ded577ab6b98909cd5f1f9127380e", "content/docs/hosted-providers.zh.mdx": "f54d80d6e65596dca6b20262abaa3dcb5af245227ffad146fcaed4bd69454dee", "content/docs/self-hosted-execution.mdx": "34877624962b24c22e8371ebd2e2f83bbf5a0d004c74d5d9b5db57f5133b692c", "content/docs/self-hosted-execution.zh.mdx": "f11d34ce30d48039376256474bc9509a508a2f6b9906f49b5cf6e396707ae5c0",