Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion docs/durable-checkpoints.md
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,17 @@ The existing loopback API and CSRF boundary expose these operations:
- `POST /api/runs/{run_id}/checkpoints/{checkpoint_id}/restore` accepts a new
`command_id` and `expected_revision`.

Readback preserves the original `acknowledged_resumable` value as history. Its
current `resumable` observation also compares the admitted authority, run and
cumulative budget, issue owner, independent Git owner and candidate branch,
original resource manifest, and the current writer witness. If the checkout is
still present, it compares its retained source and index under that witness.
Existing evidence and session files must match their sealed state; missing files
and an absent checkout can still be restored from the sealed artifacts. Changed or
unavailable proof appears in `resume_blockers`. This is a read-only observation,
not a reservation or permission to resume: restoration must reacquire custody
and repeat its own admission checks.

Creation requires a stopped run, reconciled external effects, closed original
launch and role leases, proven registered workspace ownership, and a fresh check
for open workspace actors. Registered native journals currently cause a typed
Expand Down Expand Up @@ -210,7 +221,10 @@ publication only when current fenced source/state reproduces its exact manifest
and every artifact. Corrupt, incomplete, foreign or inconsistent bytes still refuse.
Resource pressure or an interrupted write cannot produce a
successful checkpoint receipt. Checkpoint directories live outside disposable
worktrees and have no automatic expiry or cleanup path.
worktrees and have no automatic expiry or cleanup path. Explicit retained-resource
pruning preserves checkpoint directories and durable session/evidence material;
a blocked candidate remains retained. No checkpoint expiration operation is
currently provided.

Restoration validates every artifact and current execution authority, budget,
owner, original resource generation and native closure again. It reconstructs a
Expand Down
88 changes: 83 additions & 5 deletions runtime/src/devflow_temporal/delivery_checkpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -875,7 +875,11 @@ def references(value):
if not home.is_relative_to(state):
raise ValueError("checkpoint session home escaped its original run")
_openers(home)
paths = session_state_paths(home, attempt["session_id"])
# The fixed fake provider has a synthetic session ID but writes no
# transcript. Preserve fixture transcripts when present; native
# providers must always supply their required session.
paths = session_state_paths(
home, attempt["session_id"], allow_absent=spec["provider"] == "fake")
selected_manifest(home, paths) # Private owned regular files, no linked ancestors.
selected.update((home / item).relative_to(state).as_posix() for item in paths)
elif attempt["session_id"] and spec["provider"] != "fake":
Expand Down Expand Up @@ -1423,11 +1427,41 @@ def read_checkpoint(store, run_id, checkpoint_id):
spec = store.effective_spec(run_id)
with store._connect() as db:
value, receipt = _read_acknowledged(db, spec, checkpoint_id)
current_effects = _frozen(db, _row(db, run_id))["delivery_effects"]
# Keep the tracking snapshot coherent without holding a SQL read
# transaction while the potentially large artifact inventory is hashed.
db.execute("BEGIN")
row = _row(db, run_id)
current = _frozen(db, row)
claim = store.state.claim_for(db, row["work_id"])
owner = dict(claim) if claim else None
db.commit()
# Readback retains the original ACK, but does not advertise a native
# checkpoint as resumable under today's admission contract. Current custody
# can have changed since the ACK; inability to inspect it is also unknown.
blockers = list(value.get("resume_blockers", []))
if digest(spec) != value["spec_sha256"]:
blockers.append({"scope": "authority", "reason": "checkpoint_authority_changed"})
if not store.owns_execution(spec):
blockers.append({"scope": "authority", "reason": "checkpoint_transport_ownership_unknown"})
if not _same_snapshot(current, value):
blockers.append({"scope": "run", "reason": "checkpoint_run_state_changed"})
if owner != value["owner"]:
blockers.append({"scope": "owner", "reason": "checkpoint_execution_owner_changed"})
try:
from .delivery_workspace_custody import source_for

source = source_for(spec)
except (OSError, ValueError, subprocess.SubprocessError):
blockers.append({"scope": "source", "reason": "checkpoint_workspace_owner_unknown"})
else:
try:
head = _git(source, "rev-parse", "refs/heads/" + value["source"]["branch"])
except (OSError, ValueError, subprocess.SubprocessError):
blockers.append({"scope": "source", "reason": "checkpoint_candidate_branch_unknown"})
else:
if head.decode().strip() != value["source"]["candidate"]["head"]:
blockers.append({"scope": "source",
"reason": "checkpoint_candidate_branch_changed"})
effect_blockers = []
feature_unknown = None
current_feature = None
Expand All @@ -1446,7 +1480,7 @@ def read_checkpoint(store, run_id, checkpoint_id):
feature_unknown = "registry_observation_unknown"
for scope, effects in (
("delivery", value["frozen"]["delivery_effects"]),
("delivery", current_effects),
("delivery", current["delivery_effects"]),
("feature", (value.get("execution_ledger") or {}).get("execution_effects", [])),
("feature", (current_feature or {}).get("execution_effects", [])),
):
Expand All @@ -1459,14 +1493,59 @@ def read_checkpoint(store, run_id, checkpoint_id):
effect_blockers.append(blocker)
if blocker not in blockers:
blockers.append(blocker)
retained_source_unknown = False
retained_state_unknown = False
try:
registry = RunResources(spec, read_only=True)
with registry.locked(read_only=True) as resources:
custody_unknown = bool(value.get("closed_journals") or resources["processes"])
entry = resources["roots"].get(spec["checkout"])
resource_changed = (
not entry
or entry.get("kind") != "checkout"
or entry.get("identity") != value["checkout_identity"]
or _hash(read_private(registry.manifest, raw=True))
!= value["resource_manifest_sha256"]
)
try:
# Follow creation/restoration's resource-before-writer lock order.
# The exclusive witness is observational and grants no later work.
with store._checkpoint_writer_context(spec):
writer_unknown = False
try:
from .delivery_retained_checkpoint import verify_retained_state

verify_retained_state(
spec, value, _root(spec) / checkpoint_id, allow_missing=True)
except (OSError, ValueError, KeyError):
retained_state_unknown = True
checkout = Path(spec["checkout"])
if os.path.lexists(checkout) and _index_baseline_supported(value):
try:
from .delivery_retained_checkpoint import verify_retained_source

_openers(checkout)
verify_retained_source(
spec, value, _root(spec) / checkpoint_id, resources)
except (OSError, ValueError, KeyError, sqlite3.Error,
subprocess.SubprocessError):
retained_source_unknown = True
except (OSError, ValueError, sqlite3.Error, subprocess.SubprocessError):
writer_unknown = True
except (OSError, ValueError, KeyError):
custody_unknown = True
resource_changed = True
writer_unknown = True
if custody_unknown:
blockers.append({"scope": "native", "reason": NativeCheckpointCustodyUnknown.code})
if resource_changed:
blockers.append({"scope": "resource", "reason": "checkpoint_resource_custody_unknown"})
if writer_unknown:
blockers.append({"scope": "writer", "reason": "checkpoint_writer_custody_unknown"})
if retained_source_unknown:
blockers.append({"scope": "source", "reason": "checkpoint_retained_source_unverified"})
if retained_state_unknown:
blockers.append({"scope": "state", "reason": "checkpoint_retained_state_unverified"})
if feature_unknown:
blockers.append({"scope": "feature", "reason": feature_unknown})
index_unknown = not _index_baseline_supported(value)
Expand All @@ -1475,8 +1554,7 @@ def read_checkpoint(store, run_id, checkpoint_id):
return {
**receipt,
"acknowledged_resumable": receipt["resumable"],
"resumable": (receipt["resumable"] and not custody_unknown
and not effect_blockers and not feature_unknown and not index_unknown),
"resumable": receipt["resumable"] and not blockers,
"resume_blockers": blockers,
"authority_sha256": value["spec_sha256"],
"owner": value["owner"],
Expand Down
10 changes: 8 additions & 2 deletions runtime/src/devflow_temporal/delivery_continuation.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,14 +67,18 @@ def selected_digest(root: Path, paths: list[str]) -> str:
)


def session_state_paths(role_home: Path, session_id: str) -> list[str]:
"""Select only provider session data; auth, profile, caches and tools are regenerated."""
def session_state_paths(
role_home: Path, session_id: str, *, allow_absent: bool = False,
) -> list[str]:
"""Select session data; allow absence only for fixed providers without transcripts."""

home_info = role_home.lstat()
if not stat.S_ISDIR(home_info.st_mode) or home_info.st_uid != os.getuid():
raise ValueError("continuation role home is not an owned directory")
codex = role_home / "codex"
if not codex.is_dir() or codex.is_symlink():
if allow_absent and not os.path.lexists(codex):
return []
raise ValueError("continuation Codex home is unavailable")
paths = [
"codex/" + file.name
Expand All @@ -89,6 +93,8 @@ def session_state_paths(role_home: Path, session_id: str) -> list[str]:
if session_id in file.name and (file.is_file() or file.is_symlink())
)
if not any(session_id in Path(path).name for path in paths):
if allow_absent:
return []
raise ValueError("continuation session transcript is missing")
return sorted(paths)

Expand Down
94 changes: 81 additions & 13 deletions runtime/src/devflow_temporal/delivery_retained_checkpoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -287,19 +287,87 @@ def _verify_recorded_source(spec, value, checkpoint_root, resources):
"branch": branch, "head": head, "index_tree": value["source"]["index_tree"]}


def verify_retained_state(spec, value, checkpoint_root):
"""Verify every declared evidence/session byte and authenticated absence."""
def _retained_state_bytes(target, expected, size):
"""Return validated bytes, or a coherently observed initial absence."""
flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_NONBLOCK
with ExitStack() as stack:
parent = os.open(target.anchor, flags)
stack.callback(os.close, parent)
directories = []

def parents_unchanged():
keys = ("st_dev", "st_ino", "st_mode", "st_uid")
try:
for owner, name, descriptor, initial in directories:
current = os.fstat(descriptor)
named = os.stat(name, dir_fd=owner, follow_symlinks=False)
if (any(getattr(initial, key) != getattr(current, key) for key in keys)
or any(getattr(current, key) != getattr(named, key) for key in keys)):
raise ValueError("retained checkpoint state parent changed")
except FileNotFoundError as exc:
raise ValueError("retained checkpoint state parent became unavailable") from exc

def open_entry(name, entry_flags):
try:
return os.open(name, entry_flags, dir_fd=parent)
except FileNotFoundError:
parents_unchanged()
try:
os.stat(name, dir_fd=parent, follow_symlinks=False)
except FileNotFoundError:
parents_unchanged()
return None
raise ValueError("retained checkpoint state entry changed") from None

for name in target.parts[1:-1]:
opened = open_entry(name, flags)
if opened is None:
return None
stack.callback(os.close, opened)
directories.append((parent, name, opened, os.fstat(opened)))
parent = opened
if expected is not None and os.fstat(parent).st_uid != os.getuid():
raise ValueError("retained checkpoint destination parent is not owned")
opened = open_entry(target.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
if opened is None:
return None
stack.callback(os.close, opened)
before = os.fstat(opened)
if expected is None:
raise ValueError("retained checkpoint no-GO absence changed")
if (not stat.S_ISREG(before.st_mode) or before.st_uid != os.getuid()
or before.st_nlink != 1 or before.st_size != size
or stat.S_IMODE(before.st_mode) != expected["mode"]
or before.st_mtime_ns != expected["mtime_ns"]):
raise ValueError("retained checkpoint evidence metadata changed")
with os.fdopen(os.dup(opened), "rb") as stream:
raw = stream.read(size + 1)
keys = ("st_dev", "st_ino", "st_mode", "st_uid", "st_nlink", "st_size",
"st_mtime_ns", "st_ctime_ns")
after = os.fstat(opened)
try:
named = os.stat(target.name, dir_fd=parent, follow_symlinks=False)
except FileNotFoundError as exc:
raise ValueError("retained checkpoint evidence became unavailable") from exc
if (len(raw) != size
or any(getattr(before, key) != getattr(after, key) for key in keys)
or any(getattr(after, key) != getattr(named, key) for key in keys)):
raise ValueError("retained checkpoint evidence changed during observation")
parents_unchanged()
return raw


def verify_retained_state(spec, value, checkpoint_root, *, allow_missing=False):
"""Verify state targets; missing files can be restored when explicitly allowed."""
for relative, item in value["state_refs"].items():
target = _path(Path(spec["state_dir"]), relative)
if item.get("kind") == "absent":
if os.path.lexists(target):
raise ValueError("retained checkpoint no-GO absence changed: " + relative)
continue
info = target.lstat()
if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_nlink != 1:
raise ValueError("retained checkpoint evidence is not an owned file: " + relative)
raw = target.read_bytes()
if (raw != read_private(checkpoint_root / item["artifact"], raw=True)
or _hash(raw) != item["sha256"] or stat.S_IMODE(info.st_mode) != item["mode"]
or info.st_mtime_ns != item["mtime_ns"]):
expected = (None if item.get("kind") == "absent" else
read_private(checkpoint_root / item["artifact"], raw=True))
raw = _retained_state_bytes(target, item if expected is not None else None,
len(expected) if expected is not None else 0)
if raw is None:
if item.get("kind") == "absent" or allow_missing:
continue
raise FileNotFoundError("retained checkpoint evidence is absent: " + relative)
if raw != expected or _hash(raw) != item["sha256"]:
raise ValueError("retained checkpoint original evidence changed: " + relative)
5 changes: 5 additions & 0 deletions runtime/src/devflow_temporal/supervisor.py
Original file line number Diff line number Diff line change
Expand Up @@ -550,6 +550,11 @@ async def _run(self, request: dict[str, Any]) -> dict[str, Any]:
from .delivery_resources import read_private, write_private

output_candidate = await writer_thread(spec, self._completed_candidate, request, result)
if spec.get("provider") == "fake":
# This fixed fixture role cannot launch candidate-controlled children.
# Bind the supervisor's joined-process observation to its result.
result.setdefault("cleanup", "confirmed")
write_private(result_path, result)
metadata = read_private(start_path)
metadata.update(output_candidate=output_candidate, result_digest=digest(result))
write_private(start_path, metadata)
Expand Down
Loading
Loading