Add safe encrypted world checkpoint and offline restore tooling

SELF_HOSTING.md

@@ -83,15 +83,20 @@ World saves are stored at:
 
 ### Back up your worlds
 
-```bash
-tar czf tinyblock-backup-$(date +%Y%m%d).tar.gz ~/.local/share/godot/app_userdata/Tiny\ Block\ Server/
-```
+Do not archive the entire Godot application-data directory while the server is
+running: it can contain unrelated state, and a plain tar is neither an atomic
+checkpoint nor encrypted. For the four production community worlds, use the
+pinned-file-descriptor capture, age encryption, and offline restore check in
+[`docs/operations/world-backups.md`](docs/operations/world-backups.md). For a
+standalone self-hosted server, stop it first and copy only the selected
+`world_<id>.json` save; protect the archive and test a restore before relying on
+it.
 
 ### Restore a world
 
-1. Stop the server.
-2. Copy the saved world file matching the `--world` ID into the data directory.
-3. Restart the server.
+1. Stop the server before a manual restore.
+2. Verify the backup and copy only the saved world file matching the `--world` ID into the data directory.
+3. Confirm the configured `--world-mode` matches the save, then restart the server.
 
 ## Update
 

docs/operations/world-backups.md

@@ -0,0 +1,126 @@
+# Tiny Block community world backups
+
+## Scope
+
+The dedicated Bare Metal host has four fresh public community worlds. Their
+primary save files are independent JSON documents:
+
+| World | ID | Mode | Host save path below `/opt/tinyblock/fresh/` |
+|---|---|---|---|
+| Pandora | `world_pandora` | `floating_islands` | `home-fresh-floating-islands/.local/share/godot/app_userdata/Tiny Block Server/worlds/world_pandora.json` |
+| B-612 | `world_b612` | `one_block` | `home-fresh-one-block/.local/share/godot/app_userdata/Tiny Block Server/worlds/world_b612.json` |
+| Wonderland | `world_wonderland` | `procedural` | `home-fresh-procedural/.local/share/godot/app_userdata/Tiny Block Server/worlds/world_wonderland.json` |
+| Skyloft | `world_skyloft` | `skyblock` | `home-fresh-skyblock/.local/share/godot/app_userdata/Tiny Block Server/worlds/world_skyloft.json` |
+
+This contract covers only those four primary world saves. It intentionally does
+not include the Tiny Block bot's achievement/structural state, Godot session
+files, tokens, or any product database. Bot files can be rewritten in place and
+need their own backup contract. Never include `backend_session.json` in a world
+archive.
+
+## Consistency and freshness
+
+`WorldStore.save_world` and `save_repaired_world` serialize to `<world>.json.tmp`,
+flush and close that file, then publish a new primary by removing and renaming
+files. Normal saves also rotate `.bak` by deleting the previous backup and
+copying the primary. The backup copy is not a safe source during that rotation.
+There are no other in-repository writers to these world primary paths.
+
+The capture tool opens each exact primary using `O_RDONLY|O_NOFOLLOW`, pins the
+opened inode, verifies it is a regular file, reads from that descriptor, checks
+the descriptor metadata again, and validates JSON schema version, exact ID, and
+mode before emitting any archive bytes. Linux keeps the opened inode readable
+after the server unlinks/replaces the pathname. If the brief remove-to-rename
+gap makes open return `ENOENT`, the tool retries a bounded number of times; it
+never reads `.tmp` or silently substitutes `.bak`. Symlinks, non-regular files,
+duplicate JSON keys, corrupt JSON, changed-in-place descriptor size/mtime, and
+mismatched world metadata fail the whole capture. `ctime` is recorded but is
+not compared because the producer's unlink-before-rename changes the old
+inode's ctime even though its already-open contents remain stable.
+
+Each world is a separate checkpoint taken at its own open time. The four saves
+are **not** one globally atomic snapshot, and the manifest records per-world
+capture start/completion times and SHA-256. During normal operation, the game
+server autosaves dirty worlds with a configured maximum interval of 60 seconds
+for ordinary worlds and 120 seconds for worlds at or above 50,000 tiles (with
+an idle delay as well). Therefore a capture represents the most recently
+published primary observed for each world; it can omit recent unsaved play,
+normally bounded by that autosave maximum. This is an operational expectation,
+not a hard durability SLA: the current Godot save path flushes the temp file but
+does not explicitly fsync the file and containing directory.
+
+## Capture and encryption
+
+Run `tools/tinyblock_world_backup.py capture` on the host that owns the save
+paths, and stream its tar output directly through the existing `age` recipient
+to a private local artifact. Do not create a plaintext tar on disk or pipe its
+contents to a terminal. Example from a Mac that has the already-authorized SSH
+alias and protected age recipient file:
+
+```sh
+set -o pipefail
+artifact_dir="$HOME/.config/hardtab/recovery/tinyblock-worlds"
+mkdir -p "$artifact_dir"
+chmod 700 "$artifact_dir"
+partial="$artifact_dir/tinyblock-worlds-$(date -u +%Y%m%dT%H%M%SZ).tar.age.part"
+final="${partial%.part}"
+if ssh production-baremetal 'python3 - capture' \
+    < tools/tinyblock_world_backup.py \
+    | age -R "$HOME/.config/hardtab/recovery/sc-sp-20261010/recipient.txt" -o "$partial"; then
+  chmod 600 "$partial"
+  mv "$partial" "$final"
+else
+  rm -f "$partial"
+  exit 1
+fi
+```
+
+The script prints only a file-count summary to stderr. The stdout tar stream is
+encrypted by `age`; the archive contains four primary files and a manifest with
+IDs, modes, file metadata, independent capture timestamps, and hashes. Protect
+the age identity as a private credential; never place it in the repository or
+the archive. The protected Mac recovery folder is off-host relative to Bare
+Metal, so this copy survives loss of the game host. A second cloud copy through
+the existing approved Yandex recovery workflow, plus a recurring schedule and
+retention policy, are still pending. That workflow's credentials are available
+only to GitHub Actions; do not copy them to a Mac or create a new bucket/key.
+
+For a durable verification record, run `verify` after extracting and restoring
+the capture. Pass `--artifact` with the encrypted `.age` file; the private JSON
+receipt then records the capture ID and time window, each world's source time
+and hash, the offline restore result, and the encrypted artifact's filename,
+size, and SHA-256. Keep the receipt beside the ciphertext; it contains metadata
+only, not world contents or keys.
+
+## Offline restore verification
+
+Do not restore over a running world. Decrypt only into a new mode-0700 scratch
+directory, validate the archive allowlist and hashes, then run the dedicated
+Godot headless verifier against those copied JSON files. The verifier calls the
+real `WorldSim.deserialize_state` for all four worlds, checks the restored ID,
+mode, non-empty tile set, and SHA-256 of the exact input bytes; the Python
+verifier rejects a Godot receipt whose hashes do not match the archive
+manifest. It does not instantiate `ServerMain`, start multiplayer, or call the
+backend client. Use a fresh isolated `HOME`/Godot user data directory. Keep the
+resulting verification receipt beside the encrypted artifact; delete the
+plaintext extraction scratch after validation.
+
+Restore evidence proves the saved documents can be deserialized by this exact
+Tiny Block source revision in isolation. It does not prove live community
+joinability or backend health. Before a real restore/cutover, validate backup
+age, source/target IDs and modes, hashes, and a complete restore test, then use
+the normal reviewed production release/runbook to switch a world.
+
+## Local tests
+
+```sh
+python3 -m unittest discover -s tools/tests -v
+godot --headless --path . --scene res://tests/server_test_runner.tscn
+```
+
+Capture-contract tests cover inode pinning across replacement, bounded retry in
+the unlink/rename gap, rejection of symlinks and temp/backup fallbacks, schema/
+ID/mode validation, and the exact archive allowlist. Synthetic fixtures test
+the capture implementation only; they are not evidence that the current four
+world saves restore. The separate offline Godot receipt must be generated from
+the actual decrypted production capture for that claim.

tests/world_backup_restore_verifier.gd

@@ -0,0 +1,105 @@
+extends Node
+
+const WorldSim = preload("res://gameplay/scripts/world.gd")
+const EXPECTED_WORLDS := {
+	"world_pandora": "floating_islands",
+	"world_b612": "one_block",
+	"world_wonderland": "procedural",
+	"world_skyloft": "skyblock",
+}
+
+
+func _ready() -> void:
+	_run_verification()
+
+
+func _run_verification() -> void:
+	var arguments := OS.get_cmdline_user_args()
+	var world_directory := _argument_value(arguments, "--world-dir")
+	var receipt_path := _argument_value(arguments, "--receipt")
+	if world_directory.is_empty() or receipt_path.is_empty():
+		push_error("Usage: -- --world-dir=<private scratch worlds directory> --receipt=<private scratch receipt path>")
+		get_tree().quit(2)
+		return
+
+	var results: Array[Dictionary] = []
+	var failed := false
+	for world_id: String in EXPECTED_WORLDS:
+		var path := world_directory.path_join("%s.json" % world_id)
+		var file := FileAccess.open(path, FileAccess.READ)
+		if file == null:
+			push_error("Unable to open offline checkpoint for %s" % world_id)
+			failed = true
+			continue
+		var parsed := JSON.new()
+		var parse_error := parsed.parse(file.get_as_text())
+		file.close()
+		if parse_error != OK or not parsed.data is Dictionary:
+			push_error("Invalid JSON checkpoint for %s" % world_id)
+			failed = true
+			continue
+		var state: Dictionary = parsed.data
+		var generation: Dictionary = state.get("generation", {}) if state.get("generation", {}) is Dictionary else {}
+		if int(state.get("schema_version", -1)) != 1:
+			push_error("Unsupported save schema for %s" % world_id)
+			failed = true
+			continue
+		if str(state.get("world_id", "")) != world_id:
+			push_error("World ID mismatch for %s" % world_id)
+			failed = true
+			continue
+		var expected_mode: String = EXPECTED_WORLDS[world_id]
+		if str(generation.get("mode", "")) != expected_mode:
+			push_error("World mode mismatch for %s" % world_id)
+			failed = true
+			continue
+
+		# Only construct the simulation and deserialize saved state. This harness
+		# never creates ServerMain, starts multiplayer, or calls BackendClient.
+		var simulation = WorldSim.new()
+		var restored: bool = simulation.deserialize_state(state)
+		var tile_count := int(simulation.tiles.size()) if restored else 0
+		var restored_id := str(simulation.world_id) if restored else ""
+		var restored_mode := str(simulation.world_mode) if restored else ""
+		if not restored or restored_id != world_id or restored_mode != expected_mode or tile_count <= 0:
+			push_error("WorldSim.deserialize_state did not restore %s from this checkpoint" % world_id)
+			failed = true
+			continue
+		results.append({
+			"world_id": world_id,
+			"mode": expected_mode,
+			"schema_version": 1,
+			"sha256": FileAccess.get_sha256(path),
+			"deserialize_state_ok": true,
+			"tile_count": tile_count,
+		})
+		print("OFFLINE_WORLD_RESTORE_OK id=%s mode=%s tiles=%d" % [world_id, expected_mode, tile_count])
+
+	if failed or results.size() != EXPECTED_WORLDS.size():
+		get_tree().quit(1)
+		return
+	var receipt := {
+		"format": "tinyblock-offline-restore-verification",
+		"validator_version": 1,
+		"godot_version": Engine.get_version_info().get("string", "unknown"),
+		"verified_at_unix": Time.get_unix_time_from_system(),
+		"network_calls_requested": 0,
+		"server_started": false,
+		"worlds": results,
+	}
+	var receipt_file := FileAccess.open(receipt_path, FileAccess.WRITE)
+	if receipt_file == null:
+		push_error("Unable to write private offline restore receipt")
+		get_tree().quit(2)
+		return
+	receipt_file.store_string(JSON.stringify(receipt, "\t"))
+	receipt_file.flush()
+	receipt_file.close()
+	get_tree().quit(0)
+
+
+func _argument_value(arguments: PackedStringArray, name: String) -> String:
+	for argument: String in arguments:
+		if argument.begins_with("%s=" % name):
+			return argument.trim_prefix("%s=" % name)
+	return ""

tests/world_backup_restore_verifier.gd.uid

@@ -0,0 +1 @@
+uid://5hetdts08fpo

tests/world_backup_restore_verifier.tscn

@@ -0,0 +1,6 @@
+[gd_scene load_steps=2 format=3]
+
+[ext_resource type="Script" path="res://tests/world_backup_restore_verifier.gd" id="1"]
+
+[node name="WorldBackupRestoreVerifier" type="Node"]
+script = ExtResource("1")