Native: offline backup-hold release and read-only inspect for restores (#973)

docs/contracts/external-payload-storage.md

@@ -261,6 +261,25 @@ curl -fsS "$SERVER/api/system/external-payload-backup-hold?owner=$BACKUP_ID" \
   -H "Authorization: Bearer $ADMIN_TOKEN" -H 'X-Durable-Workflow-Control-Plane-Version: 2'
 ```
 
+A restore that is still offline, with no Server process running, uses the
+native image's two console commands instead. They read only the database
+settings (`DB_CONNECTION` and its connection variables), open only an already
+initialized native database, never create or change its schema, print the
+same JSON report, and start no listener or workers:
+
+```sh
+# The hold as stored, read-only: no lock, no row created, no write.
+docker run --rm --env-file db.env "$SERVER_IMAGE" backup-hold inspect
+# Release the restored hold by its recorded owner; repeating it is harmless.
+docker run --rm --env-file db.env "$SERVER_IMAGE" \
+  backup-hold release --owner="$BACKUP_ID"
+```
+
+On refusal they print the reason (`backup_hold_not_owned`,
+`native_schema_not_initialized`, or the owner validation message) and exit
+non-zero. Acquire, renew and owner-scoped status stay on the endpoints, while
+the Server runs.
+
 Each answers the command's JSON report. Where a command exits unsuccessfully,
 the endpoint answers 409: `backup_hold_owned_elsewhere` when another backup
 holds the hold, and `backup_hold_not_owned` when the owner's hold expired, was

docs/self-hosted-backup-and-restore.md

@@ -4,8 +4,11 @@ This procedure applies to the PHP 2.x Compose stack, `docker-compose.php.yml`.
 The published Compose file runs the native server from 3.0.0.
 
 On native 3.x Server, take and release the backup hold through its admin
-endpoints instead of `php artisan external-payloads:backup-hold`. The owner
-UUID, TTL limits and status checks are the same; see
+endpoints instead of `php artisan external-payloads:backup-hold`. To release a
+restored hold before any Server process starts, run the image's offline
+`backup-hold release --owner=<uuid>` command against the restored database
+(`backup-hold inspect` reads the hold without changing anything). The
+owner UUID, TTL limits and status checks are the same; see
 [coordinating online backups](contracts/external-payload-storage.md#coordinating-online-backups).
 
 This procedure applies to the published single-node Compose stack with MySQL,

rust/src/main.rs

@@ -245,6 +245,42 @@ fn main() -> Result<(), Box<dyn Error>> {
         .block_on(run())
 }
 
+/// `backup-hold release --owner <uuid>` and `backup-hold inspect`: what a
+/// restore runs before any Server process starts (Server #973). It opens only
+/// an initialized native database from the DB settings, `inspect` read-only,
+/// prints the v1 report, and on refusal prints the reason and exits non-zero.
+async fn backup_hold_command() -> Result<(), Box<dyn Error>> {
+    use durable_workflow_server::runtime::backup_hold;
+    let args: Vec<String> = env::args().skip(2).collect();
+    let backend = env::var("DB_CONNECTION").unwrap_or_default();
+    let result = match backup_hold::command(&args) {
+        Err(error) => Err(error),
+        Ok(command) => match backend.as_str() {
+            "sqlite" => {
+                let path = env::var("DB_DATABASE").map_err(|_| "DB_DATABASE is required")?;
+                backup_hold::console_sqlite(&path, &command).await
+            }
+            "pgsql" => backup_hold::console_postgres(postgres_options()?, &command).await,
+            "mysql" | "mariadb" => backup_hold::console_mysql(mysql_options()?, &command).await,
+            _ => {
+                return Err(
+                    "backup-hold requires DB_CONNECTION=sqlite, pgsql, mysql or mariadb".into(),
+                );
+            }
+        },
+    };
+    match result {
+        Ok(report) => {
+            println!("{report}");
+            Ok(())
+        }
+        Err(error) => {
+            eprintln!("{}", backup_hold::console_failure(&error));
+            Err("backup hold command refused".into())
+        }
+    }
+}
+
 /// A release build carries the version of its `native-` tag, built in as
 /// `DW_SERVER_VERSION`, and starts without the opt-in. A development build
 /// (no release version) still requires `DW_RUST_EXPERIMENTAL=1`, so an
@@ -257,6 +293,9 @@ async fn run() -> Result<(), Box<dyn Error>> {
     if !release_build() && env::var("DW_RUST_EXPERIMENTAL").as_deref() != Ok("1") {
         return Err("Rust development build requires DW_RUST_EXPERIMENTAL=1; release builds do not".into());
     }
+    if env::args().nth(1).as_deref() == Some("backup-hold") {
+        return backup_hold_command().await;
+    }
     if let Some(command) = env::args().nth(1) {
         if env::args().count() != 2
             || !matches!(

rust/src/runtime/backup_hold.rs

@@ -187,6 +187,75 @@ fn sql_time(at: DateTime<Utc>) -> String {
     at.format("%Y-%m-%d %H:%M:%S").to_string()
 }
 
+/// One offline `backup-hold` invocation (Server #973): only what a restore
+/// needs before any Server process starts. Live backups acquire, renew and
+/// check their hold through #953's endpoints.
+#[derive(Debug, PartialEq, Eq)]
+pub enum Command {
+    /// `release --owner <uuid>`: #953's release, by the hold's recorded
+    /// owner, idempotent, a lapsed restored hold included.
+    Release { owner: String },
+    /// `inspect`: the hold as stored, read without a lock or a write.
+    Inspect,
+}
+
+impl Command {
+    /// Whether the command writes; `inspect` opens the database read-only.
+    fn writes(&self) -> bool {
+        matches!(self, Self::Release { .. })
+    }
+}
+
+/// Parse the arguments after `backup-hold`: `release --owner <uuid>` (or
+/// `--owner=<uuid>`), or `inspect` alone.
+pub fn command(args: &[String]) -> Result<Command> {
+    let invalid = || super::refuse(StatusCode::BAD_REQUEST, "backup_hold_command_invalid");
+    match args {
+        [action, rest @ ..] if action == "release" => {
+            let owner = match rest {
+                [option] => option.strip_prefix("--owner=").map(str::to_owned),
+                [flag, value] if flag == "--owner" => Some(value.clone()),
+                _ => None,
+            };
+            owner.map(|owner| Command::Release { owner }).ok_or_else(invalid)
+        }
+        [action] if action == "inspect" => Ok(Command::Inspect),
+        [action, ..] if action == "inspect" => Err(invalid()),
+        _ => Err(super::refuse(StatusCode::BAD_REQUEST, "backup_hold_action_invalid")),
+    }
+}
+
+/// `backup-hold` against a native SQLite database that already exists.
+pub async fn console_sqlite(database: &str, command: &Command) -> Result<Value> {
+    let store = super::store::Store::new(super::sqlite::open_existing(database, command.writes()).await?);
+    store.run_backup_hold_command(command).await
+}
+
+/// `backup-hold` against an initialized native PostgreSQL database.
+pub async fn console_postgres(options: sqlx::postgres::PgConnectOptions, command: &Command) -> Result<Value> {
+    let store = super::store::Store::new(super::postgres::PostgresStorage::open_existing(options, command.writes()).await?);
+    store.run_backup_hold_command(command).await
+}
+
+/// `backup-hold` against an initialized native MySQL or MariaDB database.
+pub async fn console_mysql(options: sqlx::mysql::MySqlConnectOptions, command: &Command) -> Result<Value> {
+    let store = super::store::Store::new(super::mysql::MySqlStorage::open_existing(options).await?);
+    store.run_backup_hold_command(command).await
+}
+
+/// The line the command prints on failure: the refusal reason, or the
+/// validation messages of an invalid owner or TTL.
+pub fn console_failure(error: &super::RuntimeError) -> String {
+    match error {
+        super::RuntimeError::Refused { reason, .. } => (*reason).to_owned(),
+        super::RuntimeError::Protocol { response, .. } => response["message"]
+            .as_str()
+            .unwrap_or("backup_hold_request_invalid")
+            .to_owned(),
+        _ => "storage_unavailable".to_owned(),
+    }
+}
+
 store_methods! {
     /// The hold row and the database's clock, without a lock: the preflight
     /// PHP's `assertReclamationAllowed` makes before a prune stages its
@@ -246,6 +315,35 @@ store_methods! {
         Ok(next.report())
     }
 
+    /// The offline command's report. Release is #953's own method; the pool
+    /// closes whatever the outcome.
+    async fn run_backup_hold_command(&self, command: &Command) -> Result<Value> {
+        let result = match command {
+            Command::Inspect => self.inspect_backup_hold().await,
+            Command::Release { owner } => match request(&json!({ "owner": owner }), Action::Release) {
+                Ok(request) => self.change_backup_hold(Action::Release, &request).await,
+                Err(error) => Err(error),
+            },
+        };
+        self.close().await;
+        result
+    }
+
+    /// The hold as stored, from plain reads in one read-only snapshot: no
+    /// lock, no row creation, no write. A missing row is no hold.
+    async fn inspect_backup_hold(&self) -> Result<Value> {
+        let mut tx = self.begin_read().await?;
+        let hold = match Self::read_backup_hold(&mut tx).await? {
+            Some(hold) => hold,
+            None => {
+                let now = Self::query(DB::BACKUP_HOLD_CLOCK).fetch_one(&mut *tx).await?;
+                Hold { owner: None, acquired_at: None, expires_at: None, now: DB::instant(&now, "database_time")? }
+            }
+        };
+        tx.rollback().await?;
+        Ok(hold.report())
+    }
+
     /// `status`: the hold as it stands, refused when a scoped owner no longer
     /// holds it.
     pub(crate) async fn backup_hold_status(&self, owner: Option<&str>) -> Result<Value> {

rust/src/runtime/mod.rs

@@ -78,7 +78,7 @@ mod memo_limits;
 mod history_meter;
 mod standalone_activities;
 mod archive;
-pub(crate) mod backup_hold;
+pub mod backup_hold;
 mod command_payloads;
 mod completion_rules;
 mod deployments;

rust/src/runtime/mysql.rs

@@ -811,6 +811,34 @@ impl MySqlStorage {
         let pool = runtime_pool_options().connect_with(connect_options).await?;
         Ok(Self { pool })
     }
+    /// Open an initialized native database for an offline console command
+    /// (Server #973). There is no bootstrap path: the pool's own connection
+    /// must find the native marker and the exact schema, or the database is
+    /// refused as it is. The check only reads, in one read-only transaction.
+    pub(super) async fn open_existing(connect_options: MySqlConnectOptions) -> Result<MySqlPool> {
+        let pool = runtime_pool_options()
+            .max_connections(1)
+            .connect_with(options(connect_options))
+            .await?;
+        let result = async {
+            let mut connection = pool.acquire().await?;
+            configure(&mut connection).await?;
+            let flavor = Flavor::detect(&mut connection).await?;
+            let mut tx = connection.begin_with("START TRANSACTION READ ONLY").await?;
+            if !scope(&mut tx).await?.iter().any(|table| table == "dw_server_schema") {
+                return Err(refuse(StatusCode::CONFLICT, "native_schema_not_initialized"));
+            }
+            verify(&mut tx, flavor).await?;
+            tx.rollback().await?;
+            Ok::<_, super::RuntimeError>(())
+        }
+        .await;
+        if let Err(error) = result {
+            pool.close().await;
+            return Err(error);
+        }
+        Ok(pool)
+    }
     pub async fn check(connect_options: MySqlConnectOptions) -> Result<()> {
         let mut connection = MySqlConnection::connect_with(&options(connect_options)).await?;
         let result = async {

rust/src/runtime/postgres.rs

@@ -390,6 +390,43 @@ impl PostgresStorage {
         Ok(Self { pool })
     }
 
+    /// Open an initialized native database for an offline console command
+    /// (Server #973). There is no bootstrap path: under the bootstrap
+    /// advisory lock, the pool's own connection must find the native marker
+    /// and the exact schema, or the database is refused as it is. A command
+    /// that only reads checks in a read-only transaction, without the lock.
+    pub(super) async fn open_existing(connect_options: PgConnectOptions, writable: bool) -> Result<PgPool> {
+        let pool = PgPoolOptions::new()
+            .max_connections(1)
+            .acquire_timeout(Duration::from_secs(5))
+            .connect_with(options(connect_options))
+            .await?;
+        let result = async {
+            let mut tx = if writable {
+                let mut tx = pool.begin().await?;
+                sqlx::query("SELECT pg_advisory_xact_lock(4924501473315676162)")
+                    .execute(&mut *tx)
+                    .await?;
+                tx
+            } else {
+                pool.begin_with("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY").await?
+            };
+            verify_scope(&mut tx).await?;
+            if !marked(&mut tx).await? {
+                return Err(refuse(StatusCode::CONFLICT, "native_schema_not_initialized"));
+            }
+            verify(&mut tx).await?;
+            tx.commit().await?;
+            Ok::<_, super::RuntimeError>(())
+        }
+        .await;
+        if let Err(error) = result {
+            pool.close().await;
+            return Err(error);
+        }
+        Ok(pool)
+    }
+
     /// Upgrade an earlier native schema version to this one, in one
     /// transaction under the bootstrap advisory lock. Returns the version
     /// found; a current database is only verified.

rust/src/runtime/sqlite.rs

@@ -26,6 +26,50 @@ pub(super) async fn open(database: &str) -> Result<SqlitePool> {
     unreachable!("the final startup attempt always returns")
 }
 
+/// Open an initialized native database for an offline console command
+/// (Server #973). There is no bootstrap path: the file is never created or
+/// switched to WAL, and the pool's own connection must find the native marker
+/// and the exact schema, or the database is refused as it is. A command that
+/// only reads opens the file read-only.
+pub(super) async fn open_existing(database: &str, writable: bool) -> Result<SqlitePool> {
+    if database.is_empty() || database.contains('?') || database.contains(':') {
+        return Err(refuse(
+            StatusCode::BAD_REQUEST,
+            "invalid_development_configuration",
+        ));
+    }
+    let path = Path::new(database);
+    if !path.exists() {
+        return Err(refuse(StatusCode::CONFLICT, "native_schema_not_initialized"));
+    }
+    let pool = SqlitePoolOptions::new()
+        .max_connections(1)
+        .acquire_timeout(Duration::from_secs(5))
+        .connect_with(
+            SqliteConnectOptions::new()
+                .filename(path)
+                .create_if_missing(false)
+                .read_only(!writable)
+                .synchronous(SqliteSynchronous::Full)
+                .foreign_keys(true)
+                .busy_timeout(Duration::from_secs(5)),
+        )
+        .await?;
+    let result = async {
+        let mut connection = pool.acquire().await?;
+        if !inspect(&mut connection).await? {
+            return Err(refuse(StatusCode::CONFLICT, "native_schema_not_initialized"));
+        }
+        Ok::<_, super::RuntimeError>(())
+    }
+    .await;
+    if let Err(error) = result {
+        pool.close().await;
+        return Err(error);
+    }
+    Ok(pool)
+}
+
 async fn open_once(database: &str) -> Result<SqlitePool> {
     if database.is_empty() || database.contains('?') || database.contains(':') {
         return Err(refuse(