From 124ba2b089e633f2243f9352d46729c5230f9e5e Mon Sep 17 00:00:00 2001 From: Daniel Nash <38335330+dannash100@users.noreply.github.com> Date: Thu, 9 Jul 2026 05:17:58 +1200 Subject: [PATCH 1/6] docs(seedling): add healthchecks spec (SDH) --- .workhorse/specs/seedling/healthchecks.md | 29 +++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .workhorse/specs/seedling/healthchecks.md diff --git a/.workhorse/specs/seedling/healthchecks.md b/.workhorse/specs/seedling/healthchecks.md new file mode 100644 index 00000000..4cdf4405 --- /dev/null +++ b/.workhorse/specs/seedling/healthchecks.md @@ -0,0 +1,29 @@ +--- +id: SDH +--- + +# Seedling healthchecks + +On a host that runs the Seedling application orchestrator, `bestool tamanu doctor` gathers Seedling-specific health checks from the local Seedling daemon and folds them into the sweep alongside the host and Tamanu checks. +The Seedling checks share the registry, outcomes, grouping, rendering, and status wire format of every other check (see [DOC](../tamanu/doctor.md)); this spec covers only what is specific to Seedling: when they apply, where their data comes from, and what each reports. + +## Seedling host context + +The Seedling checks apply when the host runs Seedling: the local Seedling daemon is reachable and a Seedling database connection is configured in the environment. +When the daemon cannot be reached, every Seedling check skips, so the same doctor invocation runs safely on a host that carries no Seedling. + +## Obtaining checks from the daemon + +The doctor reads Seedling health from the daemon's local control interface, authenticating as a client the daemon trusts. +The daemon is the single source of truth for its own health: the doctor reports what the daemon returns rather than inspecting Seedling's containers or database itself. +When the daemon is reachable but cannot answer a given check, that check is reported as broken rather than failing, so a daemon-side fault is not mistaken for an unhealthy system. + +## Checks + +The doctor derives one check per Seedling subsystem the daemon reports on. + +A reverse-proxy check reports whether the proxy that fronts application traffic is running, and fails when it is not. +A resolver check reports whether the Seedling DNS resolver is running, and fails when it is not. +An applications check reports the health of the Seedling-managed applications: it passes when every app is in its expected running state, warns when some have not yet converged, and fails when an app that should be running is not. + +Each check carries a one-line summary, and a reason whenever it does not pass, in line with [DOC](../tamanu/doctor.md). From 2ee3624e4cb9d66446bfa15cc060fb1637230ec8 Mon Sep 17 00:00:00 2001 From: Daniel Nash <38335330+dannash100@users.noreply.github.com> Date: Thu, 9 Jul 2026 07:05:38 +1200 Subject: [PATCH 2/6] feat(seedling): doctor healthcheck via the OI API --- Cargo.lock | 77 +++++++++- crates/alertd/Cargo.toml | 1 + crates/alertd/src/doctor/checks.rs | 4 + crates/alertd/src/doctor/checks/seedling.rs | 147 ++++++++++++++++++++ 4 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 crates/alertd/src/doctor/checks/seedling.rs diff --git a/Cargo.lock b/Cargo.lock index 7b5bc878..417136e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -654,6 +654,7 @@ dependencies = [ "rustix 1.1.4", "rustls", "sd-notify", + "seedling-protocol", "serde", "serde_json", "sysinfo", @@ -1848,6 +1849,7 @@ dependencies = [ "cfg-if", "cpufeatures 0.2.17", "curve25519-dalek-derive", + "digest 0.10.7", "fiat-crypto", "rustc_version", "subtle", @@ -2176,6 +2178,31 @@ dependencies = [ "spki", ] +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core 0.6.4", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + [[package]] name = "edit" version = "0.1.5" @@ -2441,6 +2468,7 @@ checksum = "4e7f34442dbe69c60fe8eaf58a8cafff81a1f278816d8ab4db255b3bef4ac3c4" dependencies = [ "getrandom 0.3.4", "libm", + "rand 0.9.4", "siphasher", ] @@ -5519,6 +5547,7 @@ checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "aws-lc-rs", "bytes", + "fastbloom", "getrandom 0.3.4", "lru-slab", "rand 0.9.4", @@ -5526,6 +5555,7 @@ dependencies = [ "rustc-hash 2.1.2", "rustls", "rustls-pki-types", + "rustls-platform-verifier 0.6.2", "slab", "thiserror 2.0.18", "tinyvec", @@ -5890,7 +5920,7 @@ dependencies = [ "quinn", "rustls", "rustls-pki-types", - "rustls-platform-verifier", + "rustls-platform-verifier 0.7.0", "serde", "serde_json", "sync_wrapper", @@ -6164,6 +6194,27 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni 0.21.1", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.52.0", +] + [[package]] name = "rustls-platform-verifier" version = "0.7.0" @@ -6366,6 +6417,29 @@ dependencies = [ "libc", ] +[[package]] +name = "seedling-protocol" +version = "0.3.0" +source = "git+https://github.com/beyondessential/seedling.git#e9e2e4687fbd8346ba6278675120747d0def98ad" +dependencies = [ + "compact_str", + "dirs", + "ed25519-dalek", + "futures-util", + "jiff", + "quinn", + "rand_core 0.6.4", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "sha2 0.11.0", + "subtle", + "tokio", + "tracing", + "uuid", +] + [[package]] name = "self-replace" version = "1.5.0" @@ -6449,6 +6523,7 @@ version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ + "indexmap", "itoa", "memchr", "serde", diff --git a/crates/alertd/Cargo.toml b/crates/alertd/Cargo.toml index 7738cf71..85f27b81 100644 --- a/crates/alertd/Cargo.toml +++ b/crates/alertd/Cargo.toml @@ -28,6 +28,7 @@ miette = { workspace = true } prometheus = "0.14.0" reqwest = { workspace = true } rustls = "0.23.40" +seedling-protocol = { git = "https://github.com/beyondessential/seedling.git" } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.145" sysinfo.workspace = true diff --git a/crates/alertd/src/doctor/checks.rs b/crates/alertd/src/doctor/checks.rs index 8af46c46..6089b37b 100644 --- a/crates/alertd/src/doctor/checks.rs +++ b/crates/alertd/src/doctor/checks.rs @@ -39,6 +39,7 @@ pub mod migrations; pub mod patient_communication_errors; pub mod pg_tuning; pub mod report_errors; +pub mod seedling; pub mod sync_facility_stale; pub mod sync_lookup; pub mod sync_restart_loop; @@ -276,6 +277,9 @@ pub fn all() -> Vec { entry!("caddy_version", caddy_version, host), entry!("caddy_certs", caddy_certs, host), entry!("http_errors", http_errors, host), + // Host/service probe: queries the local Seedling daemon's OI when the + // host runs Seedling, and skips otherwise (see SDH). + entry!("seedling", seedling, host), entry!("tailscale", tailscale, host, off_wire), entry!("tailscale_config", tailscale_config, host), // Reports the host's LAN and best-guess WAN addresses as status facts diff --git a/crates/alertd/src/doctor/checks/seedling.rs b/crates/alertd/src/doctor/checks/seedling.rs new file mode 100644 index 00000000..eadcb815 --- /dev/null +++ b/crates/alertd/src/doctor/checks/seedling.rs @@ -0,0 +1,147 @@ +//! Seedling healthcheck. +//! +//! On a Seedling host, queries the local Seedling daemon over its OI control +//! interface and reports its subsystem health. Skips cleanly when the host +//! runs no Seedling, and reports broken (not failing) when the daemon cannot +//! be reached or has not authorised this client — a daemon-side condition, not +//! an unhealthy system. +//! +//! spec: SDH + +use std::{ + net::{Ipv4Addr, SocketAddr}, + path::{Path, PathBuf}, +}; + +use seedling_protocol::{ + actor::Actor, + client::{ClientAuth, OiClient}, + keys::{self, ClientIdentity}, +}; +use serde_json::{Value, json}; + +use super::SweepContext; +use crate::doctor::check::Check; + +const CHECK_NAME: &str = "seedling"; + +/// The Seedling daemon's OI listens here on loopback (mirrors the daemon's +/// `oi::DEFAULT_PORT`). +const OI_PORT: u16 = 7891; + +pub async fn run(_ctx: SweepContext) -> Check { + let Some(data_dir) = seedling_data_dir() else { + return Check::skip( + CHECK_NAME, + "no Seedling on this host", + "no Seedling data directory is configured in the environment", + ); + }; + + let server_key = data_dir.join("oi.key"); + if !server_key.exists() { + return Check::skip( + CHECK_NAME, + "no Seedling on this host", + format!("no Seedling OI key at {}", server_key.display()), + ); + } + + match probe(&server_key).await { + Ok(health) => health.into_check(), + Err(reason) => Check::broken(CHECK_NAME, "could not query the Seedling daemon", reason), + } +} + +/// Where the Seedling daemon keeps its data. The daemon takes this as a launch +/// argument, so the doctor reads it from the same place the deployment sets it. +fn seedling_data_dir() -> Option { + std::env::var_os("SEEDLING_DATA_DIR").map(PathBuf::from) +} + +/// This doctor's own client identity for the OI. Its fingerprint must be in the +/// daemon's authorised set for a query to be admitted. +fn client_key_path() -> PathBuf { + dirs::state_dir() + .or_else(dirs::data_local_dir) + .unwrap_or_else(|| PathBuf::from(".")) + .join("bestool") + .join("seedling-oi-client.key") +} + +async fn probe(server_key_path: &Path) -> Result { + // Pin the local daemon's fingerprint from its persisted key. The key is + // present (checked by the caller), so this loads rather than generates. + let server_key = keys::load_or_generate(server_key_path).map_err(|e| e.to_string())?; + let server_fingerprint = keys::fingerprint(&keys::spki_der(&server_key)); + + let (identity, _is_new) = + ClientIdentity::load_or_generate(&client_key_path()).map_err(|e| e.to_string())?; + + let actor = Actor { + kind: Some("bestool".to_owned()), + id: Some("doctor".to_owned()), + display: Some("bestool doctor".to_owned()), + session: Some(identity.fingerprint[..8].to_owned()), + }; + + let addr = SocketAddr::from((Ipv4Addr::LOCALHOST, OI_PORT)); + let client = OiClient::connect( + addr, + ClientAuth::Fingerprint(server_fingerprint), + &identity, + actor, + ) + .await + .map_err(|e| e.to_string())?; + + let infra = client + .request("/infra/status", json!({})) + .await + .map_err(|e| e.to_string())?; + let status = client + .request("/server/status", json!({})) + .await + .map_err(|e| e.to_string())?; + + Ok(Health { infra, status }) +} + +struct Health { + infra: Value, + status: Value, +} + +impl Health { + fn into_check(self) -> Check { + let str_field = |v: &Value, k: &str| { + v.get(k) + .and_then(Value::as_str) + .unwrap_or("unknown") + .to_owned() + }; + let proxy = str_field(&self.infra, "proxy"); + let resolver = str_field(&self.infra, "resolver"); + let apps_total = self + .status + .get("apps_total") + .and_then(Value::as_u64) + .unwrap_or(0); + + let summary = format!("proxy {proxy}, resolver {resolver}, {apps_total} apps"); + + let mut down = Vec::new(); + if proxy != "running" { + down.push(format!("reverse proxy is {proxy}")); + } + if resolver != "running" { + down.push(format!("DNS resolver is {resolver}")); + } + + if down.is_empty() { + Check::pass(CHECK_NAME, summary) + } else { + Check::fail(CHECK_NAME, summary, down.join("; ")) + } + } +} From e78846e79d47b769cb76c6f7456a9f2052a27406 Mon Sep 17 00:00:00 2001 From: Daniel Nash <38335330+dannash100@users.noreply.github.com> Date: Thu, 9 Jul 2026 07:21:32 +1200 Subject: [PATCH 3/6] feat(seedling): per-subsystem doctor checks (proxy/resolver/apps) --- crates/alertd/src/doctor/checks.rs | 25 ++- crates/alertd/src/doctor/checks/seedling.rs | 160 +++++++++++++------- crates/alertd/src/doctor/sweep.rs | 3 + 3 files changed, 128 insertions(+), 60 deletions(-) diff --git a/crates/alertd/src/doctor/checks.rs b/crates/alertd/src/doctor/checks.rs index 6089b37b..f0667262 100644 --- a/crates/alertd/src/doctor/checks.rs +++ b/crates/alertd/src/doctor/checks.rs @@ -63,6 +63,9 @@ pub mod version_drift; pub struct SweepContext { pub tamanu: Option, pub http_client: reqwest::Client, + /// The local Seedling daemon's status, resolved once per sweep and shared + /// across the Seedling checks. `None` on hosts with no Seedling. + pub seedling: seedling::Probe, } /// Shared context handed to every Tamanu-dependent check. @@ -277,9 +280,23 @@ pub fn all() -> Vec { entry!("caddy_version", caddy_version, host), entry!("caddy_certs", caddy_certs, host), entry!("http_errors", http_errors, host), - // Host/service probe: queries the local Seedling daemon's OI when the - // host runs Seedling, and skips otherwise (see SDH). - entry!("seedling", seedling, host), + // Seedling host/service probes: resolved once per sweep from the local + // Seedling daemon's OI (see SDH), and skip when the host runs no Seedling. + CheckEntry { + name: "seedling_proxy", + on_wire: true, + run: |ctx| Box::pin(seedling::proxy(ctx)), + }, + CheckEntry { + name: "seedling_resolver", + on_wire: true, + run: |ctx| Box::pin(seedling::resolver(ctx)), + }, + CheckEntry { + name: "seedling_apps", + on_wire: true, + run: |ctx| Box::pin(seedling::apps(ctx)), + }, entry!("tailscale", tailscale, host, off_wire), entry!("tailscale_config", tailscale_config, host), // Reports the host's LAN and best-guess WAN addresses as status facts @@ -403,6 +420,7 @@ mod tests { SweepContext { tamanu: None, http_client: reqwest::Client::new(), + seedling: None, } } @@ -431,6 +449,7 @@ mod tests { has_install: false, }), http_client: reqwest::Client::new(), + seedling: None, } } diff --git a/crates/alertd/src/doctor/checks/seedling.rs b/crates/alertd/src/doctor/checks/seedling.rs index eadcb815..005d0c69 100644 --- a/crates/alertd/src/doctor/checks/seedling.rs +++ b/crates/alertd/src/doctor/checks/seedling.rs @@ -1,10 +1,11 @@ -//! Seedling healthcheck. +//! Seedling healthchecks. //! -//! On a Seedling host, queries the local Seedling daemon over its OI control -//! interface and reports its subsystem health. Skips cleanly when the host -//! runs no Seedling, and reports broken (not failing) when the daemon cannot -//! be reached or has not authorised this client — a daemon-side condition, not -//! an unhealthy system. +//! On a Seedling host the doctor queries the local Seedling daemon over its OI +//! control interface once per sweep ([`probe`]) and derives one check per +//! subsystem: reverse proxy, DNS resolver, and managed apps. The checks skip +//! when the host runs no Seedling, and report broken (not failing) when the +//! daemon cannot be reached or has not authorised this client — a daemon-side +//! condition, not an unhealthy system. //! //! spec: SDH @@ -23,34 +24,31 @@ use serde_json::{Value, json}; use super::SweepContext; use crate::doctor::check::Check; -const CHECK_NAME: &str = "seedling"; - /// The Seedling daemon's OI listens here on loopback (mirrors the daemon's /// `oi::DEFAULT_PORT`). const OI_PORT: u16 = 7891; -pub async fn run(_ctx: SweepContext) -> Check { - let Some(data_dir) = seedling_data_dir() else { - return Check::skip( - CHECK_NAME, - "no Seedling on this host", - "no Seedling data directory is configured in the environment", - ); - }; +/// One sweep's view of the local Seedling daemon. +#[derive(Clone)] +pub struct SeedlingStatus { + infra: Value, + status: Value, +} - let server_key = data_dir.join("oi.key"); - if !server_key.exists() { - return Check::skip( - CHECK_NAME, - "no Seedling on this host", - format!("no Seedling OI key at {}", server_key.display()), - ); - } +/// Resolved once per sweep and shared across the Seedling checks: +/// - `None` — no Seedling on this host (checks skip); +/// - `Some(Err(reason))` — Seedling is present but the daemon couldn't be +/// queried (checks report broken); +/// - `Some(Ok(status))` — the daemon answered. +pub type Probe = Option>; - match probe(&server_key).await { - Ok(health) => health.into_check(), - Err(reason) => Check::broken(CHECK_NAME, "could not query the Seedling daemon", reason), +/// Query the local Seedling daemon, if this is a Seedling host. +pub async fn probe() -> Probe { + let server_key = seedling_data_dir()?.join("oi.key"); + if !server_key.exists() { + return None; } + Some(query(&server_key).await) } /// Where the Seedling daemon keeps its data. The daemon takes this as a launch @@ -69,7 +67,7 @@ fn client_key_path() -> PathBuf { .join("seedling-oi-client.key") } -async fn probe(server_key_path: &Path) -> Result { +async fn query(server_key_path: &Path) -> Result { // Pin the local daemon's fingerprint from its persisted key. The key is // present (checked by the caller), so this loads rather than generates. let server_key = keys::load_or_generate(server_key_path).map_err(|e| e.to_string())?; @@ -104,44 +102,92 @@ async fn probe(server_key_path: &Path) -> Result { .await .map_err(|e| e.to_string())?; - Ok(Health { infra, status }) + Ok(SeedlingStatus { infra, status }) } -struct Health { - infra: Value, - status: Value, +/// Turn the shared probe into a check, handling the skip/broken cases that are +/// identical across the three subsystems and delegating the healthy/unhealthy +/// call to `f`. +fn resolve( + ctx: &SweepContext, + name: &'static str, + f: impl FnOnce(&SeedlingStatus) -> Check, +) -> Check { + match &ctx.seedling { + None => Check::skip( + name, + "no Seedling on this host", + "no reachable Seedling daemon on this host", + ), + Some(Err(reason)) => { + Check::broken(name, "could not query the Seedling daemon", reason.clone()) + } + Some(Ok(status)) => f(status), + } +} + +fn str_field(v: &Value, key: &str) -> String { + v.get(key) + .and_then(Value::as_str) + .unwrap_or("unknown") + .to_owned() } -impl Health { - fn into_check(self) -> Check { - let str_field = |v: &Value, k: &str| { - v.get(k) - .and_then(Value::as_str) - .unwrap_or("unknown") - .to_owned() - }; - let proxy = str_field(&self.infra, "proxy"); - let resolver = str_field(&self.infra, "resolver"); - let apps_total = self +pub async fn proxy(ctx: SweepContext) -> Check { + resolve(&ctx, "seedling_proxy", |s| { + let state = str_field(&s.infra, "proxy"); + if state == "running" { + Check::pass("seedling_proxy", "reverse proxy running") + } else { + Check::fail( + "seedling_proxy", + format!("reverse proxy {state}"), + "the Seedling reverse proxy is not running", + ) + } + }) +} + +pub async fn resolver(ctx: SweepContext) -> Check { + resolve(&ctx, "seedling_resolver", |s| { + let state = str_field(&s.infra, "resolver"); + if state == "running" { + Check::pass("seedling_resolver", "DNS resolver running") + } else { + Check::fail( + "seedling_resolver", + format!("DNS resolver {state}"), + "the Seedling DNS resolver is not running", + ) + } + }) +} + +pub async fn apps(ctx: SweepContext) -> Check { + resolve(&ctx, "seedling_apps", |s| { + let total = s .status .get("apps_total") .and_then(Value::as_u64) .unwrap_or(0); + let running = s + .status + .get("apps_by_status") + .and_then(Value::as_object) + .and_then(|m| m.get("running")) + .and_then(Value::as_u64) + .unwrap_or(0); - let summary = format!("proxy {proxy}, resolver {resolver}, {apps_total} apps"); - - let mut down = Vec::new(); - if proxy != "running" { - down.push(format!("reverse proxy is {proxy}")); - } - if resolver != "running" { - down.push(format!("DNS resolver is {resolver}")); - } - - if down.is_empty() { - Check::pass(CHECK_NAME, summary) + if total == 0 { + Check::pass("seedling_apps", "no apps deployed") + } else if running >= total { + Check::pass("seedling_apps", format!("{total} apps running")) } else { - Check::fail(CHECK_NAME, summary, down.join("; ")) + Check::warning( + "seedling_apps", + format!("{running}/{total} apps running"), + format!("{} app(s) not running", total - running), + ) } - } + }) } diff --git a/crates/alertd/src/doctor/sweep.rs b/crates/alertd/src/doctor/sweep.rs index 39d1715b..52674a44 100644 --- a/crates/alertd/src/doctor/sweep.rs +++ b/crates/alertd/src/doctor/sweep.rs @@ -148,9 +148,12 @@ pub async fn perform_sweep( .filter(|t| t.has_install) .map(|t| t.root.display().to_string()); + // Resolved once so the per-subsystem Seedling checks share a single OI query. + let seedling = checks::seedling::probe().await; let check_ctx = SweepContext { tamanu: tamanu_ctx, http_client, + seedling, }; let registry = checks::all(); From 009f1e671019d784902eafe115268ac393678a12 Mon Sep 17 00:00:00 2001 From: Daniel Nash <38335330+dannash100@users.noreply.github.com> Date: Thu, 9 Jul 2026 07:44:09 +1200 Subject: [PATCH 4/6] fix(seedling): probe timeout, skip reason, spec alignment --- .workhorse/specs/seedling/healthchecks.md | 2 +- crates/alertd/src/doctor/checks/seedling.rs | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.workhorse/specs/seedling/healthchecks.md b/.workhorse/specs/seedling/healthchecks.md index 4cdf4405..edf027db 100644 --- a/.workhorse/specs/seedling/healthchecks.md +++ b/.workhorse/specs/seedling/healthchecks.md @@ -24,6 +24,6 @@ The doctor derives one check per Seedling subsystem the daemon reports on. A reverse-proxy check reports whether the proxy that fronts application traffic is running, and fails when it is not. A resolver check reports whether the Seedling DNS resolver is running, and fails when it is not. -An applications check reports the health of the Seedling-managed applications: it passes when every app is in its expected running state, warns when some have not yet converged, and fails when an app that should be running is not. +An applications check reports the health of the Seedling-managed applications: it passes when every app is running (or none are deployed), and warns when some are not running. Each check carries a one-line summary, and a reason whenever it does not pass, in line with [DOC](../tamanu/doctor.md). diff --git a/crates/alertd/src/doctor/checks/seedling.rs b/crates/alertd/src/doctor/checks/seedling.rs index 005d0c69..6c447178 100644 --- a/crates/alertd/src/doctor/checks/seedling.rs +++ b/crates/alertd/src/doctor/checks/seedling.rs @@ -48,7 +48,14 @@ pub async fn probe() -> Probe { if !server_key.exists() { return None; } - Some(query(&server_key).await) + // The connect has its own 5s timeout but the requests don't; the outer + // timeout keeps a wedged daemon from stalling the whole sweep. + Some( + match tokio::time::timeout(std::time::Duration::from_secs(10), query(&server_key)).await { + Ok(result) => result, + Err(_) => Err("timed out querying the Seedling daemon".into()), + }, + ) } /// Where the Seedling daemon keeps its data. The daemon takes this as a launch @@ -117,7 +124,7 @@ fn resolve( None => Check::skip( name, "no Seedling on this host", - "no reachable Seedling daemon on this host", + "no Seedling data directory (with an OI key) is configured on this host", ), Some(Err(reason)) => { Check::broken(name, "could not query the Seedling daemon", reason.clone()) From 2585939eef93683991e5b793c83d82b4cf1d31e5 Mon Sep 17 00:00:00 2001 From: Daniel Nash <38335330+dannash100@users.noreply.github.com> Date: Thu, 9 Jul 2026 09:56:55 +1200 Subject: [PATCH 5/6] feat(seedling): pin the daemon from its published fingerprint --- .workhorse/specs/seedling/healthchecks.md | 4 +- crates/alertd/src/doctor/checks/seedling.rs | 42 +++++++++++++++------ 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/.workhorse/specs/seedling/healthchecks.md b/.workhorse/specs/seedling/healthchecks.md index edf027db..3554144b 100644 --- a/.workhorse/specs/seedling/healthchecks.md +++ b/.workhorse/specs/seedling/healthchecks.md @@ -9,8 +9,8 @@ The Seedling checks share the registry, outcomes, grouping, rendering, and statu ## Seedling host context -The Seedling checks apply when the host runs Seedling: the local Seedling daemon is reachable and a Seedling database connection is configured in the environment. -When the daemon cannot be reached, every Seedling check skips, so the same doctor invocation runs safely on a host that carries no Seedling. +The Seedling checks apply when the host runs Seedling: the daemon's data directory is configured in the environment and the daemon's control interface is present there. +When no Seedling is configured, every Seedling check skips, so the same doctor invocation runs safely on a host that carries none. ## Obtaining checks from the daemon diff --git a/crates/alertd/src/doctor/checks/seedling.rs b/crates/alertd/src/doctor/checks/seedling.rs index 6c447178..cebce07b 100644 --- a/crates/alertd/src/doctor/checks/seedling.rs +++ b/crates/alertd/src/doctor/checks/seedling.rs @@ -44,20 +44,45 @@ pub type Probe = Option>; /// Query the local Seedling daemon, if this is a Seedling host. pub async fn probe() -> Probe { - let server_key = seedling_data_dir()?.join("oi.key"); - if !server_key.exists() { - return None; - } + let data_dir = seedling_data_dir()?; + let fingerprint = match server_fingerprint(&data_dir)? { + Ok(fp) => fp, + Err(reason) => return Some(Err(reason)), + }; // The connect has its own 5s timeout but the requests don't; the outer // timeout keeps a wedged daemon from stalling the whole sweep. Some( - match tokio::time::timeout(std::time::Duration::from_secs(10), query(&server_key)).await { + match tokio::time::timeout(std::time::Duration::from_secs(10), query(fingerprint)).await { Ok(result) => result, Err(_) => Err("timed out querying the Seedling daemon".into()), }, ) } +/// The daemon's OI fingerprint to pin, from the public `oi.fingerprint` file +/// the daemon publishes beside its key; for daemons that predate the file, +/// derived from the key itself. `None` when neither exists — no Seedling here. +fn server_fingerprint(data_dir: &Path) -> Option> { + let fingerprint_path = data_dir.join("oi.fingerprint"); + if fingerprint_path.exists() { + return Some( + std::fs::read_to_string(&fingerprint_path) + .map(|s| s.trim().to_owned()) + .map_err(|e| e.to_string()), + ); + } + let key_path = data_dir.join("oi.key"); + if !key_path.exists() { + return None; + } + // The key is present (just checked), so this loads rather than generates. + Some( + keys::load_or_generate(&key_path) + .map(|key| keys::fingerprint(&keys::spki_der(&key))) + .map_err(|e| e.to_string()), + ) +} + /// Where the Seedling daemon keeps its data. The daemon takes this as a launch /// argument, so the doctor reads it from the same place the deployment sets it. fn seedling_data_dir() -> Option { @@ -74,12 +99,7 @@ fn client_key_path() -> PathBuf { .join("seedling-oi-client.key") } -async fn query(server_key_path: &Path) -> Result { - // Pin the local daemon's fingerprint from its persisted key. The key is - // present (checked by the caller), so this loads rather than generates. - let server_key = keys::load_or_generate(server_key_path).map_err(|e| e.to_string())?; - let server_fingerprint = keys::fingerprint(&keys::spki_der(&server_key)); - +async fn query(server_fingerprint: String) -> Result { let (identity, _is_new) = ClientIdentity::load_or_generate(&client_key_path()).map_err(|e| e.to_string())?; From 639b3caded28df31a1839af2dcacec94c302a199 Mon Sep 17 00:00:00 2001 From: Daniel Nash <38335330+dannash100@users.noreply.github.com> Date: Fri, 17 Jul 2026 10:23:35 +1200 Subject: [PATCH 6/6] fix(seedling): harden the doctor probe - read the daemon's key strictly read-only instead of load_or_generate, which could create a fresh key at the daemon's path if the file vanished between the existence check and the load - report missing fields in daemon responses as broken per SDH, instead of failing (proxy/resolver "unknown") or silently passing (apps defaulting to zero total) - pin seedling-protocol to the v0.3.0 tag it was already locked at --- Cargo.lock | 3 +- crates/alertd/Cargo.toml | 4 +- crates/alertd/src/doctor/checks/seedling.rs | 80 +++++++++++---------- 3 files changed, 46 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 417136e1..8aa62716 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -644,6 +644,7 @@ dependencies = [ "bytes", "dirs", "duct", + "ed25519-dalek", "futures", "hickory-resolver", "jiff", @@ -6420,7 +6421,7 @@ dependencies = [ [[package]] name = "seedling-protocol" version = "0.3.0" -source = "git+https://github.com/beyondessential/seedling.git#e9e2e4687fbd8346ba6278675120747d0def98ad" +source = "git+https://github.com/beyondessential/seedling.git?tag=v0.3.0#e9e2e4687fbd8346ba6278675120747d0def98ad" dependencies = [ "compact_str", "dirs", diff --git a/crates/alertd/Cargo.toml b/crates/alertd/Cargo.toml index 85f27b81..b56b8d4e 100644 --- a/crates/alertd/Cargo.toml +++ b/crates/alertd/Cargo.toml @@ -20,6 +20,8 @@ bestool-tamanu = { version = "0.16.1", path = "../tamanu", features = ["canopy-r bytes = "1.9.0" dirs = "6.0.0" duct = "1.1.0" +# pkcs8 to parse the Seedling daemon's key read-only (see doctor/checks/seedling.rs) +ed25519-dalek = { version = "2.2.0", features = ["pkcs8"] } futures = { workspace = true } hickory-resolver = "0.26.1" jiff = { version = "0.2.28", features = ["serde"] } @@ -28,7 +30,7 @@ miette = { workspace = true } prometheus = "0.14.0" reqwest = { workspace = true } rustls = "0.23.40" -seedling-protocol = { git = "https://github.com/beyondessential/seedling.git" } +seedling-protocol = { git = "https://github.com/beyondessential/seedling.git", tag = "v0.3.0" } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.145" sysinfo.workspace = true diff --git a/crates/alertd/src/doctor/checks/seedling.rs b/crates/alertd/src/doctor/checks/seedling.rs index cebce07b..f87af078 100644 --- a/crates/alertd/src/doctor/checks/seedling.rs +++ b/crates/alertd/src/doctor/checks/seedling.rs @@ -14,6 +14,7 @@ use std::{ path::{Path, PathBuf}, }; +use ed25519_dalek::{SigningKey, pkcs8::DecodePrivateKey}; use seedling_protocol::{ actor::Actor, client::{ClientAuth, OiClient}, @@ -75,11 +76,14 @@ fn server_fingerprint(data_dir: &Path) -> Option> { if !key_path.exists() { return None; } - // The key is present (just checked), so this loads rather than generates. + // Strictly read-only: never `keys::load_or_generate` here, which would + // create a fresh key at the daemon's path if the file vanished between + // the check above and the load — silently replacing the daemon's identity. Some( - keys::load_or_generate(&key_path) - .map(|key| keys::fingerprint(&keys::spki_der(&key))) - .map_err(|e| e.to_string()), + std::fs::read(&key_path) + .map_err(|e| e.to_string()) + .and_then(|der| SigningKey::from_pkcs8_der(&der).map_err(|e| e.to_string())) + .map(|key| keys::fingerprint(&keys::spki_der(&key))), ) } @@ -153,55 +157,53 @@ fn resolve( } } -fn str_field(v: &Value, key: &str) -> String { - v.get(key) - .and_then(Value::as_str) - .unwrap_or("unknown") - .to_owned() +/// A subsystem's state from `/infra/status`. A response missing the field is a +/// daemon that cannot answer this check, which per SDH is broken, not failing. +fn infra_check(s: &SeedlingStatus, name: &'static str, key: &str, label: &str) -> Check { + match s.infra.get(key).and_then(Value::as_str) { + None => Check::broken( + name, + format!("daemon did not report the {label}"), + format!("`/infra/status` response has no string `{key}` field"), + ), + Some("running") => Check::pass(name, format!("{label} running")), + Some(state) => Check::fail( + name, + format!("{label} {state}"), + format!("the Seedling {label} is not running"), + ), + } } pub async fn proxy(ctx: SweepContext) -> Check { resolve(&ctx, "seedling_proxy", |s| { - let state = str_field(&s.infra, "proxy"); - if state == "running" { - Check::pass("seedling_proxy", "reverse proxy running") - } else { - Check::fail( - "seedling_proxy", - format!("reverse proxy {state}"), - "the Seedling reverse proxy is not running", - ) - } + infra_check(s, "seedling_proxy", "proxy", "reverse proxy") }) } pub async fn resolver(ctx: SweepContext) -> Check { resolve(&ctx, "seedling_resolver", |s| { - let state = str_field(&s.infra, "resolver"); - if state == "running" { - Check::pass("seedling_resolver", "DNS resolver running") - } else { - Check::fail( - "seedling_resolver", - format!("DNS resolver {state}"), - "the Seedling DNS resolver is not running", - ) - } + infra_check(s, "seedling_resolver", "resolver", "DNS resolver") }) } pub async fn apps(ctx: SweepContext) -> Check { resolve(&ctx, "seedling_apps", |s| { - let total = s - .status - .get("apps_total") - .and_then(Value::as_u64) - .unwrap_or(0); - let running = s - .status - .get("apps_by_status") - .and_then(Value::as_object) - .and_then(|m| m.get("running")) + let (Some(total), Some(by_status)) = ( + s.status.get("apps_total").and_then(Value::as_u64), + s.status.get("apps_by_status").and_then(Value::as_object), + ) else { + // Missing counts are a daemon that cannot answer: broken, per SDH. + return Check::broken( + "seedling_apps", + "daemon did not report app counts", + "`/server/status` response is missing `apps_total` or `apps_by_status`", + ); + }; + // The daemon only lists statuses that occur, so no `running` entry + // means zero apps running, not a malformed response. + let running = by_status + .get("running") .and_then(Value::as_u64) .unwrap_or(0);