diff --git a/.workhorse/specs/seedling/healthchecks.md b/.workhorse/specs/seedling/healthchecks.md new file mode 100644 index 00000000..3554144b --- /dev/null +++ b/.workhorse/specs/seedling/healthchecks.md @@ -0,0 +1,29 @@ +--- +id: SDH +--- + +# Seedling healthchecks + +On a host that runs the Seedling application orchestrator, `bestool tamanu doctor` gathers Seedling-specific health checks from the local Seedling daemon and folds them into the sweep alongside the host and Tamanu checks. +The Seedling checks share the registry, outcomes, grouping, rendering, and status wire format of every other check (see [DOC](../tamanu/doctor.md)); this spec covers only what is specific to Seedling: when they apply, where their data comes from, and what each reports. + +## Seedling host context + +The Seedling checks apply when the host runs Seedling: the daemon's data directory is configured in the environment and the daemon's control interface is present there. +When no Seedling is configured, every Seedling check skips, so the same doctor invocation runs safely on a host that carries none. + +## Obtaining checks from the daemon + +The doctor reads Seedling health from the daemon's local control interface, authenticating as a client the daemon trusts. +The daemon is the single source of truth for its own health: the doctor reports what the daemon returns rather than inspecting Seedling's containers or database itself. +When the daemon is reachable but cannot answer a given check, that check is reported as broken rather than failing, so a daemon-side fault is not mistaken for an unhealthy system. + +## Checks + +The doctor derives one check per Seedling subsystem the daemon reports on. + +A reverse-proxy check reports whether the proxy that fronts application traffic is running, and fails when it is not. +A resolver check reports whether the Seedling DNS resolver is running, and fails when it is not. +An applications check reports the health of the Seedling-managed applications: it passes when every app is running (or none are deployed), and warns when some are not running. + +Each check carries a one-line summary, and a reason whenever it does not pass, in line with [DOC](../tamanu/doctor.md). diff --git a/Cargo.lock b/Cargo.lock index 7b5bc878..8aa62716 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -644,6 +644,7 @@ dependencies = [ "bytes", "dirs", "duct", + "ed25519-dalek", "futures", "hickory-resolver", "jiff", @@ -654,6 +655,7 @@ dependencies = [ "rustix 1.1.4", "rustls", "sd-notify", + "seedling-protocol", "serde", "serde_json", "sysinfo", @@ -1848,6 +1850,7 @@ dependencies = [ "cfg-if", "cpufeatures 0.2.17", "curve25519-dalek-derive", + "digest 0.10.7", "fiat-crypto", "rustc_version", "subtle", @@ -2176,6 +2179,31 @@ dependencies = [ "spki", ] +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core 0.6.4", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + [[package]] name = "edit" version = "0.1.5" @@ -2441,6 +2469,7 @@ checksum = "4e7f34442dbe69c60fe8eaf58a8cafff81a1f278816d8ab4db255b3bef4ac3c4" dependencies = [ "getrandom 0.3.4", "libm", + "rand 0.9.4", "siphasher", ] @@ -5519,6 +5548,7 @@ checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "aws-lc-rs", "bytes", + "fastbloom", "getrandom 0.3.4", "lru-slab", "rand 0.9.4", @@ -5526,6 +5556,7 @@ dependencies = [ "rustc-hash 2.1.2", "rustls", "rustls-pki-types", + "rustls-platform-verifier 0.6.2", "slab", "thiserror 2.0.18", "tinyvec", @@ -5890,7 +5921,7 @@ dependencies = [ "quinn", "rustls", "rustls-pki-types", - "rustls-platform-verifier", + "rustls-platform-verifier 0.7.0", "serde", "serde_json", "sync_wrapper", @@ -6164,6 +6195,27 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni 0.21.1", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.52.0", +] + [[package]] name = "rustls-platform-verifier" version = "0.7.0" @@ -6366,6 +6418,29 @@ dependencies = [ "libc", ] +[[package]] +name = "seedling-protocol" +version = "0.3.0" +source = "git+https://github.com/beyondessential/seedling.git?tag=v0.3.0#e9e2e4687fbd8346ba6278675120747d0def98ad" +dependencies = [ + "compact_str", + "dirs", + "ed25519-dalek", + "futures-util", + "jiff", + "quinn", + "rand_core 0.6.4", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "sha2 0.11.0", + "subtle", + "tokio", + "tracing", + "uuid", +] + [[package]] name = "self-replace" version = "1.5.0" @@ -6449,6 +6524,7 @@ version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ + "indexmap", "itoa", "memchr", "serde", diff --git a/crates/alertd/Cargo.toml b/crates/alertd/Cargo.toml index 7738cf71..b56b8d4e 100644 --- a/crates/alertd/Cargo.toml +++ b/crates/alertd/Cargo.toml @@ -20,6 +20,8 @@ bestool-tamanu = { version = "0.16.1", path = "../tamanu", features = ["canopy-r bytes = "1.9.0" dirs = "6.0.0" duct = "1.1.0" +# pkcs8 to parse the Seedling daemon's key read-only (see doctor/checks/seedling.rs) +ed25519-dalek = { version = "2.2.0", features = ["pkcs8"] } futures = { workspace = true } hickory-resolver = "0.26.1" jiff = { version = "0.2.28", features = ["serde"] } @@ -28,6 +30,7 @@ miette = { workspace = true } prometheus = "0.14.0" reqwest = { workspace = true } rustls = "0.23.40" +seedling-protocol = { git = "https://github.com/beyondessential/seedling.git", tag = "v0.3.0" } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.145" sysinfo.workspace = true diff --git a/crates/alertd/src/doctor/checks.rs b/crates/alertd/src/doctor/checks.rs index 8af46c46..f0667262 100644 --- a/crates/alertd/src/doctor/checks.rs +++ b/crates/alertd/src/doctor/checks.rs @@ -39,6 +39,7 @@ pub mod migrations; pub mod patient_communication_errors; pub mod pg_tuning; pub mod report_errors; +pub mod seedling; pub mod sync_facility_stale; pub mod sync_lookup; pub mod sync_restart_loop; @@ -62,6 +63,9 @@ pub mod version_drift; pub struct SweepContext { pub tamanu: Option, pub http_client: reqwest::Client, + /// The local Seedling daemon's status, resolved once per sweep and shared + /// across the Seedling checks. `None` on hosts with no Seedling. + pub seedling: seedling::Probe, } /// Shared context handed to every Tamanu-dependent check. @@ -276,6 +280,23 @@ pub fn all() -> Vec { entry!("caddy_version", caddy_version, host), entry!("caddy_certs", caddy_certs, host), entry!("http_errors", http_errors, host), + // Seedling host/service probes: resolved once per sweep from the local + // Seedling daemon's OI (see SDH), and skip when the host runs no Seedling. + CheckEntry { + name: "seedling_proxy", + on_wire: true, + run: |ctx| Box::pin(seedling::proxy(ctx)), + }, + CheckEntry { + name: "seedling_resolver", + on_wire: true, + run: |ctx| Box::pin(seedling::resolver(ctx)), + }, + CheckEntry { + name: "seedling_apps", + on_wire: true, + run: |ctx| Box::pin(seedling::apps(ctx)), + }, entry!("tailscale", tailscale, host, off_wire), entry!("tailscale_config", tailscale_config, host), // Reports the host's LAN and best-guess WAN addresses as status facts @@ -399,6 +420,7 @@ mod tests { SweepContext { tamanu: None, http_client: reqwest::Client::new(), + seedling: None, } } @@ -427,6 +449,7 @@ mod tests { has_install: false, }), http_client: reqwest::Client::new(), + seedling: None, } } diff --git a/crates/alertd/src/doctor/checks/seedling.rs b/crates/alertd/src/doctor/checks/seedling.rs new file mode 100644 index 00000000..f87af078 --- /dev/null +++ b/crates/alertd/src/doctor/checks/seedling.rs @@ -0,0 +1,222 @@ +//! Seedling healthchecks. +//! +//! On a Seedling host the doctor queries the local Seedling daemon over its OI +//! control interface once per sweep ([`probe`]) and derives one check per +//! subsystem: reverse proxy, DNS resolver, and managed apps. The checks skip +//! when the host runs no Seedling, and report broken (not failing) when the +//! daemon cannot be reached or has not authorised this client — a daemon-side +//! condition, not an unhealthy system. +//! +//! spec: SDH + +use std::{ + net::{Ipv4Addr, SocketAddr}, + path::{Path, PathBuf}, +}; + +use ed25519_dalek::{SigningKey, pkcs8::DecodePrivateKey}; +use seedling_protocol::{ + actor::Actor, + client::{ClientAuth, OiClient}, + keys::{self, ClientIdentity}, +}; +use serde_json::{Value, json}; + +use super::SweepContext; +use crate::doctor::check::Check; + +/// The Seedling daemon's OI listens here on loopback (mirrors the daemon's +/// `oi::DEFAULT_PORT`). +const OI_PORT: u16 = 7891; + +/// One sweep's view of the local Seedling daemon. +#[derive(Clone)] +pub struct SeedlingStatus { + infra: Value, + status: Value, +} + +/// Resolved once per sweep and shared across the Seedling checks: +/// - `None` — no Seedling on this host (checks skip); +/// - `Some(Err(reason))` — Seedling is present but the daemon couldn't be +/// queried (checks report broken); +/// - `Some(Ok(status))` — the daemon answered. +pub type Probe = Option>; + +/// Query the local Seedling daemon, if this is a Seedling host. +pub async fn probe() -> Probe { + let data_dir = seedling_data_dir()?; + let fingerprint = match server_fingerprint(&data_dir)? { + Ok(fp) => fp, + Err(reason) => return Some(Err(reason)), + }; + // The connect has its own 5s timeout but the requests don't; the outer + // timeout keeps a wedged daemon from stalling the whole sweep. + Some( + match tokio::time::timeout(std::time::Duration::from_secs(10), query(fingerprint)).await { + Ok(result) => result, + Err(_) => Err("timed out querying the Seedling daemon".into()), + }, + ) +} + +/// The daemon's OI fingerprint to pin, from the public `oi.fingerprint` file +/// the daemon publishes beside its key; for daemons that predate the file, +/// derived from the key itself. `None` when neither exists — no Seedling here. +fn server_fingerprint(data_dir: &Path) -> Option> { + let fingerprint_path = data_dir.join("oi.fingerprint"); + if fingerprint_path.exists() { + return Some( + std::fs::read_to_string(&fingerprint_path) + .map(|s| s.trim().to_owned()) + .map_err(|e| e.to_string()), + ); + } + let key_path = data_dir.join("oi.key"); + if !key_path.exists() { + return None; + } + // Strictly read-only: never `keys::load_or_generate` here, which would + // create a fresh key at the daemon's path if the file vanished between + // the check above and the load — silently replacing the daemon's identity. + Some( + std::fs::read(&key_path) + .map_err(|e| e.to_string()) + .and_then(|der| SigningKey::from_pkcs8_der(&der).map_err(|e| e.to_string())) + .map(|key| keys::fingerprint(&keys::spki_der(&key))), + ) +} + +/// Where the Seedling daemon keeps its data. The daemon takes this as a launch +/// argument, so the doctor reads it from the same place the deployment sets it. +fn seedling_data_dir() -> Option { + std::env::var_os("SEEDLING_DATA_DIR").map(PathBuf::from) +} + +/// This doctor's own client identity for the OI. Its fingerprint must be in the +/// daemon's authorised set for a query to be admitted. +fn client_key_path() -> PathBuf { + dirs::state_dir() + .or_else(dirs::data_local_dir) + .unwrap_or_else(|| PathBuf::from(".")) + .join("bestool") + .join("seedling-oi-client.key") +} + +async fn query(server_fingerprint: String) -> Result { + let (identity, _is_new) = + ClientIdentity::load_or_generate(&client_key_path()).map_err(|e| e.to_string())?; + + let actor = Actor { + kind: Some("bestool".to_owned()), + id: Some("doctor".to_owned()), + display: Some("bestool doctor".to_owned()), + session: Some(identity.fingerprint[..8].to_owned()), + }; + + let addr = SocketAddr::from((Ipv4Addr::LOCALHOST, OI_PORT)); + let client = OiClient::connect( + addr, + ClientAuth::Fingerprint(server_fingerprint), + &identity, + actor, + ) + .await + .map_err(|e| e.to_string())?; + + let infra = client + .request("/infra/status", json!({})) + .await + .map_err(|e| e.to_string())?; + let status = client + .request("/server/status", json!({})) + .await + .map_err(|e| e.to_string())?; + + Ok(SeedlingStatus { infra, status }) +} + +/// Turn the shared probe into a check, handling the skip/broken cases that are +/// identical across the three subsystems and delegating the healthy/unhealthy +/// call to `f`. +fn resolve( + ctx: &SweepContext, + name: &'static str, + f: impl FnOnce(&SeedlingStatus) -> Check, +) -> Check { + match &ctx.seedling { + None => Check::skip( + name, + "no Seedling on this host", + "no Seedling data directory (with an OI key) is configured on this host", + ), + Some(Err(reason)) => { + Check::broken(name, "could not query the Seedling daemon", reason.clone()) + } + Some(Ok(status)) => f(status), + } +} + +/// A subsystem's state from `/infra/status`. A response missing the field is a +/// daemon that cannot answer this check, which per SDH is broken, not failing. +fn infra_check(s: &SeedlingStatus, name: &'static str, key: &str, label: &str) -> Check { + match s.infra.get(key).and_then(Value::as_str) { + None => Check::broken( + name, + format!("daemon did not report the {label}"), + format!("`/infra/status` response has no string `{key}` field"), + ), + Some("running") => Check::pass(name, format!("{label} running")), + Some(state) => Check::fail( + name, + format!("{label} {state}"), + format!("the Seedling {label} is not running"), + ), + } +} + +pub async fn proxy(ctx: SweepContext) -> Check { + resolve(&ctx, "seedling_proxy", |s| { + infra_check(s, "seedling_proxy", "proxy", "reverse proxy") + }) +} + +pub async fn resolver(ctx: SweepContext) -> Check { + resolve(&ctx, "seedling_resolver", |s| { + infra_check(s, "seedling_resolver", "resolver", "DNS resolver") + }) +} + +pub async fn apps(ctx: SweepContext) -> Check { + resolve(&ctx, "seedling_apps", |s| { + let (Some(total), Some(by_status)) = ( + s.status.get("apps_total").and_then(Value::as_u64), + s.status.get("apps_by_status").and_then(Value::as_object), + ) else { + // Missing counts are a daemon that cannot answer: broken, per SDH. + return Check::broken( + "seedling_apps", + "daemon did not report app counts", + "`/server/status` response is missing `apps_total` or `apps_by_status`", + ); + }; + // The daemon only lists statuses that occur, so no `running` entry + // means zero apps running, not a malformed response. + let running = by_status + .get("running") + .and_then(Value::as_u64) + .unwrap_or(0); + + if total == 0 { + Check::pass("seedling_apps", "no apps deployed") + } else if running >= total { + Check::pass("seedling_apps", format!("{total} apps running")) + } else { + Check::warning( + "seedling_apps", + format!("{running}/{total} apps running"), + format!("{} app(s) not running", total - running), + ) + } + }) +} diff --git a/crates/alertd/src/doctor/sweep.rs b/crates/alertd/src/doctor/sweep.rs index 39d1715b..52674a44 100644 --- a/crates/alertd/src/doctor/sweep.rs +++ b/crates/alertd/src/doctor/sweep.rs @@ -148,9 +148,12 @@ pub async fn perform_sweep( .filter(|t| t.has_install) .map(|t| t.root.display().to_string()); + // Resolved once so the per-subsystem Seedling checks share a single OI query. + let seedling = checks::seedling::probe().await; let check_ctx = SweepContext { tamanu: tamanu_ctx, http_client, + seedling, }; let registry = checks::all();