From 26b79ec4f14f33b383a979dfd019f983ad39f980 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Fri, 3 Jul 2026 11:09:52 -0700 Subject: [PATCH 01/24] tufaceous v2 for wicket and installinator --- Cargo.lock | 199 +++-- Cargo.toml | 14 +- clients/wicketd-client/src/lib.rs | 2 +- common/Cargo.toml | 1 + common/src/update/install_manifest.rs | 11 + common/src/update/mupdate_override.rs | 18 +- gateway-types/versions/Cargo.toml | 2 +- gateway-types/versions/src/initial/update.rs | 5 +- gateway/Cargo.toml | 2 +- gateway/src/http_entrypoints.rs | 2 +- installinator-api/Cargo.toml | 2 +- installinator-api/src/lib.rs | 17 +- installinator-common/src/progress.rs | 28 +- installinator/Cargo.toml | 9 +- installinator/src/artifact.rs | 71 +- installinator/src/dispatch.rs | 572 +++++++------- installinator/src/fetch.rs | 58 +- installinator/src/mock_peers.rs | 13 +- installinator/src/test_helpers.rs | 12 - installinator/src/write.rs | 124 +-- ipcc/Cargo.toml | 2 +- ipcc/src/lib.rs | 2 +- openapi/wicketd.json | 81 +- update-common/Cargo.toml | 15 +- update-common/src/lib.rs | 43 ++ wicket-common/Cargo.toml | 2 +- wicket-common/src/artifact.rs | 33 + wicket-common/src/lib.rs | 1 + wicket-common/src/rack_setup.rs | 2 +- wicket-common/src/rack_update.rs | 3 +- wicket-common/src/update_events.rs | 39 +- wicket/Cargo.toml | 2 +- wicket/src/cli/rack_update.rs | 22 +- wicket/src/events.rs | 14 +- wicket/src/state/inventory.rs | 25 - wicket/src/state/mod.rs | 5 +- wicket/src/state/update.rs | 38 +- wicket/src/ui/panes/update.rs | 144 ++-- wicket/src/wicketd.rs | 17 +- wicketd-api/Cargo.toml | 1 - wicketd-api/src/lib.rs | 13 +- wicketd/Cargo.toml | 6 +- wicketd/src/artifacts/server.rs | 13 +- wicketd/src/artifacts/store.rs | 90 +-- wicketd/src/installinator_progress.rs | 2 +- wicketd/src/update_tracker.rs | 751 +++++++------------ wicketd/tests/integration_tests/updates.rs | 269 +++---- workspace-hack/Cargo.toml | 10 +- 48 files changed, 1231 insertions(+), 1576 deletions(-) create mode 100644 wicket-common/src/artifact.rs diff --git a/Cargo.lock b/Cargo.lock index 126210bd759..ecf6303997e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -931,7 +931,7 @@ dependencies = [ "sled-agent-types", "sled-agent-types-versions", "sled-hardware-types", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", ] [[package]] @@ -3082,6 +3082,15 @@ dependencies = [ "ctutils", ] +[[package]] +name = "digest-io" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2de63d600bc7fab91180bc17385f29b342468dc8ef2af09dceba450a293de3da" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "dirs-next" version = "2.0.0" @@ -4270,7 +4279,7 @@ dependencies = [ "termios", "tokio", "tokio-tungstenite 0.23.1", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "uuid", ] @@ -4407,7 +4416,7 @@ dependencies = [ "serde", "test-strategy", "thiserror 2.0.18", - "tufaceous-artifact", + "tufaceous-artifact 0.2.0", "uuid", ] @@ -5777,6 +5786,7 @@ dependencies = [ "clap", "daft", "display-error-chain", + "flate2", "futures", "hex", "hex-literal 0.4.1", @@ -5808,12 +5818,12 @@ dependencies = [ "slog-error-chain", "slog-term", "smf 0.2.3", + "tar", "test-strategy", "thiserror 2.0.18", "tokio", "tokio-stream", - "tufaceous-artifact", - "tufaceous-lib", + "tufaceous-artifact 0.2.0", "update-engine", ] @@ -5832,7 +5842,7 @@ dependencies = [ "schemars 0.8.22", "serde", "slog", - "tufaceous-artifact", + "tufaceous-artifact 0.2.0", "update-engine", "uuid", ] @@ -6005,7 +6015,7 @@ dependencies = [ "slog-error-chain", "test-strategy", "thiserror 2.0.18", - "tufaceous-artifact", + "tufaceous-artifact 0.2.0", ] [[package]] @@ -7334,7 +7344,7 @@ dependencies = [ "thiserror 2.0.18", "tokio", "trust-quorum-protocol", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "uuid", ] @@ -7432,7 +7442,7 @@ dependencies = [ "tokio", "trust-quorum-protocol", "trust-quorum-types", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "url", "usdt 0.5.0", "uuid", @@ -7496,7 +7506,7 @@ dependencies = [ "serde_json", "slog-error-chain", "trust-quorum-types", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "uuid", ] @@ -7591,7 +7601,7 @@ dependencies = [ "swrite", "thiserror 2.0.18", "tokio", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "typed-rng", "uuid", ] @@ -7725,7 +7735,7 @@ dependencies = [ "tokio-stream", "tokio-util", "trust-quorum-types", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "uuid", ] @@ -7769,7 +7779,7 @@ dependencies = [ "omicron-uuid-kinds", "omicron-workspace-hack", "sled-agent-types", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", ] [[package]] @@ -7915,8 +7925,8 @@ dependencies = [ "tokio", "toml 0.8.23", "transient-dns-server", - "tufaceous", - "tufaceous-artifact", + "tufaceous 0.1.0", + "tufaceous-artifact 0.1.0", "tufaceous-lib", "typed-rng", "update-common", @@ -7992,7 +8002,7 @@ dependencies = [ "swrite", "sync-ptr", "thiserror 2.0.18", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "typed-rng", "uuid", ] @@ -8219,7 +8229,7 @@ dependencies = [ "tough", "trust-quorum-protocol", "trust-quorum-types", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "unicode-width 0.1.14", "update-engine", "url", @@ -8259,7 +8269,7 @@ dependencies = [ "strum 0.27.2", "thiserror 2.0.18", "tough", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "url", "uuid", ] @@ -8811,7 +8821,8 @@ dependencies = [ "thiserror 2.0.18", "tokio", "toml 0.8.23", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", + "tufaceous-artifact 0.2.0", "uuid", ] @@ -8967,7 +8978,7 @@ dependencies = [ "tokio-stream", "tokio-tungstenite 0.23.1", "toml 0.8.23", - "tufaceous-artifact", + "tufaceous-artifact 0.2.0", "uuid", "zip 4.6.1", ] @@ -9235,8 +9246,8 @@ dependencies = [ "tokio-util", "tough", "trust-quorum-types", - "tufaceous", - "tufaceous-artifact", + "tufaceous 0.1.0", + "tufaceous-artifact 0.1.0", "tufaceous-lib", "update-common", "update-engine", @@ -9379,7 +9390,7 @@ dependencies = [ "textwrap 0.16.2", "tokio", "trust-quorum-types", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "unicode-width 0.1.14", "update-engine", "url", @@ -9519,7 +9530,7 @@ dependencies = [ "tar", "tokio", "toml 0.8.23", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "tufaceous-lib", "update-common", ] @@ -9558,7 +9569,7 @@ dependencies = [ "slog-error-chain", "tokio", "tokio-util", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "update-common", ] @@ -9694,8 +9705,8 @@ dependencies = [ "trust-quorum", "trust-quorum-protocol", "trust-quorum-types", - "tufaceous-artifact", - "tufaceous-brand-metadata", + "tufaceous-artifact 0.1.0", + "tufaceous-brand-metadata 0.1.0", "usdt 0.5.0", "uuid", "walkdir", @@ -9771,11 +9782,11 @@ dependencies = [ "aws-lc-sys", "base16ct 0.2.0", "base64 0.22.1", - "base64ct", "bitflags 1.3.2", "bitflags 2.11.0", "bstr", "buf-list", + "byte-wrapper", "bytes", "camino", "chacha20 0.10.0", @@ -9855,7 +9866,6 @@ dependencies = [ "once_cell", "openapiv3", "peg-runtime", - "pem-rfc7468 0.7.0", "percent-encoding", "petgraph 0.6.5", "petgraph 0.8.3", @@ -9864,7 +9874,6 @@ dependencies = [ "ppv-lite86", "predicates", "proc-macro2", - "proptest", "quote", "rand 0.8.6", "rand 0.9.2", @@ -9916,6 +9925,7 @@ dependencies = [ "toml_parser", "tough", "tracing", + "tufaceous-artifact 0.2.0", "typenum", "url", "usdt 0.6.0", @@ -12146,6 +12156,12 @@ dependencies = [ "bitflags 2.11.0", ] +[[package]] +name = "rawzip" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d9575f44c8cf85bc843ad666dcdf20d05a7753772bef56eb2a5140282b32150" + [[package]] name = "rayon" version = "1.11.0" @@ -12224,8 +12240,8 @@ dependencies = [ "tabled 0.15.0", "tokio", "toml 0.8.23", - "tufaceous", - "tufaceous-artifact", + "tufaceous 0.1.0", + "tufaceous-artifact 0.1.0", "tufaceous-lib", "update-common", "uuid", @@ -12258,7 +12274,7 @@ dependencies = [ "slog", "swrite", "tokio", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", ] [[package]] @@ -12429,7 +12445,7 @@ dependencies = [ "omicron-workspace-hack", "schemars 0.8.22", "serde", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", ] [[package]] @@ -13813,7 +13829,7 @@ dependencies = [ "sled-hardware-types", "slog-error-chain", "trust-quorum-types", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "uuid", "x509-cert", ] @@ -13897,7 +13913,7 @@ dependencies = [ "thiserror 2.0.18", "tokio", "trust-quorum-types", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "xshell", "zone", ] @@ -14048,7 +14064,7 @@ dependencies = [ "slog", "slog-error-chain", "thiserror 2.0.18", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", ] [[package]] @@ -14065,7 +14081,7 @@ dependencies = [ "sha2 0.10.9", "sled-agent-types", "sled-storage", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", ] [[package]] @@ -14095,7 +14111,7 @@ dependencies = [ "swrite", "test-strategy", "thiserror 2.0.18", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "uuid", ] @@ -14139,7 +14155,7 @@ dependencies = [ "thiserror 2.0.18", "toml 0.8.23", "trust-quorum-types-versions", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "uuid", ] @@ -14924,7 +14940,7 @@ dependencies = [ "slog-error-chain", "tokio", "tokio-util", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "uuid", "zip 4.6.1", ] @@ -15134,9 +15150,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tar" -version = "0.4.45" +version = "0.4.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" dependencies = [ "filetime", "libc", @@ -16222,7 +16238,7 @@ dependencies = [ [[package]] name = "tufaceous" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#a7d440f5a111c7e3504e8eb125c105d7baf0deab" +source = "git+https://github.com/oxidecomputer/tufaceous?branch=v1#95395cecb019b851519232468472212efce1e42d" dependencies = [ "anyhow", "camino", @@ -16236,14 +16252,48 @@ dependencies = [ "slog-envlogger", "slog-term", "tokio", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "tufaceous-lib", ] +[[package]] +name = "tufaceous" +version = "0.2.0" +source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#6e2d1803e273de403848d62781eaec37996fd2ce" +dependencies = [ + "atomicwrites", + "aws-lc-rs", + "bytes", + "camino", + "camino-tempfile", + "chrono", + "ciborium", + "daft", + "digest-io", + "flate2", + "futures-util", + "hex", + "hubtools 0.4.7 (git+https://github.com/oxidecomputer/hubtools.git?rev=2b1ef9b38d75563ea800baa3b17327eec17b1b7a)", + "rats-corim 0.1.0 (git+https://github.com/oxidecomputer/rats-corim.git?rev=f0d5d5168d3d31487a56df32c676b0c6240bcc6b)", + "rawzip", + "semver 1.0.28", + "serde", + "serde_json", + "sha2 0.11.0", + "slog", + "tar", + "thiserror 2.0.18", + "tokio", + "tough", + "tufaceous-artifact 0.2.0", + "tufaceous-brand-metadata 0.2.0", + "url", +] + [[package]] name = "tufaceous-artifact" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#a7d440f5a111c7e3504e8eb125c105d7baf0deab" +source = "git+https://github.com/oxidecomputer/tufaceous?branch=v1#95395cecb019b851519232468472212efce1e42d" dependencies = [ "daft", "hex", @@ -16257,22 +16307,49 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "tufaceous-artifact" +version = "0.2.0" +source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#6e2d1803e273de403848d62781eaec37996fd2ce" +dependencies = [ + "byte-wrapper", + "daft", + "hubtools 0.4.7 (git+https://github.com/oxidecomputer/hubtools.git?rev=2b1ef9b38d75563ea800baa3b17327eec17b1b7a)", + "proptest", + "schemars 0.8.22", + "serde", + "serde_json", + "test-strategy", + "thiserror 2.0.18", +] + [[package]] name = "tufaceous-brand-metadata" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#a7d440f5a111c7e3504e8eb125c105d7baf0deab" +source = "git+https://github.com/oxidecomputer/tufaceous?branch=v1#95395cecb019b851519232468472212efce1e42d" dependencies = [ "semver 1.0.28", "serde", "serde_json", "tar", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", +] + +[[package]] +name = "tufaceous-brand-metadata" +version = "0.2.0" +source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#6e2d1803e273de403848d62781eaec37996fd2ce" +dependencies = [ + "serde", + "serde_json", + "tar", + "tufaceous-artifact 0.2.0", ] [[package]] name = "tufaceous-lib" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#a7d440f5a111c7e3504e8eb125c105d7baf0deab" +source = "git+https://github.com/oxidecomputer/tufaceous?branch=v1#95395cecb019b851519232468472212efce1e42d" dependencies = [ "anyhow", "async-trait", @@ -16305,8 +16382,8 @@ dependencies = [ "tokio", "toml 0.8.23", "tough", - "tufaceous-artifact", - "tufaceous-brand-metadata", + "tufaceous-artifact 0.1.0", + "tufaceous-brand-metadata 0.1.0", "url", "zip 4.6.1", ] @@ -16604,9 +16681,10 @@ dependencies = [ "tokio", "tokio-util", "tough", - "tufaceous", - "tufaceous-artifact", - "tufaceous-brand-metadata", + "tufaceous 0.1.0", + "tufaceous 0.2.0", + "tufaceous-artifact 0.1.0", + "tufaceous-brand-metadata 0.1.0", "tufaceous-lib", ] @@ -17241,7 +17319,7 @@ dependencies = [ "toml 0.8.23", "toml_edit 0.22.27", "transceiver-controller 0.1.1 (git+https://github.com/oxidecomputer/transceiver-control)", - "tufaceous-artifact", + "tufaceous-artifact 0.2.0", "tui-tree-widget", "unicode-width 0.1.14", "update-engine", @@ -17277,7 +17355,7 @@ dependencies = [ "tokio", "toml 0.8.23", "transceiver-controller 0.1.1 (git+https://github.com/oxidecomputer/transceiver-control)", - "tufaceous-artifact", + "tufaceous-artifact 0.2.0", "update-engine", ] @@ -17317,6 +17395,7 @@ dependencies = [ "bytes", "camino", "camino-tempfile", + "chrono", "clap", "debug-ignore", "display-error-chain", @@ -17346,6 +17425,7 @@ dependencies = [ "internal-dns-resolver", "internal-dns-types", "itertools 0.14.0", + "lpc55_areas", "maplit", "omicron-certificates", "omicron-common", @@ -17383,9 +17463,9 @@ dependencies = [ "toml 0.8.23", "tough", "transceiver-controller 0.1.1 (git+https://github.com/oxidecomputer/transceiver-control)", - "tufaceous", - "tufaceous-artifact", - "tufaceous-lib", + "tufaceous 0.1.0", + "tufaceous 0.2.0", + "tufaceous-artifact 0.2.0", "update-common", "update-engine", "uuid", @@ -17411,7 +17491,6 @@ dependencies = [ "serde", "sled-hardware-types", "slog", - "tufaceous-artifact", "wicket-common", ] diff --git a/Cargo.toml b/Cargo.toml index f8828bf2cc6..5950d97630f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -612,6 +612,7 @@ linear-map = "1.2.0" live-tests-macros = { path = "live-tests/macros" } lldpd_client = { git = "https://github.com/oxidecomputer/lldp", package = "lldpd-client" } lldp_protocol = { git = "https://github.com/oxidecomputer/lldp", package = "protocol" } +lpc55_areas = { git = "https://github.com/oxidecomputer/lpc55_support" } omicron-deployment-graph = { path = "dev-tools/omicron-deployment-graph" } macaddr = { version = "1.0.1", features = ["serde_std"] } maplit = "1.0.2" @@ -868,10 +869,12 @@ tough = { version = "0.22.0" } transceiver-controller = { git = "https://github.com/oxidecomputer/transceiver-control", features = [ "api-traits" ] } transient-dns-server = { path = "dns-server/transient" } trybuild = "1.0.106" -tufaceous = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main" } -tufaceous-artifact = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main", features = ["proptest", "schemars"] } -tufaceous-brand-metadata = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main" } -tufaceous-lib = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main" } +tufaceous = { git = "https://github.com/oxidecomputer/tufaceous", branch = "v1" } +tufaceous-artifact = { git = "https://github.com/oxidecomputer/tufaceous", branch = "v1", features = ["proptest", "schemars"] } +tufaceous-brand-metadata = { git = "https://github.com/oxidecomputer/tufaceous", branch = "v1" } +tufaceous-lib = { git = "https://github.com/oxidecomputer/tufaceous", branch = "v1" } +tufaceous-v2 = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main", package = "tufaceous" } +tufaceous-artifact-v2 = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main", package = "tufaceous-artifact", features = ["proptest", "schemars"] } tui-tree-widget = "0.23.1" typed-rng = { path = "typed-rng" } typify = "0.3.0" @@ -1110,7 +1113,8 @@ opt-level = 3 # tufaceous-artifact = { path = "../tufaceous/artifact" } # tufaceous-brand-metadata = { path = "../tufaceous/brand-metadata" } # tufaceous-lib = { path = "../tufaceous/lib" } -# Extra slash here to pretend to be a different source. +# tufaceous-v2 = { path = "../tufaceous/lib", package = "tufaceous" } +# tufaceous-artifact-v2 = { path = "../tufaceous/artifact", package = "tufaceous-artifact" } # [patch."https://github.com/oxidecomputer/typify"] # typify = { path = "../typify/typify" } diff --git a/clients/wicketd-client/src/lib.rs b/clients/wicketd-client/src/lib.rs index 89d2f7b6e66..067c6de79d7 100644 --- a/clients/wicketd-client/src/lib.rs +++ b/clients/wicketd-client/src/lib.rs @@ -37,7 +37,7 @@ progenitor::generate_api!( replace = { AbortUpdateOptions = wicket_common::rack_update::AbortUpdateOptions, AllowedSourceIps = omicron_common::api::internal::shared::AllowedSourceIps, - ArtifactId = omicron_common::update::ArtifactId, + ArtifactId = wicket_common::artifact::ArtifactId, Baseboard = sled_hardware_types::Baseboard, BgpAuthKey = wicket_common::rack_setup::BgpAuthKey, BgpAuthKeyId = wicket_common::rack_setup::BgpAuthKeyId, diff --git a/common/Cargo.toml b/common/Cargo.toml index 391e7439dc7..9caab7b4810 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -55,6 +55,7 @@ progenitor-extras.workspace = true omicron-workspace-hack.workspace = true regress.workspace = true tufaceous-artifact.workspace = true +tufaceous-artifact-v2.workspace = true [dev-dependencies] camino-tempfile.workspace = true diff --git a/common/src/update/install_manifest.rs b/common/src/update/install_manifest.rs index 009ba6358af..1d532bdfdc6 100644 --- a/common/src/update/install_manifest.rs +++ b/common/src/update/install_manifest.rs @@ -85,6 +85,17 @@ pub struct OmicronInstallMetadata { pub hash: ArtifactHash, } +impl OmicronInstallMetadata { + // TODO(iliana): remove when cleaning up Tufaceous v1 + pub fn new_v2( + file_name: String, + file_size: u64, + hash: tufaceous_artifact_v2::ArtifactHash, + ) -> Self { + Self { file_name, file_size, hash: ArtifactHash(hash.0) } + } +} + impl IdOrdItem for OmicronInstallMetadata { type Key<'a> = &'a str; diff --git a/common/src/update/mupdate_override.rs b/common/src/update/mupdate_override.rs index bf3b6434072..d867cadfebe 100644 --- a/common/src/update/mupdate_override.rs +++ b/common/src/update/mupdate_override.rs @@ -8,7 +8,7 @@ use std::collections::BTreeSet; use omicron_uuid_kinds::MupdateOverrideUuid; use serde::{Deserialize, Serialize}; -use tufaceous_artifact::ArtifactHashId; +use tufaceous_artifact_v2::ArtifactHash; /// MUPdate override information, typically serialized as JSON (RFD 556). /// @@ -23,10 +23,24 @@ pub struct MupdateOverrideInfo { /// /// Currently includes the host phase 2 and composite control plane /// artifacts. - pub hash_ids: BTreeSet, + pub hash_ids: BTreeSet, } impl MupdateOverrideInfo { /// The name of the file on the install dataset. pub const FILE_NAME: &'static str = "mupdate-override.json"; } + +#[derive( + Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Deserialize, Serialize, +)] +/// An artifact hash written to the install dataset. This is used only for +/// debugging. +pub struct MupdateOverrideHashId { + /// The kind of artifact this is. This is not intended to be mapped to an + /// artifact in a repository. + pub kind: String, + + /// The hash of the artifact. + pub hash: ArtifactHash, +} diff --git a/gateway-types/versions/Cargo.toml b/gateway-types/versions/Cargo.toml index 9e05cfa5ecd..2a9db6621ce 100644 --- a/gateway-types/versions/Cargo.toml +++ b/gateway-types/versions/Cargo.toml @@ -22,7 +22,7 @@ schemars.workspace = true serde.workspace = true test-strategy = { workspace = true, optional = true } thiserror.workspace = true -tufaceous-artifact.workspace = true +tufaceous-artifact-v2.workspace = true uuid.workspace = true [dev-dependencies] diff --git a/gateway-types/versions/src/initial/update.rs b/gateway-types/versions/src/initial/update.rs index fd2f7a721d0..98db8cee722 100644 --- a/gateway-types/versions/src/initial/update.rs +++ b/gateway-types/versions/src/initial/update.rs @@ -8,7 +8,7 @@ use omicron_uuid_kinds::MupdateUuid; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::time::Duration; -use tufaceous_artifact::ArtifactHash; +use tufaceous_artifact_v2::ArtifactHash; use uuid::Uuid; /// The error type returned by the `sp_component_reset()` MGS endpoint. @@ -97,7 +97,9 @@ pub struct UpdatePreparationProgress { #[serde(rename_all = "snake_case")] pub struct InstallinatorImageId { pub update_id: MupdateUuid, + #[schemars(schema_with = "ArtifactHash::v1_json_schema")] pub host_phase_2: ArtifactHash, + #[schemars(schema_with = "ArtifactHash::v1_json_schema")] pub control_plane: ArtifactHash, } @@ -116,6 +118,7 @@ pub enum HostPhase2Progress { /// Identity of a host phase2 recovery image. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct HostPhase2RecoveryImageId { + #[schemars(schema_with = "ArtifactHash::v1_json_schema")] pub sha256_hash: ArtifactHash, } diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index 287a866a51d..067e5e41e1d 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -42,7 +42,7 @@ tokio = { workspace = true, features = ["full"] } tokio-stream.workspace = true tokio-tungstenite.workspace = true toml.workspace = true -tufaceous-artifact.workspace = true +tufaceous-artifact-v2.workspace = true uuid.workspace = true omicron-workspace-hack.workspace = true omicron-uuid-kinds.workspace = true diff --git a/gateway/src/http_entrypoints.rs b/gateway/src/http_entrypoints.rs index da748f721be..c648af29863 100644 --- a/gateway/src/http_entrypoints.rs +++ b/gateway/src/http_entrypoints.rs @@ -69,7 +69,7 @@ use std::io::Cursor; use std::num::NonZeroU8; use std::str; use std::sync::Arc; -use tufaceous_artifact::ArtifactHash; +use tufaceous_artifact_v2::ArtifactHash; // TODO // The gateway service will get asynchronous notifications both from directly diff --git a/installinator-api/Cargo.toml b/installinator-api/Cargo.toml index eefd3034531..14ba06d21fe 100644 --- a/installinator-api/Cargo.toml +++ b/installinator-api/Cargo.toml @@ -19,6 +19,6 @@ omicron-workspace-hack.workspace = true schemars.workspace = true serde.workspace = true slog.workspace = true -tufaceous-artifact.workspace = true +tufaceous-artifact-v2.workspace = true update-engine.workspace = true uuid.workspace = true diff --git a/installinator-api/src/lib.rs b/installinator-api/src/lib.rs index 36148a429c3..67acccdb661 100644 --- a/installinator-api/src/lib.rs +++ b/installinator-api/src/lib.rs @@ -18,7 +18,9 @@ use dropshot_api_manager_types::api_versions; use hyper::header; use installinator_common_versions::latest; use omicron_uuid_kinds::MupdateUuid; -use tufaceous_artifact::ArtifactHashId; +use schemars::JsonSchema; +use serde::Deserialize; +use tufaceous_artifact_v2::ArtifactHash; use update_engine::{NestedSpec, events::EventReport}; api_versions!([ @@ -40,7 +42,7 @@ pub trait InstallinatorApi { }] async fn get_artifact_by_hash( rqctx: RequestContext, - path: Path, + path: Path, ) -> Result>, HttpError>; /// Report progress and completion to the server. @@ -61,6 +63,17 @@ pub trait InstallinatorApi { ) -> Result; } +#[derive(Deserialize, JsonSchema)] +pub struct GetArtifactPathParams { + /// This parameter is ignored and is present only to avoid needing a new + /// API version. Recommend using "any". + pub kind: String, + + /// The hash of the artifact. + #[schemars(schema_with = "ArtifactHash::v1_json_schema")] + pub hash: ArtifactHash, +} + /// Add a content length header to a response. /// /// Intended to be called by `get_artifact_by_hash` implementations. diff --git a/installinator-common/src/progress.rs b/installinator-common/src/progress.rs index 7bd663e296d..3f5914d6a04 100644 --- a/installinator-common/src/progress.rs +++ b/installinator-common/src/progress.rs @@ -49,16 +49,15 @@ pub enum InstallinatorComponent { /// The host phase 2 component. HostPhase2, - /// The control plane component. - ControlPlane, + /// The control plane zone component. + ControlPlaneZone, /// The measurement corpus component. MeasurementCorpus, - /// A component that means "both the host and the control plane", used for - /// writes for now. It is possible that this component will go away in the - /// future. - Both, + /// A component that means "all components", used for downloads and writes + /// for now. It is possible that this component will go away in the future. + All, /// Future variants that might be unknown. #[serde(other, deserialize_with = "deserialize_ignore_any")] @@ -72,18 +71,7 @@ pub enum InstallinatorComponent { #[serde(rename_all = "snake_case")] pub enum InstallinatorStepId { Download, - Format, Scan, - // There are multiple "composite" artifacts in the tuf repository the user - // gives to wicketd: the RoT (A/B images), the host (phase1/phase2), and the - // control plane (the collection of zones). wicketd handles unpacking the - // RoT and host composite artifacts, because it needs to give pieces from - // inside them to MGS. However, it does not unpack the control plane - // artifact: only installinator needs access to the zone images inside, so - // we have an explicit step here for that unpacking. If the user uploads a - // tuf repository with a malformed control plane composite artifact, this - // step is the point at which we'd discover that and fail. - UnpackControlPlaneArtifact, Write, } @@ -126,15 +114,11 @@ pub enum InstallinatorCompletionMetadata { disks_found: usize, }, - ControlPlaneZones { - /// Number of zone images that will be installed. - zones_to_install: usize, - }, - Download { /// The address the artifact was downloaded from. address: SocketAddr, }, + Write { /// The output of the write operation. output: WriteOutput, diff --git a/installinator/Cargo.toml b/installinator/Cargo.toml index a436b105517..c871119ddd9 100644 --- a/installinator/Cargo.toml +++ b/installinator/Cargo.toml @@ -18,6 +18,7 @@ cancel-safe-futures.workspace = true clap.workspace = true daft.workspace = true display-error-chain.workspace = true +flate2.workspace = true futures.workspace = true hex.workspace = true http.workspace = true @@ -31,10 +32,11 @@ libc.workspace = true omicron-common.workspace = true omicron-ddm-admin-client.workspace = true omicron-uuid-kinds.workspace = true +omicron-workspace-hack.workspace = true oxide-tokio-rt.workspace = true reqwest.workspace = true -sha2.workspace = true serde_json.workspace = true +sha2.workspace = true sled-hardware.workspace = true sled-hardware-types.workspace = true sled-storage.workspace = true @@ -44,12 +46,11 @@ slog-envlogger.workspace = true slog-error-chain.workspace = true slog-term.workspace = true smf.workspace = true +tar.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["full"] } -tufaceous-artifact.workspace = true -tufaceous-lib.workspace = true +tufaceous-artifact-v2.workspace = true update-engine.workspace = true -omicron-workspace-hack.workspace = true [dev-dependencies] omicron-test-utils.workspace = true diff --git a/installinator/src/artifact.rs b/installinator/src/artifact.rs index 65b44153dab..bb532d39131 100644 --- a/installinator/src/artifact.rs +++ b/installinator/src/artifact.rs @@ -12,9 +12,7 @@ use installinator_common::EventReport; use ipcc::{InstallinatorImageId, Ipcc}; use omicron_uuid_kinds::MupdateUuid; use tokio::sync::mpsc; -use tufaceous_artifact::{ - ArtifactHash, ArtifactHashId, ArtifactKind, KnownArtifactKind, -}; +use tufaceous_artifact_v2::ArtifactHash; use crate::{errors::HttpError, fetch::FetchReceiver}; @@ -83,55 +81,6 @@ impl LookupId { } } -#[derive(Clone, Debug)] -pub(crate) struct MeasurementArtifact { - pub(crate) hash: ArtifactHash, - pub(crate) name: String, -} - -pub(crate) struct MeasurementArtifactHashId { - pub(crate) hash: ArtifactHashId, - pub(crate) name: String, -} - -/// The host phase 2 and control plane hashes to download. -#[derive(Clone, Debug)] -pub(crate) struct ArtifactsToDownload { - pub(crate) host_phase_2: ArtifactHash, - pub(crate) control_plane: ArtifactHash, - pub(crate) measurement_corpus: Vec, -} - -impl ArtifactsToDownload { - pub(crate) fn host_phase_2_id(&self) -> ArtifactHashId { - ArtifactHashId { - kind: ArtifactKind::HOST_PHASE_2, - hash: self.host_phase_2, - } - } - - pub(crate) fn control_plane_id(&self) -> ArtifactHashId { - ArtifactHashId { - kind: KnownArtifactKind::ControlPlane.into(), - hash: self.control_plane, - } - } - - pub(crate) fn measurement_corpus(&self) -> Vec { - self.measurement_corpus - .clone() - .into_iter() - .map(|x| MeasurementArtifactHashId { - name: x.name, - hash: ArtifactHashId { - kind: KnownArtifactKind::MeasurementCorpus.into(), - hash: x.hash, - }, - }) - .collect() - } -} - #[derive(Debug)] pub(crate) struct ArtifactClient { log: slog::Logger, @@ -185,14 +134,11 @@ impl ArtifactClient { pub(crate) async fn fetch( &self, - artifact_hash_id: ArtifactHashId, + id: ArtifactLookupId, ) -> Result<(u64, FetchReceiver), HttpError> { let artifact_bytes = self .client - .get_artifact_by_hash( - artifact_hash_id.kind.as_str(), - &artifact_hash_id.hash.to_string(), - ) + .get_artifact_by_hash(id.kind, &id.hash.to_string()) .await?; slog::debug!( @@ -241,3 +187,14 @@ impl ArtifactClient { .map(|resp| resp.into_inner()) } } + +/// Wicket prior to Tufaceous v1 looks up artifacts by both kind and hash. The +/// kind must be correct in order for an artifact to be retrieved. When the +/// Tufaceous v2 transition is complete, only the hash will be used to look up +/// artifacts, and the `kind` value at the API layer can be replaced with any +/// string. +#[derive(Debug, Clone, Copy)] +pub(crate) struct ArtifactLookupId { + pub(crate) kind: &'static str, + pub(crate) hash: ArtifactHash, +} diff --git a/installinator/src/dispatch.rs b/installinator/src/dispatch.rs index fa7376a6390..82cfc9aea15 100644 --- a/installinator/src/dispatch.rs +++ b/installinator/src/dispatch.rs @@ -3,34 +3,36 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use std::time::Duration; +use std::{collections::BTreeSet, io::Read}; -use anyhow::{Context, Result, bail}; +use anyhow::{Context, Result, bail, ensure}; use buf_list::{BufList, Cursor}; use camino::{Utf8Path, Utf8PathBuf}; use clap::{Args, Parser, Subcommand}; +use flate2::bufread::GzDecoder; use installinator_common::{ InstallinatorCompletionMetadata, InstallinatorComponent, InstallinatorSpec, InstallinatorStepId, StepContext, StepHandle, StepProgress, StepSuccess, StepWarning, UpdateEngine, }; use omicron_common::FileKv; +use omicron_common::update::MupdateOverrideHashId; use sha2::{Digest, Sha256}; +use sled_hardware::{DataLinks, is_oxide_sled}; use slog::{Drain, error, warn}; -use tufaceous_artifact::{ - ArtifactHash, ArtifactHashId, InstallinatorArtifactKind, - InstallinatorDocument, KnownArtifactKind, +use tufaceous_artifact_v2::{ + ArtifactHash, InstallinatorArtifact, InstallinatorArtifactKind, + InstallinatorDocument, }; -use tufaceous_lib::ControlPlaneZoneImages; use update_engine::StepResult; use crate::{ - ArtifactWriter, MeasurementToWrite, WriteDestination, - artifact::{ArtifactIdOpts, ArtifactsToDownload, MeasurementArtifact}, + ArtifactWriter, MeasurementToWrite, WriteDestination, ZoneToWrite, + artifact::{ArtifactIdOpts, ArtifactLookupId}, fetch::{FetchArtifactBackend, FetchedArtifact, HttpFetchBackend}, peers::{DiscoveryMechanism, LastKnownPeer}, reporter::{HttpProgressBackend, ProgressReporter, ReportProgressBackend}, }; -use sled_hardware::{DataLinks, is_oxide_sled}; /// Installinator app. #[derive(Debug, Parser)] @@ -222,46 +224,25 @@ impl InstallOpts { let engine = UpdateEngine::new(log, event_sender); - let to_download = engine + let artifacts = engine .new_step( InstallinatorComponent::InstallinatorDocument, InstallinatorStepId::Download, "Downloading installinator document", async move |cx| { - let installinator_doc_id = ArtifactHashId { - kind: KnownArtifactKind::InstallinatorDocument.into(), + let installinator_doc_id = ArtifactLookupId { + kind: "installinator_document", hash: lookup_id.document, }; let installinator_doc = fetch_artifact( &cx, - &installinator_doc_id, + installinator_doc_id, discovery, &last_known_peer, log, ) .await?; - // Check that the sha256 of the data we got from - // wicket matches the data we asked for. - // - // If this fails, we fail the entire installation - // rather than trying to fetch the artifact again, - // because we're fetching data from wicketd (cached - // in a temp dir) over TCP to ourselves (in memory), - // so the only cases where this could fail are - // disturbing enough (memory corruption, corruption - // under TCP, or wicketd gave us something other - // than what we requested) we want to know - // immediately and not retry: it's likely an - // operator could miss any warnings we emit if a - // retry succeeds. - check_downloaded_artifact_hash( - "installinator document", - installinator_doc.artifact.clone(), - installinator_doc_id.hash, - ) - .await?; - // Read the document as JSON. // // Going via the reader is slightly less efficient @@ -271,221 +252,53 @@ impl InstallOpts { let json: InstallinatorDocument = serde_json::from_reader( buf_list::Cursor::new(&installinator_doc.artifact), ) - .context( - "error deserializing \ - installinator document", - )?; - - // Every valid installinator document must have the - // host phase 2 and control plane hashes. - let mut host_phase_2_hash = None; - let mut control_plane_hash = None; - let mut measurement_corpus = vec![]; - for artifact in &json.artifacts { - match artifact.kind { - InstallinatorArtifactKind::HostPhase2 => { - host_phase_2_hash = Some(artifact.hash); - } - InstallinatorArtifactKind::ControlPlane => { - control_plane_hash = Some(artifact.hash); - } - InstallinatorArtifactKind::MeasurementCorpus => { - measurement_corpus.push(MeasurementArtifact { - name: artifact.name.clone(), - hash: artifact.hash, - }); - } - } - } - - // Return an error if either the host phase 2 or the - // control plane artifacts are missing. - let mut missing = Vec::new(); - if host_phase_2_hash.is_none() { - missing.push(InstallinatorArtifactKind::HostPhase2); - } - if control_plane_hash.is_none() { - missing.push(InstallinatorArtifactKind::ControlPlane); - } - if !missing.is_empty() { - bail!( - "installinator document missing \ - required artifacts: {:?}", - missing - ); - } - - StepSuccess::new(ArtifactsToDownload { - host_phase_2: host_phase_2_hash - .expect("host phase 2 is Some, checked above"), - control_plane: control_plane_hash - .expect("control plane is Some, checked above"), - measurement_corpus, - }) - .into() - }, - ) - .register() - .into_shared(); - - let to_download_2 = to_download.clone(); - let to_download_3 = to_download_2.clone(); - let to_download_corpus = to_download_3.clone(); - let host_phase_2_artifact = engine - .new_step( - InstallinatorComponent::HostPhase2, - InstallinatorStepId::Download, - "Downloading host phase 2 artifact", - async move |cx| { - let to_download = to_download.into_value(cx.token()).await; - let host_phase_2_id = to_download.host_phase_2_id(); - - let host_phase_2_artifact = fetch_artifact( - &cx, - &host_phase_2_id, - discovery, - last_known_peer, - log, - ) - .await?; - - // Check that the sha256 of the data we got from wicket - // matches the data we asked for. We do not retry this for - // the same reasons described above when checking the - // installinator document. - check_downloaded_artifact_hash( - "host phase 2", - host_phase_2_artifact.artifact.clone(), - host_phase_2_id.hash, - ) - .await?; - - let address = host_phase_2_artifact.peer.address(); + .context("error deserializing installinator document")?; + + // We expect there is exactly one host phase 2 image. + let host_phase_2_count = json + .artifacts + .iter() + .filter(|artifact| { + matches!( + artifact.kind, + InstallinatorArtifactKind::HostPhase2 + ) + }) + .count(); + ensure!( + host_phase_2_count == 1, + "expected a single host phase 2 \ + image but found {host_phase_2_count}" + ); - StepSuccess::new(host_phase_2_artifact) - .with_metadata( - InstallinatorCompletionMetadata::Download { - address, - }, - ) - .into() + StepSuccess::new(json.artifacts).into() }, ) .register(); - let control_plane_artifact = engine + let downloads = engine .new_step( - InstallinatorComponent::ControlPlane, + InstallinatorComponent::All, InstallinatorStepId::Download, - "Downloading control plane artifact", + "Downloading artifacts", async move |cx| { - let to_download = - to_download_2.into_value(cx.token()).await; - let control_plane_id = to_download.control_plane_id(); - - let control_plane_artifact = fetch_artifact( + download_all_artifacts( &cx, - &control_plane_id, + &log, + artifacts, discovery, last_known_peer, - log, - ) - .await?; - - // Check that the sha256 of the data we got from wicket - // matches the data we asked for. We do not retry this for - // the same reasons described above when checking the - // downloaded host phase 2 artifact. - check_downloaded_artifact_hash( - "control plane", - control_plane_artifact.artifact.clone(), - control_plane_id.hash, ) - .await?; - - let address = control_plane_artifact.peer.address(); - - StepSuccess::new(control_plane_artifact) - .with_metadata( - InstallinatorCompletionMetadata::Download { - address, - }, - ) - .into() + .await }, ) .register(); - struct MidCorpus { - name: String, - artifact: FetchedArtifact, - } - let measurement_corpus = - engine - .new_step( - InstallinatorComponent::MeasurementCorpus, - InstallinatorStepId::Download, - "Downloading all measurements", - async move |cx| { - let to_download = - to_download_corpus.into_value(cx.token()).await; - let all_corpus = to_download.measurement_corpus(); - - let mut all = vec![]; - for c in all_corpus { - cx.with_nested_engine(|engine| { - let result = engine.new_step( - InstallinatorComponent::MeasurementCorpus, - InstallinatorStepId::Download, - format!( - "Downloading single measurement {}", - c.hash.hash - ), - async move |cx2| { - let measurement_artifact = - fetch_artifact( - &cx2, &c.hash, discovery, last_known_peer, log, - ) - .await?; - - // Check that the sha256 of the data we got from wicket - // matches the data we asked for. We do not retry this for - // the same reasons described above when checking the - // downloaded host phase 2 artifact. - check_downloaded_artifact_hash( - "measurement corpus", - measurement_artifact.artifact.clone(), - c.hash.hash, - ) - .await?; - - let address = - measurement_artifact.peer.address(); - - StepSuccess::new(MidCorpus { name: c.name, artifact: measurement_artifact }) - .with_metadata( - InstallinatorCompletionMetadata::Download { - address, - }, - ) - .into() - }, - ).register(); - all.push(result); - Ok(()) - }) - .await?; - } - StepSuccess::new(all).into() - }, - ) - .register(); - let destination = if self.install_on_gimlet { let log = log.clone(); engine .new_step( - InstallinatorComponent::Both, + InstallinatorComponent::All, InstallinatorStepId::Scan, "Scanning hardware to find M.2 disks", async move |cx| scan_hardware_with_retries(&cx, &log).await, @@ -501,64 +314,76 @@ impl InstallOpts { )?) }; - let control_plane_zones = engine - .new_step( - InstallinatorComponent::ControlPlane, - InstallinatorStepId::UnpackControlPlaneArtifact, - "Unpacking composite control plane artifact", - async move |cx| { - let control_plane_artifact = - control_plane_artifact.into_value(cx.token()).await; - let zones = tokio::task::spawn_blocking(|| { - ControlPlaneZoneImages::extract(Cursor::new( - control_plane_artifact.artifact, - )) - }) - .await - .unwrap()?; - - let zones_to_install = zones.zones.len(); - StepSuccess::new(zones).with_metadata(InstallinatorCompletionMetadata::ControlPlaneZones { - zones_to_install, - }).into() - }, - ) - .register(); - engine .new_step( - InstallinatorComponent::Both, + InstallinatorComponent::All, InstallinatorStepId::Write, "Writing host, control plane, and measurement artifacts", async move |cx| { let destination = destination.into_value(cx.token()).await; - let to_download = - to_download_3.into_value(cx.token()).await; - let host_phase_2_id = to_download.host_phase_2_id(); - let control_plane_id = to_download.control_plane_id(); - let host_phase_2_artifact = - host_phase_2_artifact.into_value(cx.token()).await; - let control_plane_zones = - control_plane_zones.into_value(cx.token()).await; - let measurement_corpus = - measurement_corpus.into_value(cx.token()).await; - - let mut all_corpus = vec![]; - for c in measurement_corpus { - let result = c.into_value(cx.token()).await; - all_corpus.push(MeasurementToWrite { - artifact: result.artifact.artifact, - name: result.name, - }); + + let mut host_phase_2 = None; + let mut zones = Vec::new(); + let mut measurement_corpus = Vec::new(); + for (artifact, step) in + downloads.into_value(cx.token()).await + { + let data = step.into_value(cx.token()).await; + match &artifact.kind { + InstallinatorArtifactKind::MeasurementCorpus => { + let id = MupdateOverrideHashId { + kind: "measurement_corpus".into(), + hash: artifact.hash, + }; + measurement_corpus.push(MeasurementToWrite { + id, + name: artifact.file_name, + artifact: data, + }); + } + InstallinatorArtifactKind::HostPhase2 => { + let id = MupdateOverrideHashId { + kind: "host_phase2".into(), + hash: artifact.hash, + }; + host_phase_2 = Some((id, data)); + } + InstallinatorArtifactKind::Zone { zone_name } => { + let id = MupdateOverrideHashId { + kind: format!("zone-{zone_name}"), + hash: artifact.hash, + }; + zones.push(ZoneToWrite { + id, + file_name: artifact.file_name, + data, + }); + } + InstallinatorArtifactKind::ControlPlane => { + zones.extend( + read_legacy_control_plane_tarball(data) + .await + .context( + "failed to read control \ + plane tarball", + )?, + ); + } + } } + // We checked there is at least one host phase 2 artifact + // when we read the installinator document. + let (host_phase_2_id, host_phase_2_data) = host_phase_2 + .expect( + "already checked presence of host phase 2 artifact", + ); let mut writer = ArtifactWriter::new( lookup_id.update_id, &host_phase_2_id, - &host_phase_2_artifact.artifact, - &control_plane_id, - &control_plane_zones, - &all_corpus, + &host_phase_2_data, + &zones, + &measurement_corpus, destination, ); @@ -615,31 +440,6 @@ impl InstallOpts { } } -async fn check_downloaded_artifact_hash( - name: &'static str, - data: BufList, - expected_hash: ArtifactHash, -) -> Result<()> { - tokio::task::spawn_blocking(move || { - let mut hasher = Sha256::new(); - for chunk in data.iter() { - hasher.update(chunk); - } - let computed_hash = ArtifactHash(hasher.finalize().into()); - if expected_hash != computed_hash { - bail!( - "downloaded {name} checksum failure: \ - expected {} but calculated {}", - hex::encode(&expected_hash), - hex::encode(&computed_hash) - ); - } - Ok(()) - }) - .await - .unwrap() -} - async fn scan_hardware_with_retries( cx: &StepContext, log: &slog::Logger, @@ -695,17 +495,85 @@ async fn scan_hardware_with_retries( .into() } +async fn download_all_artifacts( + cx: &StepContext, + log: &slog::Logger, + artifacts: StepHandle, InstallinatorSpec>, + discovery: &DiscoveryMechanism, + last_known_peer: &LastKnownPeer, +) -> Result< + StepResult< + Vec<(InstallinatorArtifact, StepHandle)>, + InstallinatorSpec, + >, +> { + let artifacts = artifacts.into_value(cx.token()).await; + let mut downloads = Vec::new(); + cx.with_nested_engine(|engine| { + for artifact in artifacts { + let kind = match &artifact.kind { + InstallinatorArtifactKind::HostPhase2 => "host_phase_2", + InstallinatorArtifactKind::MeasurementCorpus => { + "measurement_corpus" + } + InstallinatorArtifactKind::Zone { .. } => "zone", + InstallinatorArtifactKind::ControlPlane => "control_plane", + }; + let id = ArtifactLookupId { kind, hash: artifact.hash }; + + let component = match &artifact.kind { + InstallinatorArtifactKind::MeasurementCorpus => { + InstallinatorComponent::MeasurementCorpus + } + InstallinatorArtifactKind::HostPhase2 => { + InstallinatorComponent::HostPhase2 + } + InstallinatorArtifactKind::Zone { .. } + | InstallinatorArtifactKind::ControlPlane => { + InstallinatorComponent::ControlPlaneZone + } + }; + let step = engine + .new_step( + component, + InstallinatorStepId::Download, + format!("Downloading {}", artifact.file_name), + async move |cx2| { + let fetched = fetch_artifact( + &cx2, + id, + discovery, + last_known_peer, + log, + ) + .await?; + StepSuccess::new(fetched.artifact) + .with_metadata( + InstallinatorCompletionMetadata::Download { + address: fetched.peer.address(), + }, + ) + .into() + }, + ) + .register(); + downloads.push((artifact, step)); + } + Ok(()) + }) + .await?; + StepSuccess::new(downloads).into() +} + async fn fetch_artifact( cx: &StepContext, - id: &ArtifactHashId, + id: ArtifactLookupId, discovery: &DiscoveryMechanism, last_known_peer: &LastKnownPeer, log: &slog::Logger, ) -> Result { - // TODO: Not sure why slog::o!("artifact" => ?id) isn't working, figure it - // out at some point. let log = log.new(slog::o!("artifact" => format!("{id:?}"))); - let artifact = FetchedArtifact::loop_fetch_from_peers( + let fetched = FetchedArtifact::loop_fetch_from_peers( cx, &log, || async { @@ -723,18 +591,48 @@ async fn fetch_artifact( id, ) .await - .with_context(|| format!("error fetching image with id {id:?}"))?; + .with_context(|| format!("error fetching artifact with id {id:?}"))?; + + // Check that the sha256 of the data we got from wicket matches the data we + // asked for. + // + // If this fails, we fail the entire installation rather than trying to + // fetch the artifact again, because we're fetching data from wicketd + // (cached in a temp dir) over TCP to ourselves (in memory), so the only + // cases where this could fail are disturbing enough (memory corruption, + // corruption under TCP, or wicketd gave us something other than what we + // requested) we want to know immediately and not retry: it's likely an + // operator could miss any warnings we emit if a retry succeeds. + let fetched = tokio::task::spawn_blocking(move || { + let mut hasher = Sha256::new(); + for chunk in &fetched.artifact { + hasher.update(&chunk); + } + let computed_hash = ArtifactHash(hasher.finalize().into()); + if id.hash != computed_hash { + bail!( + "downloaded {} checksum failure: \ + expected {} but calculated {}", + id.kind, + hex::encode(&id.hash), + hex::encode(&computed_hash) + ); + } + Ok(fetched) + }) + .await + .unwrap()?; - last_known_peer.set(artifact.peer); + last_known_peer.set(fetched.peer); slog::info!( log, "fetched {} bytes from {}", - artifact.artifact.num_bytes(), - artifact.peer, + fetched.artifact.num_bytes(), + fetched.peer, ); - Ok(artifact) + Ok(fetched) } pub(crate) fn stderr_env_drain( @@ -752,3 +650,59 @@ pub(crate) fn stderr_env_drain( } builder.build() } + +/// If the Wicket on the other end is older and using Tufaceous v1, control +/// plane zones won't be split into their own files. We have to read the +/// individual zone artifacts out of the tarball we've downloaded. +/// +/// This function can be removed at the same time the +/// `InstallinatorArtifactKind::ControlPlane` variant is removed at the end of +/// the transition. +async fn read_legacy_control_plane_tarball( + data: BufList, +) -> Result> { + // None of this is blocking I/O since it's copying and hashing memory, but + // the tarball is large enough that we should probably do this on its own + // worker thread. + tokio::task::spawn_blocking(move || { + let mut zones = Vec::new(); + let mut archive = tar::Archive::new(GzDecoder::new(Cursor::new(data))); + for entry in archive.entries()? { + let mut entry = entry?; + let path = entry.header().path()?; + if !path.starts_with("zones/") { + continue; + } + let path = Utf8PathBuf::try_from(path.into_owned())?; + let Some(file_name) = path.file_name() else { continue }; + + let mut buf = [0; 8192]; + let mut data = BufList::new(); + let mut hasher = Sha256::new(); + loop { + let n = entry + .read(&mut buf) + .with_context(|| format!("failed to read from {path}"))?; + if n == 0 { + break; + } + data.push_chunk(&buf[..n]); + hasher.update(&buf[..n]); + } + + zones.push(ZoneToWrite { + id: MupdateOverrideHashId { + // We normally write the zone name as found in oxide.json + // but this string is for debugging and reading the inner + // tarball is annoying. + kind: format!("zone-{file_name}"), + hash: ArtifactHash(hasher.finalize().into()), + }, + file_name: file_name.to_owned(), + data, + }); + } + Ok(zones) + }) + .await? +} diff --git a/installinator/src/fetch.rs b/installinator/src/fetch.rs index 5a77c3c97b9..db11dcc8e95 100644 --- a/installinator/src/fetch.rs +++ b/installinator/src/fetch.rs @@ -16,11 +16,10 @@ use installinator_common::{ InstallinatorProgressMetadata, StepContext, StepProgress, }; use tokio::{sync::mpsc, time::Instant}; -use tufaceous_artifact::ArtifactHashId; use update_engine::events::ProgressUnits; use crate::{ - artifact::ArtifactClient, + artifact::{ArtifactClient, ArtifactLookupId}, errors::{ArtifactFetchError, DiscoverPeersError, HttpError}, peers::{PeerAddress, PeerAddresses}, }; @@ -41,7 +40,7 @@ impl FetchedArtifact { cx: &StepContext, log: &slog::Logger, mut discover_fn: F, - artifact_hash_id: &ArtifactHashId, + id: ArtifactLookupId, ) -> Result where F: FnMut() -> Fut, @@ -84,7 +83,7 @@ impl FetchedArtifact { peers.len(), peers.display(), ); - match fetch_backend.fetch_artifact(&cx, artifact_hash_id).await { + match fetch_backend.fetch_artifact(&cx, id).await { Some((peer, artifact)) => { return Ok(Self { attempt, peer, artifact }); } @@ -144,15 +143,13 @@ impl FetchArtifactBackend { pub(crate) async fn fetch_artifact( &self, cx: &StepContext, - artifact_hash_id: &ArtifactHashId, + id: ArtifactLookupId, ) -> Option<(PeerAddress, BufList)> { // TODO: do we want a check phase that happens before the download? let peers = self.peers(); let mut remaining_peers = peers.len(); - let log = self.log.new( - slog::o!("artifact_hash_id" => format!("{artifact_hash_id:?}")), - ); + let log = self.log.new(slog::o!("id" => format!("{id:?}"))); slog::debug!(log, "start fetch from peers"; "remaining_peers" => remaining_peers); @@ -166,7 +163,7 @@ impl FetchArtifactBackend { // Attempt to download data from this peer. let start = Instant::now(); - match self.fetch_from_peer(cx, peer, artifact_hash_id).await { + match self.fetch_from_peer(cx, peer, id).await { Ok(artifact_bytes) => { let elapsed = start.elapsed(); slog::info!( @@ -198,30 +195,27 @@ impl FetchArtifactBackend { &self, cx: &StepContext, peer: PeerAddress, - artifact_hash_id: &ArtifactHashId, + id: ArtifactLookupId, ) -> Result { let log = self.log.new(slog::o!("peer" => peer.to_string())); - let (total_bytes, mut receiver) = match self - .imp - .fetch_from_peer_impl(peer, artifact_hash_id.clone()) - .await - { - Ok(x) => x, - Err(error) => { - cx.send_progress(StepProgress::Reset { - metadata: InstallinatorProgressMetadata::Download { + let (total_bytes, mut receiver) = + match self.imp.fetch_from_peer_impl(peer, id).await { + Ok(x) => x, + Err(error) => { + cx.send_progress(StepProgress::Reset { + metadata: InstallinatorProgressMetadata::Download { + peer: peer.address(), + }, + message: error.to_string().into(), + }) + .await; + return Err(ArtifactFetchError::HttpError { peer: peer.address(), - }, - message: error.to_string().into(), - }) - .await; - return Err(ArtifactFetchError::HttpError { - peer: peer.address(), - error, - }); - } - }; + error, + }); + } + }; let mut artifact_bytes = BufList::new(); let mut downloaded_bytes = 0u64; @@ -319,7 +313,7 @@ pub(crate) trait FetchArtifactImpl: fmt::Debug + Send + Sync { async fn fetch_from_peer_impl( &self, peer: PeerAddress, - artifact_hash_id: ArtifactHashId, + id: ArtifactLookupId, ) -> Result<(u64, FetchReceiver), HttpError>; } @@ -351,10 +345,10 @@ impl FetchArtifactImpl for HttpFetchBackend { async fn fetch_from_peer_impl( &self, peer: PeerAddress, - artifact_hash_id: ArtifactHashId, + id: ArtifactLookupId, ) -> Result<(u64, FetchReceiver), HttpError> { // TODO: be able to fetch from sled-agent clients as well let artifact_client = ArtifactClient::new(peer.address(), &self.log); - artifact_client.fetch(artifact_hash_id).await + artifact_client.fetch(id).await } } diff --git a/installinator/src/mock_peers.rs b/installinator/src/mock_peers.rs index fd14dab3030..5c4d6f6e86f 100644 --- a/installinator/src/mock_peers.rs +++ b/installinator/src/mock_peers.rs @@ -24,10 +24,10 @@ use proptest::{collection::vec_deque, prelude::*}; use reqwest::StatusCode; use test_strategy::Arbitrary; use tokio::sync::mpsc; -use tufaceous_artifact::ArtifactHashId; use update_engine::events::StepEventIsTerminal; use crate::{ + artifact::ArtifactLookupId, errors::{DiscoverPeersError, HttpError}, fetch::{FetchArtifactImpl, FetchReceiver}, peers::{PeerAddress, PeerAddresses}, @@ -252,7 +252,7 @@ impl FetchArtifactImpl for MockFetchBackend { &self, peer: PeerAddress, // We don't (yet) use the artifact ID in MockPeers - _artifact_hash_id: ArtifactHashId, + _hash: ArtifactLookupId, ) -> Result<(u64, FetchReceiver), HttpError> { let peer_data = self .get(peer) @@ -690,7 +690,7 @@ mod tests { errors::DiscoverPeersError, fetch::{FetchArtifactBackend, FetchedArtifact}, reporter::{ProgressReporter, ReportProgressBackend}, - test_helpers::{dummy_artifact_hash_id, with_test_runtime}, + test_helpers::with_test_runtime, }; use bytes::Buf; @@ -703,7 +703,7 @@ mod tests { use omicron_test_utils::dev::test_setup_log; use test_strategy::proptest; use tokio_stream::wrappers::UnboundedReceiverStream; - use tufaceous_artifact::KnownArtifactKind; + use tufaceous_artifact_v2::ArtifactHash; // The #[proptest] macro doesn't currently with with #[tokio::test] sadly. #[proptest] @@ -846,7 +846,10 @@ mod tests { anyhow::anyhow!("ran out of attempts"), )), }, - &dummy_artifact_hash_id(KnownArtifactKind::ControlPlane), + ArtifactLookupId { + kind: "test-control-plane", + hash: ArtifactHash([0; 32]), + }, ) .await } diff --git a/installinator/src/test_helpers.rs b/installinator/src/test_helpers.rs index d49827dbc8d..6c61ccebdbe 100644 --- a/installinator/src/test_helpers.rs +++ b/installinator/src/test_helpers.rs @@ -3,18 +3,6 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use futures::Future; -use tufaceous_artifact::{ArtifactHash, ArtifactHashId, KnownArtifactKind}; - -pub(crate) fn dummy_artifact_hash_id( - kind: KnownArtifactKind, -) -> ArtifactHashId { - ArtifactHashId { - kind: kind.into(), - hash: ArtifactHash( - hex_literal::hex!("b5bb9d8014a0f9b1d61e21e796d78dcc" "df1352f23cd32812f4850b878ae4944c"), - ), - } -} pub(crate) fn with_test_runtime(fut: Fut) -> T where diff --git a/installinator/src/write.rs b/installinator/src/write.rs index 1d00899907d..afa5a992a0c 100644 --- a/installinator/src/write.rs +++ b/installinator/src/write.rs @@ -12,7 +12,7 @@ use std::{ use anyhow::{Context, Result, anyhow, ensure}; use async_trait::async_trait; use buf_list::BufList; -use bytes::{Buf, Bytes}; +use bytes::Buf; use camino::{Utf8Path, Utf8PathBuf}; use iddqd::IdOrdMap; use illumos_utils::zpool::{Zpool, ZpoolName}; @@ -24,7 +24,7 @@ use installinator_common::{ use omicron_common::{ disk::M2Slot, update::{ - MupdateOverrideInfo, OmicronInstallManifest, + MupdateOverrideHashId, MupdateOverrideInfo, OmicronInstallManifest, OmicronInstallManifestSource, OmicronInstallMetadata, }, }; @@ -38,8 +38,7 @@ use tokio::{ io::{AsyncWrite, AsyncWriteExt}, task::JoinSet, }; -use tufaceous_artifact::{ArtifactHash, ArtifactHashId}; -use tufaceous_lib::ControlPlaneZoneImages; +use tufaceous_artifact_v2::ArtifactHash; use update_engine::{ StepSpec, errors::NestedEngineError, events::ProgressUnits, }; @@ -228,13 +227,10 @@ pub(crate) struct ArtifactWriter<'a> { impl<'a> ArtifactWriter<'a> { pub(crate) fn new( mupdate_id: MupdateUuid, - host_phase_2_id: &'a ArtifactHashId, + host_phase_2_id: &'a MupdateOverrideHashId, host_phase_2_data: &'a BufList, - control_plane_id: &'a ArtifactHashId, - control_plane_zones: &'a ControlPlaneZoneImages, - + control_plane_zones: &'a [ZoneToWrite], measurement_corpus: &'a [MeasurementToWrite], - destination: WriteDestination, ) -> Self { let drives = destination @@ -256,7 +252,6 @@ impl<'a> ArtifactWriter<'a> { artifacts: ArtifactsToWrite { host_phase_2_id, host_phase_2_data, - control_plane_id, control_plane_zones, mupdate_id, mupdate_override_uuid, @@ -567,10 +562,9 @@ impl SlotWriteContext<'_> { #[derive(Copy, Clone)] struct ArtifactsToWrite<'a> { - host_phase_2_id: &'a ArtifactHashId, + host_phase_2_id: &'a MupdateOverrideHashId, host_phase_2_data: &'a BufList, - control_plane_id: &'a ArtifactHashId, - control_plane_zones: &'a ControlPlaneZoneImages, + control_plane_zones: &'a [ZoneToWrite], mupdate_id: MupdateUuid, mupdate_override_uuid: MupdateOverrideUuid, measurement_corpus: &'a [MeasurementToWrite], @@ -625,7 +619,6 @@ impl ArtifactsToWrite<'_> { measurement_directory: &destinations.measurement_dir, zones: self.control_plane_zones, host_phase_2_id: self.host_phase_2_id, - control_plane_id: self.control_plane_id, measurement_corpus: self.measurement_corpus, mupdate_id: self.mupdate_id, mupdate_override_uuid: self.mupdate_override_uuid, @@ -639,25 +632,28 @@ impl ArtifactsToWrite<'_> { Ok(()) }) .await - .map_err(|error| { - warn!( - log, "{error:?}"; - "artifact_id" => ?self.control_plane_id, - ); - error + .inspect_err(|error| { + warn!(log, "{error:?}"); })?; info!( log, "finished writing {} control plane zones", - self.control_plane_zones.zones.len() + self.control_plane_zones.len() ); StepSuccess::new(()).into() } } +pub(crate) struct ZoneToWrite { + pub(crate) id: MupdateOverrideHashId, + pub(crate) file_name: String, + pub(crate) data: BufList, +} + pub(crate) struct MeasurementToWrite { + pub(crate) id: MupdateOverrideHashId, pub(crate) name: String, pub(crate) artifact: BufList, } @@ -667,9 +663,8 @@ struct ControlPlaneZoneWriteContext<'a> { clean_output_directory: bool, output_directory: &'a Utf8Path, measurement_directory: &'a Utf8Path, - zones: &'a ControlPlaneZoneImages, - host_phase_2_id: &'a ArtifactHashId, - control_plane_id: &'a ArtifactHashId, + zones: &'a [ZoneToWrite], + host_phase_2_id: &'a MupdateOverrideHashId, measurement_corpus: &'a [MeasurementToWrite], mupdate_id: MupdateUuid, mupdate_override_uuid: MupdateOverrideUuid, @@ -793,19 +788,19 @@ impl ControlPlaneZoneWriteContext<'_> { ) .register(); - for (name, data) in &self.zones.zones { - let out_path = self.output_directory.join(name); + for ZoneToWrite { file_name, data, .. } in self.zones { + let out_path = self.output_directory.join(file_name); transport = engine .new_step( WriteComponent::ControlPlane, - ControlPlaneZonesStepId::Zone { name: name.clone() }, - format!("Writing zone {name}"), + ControlPlaneZonesStepId::Zone { name: file_name.clone() }, + format!("Writing zone {file_name}"), async move |cx| { let transport = transport.into_value(cx.token()).await; write_artifact_impl( WriteComponent::ControlPlane, slot, - data.clone().into(), + data.clone(), &out_path, transport, &cx, @@ -931,10 +926,10 @@ impl ControlPlaneZoneWriteContext<'_> { &self, mupdate_override_uuid: MupdateOverrideUuid, ) -> BufList { - let hash_ids = - [self.host_phase_2_id.clone(), self.control_plane_id.clone()] - .into_iter() - .collect(); + let mut hash_ids = BTreeSet::new(); + hash_ids.insert(self.host_phase_2_id.clone()); + hash_ids.extend(self.zones.iter().map(|z| z.id.clone())); + hash_ids.extend(self.measurement_corpus.iter().map(|z| z.id.clone())); let mupdate_override = MupdateOverrideInfo { mupdate_uuid: mupdate_override_uuid, @@ -995,11 +990,11 @@ async fn compute_measurement_hashes( hasher.update(&d); } let hash = hasher.finalize(); - OmicronInstallMetadata { + OmicronInstallMetadata::new_v2( file_name, - file_size: u64::try_from(data.num_bytes()).unwrap(), - hash: ArtifactHash(hash.into()), - } + u64::try_from(data.num_bytes()).unwrap(), + ArtifactHash(hash.into()), + ) }); } @@ -1022,23 +1017,27 @@ async fn compute_measurement_hashes( /// /// Panics if the runtime shuts down causing a task abort, or a task panics. async fn compute_zone_hashes( - images: &ControlPlaneZoneImages, + images: &[ZoneToWrite], ) -> IdOrdMap { let mut tasks = JoinSet::new(); - for (file_name, data) in &images.zones { + for ZoneToWrite { file_name, data, .. } in images { let file_name = file_name.clone(); - // data is a Bytes so is cheap to clone. - let data: Bytes = data.clone(); + // data is a BufList so is relatively cheap to clone. + let data: BufList = data.clone(); // Compute hashes in parallel. tasks.spawn_blocking(move || { + let mut len = 0; let mut hasher = Sha256::new(); - hasher.update(&data); + for buf in data { + len += buf.len(); + hasher.update(&buf); + } let hash = hasher.finalize(); - OmicronInstallMetadata { + OmicronInstallMetadata::new_v2( file_name, - file_size: u64::try_from(data.len()).unwrap(), - hash: ArtifactHash(hash.into()), - } + u64::try_from(len).unwrap(), + ArtifactHash(hash.into()), + ) }); } @@ -1269,7 +1268,7 @@ mod tests { use std::{collections::VecDeque, sync::Arc}; use super::*; - use crate::test_helpers::{dummy_artifact_hash_id, with_test_runtime}; + use crate::test_helpers::with_test_runtime; use anyhow::Result; use bytes::{Buf, Bytes}; @@ -1292,7 +1291,6 @@ mod tests { use tokio::io::AsyncReadExt; use tokio::sync::Mutex; use tokio_stream::wrappers::ReceiverStream; - use tufaceous_artifact::{ArtifactKind, KnownArtifactKind}; #[proptest(ProptestConfig { cases: 32, ..ProptestConfig::default() })] fn proptest_write_artifact( @@ -1403,8 +1401,8 @@ mod tests { let artifact_measurement_corpus: BufList = data3.into_iter().map(Bytes::from).collect(); - let host_id = ArtifactHashId { - kind: ArtifactKind::HOST_PHASE_2, + let host_id = MupdateOverrideHashId { + kind: "host_phase2".into(), hash: { // The `validate_written_host_phase_2_hash()` will fail unless // we give the actual hash of the host phase 2 data, so compute @@ -1419,8 +1417,6 @@ mod tests { ArtifactHash(hasher.finalize().into()) }, }; - let control_plane_id = - dummy_artifact_hash_id(KnownArtifactKind::ControlPlane); // XXX: note we don't assert on the number of attempts it took to write // just the host image at the moment. @@ -1465,14 +1461,23 @@ mod tests { // Assemble our one control plane artifact into a 1-long list of zone // images. - let control_plane_zone_images = ControlPlaneZoneImages { - zones: vec![( - destination_control_plane.file_name().unwrap().to_string(), - artifact_control_plane.iter().flatten().copied().collect(), - )], - }; + let control_plane_zone_images = vec![ZoneToWrite { + id: MupdateOverrideHashId { + kind: "zone".into(), + hash: ArtifactHash([0; 32]), + }, + file_name: destination_control_plane + .file_name() + .unwrap() + .to_string(), + data: artifact_control_plane.clone(), + }]; let all_corpus = vec![MeasurementToWrite { + id: MupdateOverrideHashId { + kind: "measurement_corpus".into(), + hash: ArtifactHash([0; 32]), + }, artifact: artifact_measurement_corpus, name: "blah".to_string(), }]; @@ -1480,7 +1485,6 @@ mod tests { MupdateUuid::new_v4(), &host_id, &artifact_host, - &control_plane_id, &control_plane_zone_images, &all_corpus, destination, @@ -1490,7 +1494,7 @@ mod tests { let log = logctx.log.clone(); engine .new_step( - InstallinatorComponent::Both, + InstallinatorComponent::All, InstallinatorStepId::Write, "Writing", async move |cx| { diff --git a/ipcc/Cargo.toml b/ipcc/Cargo.toml index cc2737e312e..3674429acb4 100644 --- a/ipcc/Cargo.toml +++ b/ipcc/Cargo.toml @@ -12,7 +12,7 @@ ciborium.workspace = true omicron-common.workspace = true omicron-uuid-kinds.workspace = true serde.workspace = true -tufaceous-artifact.workspace = true +tufaceous-artifact-v2.workspace = true thiserror.workspace = true omicron-workspace-hack.workspace = true libipcc.workspace = true diff --git a/ipcc/src/lib.rs b/ipcc/src/lib.rs index 6cbcdc23516..7edad25ec6e 100644 --- a/ipcc/src/lib.rs +++ b/ipcc/src/lib.rs @@ -13,7 +13,7 @@ use serde::Deserialize; use serde::Serialize; use slog_error_chain::InlineErrorChain; use thiserror::Error; -use tufaceous_artifact::ArtifactHash; +use tufaceous_artifact_v2::ArtifactHash; #[cfg(test)] use proptest::arbitrary::any; diff --git a/openapi/wicketd.json b/openapi/wicketd.json index aafb3164430..cc1f7399046 100644 --- a/openapi/wicketd.json +++ b/openapi/wicketd.json @@ -920,48 +920,29 @@ "media_lane_count" ] }, - "ArtifactHashId": { - "description": "A hash-based identifier for an artifact or deployment unit: the kind and hash.", - "type": "object", - "properties": { - "hash": { - "description": "The hash of the artifact.", - "type": "string", - "format": "hex string (32 bytes)" - }, - "kind": { - "description": "The kind of artifact this is.", - "type": "string" - } - }, - "required": [ - "hash", - "kind" - ] - }, "ArtifactId": { - "description": "An identifier for an artifact.", "type": "object", "properties": { - "kind": { - "description": "The kind of artifact this is.", - "type": "string" - }, - "name": { - "description": "The artifact's name.", - "type": "string" + "tags": { + "type": "object", + "additionalProperties": { + "type": "string" + } }, "version": { - "description": "The artifact's version.", - "type": "string" + "$ref": "#/components/schemas/ArtifactVersion" } }, "required": [ - "kind", - "name", + "tags", "version" ] }, + "ArtifactVersion": { + "description": "An artifact version.\n\nThis is a freeform identifier with some basic validation. It may be the serialized form of a semver version, or a custom identifier that uses the same character set as a semver, plus `_`.\n\nThe exact pattern accepted is `^[a-zA-Z0-9._+-]{1,63}$`.\n\n# Ord implementation\n\n`ArtifactVersion`s are not intended to be sorted, just compared for equality. `ArtifactVersion` implements `Ord` only for storage within sorted collections.", + "type": "string", + "pattern": "^[a-zA-Z0-9._+-]{1,63}$" + }, "Aux1Monitor": { "description": "The first auxlliary CMIS monitor.", "oneOf": [ @@ -1229,8 +1210,15 @@ }, "sha256": { "description": "A SHA-256 digest of the key.", + "x-rust-type": { + "crate": "byte-wrapper", + "path": "byte_wrapper::HexArray::<32>", + "version": "0.1.0" + }, "type": "string", - "format": "hex string (32 bytes)" + "pattern": "^[0-9a-fA-F]{64}$", + "minLength": 64, + "maxLength": 64 } }, "required": [ @@ -1968,7 +1956,7 @@ "description": "Map of artifacts we ingested from the most-recently-uploaded TUF repository to a list of artifacts we're serving over the bootstrap network. In some cases the list of artifacts being served will have length 1 (when we're serving the artifact directly); in other cases the artifact in the TUF repo contains multiple nested artifacts inside it (e.g., RoT artifacts contain both A and B images), and we serve the list of extracted artifacts but not the original combination.\n\nConceptually, this is a `BTreeMap>`, but JSON requires string keys for maps, so we give back a vec of pairs instead.", "type": "array", "items": { - "$ref": "#/components/schemas/InstallableArtifacts" + "$ref": "#/components/schemas/ArtifactId" } }, "event_reports": { @@ -2138,33 +2126,6 @@ } } }, - "InstallableArtifacts": { - "type": "object", - "properties": { - "artifact_id": { - "$ref": "#/components/schemas/ArtifactId" - }, - "installable": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ArtifactHashId" - } - }, - "sign": { - "nullable": true, - "type": "array", - "items": { - "type": "integer", - "format": "uint8", - "minimum": 0 - } - } - }, - "required": [ - "artifact_id", - "installable" - ] - }, "IpNet": { "x-rust-type": { "crate": "oxnet", diff --git a/update-common/Cargo.toml b/update-common/Cargo.toml index 89e08d816fa..a736f5ee75e 100644 --- a/update-common/Cargo.toml +++ b/update-common/Cargo.toml @@ -16,24 +16,25 @@ chrono.workspace = true debug-ignore.workspace = true display-error-chain.workspace = true dropshot.workspace = true +flate2.workspace = true +fs-err = { workspace = true, features = ["tokio"] } futures.workspace = true hex.workspace = true hubtools.workspace = true omicron-common.workspace = true +omicron-workspace-hack.workspace = true +semver.workspace = true sha2.workspace = true slog.workspace = true +tar.workspace = true thiserror.workspace = true tokio.workspace = true tokio-util.workspace = true tough.workspace = true -tufaceous-lib.workspace = true -omicron-workspace-hack.workspace = true -tufaceous-brand-metadata.workspace = true -tar.workspace = true -flate2.workspace = true -fs-err = { workspace = true, features = ["tokio"] } -semver.workspace = true tufaceous-artifact.workspace = true +tufaceous-brand-metadata.workspace = true +tufaceous-lib.workspace = true +tufaceous-v2.workspace = true [dev-dependencies] clap.workspace = true diff --git a/update-common/src/lib.rs b/update-common/src/lib.rs index b1f0d884840..be5f3651d00 100644 --- a/update-common/src/lib.rs +++ b/update-common/src/lib.rs @@ -6,3 +6,46 @@ pub mod artifacts; pub mod errors; + +use std::error::Error as _; + +use display_error_chain::DisplayErrorChain; +use dropshot::HttpError; +use tufaceous_v2::error::Error; +use tufaceous_v2::error::ErrorKind; + +pub trait ErrorExt { + fn to_http_error(&self) -> HttpError; +} + +impl ErrorExt for Error { + fn to_http_error(&self) -> HttpError { + self.0.to_http_error() + } +} + +impl ErrorExt for ErrorKind { + fn to_http_error(&self) -> HttpError { + // `HttpError` from reading a stream is bubbled up, if possible. + if let Some(source) = self.source() + && let Some(error) = source.downcast_ref::() + { + return HttpError { + status_code: error.status_code, + error_code: error.error_code.clone(), + external_message: error.external_message.clone(), + internal_message: error.internal_message.clone(), + headers: error.headers.clone(), + }; + } + + let message = DisplayErrorChain::new(self).to_string(); + if self.is_repository_error() { + // This error is definitely because of bad repository contents. + HttpError::for_bad_request(None, message) + } else { + // This error is probably not due to bad repository contents. + HttpError::for_unavail(None, message) + } + } +} diff --git a/wicket-common/Cargo.toml b/wicket-common/Cargo.toml index b1283e585bd..dcb22a7f623 100644 --- a/wicket-common/Cargo.toml +++ b/wicket-common/Cargo.toml @@ -30,7 +30,7 @@ slog-error-chain.workspace = true thiserror.workspace = true tokio.workspace = true transceiver-controller.workspace = true -tufaceous-artifact.workspace = true +tufaceous-artifact-v2.workspace = true update-engine.workspace = true [dev-dependencies] diff --git a/wicket-common/src/artifact.rs b/wicket-common/src/artifact.rs new file mode 100644 index 00000000000..5fb928f3a12 --- /dev/null +++ b/wicket-common/src/artifact.rs @@ -0,0 +1,33 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeMap; + +use schemars::JsonSchema; +use serde::Deserialize; +use serde::Serialize; +use tufaceous_artifact_v2::Artifact; +use tufaceous_artifact_v2::ArtifactVersion; + +#[derive( + Debug, + Clone, + PartialEq, + Eq, + PartialOrd, + Ord, + Deserialize, + Serialize, + JsonSchema, +)] +pub struct ArtifactId { + pub tags: BTreeMap, + pub version: ArtifactVersion, +} + +impl ArtifactId { + pub fn new(artifact: &Artifact) -> Self { + Self { tags: artifact.tags.clone(), version: artifact.version.clone() } + } +} diff --git a/wicket-common/src/lib.rs b/wicket-common/src/lib.rs index 70fa2cd5e6e..0dfe996ec6b 100644 --- a/wicket-common/src/lib.rs +++ b/wicket-common/src/lib.rs @@ -4,6 +4,7 @@ use std::time::Duration; +pub mod artifact; pub mod example; pub mod inventory; pub mod multirack_setup; diff --git a/wicket-common/src/rack_setup.rs b/wicket-common/src/rack_setup.rs index 6e968435c6b..2c753a6bf12 100644 --- a/wicket-common/src/rack_setup.rs +++ b/wicket-common/src/rack_setup.rs @@ -36,7 +36,7 @@ use std::fmt; use std::net::IpAddr; use std::net::Ipv6Addr; use std::str::FromStr; -use tufaceous_artifact::ArtifactHash; +use tufaceous_artifact_v2::ArtifactHash; use crate::inventory::SpIdentifier; diff --git a/wicket-common/src/rack_update.rs b/wicket-common/src/rack_update.rs index ee21ae10f52..521dfa7a7fc 100644 --- a/wicket-common/src/rack_update.rs +++ b/wicket-common/src/rack_update.rs @@ -7,11 +7,10 @@ use std::{collections::BTreeSet, time::Duration}; use semver::Version; use dropshot::HttpError; -use omicron_common::update::ArtifactId; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::inventory::SpIdentifier; +use crate::{artifact::ArtifactId, inventory::SpIdentifier}; #[derive(Clone, Debug, Default, JsonSchema, Deserialize, Serialize)] pub struct StartUpdateOptions { diff --git a/wicket-common/src/update_events.rs b/wicket-common/src/update_events.rs index 73b78b775e1..cb2ccde874a 100644 --- a/wicket-common/src/update_events.rs +++ b/wicket-common/src/update_events.rs @@ -3,16 +3,18 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use gateway_client::types::PowerState; -use omicron_common::update::ArtifactId; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; use std::fmt; use std::sync::Arc; use thiserror::Error; +use tufaceous_artifact_v2::DisplayTags; use update_engine::StepSpec; use update_engine::errors::NestedEngineError; +use crate::artifact::ArtifactId; + #[derive(JsonSchema)] pub enum WicketdEngineSpec {} @@ -162,18 +164,13 @@ pub enum UpdateTerminalError { #[source] error: gateway_client::Error, }, - #[error("getting RoT CMPA failed")] - GetRotCmpaFailed { + #[error("getting RoT bootloader caboose failed")] + GetRotBootloaderCabooseFailed { #[source] - error: anyhow::Error, - }, - #[error("getting RoT CFPA failed")] - GetRotCfpaFailed { - #[source] - error: anyhow::Error, + error: gateway_client::Error, }, - #[error("failed to find correctly-signed RoT image")] - FailedFindingSignedRotImage { + #[error("getting RoT CMPA failed")] + GetRotCmpaFailed { #[source] error: anyhow::Error, }, @@ -182,8 +179,12 @@ pub enum UpdateTerminalError { #[source] error: gateway_client::Error, }, - #[error("TUF repository missing SP image for board {board}")] - MissingSpImageForBoard { board: String }, + #[error("failed to find artifact matching {tags}")] + MissingArtifact { + tags: tufaceous_artifact_v2::KnownArtifactTags, + #[source] + error: tufaceous_artifact_v2::artifact_set::GetError, + }, #[error("setting installinator image ID failed")] SetInstallinatorImageIdFailed { #[source] @@ -239,8 +240,9 @@ impl update_engine::AsError for UpdateTerminalError { #[derive(Debug, Error)] pub enum SpComponentUpdateTerminalError { #[error( - "SP component update failed at stage \"{stage}\" for {}", - display_artifact_id(.artifact) + "SP component update failed at stage \"{stage}\" for {} version {}", + DisplayTags::from(&.artifact.tags), + .artifact.version, )] SpComponentUpdateFailed { stage: SpComponentUpdateStage, @@ -293,13 +295,6 @@ impl update_engine::AsError for SpComponentUpdateTerminalError { } } -fn display_artifact_id(artifact: &ArtifactId) -> String { - format!( - "{}:{} (version {})", - artifact.kind, artifact.name, artifact.version - ) -} - #[derive( Clone, Copy, Debug, Eq, PartialEq, JsonSchema, Deserialize, Serialize, )] diff --git a/wicket/Cargo.toml b/wicket/Cargo.toml index 2ad8b48245a..060776e4105 100644 --- a/wicket/Cargo.toml +++ b/wicket/Cargo.toml @@ -54,7 +54,7 @@ wicket-common.workspace = true wicketd-client.workspace = true omicron-workspace-hack.workspace = true semver.workspace = true -tufaceous-artifact.workspace = true +tufaceous-artifact-v2.workspace = true slog-error-chain.workspace = true [dev-dependencies] diff --git a/wicket/src/cli/rack_update.rs b/wicket/src/cli/rack_update.rs index afc1847a9c7..e6803cd8ed8 100644 --- a/wicket/src/cli/rack_update.rs +++ b/wicket/src/cli/rack_update.rs @@ -25,9 +25,9 @@ use crate::{ use anyhow::{Context, Result, anyhow, bail}; use camino::Utf8PathBuf; use clap::{Args, Subcommand, ValueEnum}; -use omicron_common::update::ArtifactId; use slog::Logger; use tokio::{sync::watch, task::JoinHandle}; +use tufaceous_artifact_v2::DisplayTags; use update_engine::{ AbortReason, EventBuffer, ExecutionStatus, FailureReason, NestedError, StepKey, StepSpec, TerminalKind, @@ -419,13 +419,8 @@ fn build_rack_update_status( response: GetArtifactsAndEventReportsResponse, selector: &ComponentIdSelector, ) -> Result { - let artifacts: Vec = response - .artifacts - .into_iter() - .map(|a| a.artifact_id) - .collect::>() - .into_iter() - .collect(); + let mut artifacts = response.artifacts; + artifacts.sort(); let event_reports = parse_event_report_map(log, response.event_reports); @@ -528,11 +523,10 @@ fn write_status_table( out: &mut dyn Write, status: &RackUpdateStatus, ) -> Result<()> { - #[derive(tabled::Tabled)] + #[derive(PartialEq, Eq, PartialOrd, Ord, tabled::Tabled)] #[tabled(rename_all = "UPPERCASE")] struct ArtifactRow { - name: String, - kind: String, + tags: String, version: String, } @@ -560,15 +554,15 @@ fn write_status_table( .unwrap_or("(none)") )?; - let artifact_rows: Vec = status + let mut artifact_rows: Vec = status .artifacts .iter() .map(|a| ArtifactRow { - name: a.name.clone(), - kind: a.kind.to_string(), + tags: DisplayTags::from(&a.tags).to_string(), version: a.version.to_string(), }) .collect(); + artifact_rows.sort_unstable(); let artifact_table = tabled::Table::new(artifact_rows) .with(tabled::settings::Style::empty()) .with(tabled::settings::Padding::new(0, 2, 0, 0)) diff --git a/wicket/src/events.rs b/wicket/src/events.rs index fd58bf28904..c3fce6bd2d8 100644 --- a/wicket/src/events.rs +++ b/wicket/src/events.rs @@ -1,15 +1,16 @@ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. + use crate::{State, keymap::Cmd, state::ComponentId}; use camino::Utf8PathBuf; use humantime::format_rfc3339; -use omicron_common::update::ArtifactId; use semver::Version; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::fs::File; use std::time::SystemTime; +use wicket_common::artifact::ArtifactId; use wicket_common::inventory::RackV1Inventory; use wicket_common::update_events::EventReport; use wicketd_client::types::{ @@ -20,15 +21,6 @@ use wicketd_client::types::{ /// Event report type returned by the get_artifacts_and_event_reports API call. pub type EventReportMap = HashMap>; -/// Represents an artifact from a TUF repo -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ArtifactData { - /// Artifact ID - pub id: ArtifactId, - /// Optional sign information (currently only used for RoT images) - pub sign: Option>, -} - /// An event that will update state /// /// This can be a keypress, mouse event, or response from a downstream service. @@ -43,7 +35,7 @@ pub enum Event { /// TUF repo artifacts unpacked by wicketd, and event reports ArtifactsAndEventReports { system_version: Option, - artifacts: Vec, + artifacts: Vec, event_reports: EventReportMap, }, diff --git a/wicket/src/state/inventory.rs b/wicket/src/state/inventory.rs index 2ba6cc51f83..85b72a6a1a7 100644 --- a/wicket/src/state/inventory.rs +++ b/wicket/src/state/inventory.rs @@ -11,7 +11,6 @@ use std::collections::BTreeMap; use std::fmt::Display; use std::iter::Iterator; use std::sync::LazyLock; -use tufaceous_artifact::KnownArtifactKind; use wicket_common::inventory::{ RackV1Inventory, RotInventory, RotSlot, SpComponentCaboose, SpComponentInfo, SpIgnition, SpState, SpType, Transceiver, @@ -295,30 +294,6 @@ impl ComponentId { self.to_string() } - pub fn sp_known_artifact_kind(&self) -> KnownArtifactKind { - match self { - ComponentId::Sled(_) => KnownArtifactKind::GimletSp, - ComponentId::Switch(_) => KnownArtifactKind::SwitchSp, - ComponentId::Psc(_) => KnownArtifactKind::PscSp, - } - } - - pub fn rot_known_artifact_kind(&self) -> KnownArtifactKind { - match self { - ComponentId::Sled(_) => KnownArtifactKind::GimletRot, - ComponentId::Switch(_) => KnownArtifactKind::SwitchRot, - ComponentId::Psc(_) => KnownArtifactKind::PscRot, - } - } - - pub fn rot_bootloader_known_artifact_kind(&self) -> KnownArtifactKind { - match self { - ComponentId::Sled(_) => KnownArtifactKind::GimletRotBootloader, - ComponentId::Switch(_) => KnownArtifactKind::SwitchRotBootloader, - ComponentId::Psc(_) => KnownArtifactKind::PscRotBootloader, - } - } - pub fn to_string_uppercase(&self) -> String { let mut s = self.to_string(); s.make_ascii_uppercase(); diff --git a/wicket/src/state/mod.rs b/wicket/src/state/mod.rs index 1b7678025dd..0ff41a82547 100644 --- a/wicket/src/state/mod.rs +++ b/wicket/src/state/mod.rs @@ -17,9 +17,8 @@ pub use inventory::{ pub use rack::{KnightRiderMode, RackState}; pub use status::ServiceStatus; pub use update::{ - ArtifactVersions, CreateClearUpdateStateOptions, CreateStartUpdateOptions, - RackUpdateState, UpdateItemState, parse_event_report_map, - update_component_title, + CreateClearUpdateStateOptions, CreateStartUpdateOptions, RackUpdateState, + UpdateItemState, parse_event_report_map, update_component_title, }; use serde::{Deserialize, Serialize}; diff --git a/wicket/src/state/update.rs b/wicket/src/state/update.rs index a47d8962490..078ca8eddd8 100644 --- a/wicket/src/state/update.rs +++ b/wicket/src/state/update.rs @@ -4,6 +4,7 @@ use anyhow::Result; use ratatui::style::Style; +use wicket_common::artifact::ArtifactId; use wicket_common::rack_update::{ClearUpdateStateOptions, StartUpdateOptions}; use wicket_common::update_events::{ EventReport, ProgressEventKind, StepEventKind, UpdateComponent, @@ -11,10 +12,7 @@ use wicket_common::update_events::{ }; use crate::helpers::{get_update_simulated_result, get_update_test_error}; -use crate::{ - events::{ArtifactData, EventReportMap}, - ui::defaults::style, -}; +use crate::{events::EventReportMap, ui::defaults::style}; use super::{ALL_COMPONENT_IDS, ComponentId, ParsableComponentId}; use semver::Version; @@ -22,24 +20,14 @@ use serde::{Deserialize, Serialize}; use slog::Logger; use std::collections::BTreeMap; use std::fmt::Display; -use tufaceous_artifact::{ArtifactVersion, KnownArtifactKind}; - -// Represents a version and the signature (optional) associated -// with a particular artifact. This allows for multiple versions -// with different versions to be present in the repo. Note -// sign is currently only used for RoT artifacts -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ArtifactVersions { - pub version: ArtifactVersion, - pub sign: Option>, -} +use tufaceous_artifact_v2::{ArtifactVersion, KnownArtifactTags}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RackUpdateState { pub items: BTreeMap, pub system_version: Option, - pub artifacts: Vec, - pub artifact_versions: BTreeMap>, + pub artifacts: Vec, + pub artifact_versions: BTreeMap>, // The update item currently selected is recorded in // state.rack_state.selected. pub status_view_displayed: bool, @@ -115,20 +103,18 @@ impl RackUpdateState { &mut self, logger: &Logger, system_version: Option, - artifacts: Vec, + artifacts: Vec, reports: EventReportMap, ) { self.system_version = system_version; self.artifacts = artifacts; self.artifact_versions.clear(); - for a in &mut self.artifacts { - if let Some(known) = a.id.kind.to_known() { - self.artifact_versions.entry(known).or_default().push( - ArtifactVersions { - version: a.id.version.clone(), - sign: a.sign.clone(), - }, - ); + for a in &self.artifacts { + if let Ok(known) = KnownArtifactTags::from_tags(a.tags.clone()) { + self.artifact_versions + .entry(known) + .or_default() + .push(a.version.clone()); } } diff --git a/wicket/src/ui/panes/update.rs b/wicket/src/ui/panes/update.rs index b479b70360b..0f2509e7cf8 100644 --- a/wicket/src/ui/panes/update.rs +++ b/wicket/src/ui/panes/update.rs @@ -8,8 +8,8 @@ use std::collections::BTreeMap; use super::{Control, PendingScroll, align_by, help_text, push_text_lines}; use crate::keymap::ShowPopupCmd; use crate::state::{ - ALL_COMPONENT_IDS, ArtifactVersions, ComponentId, Inventory, - UpdateItemState, update_component_title, + ALL_COMPONENT_IDS, ComponentId, Inventory, UpdateItemState, + update_component_title, }; use crate::ui::defaults::style; use crate::ui::widgets::{ @@ -27,7 +27,10 @@ use ratatui::widgets::{ Row, Table, }; use slog::{Logger, info, o}; -use tufaceous_artifact::KnownArtifactKind; +use tufaceous_artifact_v2::{ + ArtifactVersion, KnownArtifactTags, RotBootloaderTags, RotKeyTableHash, + RotTags, +}; use tui_tree_widget::{Tree, TreeItem, TreeState}; use update_engine::display::ProgressRatioDisplay; use update_engine::{ @@ -2397,88 +2400,69 @@ fn all_installed_versions( fn artifact_version( id: &ComponentId, update_component: UpdateComponent, - versions: &BTreeMap>, + versions: &BTreeMap>, inventory: &Inventory, ) -> String { - let (artifact, multiple) = match (id, update_component) { - (ComponentId::Sled(_), UpdateComponent::RotBootloader) => { - (KnownArtifactKind::GimletRotBootloader, true) - } - (ComponentId::Sled(_), UpdateComponent::Rot) => { - (KnownArtifactKind::GimletRot, true) - } - (ComponentId::Sled(_), UpdateComponent::Sp) => { - (KnownArtifactKind::GimletSp, false) - } - (ComponentId::Sled(_), UpdateComponent::Host) => { - (KnownArtifactKind::Host, false) - } - (ComponentId::Switch(_), UpdateComponent::RotBootloader) => { - (KnownArtifactKind::SwitchRotBootloader, true) - } - (ComponentId::Switch(_), UpdateComponent::Rot) => { - (KnownArtifactKind::SwitchRot, true) - } - (ComponentId::Switch(_), UpdateComponent::Sp) => { - (KnownArtifactKind::SwitchSp, false) - } - (ComponentId::Psc(_), UpdateComponent::RotBootloader) => { - (KnownArtifactKind::PscRotBootloader, true) - } - (ComponentId::Psc(_), UpdateComponent::Rot) => { - (KnownArtifactKind::PscRot, true) - } - (ComponentId::Psc(_), UpdateComponent::Sp) => { - (KnownArtifactKind::PscSp, false) - } - - // Switches and PSCs do not have a host. - (ComponentId::Switch(_), UpdateComponent::Host) - | (ComponentId::Psc(_), UpdateComponent::Host) => { - return "N/A".to_string(); + // NOTE: This function currently assumes that all artifacts in the repo of + // the same general kind (SP, ROT, bootloader) are of the same version; we + // don't filter on board or slot. + match update_component { + UpdateComponent::RotBootloader => { + let Some(sign) = inventory + .get_inventory(id) + .and_then(|component| component.rot_sign()) + .and_then(|sign| String::from_utf8(sign).ok()) + else { + return "UNKNOWN (MISSING SIGN)".to_string(); + }; + versions + .iter() + .find_map(|(tags, versions)| match tags { + KnownArtifactTags::RotBootloader(RotBootloaderTags { + rot_rkth: Some(RotKeyTableHash(rot_rkth)), + .. + }) if rot_rkth == &sign => versions.first(), + _ => None, + }) + .map(ArtifactVersion::as_str) + .unwrap_or("NO MATCH") + .to_string() } - }; - match versions.get(&artifact) { - None => "UNKNOWN".to_string(), - Some(artifact_versions) => { - let component = match inventory.get_inventory(id) { - Some(c) => c, - None => return "UNKNOWN".to_string(), + UpdateComponent::Rot => { + let Some(sign) = inventory + .get_inventory(id) + .and_then(|component| component.rot_sign()) + .and_then(|sign| String::from_utf8(sign).ok()) + else { + return "UNKNOWN (MISSING SIGN)".to_string(); }; - let cnt = artifact_versions.len(); - // We loop through all possible artifact versions for a - // given artifact type. Right now only RoT artifacts - // will have a sign value. - for a in artifact_versions { - match (&a.sign, component.rot_sign()) { - // No sign anywhere, this is for SP components - (None, None) => return a.version.to_string(), - // if we have a version that's tagged with sign data but - // we can't read from the caboose check if we can fall - // back to just returning the version. This matches - // very old repositories - (Some(_), None) => { - if multiple && cnt > 1 { - return "UNKNOWN (MISSING SIGN)".to_string(); - } else { - return a.version.to_string(); - } - } - // If something isn't tagged with a sign just - // pass on the version. This should only match - // very old repositories/testing configurations - (None, Some(_)) => return a.version.to_string(), - // The interesting case to make sure the sign - // matches both the component and the caboose - (Some(s), Some(c)) => { - if *s == c { - return a.version.to_string(); - } - } - } - } - "NO MATCH".to_string() + versions + .iter() + .find_map(|(tags, versions)| match tags { + KnownArtifactTags::Rot(RotTags { + rot_rkth: Some(RotKeyTableHash(rot_rkth)), + .. + }) if rot_rkth == &sign => versions.first(), + _ => None, + }) + .map(ArtifactVersion::as_str) + .unwrap_or("NO MATCH") + .to_string() } + UpdateComponent::Sp => versions + .iter() + .find_map(|(tags, versions)| match tags { + KnownArtifactTags::Sp(_) => versions.first(), + _ => None, + }) + .map(ArtifactVersion::as_str) + .unwrap_or("NO MATCH") + .to_string(), + UpdateComponent::Host => match id { + ComponentId::Sled(_) => "UNKNOWN".to_string(), + // Switches and PSCs do not have a host. + ComponentId::Switch(_) | ComponentId::Psc(_) => "N/A".to_string(), + }, } } diff --git a/wicket/src/wicketd.rs b/wicket/src/wicketd.rs index f9a5bdec5c7..745de1dc6cc 100644 --- a/wicket/src/wicketd.rs +++ b/wicket/src/wicketd.rs @@ -19,7 +19,6 @@ use wicketd_client::types::{ GetLocationResponse, IgnitionCommand, StartUpdateParams, }; -use crate::events::{ArtifactData, EventReportMap}; use crate::keymap::ShowPopupCmd; use crate::state::ComponentId; use crate::{Cmd, Event}; @@ -448,20 +447,10 @@ impl WicketdManager { Ok(val) => { // TODO: Only send on changes let rsp = val.into_inner(); - let artifacts = rsp - .artifacts - .into_iter() - .map(|artifact| ArtifactData { - id: artifact.artifact_id, - sign: artifact.sign, - }) - .collect(); - let system_version = rsp.system_version; - let event_reports: EventReportMap = rsp.event_reports; let _ = tx.send(Event::ArtifactsAndEventReports { - system_version, - artifacts, - event_reports, + system_version: rsp.system_version, + artifacts: rsp.artifacts, + event_reports: rsp.event_reports, }); } Err(e) => { diff --git a/wicketd-api/Cargo.toml b/wicketd-api/Cargo.toml index 474b19b562d..898f2303ec6 100644 --- a/wicketd-api/Cargo.toml +++ b/wicketd-api/Cargo.toml @@ -19,5 +19,4 @@ semver.workspace = true serde.workspace = true sled-hardware-types.workspace = true slog.workspace = true -tufaceous-artifact.workspace = true wicket-common.workspace = true diff --git a/wicketd-api/src/lib.rs b/wicketd-api/src/lib.rs index 813dd15ebe1..741b1266a3a 100644 --- a/wicketd-api/src/lib.rs +++ b/wicketd-api/src/lib.rs @@ -11,7 +11,6 @@ use dropshot::RequestContext; use dropshot::StreamingBody; use dropshot::TypedBody; use gateway_client::types::IgnitionCommand; -use omicron_common::update::ArtifactId; use omicron_uuid_kinds::RackInitUuid; use omicron_uuid_kinds::RackResetUuid; use schemars::JsonSchema; @@ -22,7 +21,7 @@ use sled_hardware_types::Baseboard; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::net::Ipv6Addr; -use tufaceous_artifact::ArtifactHashId; +use wicket_common::artifact::ArtifactId; use wicket_common::inventory::RackV1Inventory; use wicket_common::inventory::SpIdentifier; use wicket_common::inventory::SpType; @@ -483,14 +482,6 @@ pub enum GetInventoryResponse { Unavailable, } -#[derive(Clone, Debug, JsonSchema, Serialize)] -#[serde(rename_all = "snake_case")] -pub struct InstallableArtifacts { - pub artifact_id: ArtifactId, - pub installable: Vec, - pub sign: Option>, -} - /// The response to a `get_artifacts` call: the system version, and the list of /// all artifacts currently held by wicketd. #[derive(Clone, Debug, JsonSchema, Serialize)] @@ -509,7 +500,7 @@ pub struct GetArtifactsAndEventReportsResponse { /// Conceptually, this is a `BTreeMap>`, but /// JSON requires string keys for maps, so we give back a vec of pairs /// instead. - pub artifacts: Vec, + pub artifacts: Vec, pub event_reports: BTreeMap>, } diff --git a/wicketd/Cargo.toml b/wicketd/Cargo.toml index e7976a18de4..a50ff141b98 100644 --- a/wicketd/Cargo.toml +++ b/wicketd/Cargo.toml @@ -45,8 +45,8 @@ slog-dtrace.workspace = true slog.workspace = true rand.workspace = true thiserror.workspace = true -tufaceous-artifact.workspace = true -tufaceous-lib.workspace = true +tufaceous-artifact-v2.workspace = true +tufaceous-v2.workspace = true tokio = { workspace = true, features = [ "full" ] } tokio-stream.workspace = true tokio-util.workspace = true @@ -76,12 +76,14 @@ wicketd-client.workspace = true omicron-workspace-hack.workspace = true semver.workspace = true slog-error-chain.workspace = true +lpc55_areas.workspace = true [[bin]] name = "wicketd" doc = false [dev-dependencies] +chrono.workspace = true expectorate.workspace = true flate2.workspace = true fs-err.workspace = true diff --git a/wicketd/src/artifacts/server.rs b/wicketd/src/artifacts/server.rs index 84c2aa6543d..0f9b0c4306d 100644 --- a/wicketd/src/artifacts/server.rs +++ b/wicketd/src/artifacts/server.rs @@ -13,12 +13,12 @@ use dropshot::Path; use dropshot::RequestContext; use dropshot::TypedBody; use futures::TryStreamExt; +use installinator_api::GetArtifactPathParams; use installinator_api::InstallinatorApi; use installinator_api::body_to_artifact_response; use installinator_common::report::ReportQuery; use slog::Logger; use slog::error; -use tufaceous_artifact::ArtifactHashId; use update_engine::NestedSpec; use update_engine::events::EventReport; @@ -54,14 +54,15 @@ impl InstallinatorApi for WicketdInstallinatorApiImpl { async fn get_artifact_by_hash( rqctx: RequestContext, - path: Path, + path: Path, ) -> Result>, HttpError> { let context = rqctx.context(); - match context.store.get_by_hash(&path.into_inner()) { - Some(data_handle) => { - let size = data_handle.file_size() as u64; - let data_stream = match data_handle.reader_stream().await { + let hash = path.into_inner().hash; + match context.store.get_installinator_artifact(hash) { + Some(handle) => { + let size = handle.artifact().length; + let data_stream = match handle.stream().await { Ok(stream) => stream, Err(err) => { error!( diff --git a/wicketd/src/artifacts/store.rs b/wicketd/src/artifacts/store.rs index 555e0dbd064..1fda57e1d8e 100644 --- a/wicketd/src/artifacts/store.rs +++ b/wicketd/src/artifacts/store.rs @@ -4,13 +4,13 @@ use semver::Version; use slog::Logger; +use std::collections::BTreeMap; use std::sync::Arc; use std::sync::Mutex; -use tufaceous_artifact::ArtifactHashId; -use update_common::artifacts::ArtifactsWithPlan; -use update_common::artifacts::ExtractedArtifactDataHandle; -use update_common::artifacts::UpdatePlan; -use wicketd_api::InstallableArtifacts; +use tufaceous_artifact_v2::ArtifactHash; +use tufaceous_v2::ArtifactHandle; +use tufaceous_v2::Repository; +use wicket_common::artifact::ArtifactId; /// The artifact store for wicketd. /// @@ -21,77 +21,67 @@ pub struct WicketdArtifactStore { log: Logger, // NOTE: this is a `std::sync::Mutex` rather than a `tokio::sync::Mutex` // because the critical sections are extremely small. - artifacts_with_plan: Arc>>, + inner: Arc>>, +} + +#[derive(Debug)] +struct Inner { + repo: Arc, + by_hash: BTreeMap, } impl WicketdArtifactStore { pub(crate) fn new(log: &Logger) -> Self { let log = log.new(slog::o!("component" => "wicketd artifact store")); - Self { log, artifacts_with_plan: Default::default() } + Self { log, inner: Default::default() } } - pub(crate) fn set_artifacts_with_plan( - &self, - artifacts_with_plan: ArtifactsWithPlan, - ) { - slog::debug!(self.log, "setting artifacts_with_plan"); - self.replace(artifacts_with_plan); + pub(crate) fn set_repository(&self, repo: Arc) { + slog::debug!(self.log, "setting repository"); + let by_hash = repo + .handles() + .map(|handle| (handle.artifact().hash, handle)) + .collect(); + self.replace(Inner { by_hash, repo }); } pub(crate) fn system_version_and_artifact_ids( &self, - ) -> Option<(Version, Vec)> { - let artifacts = self.artifacts_with_plan.lock().unwrap(); - let artifacts = artifacts.as_ref()?; - let system_version = artifacts.plan().system_version.clone(); - - let artifact_ids = artifacts - .by_id() - .iter() - .map(|(k, v)| InstallableArtifacts { - artifact_id: k.clone(), - installable: v.clone(), - sign: artifacts.rot_by_sign().get(&k).cloned(), - }) - .collect(); + ) -> Option<(Version, Vec)> { + let guard = self.inner.lock().unwrap(); + let inner = guard.as_ref()?; + let system_version = inner.repo.system_version().clone(); + let artifact_ids = + inner.repo.artifacts().iter().map(ArtifactId::new).collect(); Some((system_version, artifact_ids)) } - /// Obtain the current plan. + /// Obtain the current repository. /// - /// Exposed for testing. - pub fn current_plan(&self) -> Option { - // We expect this hashmap to be relatively small (order ~10), and - // cloning both ArtifactIds and ExtractedArtifactDataHandles are cheap. - self.artifacts_with_plan - .lock() - .unwrap() - .as_ref() - .map(|artifacts| artifacts.plan().clone()) + /// Exposed as `pub` for testing. + pub fn current_repository(&self) -> Option> { + Some(Arc::clone(&self.inner.lock().unwrap().as_ref()?.repo)) } // --- // Helper methods // --- - pub(super) fn get_by_hash( + pub(super) fn get_installinator_artifact( &self, - id: &ArtifactHashId, - ) -> Option { - self.artifacts_with_plan.lock().unwrap().as_ref()?.get_by_hash(id) + hash: ArtifactHash, + ) -> Option { + self.inner.lock().unwrap().as_ref()?.by_hash.get(&hash).cloned() } // `pub` to allow use in integration tests. - pub fn contains_by_hash(&self, id: &ArtifactHashId) -> bool { - self.get_by_hash(id).is_some() + pub fn contains_installinator_artifact(&self, hash: ArtifactHash) -> bool { + self.get_installinator_artifact(hash).is_some() } - /// Replaces the artifact hash map, returning the previous map. - fn replace( - &self, - new_artifacts: ArtifactsWithPlan, - ) -> Option { - let mut artifacts = self.artifacts_with_plan.lock().unwrap(); - std::mem::replace(&mut *artifacts, Some(new_artifacts)) + /// Replaces the inner value. + fn replace(&self, inner: Inner) { + let mut guard = self.inner.lock().unwrap(); + *guard = Some(inner); } } diff --git a/wicketd/src/installinator_progress.rs b/wicketd/src/installinator_progress.rs index e99a77ed3e6..1c1160f4040 100644 --- a/wicketd/src/installinator_progress.rs +++ b/wicketd/src/installinator_progress.rs @@ -369,7 +369,7 @@ mod tests { ) .expect("serialized step ID"), component: serde_json::to_value( - InstallinatorComponent::Both, + InstallinatorComponent::All, ) .expect("serialized component"), description: "Fake step".into(), diff --git a/wicketd/src/update_tracker.rs b/wicketd/src/update_tracker.rs index 84fae473232..8149eefa93e 100644 --- a/wicketd/src/update_tracker.rs +++ b/wicketd/src/update_tracker.rs @@ -16,23 +16,21 @@ use bytes::Bytes; use display_error_chain::DisplayErrorChain; use dropshot::HttpError; use futures::Stream; -use futures::TryFutureExt; use gateway_client::types::GetRotBootInfoParams; use gateway_client::types::HostPhase2Progress; use gateway_client::types::HostPhase2RecoveryImageId; use gateway_client::types::HostStartupOptions; use gateway_client::types::InstallinatorImageId; use gateway_client::types::PowerState; -use gateway_client::types::RotCfpaSlot; use gateway_client::types::SpComponentFirmwareSlot; use gateway_client::types::SpUpdateStatus; use gateway_messages::ROT_PAGE_SIZE; use gateway_messages::SpComponent; use gateway_types::rot::RotImageError; use gateway_types::rot::RotState; -use hubtools::RawHubrisArchive; use installinator_common::InstallinatorCompletionMetadata; use installinator_common::WriteOutput; +use lpc55_areas::CMPAPage; use omicron_common::disk::M2Slot; use omicron_uuid_kinds::MupdateUuid; use semver::Version; @@ -52,25 +50,33 @@ use std::sync::atomic::AtomicBool; use std::time::Duration; use std::time::Instant; use thiserror::Error; -use tokio::io::AsyncReadExt; use tokio::sync::Mutex; use tokio::sync::oneshot; use tokio::sync::watch; use tokio::task::JoinHandle; -use tokio_util::io::StreamReader; -use tufaceous_artifact::ArtifactHash; -use tufaceous_artifact::ArtifactVersion; -use update_common::artifacts::ArtifactIdData; -use update_common::artifacts::ArtifactsWithPlan; -use update_common::artifacts::ControlPlaneZonesMode; -use update_common::artifacts::UpdatePlan; -use update_common::artifacts::VerificationMode; +use tufaceous_artifact_v2::ArtifactHash; +use tufaceous_artifact_v2::ArtifactVersion; +use tufaceous_artifact_v2::KnownArtifactTags; +use tufaceous_artifact_v2::OsBoard; +use tufaceous_artifact_v2::OsPhase1Tags; +use tufaceous_artifact_v2::OsPhase2Tags; +use tufaceous_artifact_v2::OsVariant; +use tufaceous_artifact_v2::RotBootloaderTags; +use tufaceous_artifact_v2::RotKeyTableHash; +use tufaceous_artifact_v2::RotSlot; +use tufaceous_artifact_v2::RotTags; +use tufaceous_artifact_v2::SpTags; +use tufaceous_v2::ArtifactHandle; +use tufaceous_v2::ExpirationEnforcement; +use tufaceous_v2::Repository; +use tufaceous_v2::RepositoryLoader; +use update_common::ErrorExt; use update_engine::AbortHandle; use update_engine::NestedSpec; use update_engine::StepSpec; use update_engine::events::ProgressUnits; use uuid::Uuid; -use wicket_common::inventory::SpComponentCaboose; +use wicket_common::artifact::ArtifactId; use wicket_common::inventory::SpIdentifier; use wicket_common::inventory::SpType; use wicket_common::rack_update::ClearUpdateStateResponse; @@ -100,6 +106,11 @@ use wicket_common::update_events::UpdateStepId; use wicket_common::update_events::UpdateTerminalError; use wicketd_api::GetArtifactsAndEventReportsResponse; +const TRAMPOLINE_PHASE_2_TAGS: KnownArtifactTags = + KnownArtifactTags::OsPhase2(OsPhase2Tags { + os_variant: OsVariant::Recovery, + }); + #[derive(Debug)] struct SpUpdateData { // See the documentation for is_finished. @@ -161,7 +172,7 @@ pub struct UpdateTracker { // Every sled update via trampoline requires MGS to serve the trampoline // phase 2 image to the sled's SP over the management network; however, that // doesn't mean we should upload the trampoline image to MGS for every sled - // update - it's always the same (for any given update plan). Therefore, we + // update - it's always the same (for any given repository). Therefore, we // separate the status of uploading the trampoline phase 2 MGS from the // status of individual SP updates: we'll start this upload the first time a // sled update starts that uses it, and any update (including that one or @@ -284,8 +295,8 @@ impl UpdateTracker { errors.push(StartUpdateError::ExistingUpdates(existing_updates)); } - let plan = update_data.artifact_store.current_plan(); - if plan.is_none() { + let repo = update_data.artifact_store.current_repository(); + if repo.is_none() { // (1), referred to below. errors.push(StartUpdateError::TufRepositoryUnavailable); } @@ -295,12 +306,12 @@ impl UpdateTracker { return Err(errors); } - let plan = - plan.expect("we'd have returned an error at (1) if plan was None"); + let repo = + repo.expect("we'd have returned an error at (1) if repo was None"); // Call the setup method now. if let Some(mut spawn_update_driver) = spawn_update_driver { - let setup_data = spawn_update_driver.setup(&plan).await; + let setup_data = spawn_update_driver.setup(&repo).await; for sp in sps { match update_data.sp_update_data.entry(sp) { @@ -311,7 +322,7 @@ impl UpdateTracker { spawn_update_driver .spawn_update_driver( sp, - plan.clone(), + repo.clone(), &setup_data, ) .await, @@ -333,20 +344,36 @@ impl UpdateTracker { fn spawn_upload_trampoline_phase_2_to_mgs( &self, - plan: &UpdatePlan, + repo: &Arc, ) -> UploadTrampolinePhase2ToMgs { - let artifact = plan.trampoline_phase_2.clone(); - let (status_tx, status_rx) = - watch::channel(UploadTrampolinePhase2ToMgsStatus::Running { - hash: artifact.data.hash(), - }); - let task = tokio::spawn(upload_trampoline_phase_2_to_mgs( - self.mgs_client.clone(), - artifact, - status_tx, - self.log.clone(), - )); - UploadTrampolinePhase2ToMgs { status: status_rx, task } + match repo.get_handle(&TRAMPOLINE_PHASE_2_TAGS) { + Ok(handle) => { + let (status_tx, status_rx) = watch::channel( + UploadTrampolinePhase2ToMgsStatus::Running { + hash: handle.artifact().hash, + }, + ); + let task = tokio::spawn(upload_trampoline_phase_2_to_mgs( + self.mgs_client.clone(), + handle, + status_tx, + self.log.clone(), + )); + UploadTrampolinePhase2ToMgs { status: status_rx, task } + } + Err(error) => { + let error = UpdateTerminalError::MissingArtifact { + tags: TRAMPOLINE_PHASE_2_TAGS, + error, + }; + let (_, status) = + watch::channel(UploadTrampolinePhase2ToMgsStatus::Failed( + Arc::new(anyhow!(error)), + )); + let task = tokio::spawn(futures::future::ready(())); + UploadTrampolinePhase2ToMgs { status, task } + } + } } /// Updates the repository stored inside the update tracker. @@ -354,20 +381,20 @@ impl UpdateTracker { &self, stream: impl Stream> + Send + 'static, ) -> Result<(), HttpError> { - // Build the ArtifactsWithPlan from the stream. - let artifacts_with_plan = ArtifactsWithPlan::from_stream( - stream, - // We don't have a good file name here because file contents are - // uploaded over stdin, so let ArtifactsWithPlan pick the name. - None, - ControlPlaneZonesMode::Composite, - VerificationMode::BlindlyTrustAnything, - &self.log, - ) - .await - .map_err(|error| error.to_http_error())?; + const VERIFY_PARALLELISM: usize = 16; + + let repo = RepositoryLoader::new() + .expiration_enforcement(ExpirationEnforcement::Unsafe) + .unsafe_blindly_trust_repo() + .v1_compatibility(true) + .load_zip_stream(stream, None, &self.log) + .await + .map_err(|err| err.to_http_error())?; + repo.verify_targets(VERIFY_PARALLELISM) + .await + .map_err(|err| err.to_http_error())?; let mut update_data = self.sp_update_data.lock().await; - update_data.set_artifacts_with_plan(artifacts_with_plan).await + update_data.set_repository(Arc::new(repo)).await } /// Gets a list of artifacts stored in the update repository. @@ -425,7 +452,7 @@ trait SpawnUpdateDriver { /// /// This is called *once*, before any calls to /// [`Self::spawn_update_driver`]. - async fn setup(&mut self, plan: &UpdatePlan) -> Self::Setup; + async fn setup(&mut self, repo: &Arc) -> Self::Setup; /// Spawn the update driver for the given SP. /// @@ -433,7 +460,7 @@ trait SpawnUpdateDriver { async fn spawn_update_driver( &mut self, sp: SpIdentifier, - plan: UpdatePlan, + repo: Arc, setup_data: &Self::Setup, ) -> SpUpdateData; } @@ -451,9 +478,11 @@ struct RealSpawnUpdateDriver<'tr> { impl SpawnUpdateDriver for RealSpawnUpdateDriver<'_> { type Setup = watch::Receiver; - async fn setup(&mut self, plan: &UpdatePlan) -> Self::Setup { - // Do we need to upload this plan's trampoline phase 2 to MGS? + async fn setup(&mut self, repo: &Arc) -> Self::Setup { + // Do we need to upload this repo's trampoline phase 2 to MGS? + let trampoline_phase_2 = + repo.artifacts().get_only(&TRAMPOLINE_PHASE_2_TAGS).ok(); let mut upload_trampoline_phase_2_to_mgs = self.update_tracker.upload_trampoline_phase_2_to_mgs.lock().await; @@ -464,21 +493,21 @@ impl SpawnUpdateDriver for RealSpawnUpdateDriver<'_> { // might still be trying to upload) and start a new one // with our current image. if prev.status.borrow().hash() - != Some(plan.trampoline_phase_2.data.hash()) + != trampoline_phase_2.map(|artifact| artifact.hash) { - // It does _not_ match - we have a new plan with a + // It does _not_ match - we have a new repo with a // different trampoline image. If the old task is // still running, cancel it, and start a new one. prev.task.abort(); *prev = self .update_tracker - .spawn_upload_trampoline_phase_2_to_mgs(&plan); + .spawn_upload_trampoline_phase_2_to_mgs(&repo); } } None => { *upload_trampoline_phase_2_to_mgs = Some( self.update_tracker - .spawn_upload_trampoline_phase_2_to_mgs(&plan), + .spawn_upload_trampoline_phase_2_to_mgs(&repo), ); } } @@ -492,7 +521,7 @@ impl SpawnUpdateDriver for RealSpawnUpdateDriver<'_> { async fn spawn_update_driver( &mut self, sp: SpIdentifier, - plan: UpdatePlan, + repo: Arc, setup_data: &Self::Setup, ) -> SpUpdateData { // Generate an ID for this update; the update tracker will send it to the @@ -526,7 +555,7 @@ impl SpawnUpdateDriver for RealSpawnUpdateDriver<'_> { let finished_indicator = SetTrueOnDrop(finished.clone()); tokio::spawn(update_driver.run( - plan, + repo, update_cx, event_buffer.clone(), ipr_start_receiver, @@ -557,12 +586,12 @@ struct FakeUpdateDriver { impl SpawnUpdateDriver for FakeUpdateDriver { type Setup = (); - async fn setup(&mut self, _plan: &UpdatePlan) -> Self::Setup {} + async fn setup(&mut self, _plan: &Arc) -> Self::Setup {} async fn spawn_update_driver( &mut self, _sp: SpIdentifier, - _plan: UpdatePlan, + _plan: Arc, _setup_data: &Self::Setup, ) -> SpUpdateData { let (sender, mut receiver) = update_engine::channel(); @@ -654,12 +683,12 @@ enum NeverUpdateDriver {} impl SpawnUpdateDriver for NeverUpdateDriver { type Setup = (); - async fn setup(&mut self, _plan: &UpdatePlan) -> Self::Setup {} + async fn setup(&mut self, _plan: &Arc) -> Self::Setup {} async fn spawn_update_driver( &mut self, _sp: SpIdentifier, - _plan: UpdatePlan, + _plan: Arc, _setup_data: &Self::Setup, ) -> SpUpdateData { unreachable!("this update driver cannot be constructed") @@ -742,9 +771,9 @@ impl UpdateTrackerData { } } - async fn set_artifacts_with_plan( + async fn set_repository( &mut self, - artifacts_with_plan: ArtifactsWithPlan, + repository: Arc, ) -> Result<(), HttpError> { // Are there any updates currently running? If so, then reject the new // repository. @@ -763,7 +792,7 @@ impl UpdateTrackerData { } // Set the new artifacts_with_plan. - self.artifact_store.set_artifacts_with_plan(artifacts_with_plan); + self.artifact_store.set_repository(repository); // Reset all running data: a new repository means starting afresh. self.sp_update_data.clear(); @@ -835,7 +864,7 @@ impl UpdateDriver { #![allow(clippy::too_many_arguments)] async fn run( self, - plan: UpdatePlan, + repo: Arc, update_cx: UpdateContext, event_buffer: Arc>, ipr_start_receiver: IprStartReceiver, @@ -843,6 +872,7 @@ impl UpdateDriver { abort_handle_sender: oneshot::Sender, finished_indicator: SetTrueOnDrop, ) { + let repo = &repo; let update_cx = &update_cx; // TODO: We currently do updates in the order RoT -> SP -> host. This is @@ -871,28 +901,6 @@ impl UpdateDriver { define_test_steps(&engine, secs); } - let (rot_a, rot_b, sp_artifacts, rot_bootloader) = - match update_cx.sp.type_ { - SpType::Sled => ( - &plan.gimlet_rot_a, - &plan.gimlet_rot_b, - &plan.gimlet_sp, - &plan.gimlet_rot_bootloader, - ), - SpType::Power => ( - &plan.psc_rot_a, - &plan.psc_rot_b, - &plan.psc_sp, - &plan.psc_rot_bootloader, - ), - SpType::Switch => ( - &plan.sidecar_rot_a, - &plan.sidecar_rot_b, - &plan.sidecar_sp, - &plan.sidecar_rot_bootloader, - ), - }; - let rot_bootloader_registrar = engine.for_component(UpdateComponent::RotBootloader); let rot_registrar = engine.for_component(UpdateComponent::Rot); @@ -904,9 +912,7 @@ impl UpdateDriver { .new_step( UpdateStepId::InterrogateRot, "Checking current RoT bootloader version", - async |_cx| { - update_cx.interrogate_rot_bootloader(rot_bootloader).await - }, + async |_cx| update_cx.interrogate_rot_bootloader(&repo).await, ) .register(); @@ -918,7 +924,7 @@ impl UpdateDriver { .new_step( UpdateStepId::InterrogateRot, "Checking current RoT version and active slot", - async |_cx| update_cx.interrogate_rot(rot_a, rot_b).await, + async |_cx| update_cx.interrogate_rot(&repo).await, ) .register(); @@ -928,7 +934,7 @@ impl UpdateDriver { let sp_firmware_slot = 0; // To update the SP, we want to know both its version and its board (so - // we can map to the correct artifact from our update plan). + // we can map to the correct artifact from our repository). let sp_artifact_and_version = sp_registrar .new_step( UpdateStepId::InterrogateSp, @@ -948,15 +954,13 @@ impl UpdateDriver { })? .into_inner(); - let Some(sp_artifact) = sp_artifacts.get(&caboose.board) - else { - return Err( - UpdateTerminalError::MissingSpImageForBoard { - board: caboose.board, - }, - ); - }; - let sp_artifact = sp_artifact.clone(); + let tags = KnownArtifactTags::Sp(SpTags { + sp_board: caboose.board.clone(), + }); + let sp_artifact = + repo.get_handle(&tags).map_err(|error| { + UpdateTerminalError::MissingArtifact { tags, error } + })?; let message = format!( "SP board {}, version {} (git commit {})", @@ -990,42 +994,59 @@ impl UpdateDriver { UpdateStepId::SpComponentUpdate, "Updating RoT bootloader", async move |cx| { - if let Some(result) = opts.test_simulate_rot_bootloader_result { + if let Some(result) = + opts.test_simulate_rot_bootloader_result + { return simulate_result(result); } let rot_bootloader_interrogation = - match rot_bootloader_interrogation.into_value(cx.token()).await { - Some(v) => v, - None => return StepSkipped::new( - (), - "Skipping bootloader update, check interrogation step", - ).into(), - }; + match rot_bootloader_interrogation + .into_value(cx.token()) + .await + { + Some(v) => v, + None => { + return StepSkipped::new( + (), + "Skipping bootloader update, \ + check interrogation step", + ) + .into(); + } + }; - let bootloader_has_this_version = rot_bootloader_interrogation - .active_version_matches_artifact_to_apply(); + let bootloader_has_this_version = + rot_bootloader_interrogation + .active_version_matches_artifact_to_apply(); - let sp_can_update = rot_bootloader_interrogation.sp_can_update_bootloader(&update_cx.mgs_client).await; + let sp_can_update = rot_bootloader_interrogation + .sp_can_update_bootloader(&update_cx.mgs_client) + .await; if !sp_can_update { return StepSkipped::new( (), - "SP version needs to be upgraded before RoT bootloader can be updated", + "SP version needs to be upgraded \ + before RoT bootloader can be updated", ) .into(); - } // If this RoT already has this version, skip the rest of // this step, UNLESS we've been told to skip this version // check. - if bootloader_has_this_version && !opts.skip_rot_bootloader_version_check { + if bootloader_has_this_version + && !opts.skip_rot_bootloader_version_check + { return StepSkipped::new( (), format!( "RoT bootloader already at version {}", - rot_bootloader_interrogation.artifact_to_apply.id.version, + rot_bootloader_interrogation + .artifact_to_apply + .artifact() + .version, ), ) .into(); @@ -1048,8 +1069,12 @@ impl UpdateDriver { StepWarning::new( (), format!( - "RoT bootloader updated despite already having version {}", - rot_bootloader_interrogation.artifact_to_apply.id.version, + "RoT bootloader updated despite \ + already having version {}", + rot_bootloader_interrogation + .artifact_to_apply + .artifact() + .version, ), ) .into() @@ -1086,7 +1111,10 @@ impl UpdateDriver { (), format!( "RoT active slot already at version {}", - rot_interrogation.artifact_to_apply.id.version + rot_interrogation + .artifact_to_apply + .artifact() + .version ), ) .into(); @@ -1110,7 +1138,10 @@ impl UpdateDriver { (), format!( "RoT updated despite already having version {}", - rot_interrogation.artifact_to_apply.id.version + rot_interrogation + .artifact_to_apply + .artifact() + .version ), ) .into() @@ -1136,7 +1167,8 @@ impl UpdateDriver { sp_artifact_and_version.into_value(cx.token()).await; let sp_has_this_version = - Some(&sp_artifact.id.version) == sp_version.as_ref(); + Some(&sp_artifact.artifact().version) + == sp_version.as_ref(); // If this SP already has this version, skip the rest of // this step, UNLESS we've been told to skip this version @@ -1146,7 +1178,7 @@ impl UpdateDriver { (), format!( "SP already at version {}", - sp_artifact.id.version + sp_artifact.artifact().version ), ) .into(); @@ -1170,7 +1202,7 @@ impl UpdateDriver { (), format!( "SP updated despite already having version {}", - sp_artifact.id.version + sp_artifact.artifact().version ), ) .into() @@ -1185,7 +1217,7 @@ impl UpdateDriver { self.register_sled_steps( update_cx, &mut engine, - &plan, + &repo, ipr_start_receiver, ); } @@ -1215,7 +1247,7 @@ impl UpdateDriver { &self, update_cx: &'a UpdateContext, engine: &mut UpdateEngine<'a>, - plan: &'a UpdatePlan, + repo: &'a Arc, ipr_start_receiver: IprStartReceiver, ) { let mut host_registrar = engine.for_component(UpdateComponent::Host); @@ -1249,7 +1281,7 @@ impl UpdateDriver { let image_id_handle = self.register_trampoline_phase1_steps( update_cx, &mut host_registrar, - plan, + repo, host_type.clone(), ); @@ -1311,7 +1343,7 @@ impl UpdateDriver { self.register_install_host_phase1_and_boot_steps( update_cx, &mut host_registrar, - plan, + repo, slots_to_update, host_type, ); @@ -1324,7 +1356,7 @@ impl UpdateDriver { &self, update_cx: &'a UpdateContext, registrar: &mut ComponentRegistrar<'_, 'a>, - plan: &'a UpdatePlan, + repo: &'a Arc, host_type: SharedStepHandle, ) -> StepHandle { // We arbitrarily choose to store the trampoline phase 1 in host boot @@ -1337,16 +1369,15 @@ impl UpdateDriver { self.register_deliver_host_phase1_steps( update_cx, registrar, - &plan.gimlet_trampoline_phase_1, - &plan.cosmo_trampoline_phase_1, - "trampoline", + repo, + OsVariant::Recovery, StepHandle::ready(trampoline_phase_1_boot_slots).into_shared(), host_type, ); // Wait (if necessary) for the trampoline phase 2 upload to MGS to // complete. We started a task to do this the first time a sled update - // was started with this plan. + // was started with this repo. let mut upload_trampoline_phase_2_to_mgs = update_cx.upload_trampoline_phase_2_to_mgs.clone(); @@ -1398,38 +1429,16 @@ impl UpdateDriver { UpdateStepId::SettingInstallinatorImageId, "Setting installinator image ID", async move |_cx| { - let installinator_image_id = - match plan.installinator_doc_hash { - Some(hash) => { - // In this case (for newer TUF repos), we set the - // host phase 2 hash to the document hash, and the - // control plane hash to all zeroes. The latter acts - // as an indication to installinator that the former - // is actually a document hash. - InstallinatorImageId { - host_phase_2: hash.to_string(), - control_plane: ArtifactHash([0; 32]) - .to_string(), - update_id: update_cx.update_id, - } - } - None => { - // For older TUF repos, we follow the previous - // logic. - // - // TODO-cleanup: Once we no longer support older TUF - // repos, we can remove this logic. - InstallinatorImageId { - host_phase_2: plan - .host_phase_2_hash - .to_string(), - control_plane: plan - .control_plane_hash - .to_string(), - update_id: update_cx.update_id, - } - } - }; + let tags = KnownArtifactTags::InstallinatorDocument; + let artifact = + repo.artifacts().get_only(&tags).map_err(|error| { + UpdateTerminalError::MissingArtifact { tags, error } + })?; + let installinator_image_id = InstallinatorImageId { + host_phase_2: artifact.hash.to_string(), + control_plane: ArtifactHash([0; 32]).to_string(), + update_id: update_cx.update_id, + }; update_cx .mgs_client .sp_installinator_image_id_set( @@ -1516,7 +1525,7 @@ impl UpdateDriver { &self, update_cx: &'a UpdateContext, registrar: &mut ComponentRegistrar<'engine, 'a>, - plan: &'a UpdatePlan, + repo: &'a Arc, slots_to_update: StepHandle>, host_type: SharedStepHandle, ) { @@ -1528,9 +1537,8 @@ impl UpdateDriver { self.register_deliver_host_phase1_steps( update_cx, registrar, - &plan.gimlet_host_phase_1, - &plan.cosmo_host_phase_1, - "host", + repo, + OsVariant::Host, slots_to_update.clone(), host_type, ); @@ -1639,9 +1647,8 @@ impl UpdateDriver { &self, update_cx: &'a UpdateContext, registrar: &mut ComponentRegistrar<'_, 'a>, - gimlet_artifact: &'a ArtifactIdData, - cosmo_artifact: &'a ArtifactIdData, - kind: &str, // "host" or "trampoline" + repo: &'a Arc, + os_variant: OsVariant, slots_to_update: SharedStepHandle>, host_type: SharedStepHandle, ) { @@ -1660,21 +1667,27 @@ impl UpdateDriver { registrar .new_step( UpdateStepId::SpComponentUpdate, - format!("Updating {kind} phase 1"), + format!("Updating {os_variant} phase 1"), async move |cx| { let host_type = host_type.into_value(cx.token()).await; let slots_to_update = slots_to_update.into_value(cx.token()).await; - - let artifact = match host_type { - OxideSled::Cosmo => cosmo_artifact, - OxideSled::Gimlet => gimlet_artifact, + let os_board = match host_type { + OxideSled::Gimlet => OsBoard::GIMLET, + OxideSled::Cosmo => OsBoard::COSMO, }; + let tags = KnownArtifactTags::OsPhase1(OsPhase1Tags { + os_board, + os_variant, + }); + let artifact = repo.get_handle(&tags).map_err(|error| { + UpdateTerminalError::MissingArtifact { tags, error } + })?; for boot_slot in slots_to_update { cx.with_nested_engine(|engine| { inner_cx - .register_steps(engine, boot_slot, artifact); + .register_steps(engine, boot_slot, &artifact); Ok(()) }) .await?; @@ -1753,7 +1766,7 @@ struct RotInterrogation { // Identifier of the target RoT's SP. sp: SpIdentifier, // Version reported by the target RoT. - artifact_to_apply: ArtifactIdData, + artifact_to_apply: ArtifactHandle, active_version: Option, } @@ -1796,7 +1809,8 @@ impl RotInterrogation { } fn active_version_matches_artifact_to_apply(&self) -> bool { - Some(&self.artifact_to_apply.id.version) == self.active_version.as_ref() + Some(&self.artifact_to_apply.artifact().version) + == self.active_version.as_ref() } } @@ -1872,9 +1886,12 @@ impl UpdateContext { InstallinatorCompletionMetadata::Write { output } => { write_output = Some(output.clone()); } - InstallinatorCompletionMetadata::HardwareScan { .. } - | InstallinatorCompletionMetadata::ControlPlaneZones { .. } - | InstallinatorCompletionMetadata::Download { .. } + InstallinatorCompletionMetadata::HardwareScan { + .. + } + | InstallinatorCompletionMetadata::Download { + .. + } | InstallinatorCompletionMetadata::Unknown => (), } } @@ -1895,72 +1912,9 @@ impl UpdateContext { async fn interrogate_rot_bootloader( &self, - rot_bootloader: &[ArtifactIdData], + repo: &Arc, ) -> Result>, UpdateTerminalError> { - // We have a known set of bootloader FWID that don't have cabooses. - static KNOWN_MISSING_CABOOSE: [&str; 18] = [ - "1122095f4a3797db8a7d6279ae889ddde0316631f1f3bc204bdc39c2d75707af", - "1525832a663024f6421c13c0f7c7d9e9b32ebf433898565a2ad8112e7d237ead", - "29fc0d31e1739865c7f3d4bb5f5b86779db92a65a2decbd59e42f6e95dd84698", - "37aa40d0ea12e1290477a84014cd03dbc6fa9817223d1546a10847510d75c383", - "53cb91f4a3fbb69efa733a9eb326bd9f71c849782b0eea4306ebc66620158d44", - "60effb7fd6c4780138887e0d65c9e9b9c8447ce4ea3ea71e08194aec2847b185", - "77b8fc4308221dfe123d93431c21b57fa896db65c015ca82e22a337c7aa7cd77", - "77c2b94e3a83fc6b3c8924d38b0d23ac7c1e7a15defa910ee3f850b41af9ca4c", - "8c58b2272fe2da219ab0757ff27398b8d4a459eb4e75c32c782f98d684269352", - "9dd79a4e7609bd4af8e39a03f77b997b35f5050409a2ecd19de1e7d16184b1f3", - "b123a0f683f4e7b60238840139c9f3dbfe2b2c61597d9cdd4e92c718f7f98bb7", - "ba08df44e7282a1daeae2d9346b99ca741bfc2649c12aa8292f413a1c84d80b7", - "bfa9adfc127886aeaa1ac58d30c07c76e89592c29fc83dfa88062e7f3a48335e", - "c23a53858e94932a95945f28730e41ae4a2d1a8db4776283245eda143b6b2994", - "e7ec5dae7ac462cc7f7561a91ef244a2ece0894ff212995fcccb1e86438cb665", - "ee688a237a480e9fd111a7f70cc4c6f9ac837dcac65a01e7cfa29f7c28545d07", - "f31442015da37523a13ffaa173b4dfe0b069c6d890cf1c9748a898001fe4110e", - "fa73f26fb73b27b5db8f425320e206df5ebf3e137475d40be76b540ea8bd2af9", - ]; - - let stage0_fwid = match self - .mgs_client - .sp_rot_boot_info( - &self.sp.type_, - self.sp.slot, - SpComponent::ROT.const_as_str(), - &GetRotBootInfoParams { - version: - gateway_messages::RotBootInfo::HIGHEST_KNOWN_VERSION, - }, - ) - .await - { - Ok(v) => match v.into_inner() { - // the minimum we will ever return is 3 - RotState::V2 { .. } => unreachable!(), - RotState::V3 { stage0_fwid, .. } => stage0_fwid, - // ugh - RotState::CommunicationFailed { message } => { - return StepWarning::new( - None, - format!( - "Failed to communicate with the RoT: {message}. Will not proceed with update." - ), - ) - .into(); - } - }, - // If we can't run `rot_boot_info` there's a chance we can't do - // antything else with stage0 either - Err(e) => return StepWarning::new( - None, - format!("Failed to run `rot_boot_info`: {e:?}. Will not proceed with update."), - ) - .into(), - }; - - let available_artifacts = rot_bootloader.to_vec(); // Read the caboose of the currently running version (always 0) - // When updating from older stage0 we may not have a caboose so an error here - // need not be fatal - // TODO make this fatal at some point let caboose = self .mgs_client .sp_component_caboose_get( @@ -1971,14 +1925,18 @@ impl UpdateContext { ) .await .map(|v| v.into_inner()) - .ok(); + .map_err(|error| { + UpdateTerminalError::GetRotBootloaderCabooseFailed { error } + })?; - let artifact_to_apply = self - .choose_rot_artifact_to_apply( - &available_artifacts, - caboose.as_ref(), - ) - .await?; + let rkth = self.get_rot_rkth().await?; + let tags = KnownArtifactTags::RotBootloader(RotBootloaderTags { + rot_board: caboose.board, + rot_rkth: Some(RotKeyTableHash::from_bytes(rkth)), + }); + let artifact_to_apply = repo.get_handle(&tags).map_err(|error| { + UpdateTerminalError::MissingArtifact { tags, error } + })?; let make_result = |active_version| { Some(RotInterrogation { @@ -1990,52 +1948,28 @@ impl UpdateContext { }) }; - match caboose { - Some(c) => { - let message = format!( - "RoT bootloader version {} (git commit {})", - c.version, c.git_commit - ); + let message = format!( + "RoT bootloader version {} (git commit {})", + caboose.version, caboose.git_commit + ); - match c.version.parse::() { - Ok(version) => StepSuccess::new(make_result(Some(version))) - .with_message(message) - .into(), - Err(err) => StepWarning::new( - make_result(None), - format!( - "{message} (failed to parse RoT bootloader version: {err})" - ), - ) - .into(), - } - } - None => { - if KNOWN_MISSING_CABOOSE.contains(&stage0_fwid.as_str()) { - StepWarning::new( - make_result(None), - format!( - "fwid {stage0_fwid} is known to be missing a caboose." - ), - ) - .into() - } else { - StepWarning::new( - None, - format!( - "fwid {stage0_fwid} is _not_ supposed to be missing a caboose. Will not proceed with update" - ), - ) - .into() - } - } + match caboose.version.parse::() { + Ok(version) => StepSuccess::new(make_result(Some(version))) + .with_message(message) + .into(), + Err(err) => StepWarning::new( + make_result(None), + format!( + "{message} (failed to parse RoT bootloader version: {err})" + ), + ) + .into(), } } async fn interrogate_rot( &self, - rot_a: &[ArtifactIdData], - rot_b: &[ArtifactIdData], + repo: &Arc, ) -> Result, UpdateTerminalError> { let rot_active_slot = self .get_component_active_slot(SpComponent::ROT.const_as_str()) @@ -2046,18 +1980,18 @@ impl UpdateContext { // Flip these around: if 0 (A) is active, we want to // update 1 (B), and vice versa. - let (active_slot_name, slot_to_update, available_artifacts) = - match rot_active_slot { - 0 => ('A', 1, rot_b), - 1 => ('B', 0, rot_a), - _ => { - return Err(UpdateTerminalError::GetRotActiveSlotFailed { - error: anyhow!( - "unexpected RoT active slot {rot_active_slot}" - ), - }); - } - }; + let (active_slot_name, slot_to_update, slot_tag) = match rot_active_slot + { + 0 => ('A', 1, RotSlot::B), + 1 => ('B', 0, RotSlot::A), + _ => { + return Err(UpdateTerminalError::GetRotActiveSlotFailed { + error: anyhow!( + "unexpected RoT active slot {rot_active_slot}" + ), + }); + } + }; // Read the caboose of the currently-active slot. let caboose = self @@ -2074,10 +2008,15 @@ impl UpdateContext { })? .into_inner(); - let available_artifacts = available_artifacts.to_vec(); - let artifact_to_apply = self - .choose_rot_artifact_to_apply(&available_artifacts, Some(&caboose)) - .await?; + let rkth = self.get_rot_rkth().await?; + let tags = KnownArtifactTags::Rot(RotTags { + rot_board: caboose.board, + rot_rkth: Some(RotKeyTableHash::from_bytes(rkth)), + rot_slot: slot_tag, + }); + let artifact_to_apply = repo.get_handle(&tags).map_err(|error| { + UpdateTerminalError::MissingArtifact { tags, error } + })?; let message = format!( "RoT slot {active_slot_name} version {} (git commit {})", @@ -2103,21 +2042,9 @@ impl UpdateContext { } } - /// Via `client`, ask the target RoT for its CMPA/CFPA pages, then loop - /// through our `available_artifacts` to find one that verifies. - /// - /// For backwards compatibility with RoTs that do not know how to return - /// their CMPA/CFPA pages, if we fail to fetch them _and_ - /// `available_artifacts` has exactly one item, we will return that one - /// item. - /// - /// This is also applicable to the RoT bootloader which follows the - /// same vaildation method - async fn choose_rot_artifact_to_apply<'a>( - &'a self, - available_artifacts: &'a Vec, - caboose: Option<&SpComponentCaboose>, - ) -> Result<&'a ArtifactIdData, UpdateTerminalError> { + /// Via `client`, ask the target RoT for its CMPA page, then extract + /// and return the RKTH. + async fn get_rot_rkth(&self) -> Result<[u8; 32], UpdateTerminalError> { let cmpa = match self .mgs_client .sp_rot_cmpa_get( @@ -2139,26 +2066,15 @@ impl UpdateContext { // that's pretty fragile. Instead we'll treat any error response // here as a "fallback to previous behavior". Err(err @ gateway_client::Error::ErrorResponse(_)) => { - if available_artifacts.len() == 1 { - info!( - self.log, - "Failed to get RoT CMPA page; \ - using only available RoT artifact"; - "err" => %err, - ); - return Ok(&available_artifacts[0]); - } else { - error!( - self.log, - "Failed to get RoT CMPA; unable to choose from \ - multiple available RoT artifacts"; - "err" => %err, - "num_rot_artifacts" => available_artifacts.len(), - ); - return Err(UpdateTerminalError::GetRotCmpaFailed { - error: err.into(), - }); - } + error!( + self.log, + "Failed to get RoT CMPA; unable to choose from \ + available RoT artifacts"; + "err" => %err, + ); + return Err(UpdateTerminalError::GetRotCmpaFailed { + error: err.into(), + }); } // For any other error (e.g., comms failures), just fail as normal. Err(err) => { @@ -2167,127 +2083,10 @@ impl UpdateContext { }); } }; - - // We have a CMPA; we also need the CFPA, but we don't bother checking - // for an `ErrorResponse` as above because succeeding in getting the - // CMPA means the RoT is new enough to support returning both. - let cfpa = self - .mgs_client - .sp_rot_cfpa_get( - &self.sp.type_, - self.sp.slot, - SpComponent::ROT.const_as_str(), - &gateway_client::types::GetCfpaParams { - slot: RotCfpaSlot::Active, - }, - ) - .await - .map_err(|err| UpdateTerminalError::GetRotCfpaFailed { - error: err.into(), - }) - .and_then(|response| { - let data = response.into_inner().base64_data; - self.decode_rot_page(&data).map_err(|error| { - UpdateTerminalError::GetRotCfpaFailed { error } - }) - })?; - - // Loop through our possible artifacts and find the first (we only - // expect one!) that verifies against the RoT's CMPA/CFPA. - for artifact in available_artifacts { - let image = artifact - .data - .reader_stream() - .and_then(|stream| async { - let mut buf = Vec::with_capacity(artifact.data.file_size()); - StreamReader::new(stream) - .read_to_end(&mut buf) - .await - .context("I/O error reading extracted archive")?; - Ok(buf) - }) - .await - .map_err(|error| { - UpdateTerminalError::FailedFindingSignedRotImage { error } - })?; - let archive = RawHubrisArchive::from_vec(image).map_err(|err| { - UpdateTerminalError::FailedFindingSignedRotImage { - error: anyhow::Error::new(err).context(format!( - "failed to read hubris archive for {:?}", - artifact.id - )), - } - })?; - if let Some(c) = caboose { - // If we get errors on any part of reading the caboose - // something has gone wrong and it's probably safer to - // bail out on updating than potentially install an - // incorrect image - let archive_caboose = - archive.read_caboose().map_err(|error| { - UpdateTerminalError::FailedFindingSignedRotImage { - error: anyhow::Error::new(error).context(format!( - "failed to read hubris archive caboose for {:?}", - artifact.id - )), - } - })?; - - let archive_board = archive_caboose.board().map_err(|error| { - UpdateTerminalError::FailedFindingSignedRotImage { - error: anyhow::Error::new(error).context(format!( - "failed to read hubris archive BORD from caboose for {:?}", - artifact.id - )) - } - - })?; - let archive_board = String::from_utf8(archive_board.into()) - .map_err(|error| { - UpdateTerminalError::FailedFindingSignedRotImage { - error: anyhow::Error::new(error).context(format!( - "utf8 error on archive board from {:?}", - artifact.id - )), - } - })?; - - if archive_board != c.board { - info!( - self.log, "RoT archive did not match caboose board"; - "artifact" => ?artifact.id, - "archive_board" => ?archive_board, - "expected" => ?c.board, - ); - continue; - } - } - match archive.verify(&cmpa, &cfpa) { - Ok(()) => { - info!( - self.log, "RoT archive verification success"; - "name" => artifact.id.name.as_str(), - "version" => %artifact.id.version, - "kind" => ?artifact.id.kind, - ); - return Ok(artifact); - } - Err(err) => { - // We log this but don't fail - we want to continue - // looking for a verifiable artifact. - info!( - self.log, "RoT archive verification failed"; - "artifact" => ?artifact.id, - "err" => %DisplayErrorChain::new(&err), - ); - } - } - } - - // If the loop above didn't find a verifiable image, we cannot proceed. - Err(UpdateTerminalError::FailedFindingSignedRotImage { - error: anyhow!("no RoT image found with valid CMPA/CFPA"), - }) + let cmpa = CMPAPage::from_bytes(&cmpa).map_err(|err| { + UpdateTerminalError::GetRotCmpaFailed { error: err.into() } + })?; + Ok(cmpa.rotkh) } /// Decode a base64-encoded RoT page we received from MGS. @@ -2690,7 +2489,7 @@ enum ComponentUpdateStage { async fn upload_trampoline_phase_2_to_mgs( mgs_client: gateway_client::Client, - artifact: ArtifactIdData, + artifact: ArtifactHandle, status: watch::Sender, log: Logger, ) { @@ -2701,7 +2500,7 @@ async fn upload_trampoline_phase_2_to_mgs( let mut attempt = 1; let final_status = loop { - let image_stream = match artifact.data.reader_stream().await { + let image_stream = match artifact.stream().await { Ok(stream) => stream, Err(err) => { error!( @@ -2709,7 +2508,8 @@ async fn upload_trampoline_phase_2_to_mgs( "err" => #%err, ); break UploadTrampolinePhase2ToMgsStatus::Failed(Arc::new( - err.context("failed to read trampoline phase 2"), + anyhow::Error::new(err) + .context("failed to read trampoline phase 2"), )); } }; @@ -2722,7 +2522,7 @@ async fn upload_trampoline_phase_2_to_mgs( { Ok(response) => { break UploadTrampolinePhase2ToMgsStatus::Done { - hash: artifact.data.hash(), + hash: artifact.artifact().hash, uploaded_image_id: response.into_inner(), }; } @@ -2773,7 +2573,7 @@ impl<'a> SpComponentUpdateContext<'a> { &self, engine: &UpdateEngine<'a, SpComponentUpdateSpec>, firmware_slot: u16, - artifact: &'a ArtifactIdData, + artifact: &'a ArtifactHandle, ) { let update_id = Uuid::new_v4(); let component = self.component; @@ -2798,14 +2598,13 @@ impl<'a> SpComponentUpdateContext<'a> { format!("Sending data to MGS (slot {firmware_slot})"), async move |_cx| { let data_stream = artifact - .data - .reader_stream() + .stream() .await .map_err(|error| { SpComponentUpdateTerminalError::SpComponentUpdateFailed { stage: SpComponentUpdateStage::Sending, - artifact: artifact.id.clone(), - error, + artifact: ArtifactId::new(artifact.artifact()), + error: anyhow!(error), } })?; @@ -2825,7 +2624,7 @@ impl<'a> SpComponentUpdateContext<'a> { .map_err(|error| { SpComponentUpdateTerminalError::SpComponentUpdateFailed { stage: SpComponentUpdateStage::Sending, - artifact: artifact.id.clone(), + artifact: ArtifactId::new(artifact.artifact()), error: anyhow!(error), } })?; @@ -2851,7 +2650,7 @@ impl<'a> SpComponentUpdateContext<'a> { .map_err(|error| { SpComponentUpdateTerminalError::SpComponentUpdateFailed { stage: SpComponentUpdateStage::Preparing, - artifact: artifact.id.clone(), + artifact: ArtifactId::new(artifact.artifact()), error, } })?; @@ -2877,7 +2676,7 @@ impl<'a> SpComponentUpdateContext<'a> { .map_err(|error| { SpComponentUpdateTerminalError::SpComponentUpdateFailed { stage: SpComponentUpdateStage::Writing, - artifact: artifact.id.clone(), + artifact: ArtifactId::new(artifact.artifact()), error, } })?; diff --git a/wicketd/tests/integration_tests/updates.rs b/wicketd/tests/integration_tests/updates.rs index 173147b812e..9493c828d18 100644 --- a/wicketd/tests/integration_tests/updates.rs +++ b/wicketd/tests/integration_tests/updates.rs @@ -17,11 +17,12 @@ use maplit::btreeset; use omicron_common::{ disk::DiskIdentity, update::{ - MupdateOverrideInfo, OmicronInstallManifest, + MupdateOverrideHashId, MupdateOverrideInfo, OmicronInstallManifest, OmicronInstallManifestSource, }, }; use omicron_uuid_kinds::{InternalZpoolUuid, MupdateUuid}; +use semver::Version; use sled_agent_config_reconciler::{ InternalDiskDetails, InternalDisksReceiver, InternalDisksWithBootDisk, }; @@ -29,8 +30,8 @@ use sled_agent_resolvable_files::ZoneImageSourceResolver; use sled_agent_types::resolvable_files::MupdateOverrideNonBootResult; use sled_storage::config::MountConfig; use tokio::sync::oneshot; -use tufaceous_artifact::{ArtifactHashId, ArtifactKind, KnownArtifactKind}; -use update_common::artifacts::UpdatePlan; +use tufaceous_artifact_v2::{InstallinatorArtifactKind, KnownArtifactTags}; +use tufaceous_v2::{Repository, edit::RepositoryEditor}; use update_engine::NestedError; use wicket::OutputKind; use wicket_common::{ @@ -61,30 +62,24 @@ static FAKE_NON_SEMVER_ZONE_FILE_NAMES: &[&str] = &[ "oximeter.tar.gz", ]; -// See documentation for extract_nested_artifact_pair in update_plan.rs for why -// multi_thread is required. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[tokio::test] async fn test_updates() { let gateway = gateway_setup::test_setup("test_updates", SpPort::One).await; let wicketd_testctx = WicketdTestContext::setup(gateway).await; let log = wicketd_testctx.log(); - let temp_dir = Utf8TempDir::new().expect("temp dir created"); - let archive_path = temp_dir.path().join("archive.zip"); - - let args = tufaceous::Args::try_parse_from([ - "tufaceous", - "assemble", - "../update-common/manifests/fake.toml", - archive_path.as_str(), - ]) - .expect("args parsed correctly"); - - args.exec(log).await.expect("assemble command completed successfully"); - - // Read the archive and upload it to the server. - let zip_bytes = - fs_err::read(&archive_path).expect("archive read correctly"); + let zip_bytes = RepositoryEditor::fake(Version::new(1, 0, 0)) + .unwrap() + .finish() + .await + .unwrap() + .generate_root() + .sign() + .await + .unwrap() + .write_zip(Vec::new(), chrono::Utc::now()) + .await + .unwrap(); wicketd_testctx .wicketd_client .put_repository(zip_bytes) @@ -92,66 +87,16 @@ async fn test_updates() { .expect("bytes read and archived"); // List out the artifacts in the repository. - let response = wicketd_testctx - .wicketd_client - .get_artifacts_and_event_reports() - .await - .expect("get_artifacts_and_event_reports succeeded") - .into_inner(); - - // We should have an artifact for every known artifact kind (except - // `Zone`), as well as the installinator document... - let expected_kinds: BTreeSet<_> = KnownArtifactKind::iter() - .filter(|k| !matches!(k, KnownArtifactKind::Zone)) - .map(ArtifactKind::from) - .collect(); - - // ... and installable artifacts that replace the top level host, - // trampoline, and RoT with their inner parts (phase1/phase2 for OS images - // and A/B images for the RoT) during import. - let mut expected_installable_kinds = expected_kinds.clone(); - for remove in [ - KnownArtifactKind::Host, - KnownArtifactKind::Trampoline, - KnownArtifactKind::GimletRot, - KnownArtifactKind::PscRot, - KnownArtifactKind::SwitchRot, - ] { - assert!(expected_installable_kinds.remove(&remove.into())); - } - for add in [ - ArtifactKind::GIMLET_HOST_PHASE_1, - ArtifactKind::COSMO_HOST_PHASE_1, - ArtifactKind::HOST_PHASE_2, - ArtifactKind::GIMLET_TRAMPOLINE_PHASE_1, - ArtifactKind::COSMO_TRAMPOLINE_PHASE_1, - ArtifactKind::TRAMPOLINE_PHASE_2, - ArtifactKind::GIMLET_ROT_IMAGE_A, - ArtifactKind::GIMLET_ROT_IMAGE_B, - ArtifactKind::PSC_ROT_IMAGE_A, - ArtifactKind::PSC_ROT_IMAGE_B, - ArtifactKind::SWITCH_ROT_IMAGE_A, - ArtifactKind::SWITCH_ROT_IMAGE_B, - ] { - assert!(expected_installable_kinds.insert(add)); - } - - // Ensure that this is a sensible result. - let mut kinds = BTreeSet::new(); - let mut installable_kinds = BTreeSet::new(); - let expected_artifact_ids: BTreeSet<_> = - response.artifacts.iter().map(|a| a.artifact_id.clone()).collect(); - for artifact in response.artifacts { - kinds.insert(artifact.artifact_id.kind); - for installable in artifact.installable { - installable_kinds.insert(installable.kind.parse().unwrap()); - } - } - assert_eq!(expected_kinds, kinds, "all expected kinds present"); - assert_eq!( - expected_installable_kinds, installable_kinds, - "all expected installable kinds present" - ); + let expected_artifact_ids = { + let mut response = wicketd_testctx + .wicketd_client + .get_artifacts_and_event_reports() + .await + .expect("get_artifacts_and_event_reports succeeded") + .into_inner(); + response.artifacts.sort_unstable(); + response.artifacts + }; let target_sp = SpIdentifier { type_: SpType::Sled, slot: 0 }; @@ -191,10 +136,10 @@ async fn test_updates() { { // Before starting, all artifacts should be present, no components should be present, // and the state should be NotStarted. - let status = get_rack_update_status(&wicketd_testctx, &[]).await; - let actual_ids: BTreeSet<_> = status.artifacts.into_iter().collect(); + let mut status = get_rack_update_status(&wicketd_testctx, &[]).await; + status.artifacts.sort_unstable(); assert_eq!( - expected_artifact_ids, actual_ids, + expected_artifact_ids, status.artifacts, "all uploaded artifacts appear in status" ); assert_eq!( @@ -350,7 +295,7 @@ async fn test_updates() { { // After clearing, status should show NotStarted and no components. // Uploaded artifacts should be unaffected by the clear. - let status = get_rack_update_status(&wicketd_testctx, &[]).await; + let mut status = get_rack_update_status(&wicketd_testctx, &[]).await; assert_eq!( status.state, UpdateState::NotStarted, @@ -360,9 +305,9 @@ async fn test_updates() { status.components.is_empty(), "no components should be present" ); - let actual_ids: BTreeSet<_> = status.artifacts.into_iter().collect(); + status.artifacts.sort_unstable(); assert_eq!( - expected_artifact_ids, actual_ids, + expected_artifact_ids, status.artifacts, "artifacts should be unaffected by clear" ); } @@ -405,9 +350,7 @@ async fn get_rack_update_status( .expect("rack-update status --json output is valid JSON") } -// See documentation for extract_nested_artifact_pair in update_plan.rs for why -// multi_thread is required. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[tokio::test] async fn test_installinator_fetch() { let gateway = gateway_setup::test_setup( "test_installinator_fetch_no_installinator_document", @@ -415,92 +358,74 @@ async fn test_installinator_fetch() { ) .await; let wicketd_testctx = WicketdTestContext::setup(gateway).await; - let log = wicketd_testctx.log(); - - let temp_dir = Utf8TempDir::new().expect("temp dir created"); - let archive_path = temp_dir.path().join("archive.zip"); - // Test ingestion of an artifact with non-semver versions. This ensures that - // wicketd for v14 and above can handle non-semver versions. - // - // --allow-non-semver can be removed once customer systems are updated to - // v14 and above. - let args = tufaceous::Args::try_parse_from([ - "tufaceous", - "assemble", - "../update-common/manifests/fake-non-semver.toml", - "--allow-non-semver", - archive_path.as_str(), - ]) - .expect("args parsed correctly"); - - args.exec(log).await.expect("assemble command completed successfully"); - - // Read the archive and upload it to the server. - let zip_bytes = - fs_err::read(&archive_path).expect("archive read correctly"); + let zip_bytes = RepositoryEditor::fake(Version::new(1, 0, 0)) + .unwrap() + .finish() + .await + .unwrap() + .generate_root() + .sign() + .await + .unwrap() + .write_zip(Vec::new(), chrono::Utc::now()) + .await + .unwrap(); wicketd_testctx .wicketd_client .put_repository(zip_bytes) .await .expect("bytes read and archived"); - let update_plan = wicketd_testctx + let repo = wicketd_testctx .server .artifact_store - .current_plan() + .current_repository() .expect("we just uploaded a repository, so there should be a plan"); - installinator_fetch_impl(&wicketd_testctx, &temp_dir, &update_plan).await; + installinator_fetch_impl(&wicketd_testctx, &repo).await; wicketd_testctx.teardown().await; } async fn installinator_fetch_impl( wicketd_testctx: &WicketdTestContext, - temp_dir: &Utf8TempDir, - update_plan: &UpdatePlan, + repo: &Repository, ) { let log = wicketd_testctx.log(); + let temp_dir = Utf8TempDir::new().expect("temp dir created"); - // Are the host phase 2 and control plane artifacts available when looked up - // by hash? - let host_phase_2_id = ArtifactHashId { - kind: ArtifactKind::HOST_PHASE_2, - hash: update_plan.host_phase_2_hash, - }; - assert!( - wicketd_testctx - .server - .artifact_store - .contains_by_hash(&host_phase_2_id), - "host phase 2 ID found by hash" - ); - - let control_plane_id = ArtifactHashId { - kind: KnownArtifactKind::ControlPlane.into(), - hash: update_plan.control_plane_hash, - }; - assert!( - wicketd_testctx - .server - .artifact_store - .contains_by_hash(&control_plane_id), - "control plane ID found by hash" - ); + let mut expected_ids = BTreeSet::new(); + for artifact in repo.artifacts() { + let Some(kind) = + artifact.known_tags().and_then(|tags| tags.to_installinator()) + else { + continue; + }; + let kind = match kind { + InstallinatorArtifactKind::Zone { zone_name } => { + format!("zone-{zone_name}") + } + InstallinatorArtifactKind::HostPhase2 => "host_phase2".into(), + InstallinatorArtifactKind::MeasurementCorpus => { + "measurement_corpus".into() + } + InstallinatorArtifactKind::ControlPlane => unreachable!(), + }; + expected_ids + .insert(MupdateOverrideHashId { kind, hash: artifact.hash }); + } - let installinator_doc_hash = update_plan - .installinator_doc_hash - .expect("expected installinator document to be present"); - let installinator_doc_id = ArtifactHashId { - kind: KnownArtifactKind::InstallinatorDocument.into(), - hash: installinator_doc_hash, - }; + let installinator_doc_hash = repo + .artifacts() + .get_only(&KnownArtifactTags::InstallinatorDocument) + .unwrap() + .hash; assert!( wicketd_testctx .server .artifact_store - .contains_by_hash(&installinator_doc_id), + .contains_installinator_artifact(installinator_doc_hash), "installinator document ID found by hash" ); @@ -600,12 +525,7 @@ async fn installinator_fetch_impl( serde_json::from_slice::(&a_override_bytes) .expect("mupdate override file successfully deserialized"); - assert_eq!( - a_override_info.hash_ids, - btreeset! { - host_phase_2_id, control_plane_id, - } - ); + assert_eq!(a_override_info.hash_ids, expected_ids); // Ensure that the B path also had the same file written out. let b_override_path = @@ -750,9 +670,7 @@ async fn installinator_fetch_impl( recv_handle.await.expect("recv_handle succeeded"); } -// See documentation for extract_nested_artifact_pair in update_plan.rs for why -// multi_thread is required. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[tokio::test] async fn test_update_races() { let gateway = gateway_setup::test_setup( "test_artifact_upload_while_updating", @@ -760,24 +678,19 @@ async fn test_update_races() { ) .await; let wicketd_testctx = WicketdTestContext::setup(gateway).await; - let log = wicketd_testctx.log(); - - let temp_dir = Utf8TempDir::new().expect("temp dir created"); - let archive_path = temp_dir.path().join("archive.zip"); - - let args = tufaceous::Args::try_parse_from([ - "tufaceous", - "assemble", - "../update-common/manifests/fake.toml", - archive_path.as_str(), - ]) - .expect("args parsed correctly"); - args.exec(log).await.expect("assemble command completed successfully"); - - // Read the archive and upload it to the server. - let zip_bytes = - fs_err::read(&archive_path).expect("archive read correctly"); + let zip_bytes = RepositoryEditor::fake(Version::new(1, 0, 0)) + .unwrap() + .finish() + .await + .unwrap() + .generate_root() + .sign() + .await + .unwrap() + .write_zip(Vec::new(), chrono::Utc::now()) + .await + .unwrap(); wicketd_testctx .wicketd_client .put_repository(zip_bytes.clone()) diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 634077d7307..dc71a282e49 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -26,11 +26,11 @@ aws-lc-rs = { version = "1.16.3", features = ["prebuilt-nasm"] } aws-lc-sys = { version = "0.40.0", default-features = false, features = ["prebuilt-nasm"] } base16ct = { version = "0.2.0", default-features = false, features = ["alloc"] } base64 = { version = "0.22.1" } -base64ct = { version = "1.8.3", default-features = false, features = ["std"] } bitflags-dff4ba8e3ae991db = { package = "bitflags", version = "1.3.2" } bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.11.0", default-features = false, features = ["serde", "std"] } bstr = { version = "1.12.1" } buf-list = { version = "1.1.2", default-features = false, features = ["tokio1"] } +byte-wrapper = { version = "0.1.0", features = ["schemars08", "serde"] } bytes = { version = "1.11.1", features = ["serde"] } camino = { version = "1.2.2", default-features = false, features = ["serde1"] } chrono = { version = "0.4.44", features = ["serde"] } @@ -97,7 +97,6 @@ num-traits = { version = "0.2.19", features = ["i128", "libm"] } once_cell = { version = "1.21.3", features = ["critical-section"] } openapiv3 = { version = "2.2.0", default-features = false, features = ["skip_serializing_defaults"] } peg-runtime = { version = "0.8.5", default-features = false, features = ["std"] } -pem-rfc7468 = { version = "0.7.0", default-features = false, features = ["std"] } percent-encoding = { version = "2.3.2" } petgraph-3b31131e45eafb45 = { package = "petgraph", version = "0.6.5", features = ["serde-1"] } petgraph-c38e5c1d305a1b54 = { package = "petgraph", version = "0.8.3", features = ["serde-1"] } @@ -106,7 +105,6 @@ postgres-types = { version = "0.2.12", default-features = false, features = ["wi ppv-lite86 = { version = "0.2.21", default-features = false, features = ["simd", "std"] } predicates = { version = "3.1.4" } proc-macro2 = { version = "1.0.106" } -proptest = { version = "1.10.0" } quote = { version = "1.0.45" } rand-274715c4dabd11b0 = { package = "rand", version = "0.9.2" } rand-c38e5c1d305a1b54 = { package = "rand", version = "0.8.6" } @@ -152,6 +150,7 @@ toml_edit-3c51e837cfc5589a = { package = "toml_edit", version = "0.22.27", featu toml_parser = { version = "1.1.2" } tough = { version = "0.22.0", default-features = false, features = ["http"] } tracing = { version = "0.1.44", features = ["log"] } +tufaceous-artifact = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main", default-features = false, features = ["hubtools", "proptest", "schemars"] } typenum = { version = "1.20.1", default-features = false, features = ["const-generics"] } url = { version = "2.5.8", features = ["serde"] } usdt = { version = "0.6.0" } @@ -173,11 +172,11 @@ aws-lc-rs = { version = "1.16.3", features = ["prebuilt-nasm"] } aws-lc-sys = { version = "0.40.0", default-features = false, features = ["prebuilt-nasm"] } base16ct = { version = "0.2.0", default-features = false, features = ["alloc"] } base64 = { version = "0.22.1" } -base64ct = { version = "1.8.3", default-features = false, features = ["std"] } bitflags-dff4ba8e3ae991db = { package = "bitflags", version = "1.3.2" } bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.11.0", default-features = false, features = ["serde", "std"] } bstr = { version = "1.12.1" } buf-list = { version = "1.1.2", default-features = false, features = ["tokio1"] } +byte-wrapper = { version = "0.1.0", features = ["schemars08", "serde"] } bytes = { version = "1.11.1", features = ["serde"] } camino = { version = "1.2.2", default-features = false, features = ["serde1"] } chrono = { version = "0.4.44", features = ["serde"] } @@ -245,7 +244,6 @@ num-traits = { version = "0.2.19", features = ["i128", "libm"] } once_cell = { version = "1.21.3", features = ["critical-section"] } openapiv3 = { version = "2.2.0", default-features = false, features = ["skip_serializing_defaults"] } peg-runtime = { version = "0.8.5", default-features = false, features = ["std"] } -pem-rfc7468 = { version = "0.7.0", default-features = false, features = ["std"] } percent-encoding = { version = "2.3.2" } petgraph-3b31131e45eafb45 = { package = "petgraph", version = "0.6.5", features = ["serde-1"] } petgraph-c38e5c1d305a1b54 = { package = "petgraph", version = "0.8.3", features = ["serde-1"] } @@ -254,7 +252,6 @@ postgres-types = { version = "0.2.12", default-features = false, features = ["wi ppv-lite86 = { version = "0.2.21", default-features = false, features = ["simd", "std"] } predicates = { version = "3.1.4" } proc-macro2 = { version = "1.0.106" } -proptest = { version = "1.10.0" } quote = { version = "1.0.45" } rand-274715c4dabd11b0 = { package = "rand", version = "0.9.2" } rand-c38e5c1d305a1b54 = { package = "rand", version = "0.8.6" } @@ -303,6 +300,7 @@ toml_edit-3c51e837cfc5589a = { package = "toml_edit", version = "0.22.27", featu toml_parser = { version = "1.1.2" } tough = { version = "0.22.0", default-features = false, features = ["http"] } tracing = { version = "0.1.44", features = ["log"] } +tufaceous-artifact = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main", default-features = false, features = ["hubtools", "proptest", "schemars"] } typenum = { version = "1.20.1", default-features = false, features = ["const-generics"] } url = { version = "2.5.8", features = ["serde"] } usdt = { version = "0.6.0" } From bbf290e6a899bd273c040495439d3b2ff7485607 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 7 Jul 2026 08:52:13 -0700 Subject: [PATCH 02/24] avoid adding ErrorExt to update-common for now update-common is used to validate that repositories generated by releng are usable. we want to keep this code around beyond the release, so it's better to remove update-common deps as we switch things over to tufaceous v2. --- Cargo.lock | 2 -- update-common/Cargo.toml | 15 ++++++------ update-common/src/lib.rs | 43 ----------------------------------- wicketd/Cargo.toml | 1 - wicketd/src/update_tracker.rs | 34 +++++++++++++++++++++++---- 5 files changed, 36 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ecf6303997e..548ac0a43cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16682,7 +16682,6 @@ dependencies = [ "tokio-util", "tough", "tufaceous 0.1.0", - "tufaceous 0.2.0", "tufaceous-artifact 0.1.0", "tufaceous-brand-metadata 0.1.0", "tufaceous-lib", @@ -17466,7 +17465,6 @@ dependencies = [ "tufaceous 0.1.0", "tufaceous 0.2.0", "tufaceous-artifact 0.2.0", - "update-common", "update-engine", "uuid", "wicket", diff --git a/update-common/Cargo.toml b/update-common/Cargo.toml index a736f5ee75e..89e08d816fa 100644 --- a/update-common/Cargo.toml +++ b/update-common/Cargo.toml @@ -16,25 +16,24 @@ chrono.workspace = true debug-ignore.workspace = true display-error-chain.workspace = true dropshot.workspace = true -flate2.workspace = true -fs-err = { workspace = true, features = ["tokio"] } futures.workspace = true hex.workspace = true hubtools.workspace = true omicron-common.workspace = true -omicron-workspace-hack.workspace = true -semver.workspace = true sha2.workspace = true slog.workspace = true -tar.workspace = true thiserror.workspace = true tokio.workspace = true tokio-util.workspace = true tough.workspace = true -tufaceous-artifact.workspace = true -tufaceous-brand-metadata.workspace = true tufaceous-lib.workspace = true -tufaceous-v2.workspace = true +omicron-workspace-hack.workspace = true +tufaceous-brand-metadata.workspace = true +tar.workspace = true +flate2.workspace = true +fs-err = { workspace = true, features = ["tokio"] } +semver.workspace = true +tufaceous-artifact.workspace = true [dev-dependencies] clap.workspace = true diff --git a/update-common/src/lib.rs b/update-common/src/lib.rs index be5f3651d00..b1f0d884840 100644 --- a/update-common/src/lib.rs +++ b/update-common/src/lib.rs @@ -6,46 +6,3 @@ pub mod artifacts; pub mod errors; - -use std::error::Error as _; - -use display_error_chain::DisplayErrorChain; -use dropshot::HttpError; -use tufaceous_v2::error::Error; -use tufaceous_v2::error::ErrorKind; - -pub trait ErrorExt { - fn to_http_error(&self) -> HttpError; -} - -impl ErrorExt for Error { - fn to_http_error(&self) -> HttpError { - self.0.to_http_error() - } -} - -impl ErrorExt for ErrorKind { - fn to_http_error(&self) -> HttpError { - // `HttpError` from reading a stream is bubbled up, if possible. - if let Some(source) = self.source() - && let Some(error) = source.downcast_ref::() - { - return HttpError { - status_code: error.status_code, - error_code: error.error_code.clone(), - external_message: error.external_message.clone(), - internal_message: error.internal_message.clone(), - headers: error.headers.clone(), - }; - } - - let message = DisplayErrorChain::new(self).to_string(); - if self.is_repository_error() { - // This error is definitely because of bad repository contents. - HttpError::for_bad_request(None, message) - } else { - // This error is probably not due to bad repository contents. - HttpError::for_unavail(None, message) - } - } -} diff --git a/wicketd/Cargo.toml b/wicketd/Cargo.toml index a50ff141b98..8d0fe355224 100644 --- a/wicketd/Cargo.toml +++ b/wicketd/Cargo.toml @@ -68,7 +68,6 @@ omicron-passwords.workspace = true omicron-uuid-kinds.workspace = true sled-agent-types.workspace = true sled-hardware-types.workspace = true -update-common.workspace = true update-engine.workspace = true wicket-common.workspace = true wicketd-api.workspace = true diff --git a/wicketd/src/update_tracker.rs b/wicketd/src/update_tracker.rs index 8149eefa93e..043d618a8d0 100644 --- a/wicketd/src/update_tracker.rs +++ b/wicketd/src/update_tracker.rs @@ -16,6 +16,7 @@ use bytes::Bytes; use display_error_chain::DisplayErrorChain; use dropshot::HttpError; use futures::Stream; +use futures::TryFutureExt; use gateway_client::types::GetRotBootInfoParams; use gateway_client::types::HostPhase2Progress; use gateway_client::types::HostPhase2RecoveryImageId; @@ -43,6 +44,7 @@ use slog::warn; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::btree_map::Entry; +use std::error::Error as _; use std::net::SocketAddrV6; use std::sync::Arc; use std::sync::Mutex as StdMutex; @@ -70,7 +72,6 @@ use tufaceous_v2::ArtifactHandle; use tufaceous_v2::ExpirationEnforcement; use tufaceous_v2::Repository; use tufaceous_v2::RepositoryLoader; -use update_common::ErrorExt; use update_engine::AbortHandle; use update_engine::NestedSpec; use update_engine::StepSpec; @@ -388,11 +389,34 @@ impl UpdateTracker { .unsafe_blindly_trust_repo() .v1_compatibility(true) .load_zip_stream(stream, None, &self.log) + .and_then(async |repo| { + repo.verify_targets(VERIFY_PARALLELISM).await?; + Ok(repo) + }) .await - .map_err(|err| err.to_http_error())?; - repo.verify_targets(VERIFY_PARALLELISM) - .await - .map_err(|err| err.to_http_error())?; + .map_err(|err| { + if let Some(source) = err.source() + && let Some(error) = source.downcast_ref::() + { + // manual Clone::clone + return HttpError { + status_code: error.status_code, + error_code: error.error_code.clone(), + external_message: error.external_message.clone(), + internal_message: error.internal_message.clone(), + headers: error.headers.clone(), + }; + } + + let message = DisplayErrorChain::new(&err).to_string(); + if err.is_repository_error() { + // This error is because of bad repository contents. + HttpError::for_bad_request(None, message) + } else { + // This error is likely not due to bad repository contents. + HttpError::for_unavail(None, message) + } + })?; let mut update_data = self.sp_update_data.lock().await; update_data.set_repository(Arc::new(repo)).await } From 1a3d0524dd796bce38e96b531caea7aea2abde7d Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 7 Jul 2026 15:16:03 -0700 Subject: [PATCH 03/24] nit --- Cargo.lock | 1 - wicketd/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 548ac0a43cb..a2ec4c172db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17462,7 +17462,6 @@ dependencies = [ "toml 0.8.23", "tough", "transceiver-controller 0.1.1 (git+https://github.com/oxidecomputer/transceiver-control)", - "tufaceous 0.1.0", "tufaceous 0.2.0", "tufaceous-artifact 0.2.0", "update-engine", diff --git a/wicketd/Cargo.toml b/wicketd/Cargo.toml index 8d0fe355224..53fafb88a45 100644 --- a/wicketd/Cargo.toml +++ b/wicketd/Cargo.toml @@ -102,5 +102,4 @@ sled-storage.workspace = true subprocess.workspace = true tar.workspace = true tokio = { workspace = true, features = ["test-util"] } -tufaceous.workspace = true wicket.workspace = true From e9fd524864a44bd6e306232f002ceca89d7527dc Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 9 Jul 2026 09:26:07 -0700 Subject: [PATCH 04/24] update tufaceous --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e71b223faff..825412e42ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16276,7 +16276,7 @@ dependencies = [ [[package]] name = "tufaceous" version = "0.2.0" -source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#6e2d1803e273de403848d62781eaec37996fd2ce" +source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#de59b9508e99da8fc39a70d29e4d69300325b220" dependencies = [ "atomicwrites", "aws-lc-rs", @@ -16327,7 +16327,7 @@ dependencies = [ [[package]] name = "tufaceous-artifact" version = "0.2.0" -source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#6e2d1803e273de403848d62781eaec37996fd2ce" +source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#de59b9508e99da8fc39a70d29e4d69300325b220" dependencies = [ "byte-wrapper", "daft", @@ -16355,7 +16355,7 @@ dependencies = [ [[package]] name = "tufaceous-brand-metadata" version = "0.2.0" -source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#6e2d1803e273de403848d62781eaec37996fd2ce" +source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#de59b9508e99da8fc39a70d29e4d69300325b220" dependencies = [ "serde", "serde_json", From 7e63b23dacf9ed318dc270fb9cbaddb1cadfa34c Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 20 Jul 2026 17:00:20 -0700 Subject: [PATCH 05/24] serve the v1 installinator document if present --- Cargo.lock | 91 +++++++++++++++++++++++++++------- wicketd/src/artifacts/store.rs | 1 + wicketd/src/update_tracker.rs | 25 +++++++--- workspace-hack/Cargo.toml | 2 + 4 files changed, 96 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e3945f538f8..29714246a4f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6161,25 +6161,37 @@ checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" [[package]] name = "jiff" -version = "0.2.21" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3e3d65f018c6ae946ab16e80944b97096ed73c35b221d1c478a6c81d8f57940" +checksum = "e184d09547b80eb7e20d141ba2fb1fbac843ca53f4cf1b31210adc4c1adc6e16" dependencies = [ + "defmt 1.0.1", + "jiff-core", "jiff-static", "jiff-tzdb-platform", "log", "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.61.2", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt 1.0.1", ] [[package]] name = "jiff-static" -version = "0.2.21" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17c2b211d863c7fde02cbea8a3c1a439b98e109286554f2860bdded7ff83818" +checksum = "323da076b7a6faf914dc677cb05a4b907742ff7375c8322c9e7f5061e5e0e9de" dependencies = [ + "jiff-core", "proc-macro2", "quote", "syn 2.0.117", @@ -8256,7 +8268,7 @@ dependencies = [ "textwrap 0.16.2", "thiserror 2.0.18", "tokio", - "tough", + "tough 0.22.0", "trust-quorum-protocol", "trust-quorum-types", "tufaceous-artifact 0.1.0", @@ -8298,7 +8310,7 @@ dependencies = [ "slog-error-chain", "strum 0.27.2", "thiserror 2.0.18", - "tough", + "tough 0.22.0", "tufaceous-artifact 0.1.0", "url", "uuid", @@ -8663,6 +8675,16 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "olpc-cjson" +version = "0.1.4" +source = "git+https://github.com/oxidecomputer/tough?branch=oxide%2Fdevelop#56967c9651efcffd7326198f04fc388c4f0f92b9" +dependencies = [ + "serde", + "serde_json", + "unicode-normalization", +] + [[package]] name = "omicron-certificates" version = "0.1.0" @@ -9261,7 +9283,7 @@ dependencies = [ "tokio", "tokio-postgres", "tokio-util", - "tough", + "tough 0.22.0", "trust-quorum-types", "tufaceous 0.1.0", "tufaceous-artifact 0.1.0", @@ -9870,6 +9892,7 @@ dependencies = [ "ipnet", "ipnetwork", "itertools 0.13.0", + "jiff", "lalrpop-util", "lazy_static", "libc", @@ -9945,7 +9968,7 @@ dependencies = [ "toml_edit 0.19.15", "toml_edit 0.22.27", "toml_parser", - "tough", + "tough 0.22.0", "tracing", "tufaceous-artifact 0.2.0", "typenum", @@ -15863,7 +15886,7 @@ dependencies = [ "globset", "hex", "log", - "olpc-cjson", + "olpc-cjson 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "pem", "percent-encoding", "reqwest 0.12.28", @@ -15881,6 +15904,39 @@ dependencies = [ "walkdir", ] +[[package]] +name = "tough" +version = "0.24.0" +source = "git+https://github.com/oxidecomputer/tough?branch=oxide%2Fdevelop#56967c9651efcffd7326198f04fc388c4f0f92b9" +dependencies = [ + "async-recursion", + "async-trait", + "aws-lc-rs", + "bytes", + "dyn-clone", + "futures", + "futures-core", + "globset", + "hex", + "jiff", + "log", + "olpc-cjson 0.1.4 (git+https://github.com/oxidecomputer/tough?branch=oxide%2Fdevelop)", + "pem", + "percent-encoding", + "rustls", + "serde", + "serde_json", + "serde_plain", + "snafu", + "tempfile", + "tokio", + "tokio-util", + "typed-path", + "untrusted 0.7.1", + "url", + "walkdir", +] + [[package]] name = "tower" version = "0.5.3" @@ -16294,7 +16350,7 @@ dependencies = [ [[package]] name = "tufaceous" version = "0.2.0" -source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#de59b9508e99da8fc39a70d29e4d69300325b220" +source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#d24aec34fcbdf5fda0db8103a1e77926bd166a86" dependencies = [ "atomicwrites", "aws-lc-rs", @@ -16309,6 +16365,7 @@ dependencies = [ "futures-util", "hex", "hubtools 0.4.7 (git+https://github.com/oxidecomputer/hubtools.git?rev=2b1ef9b38d75563ea800baa3b17327eec17b1b7a)", + "jiff", "rats-corim 0.1.0 (git+https://github.com/oxidecomputer/rats-corim.git?rev=f0d5d5168d3d31487a56df32c676b0c6240bcc6b)", "rawzip", "semver 1.0.28", @@ -16319,7 +16376,7 @@ dependencies = [ "tar", "thiserror 2.0.18", "tokio", - "tough", + "tough 0.24.0", "tufaceous-artifact 0.2.0", "tufaceous-brand-metadata 0.2.0", "url", @@ -16345,7 +16402,7 @@ dependencies = [ [[package]] name = "tufaceous-artifact" version = "0.2.0" -source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#de59b9508e99da8fc39a70d29e4d69300325b220" +source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#d24aec34fcbdf5fda0db8103a1e77926bd166a86" dependencies = [ "byte-wrapper", "daft", @@ -16373,7 +16430,7 @@ dependencies = [ [[package]] name = "tufaceous-brand-metadata" version = "0.2.0" -source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#de59b9508e99da8fc39a70d29e4d69300325b220" +source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#d24aec34fcbdf5fda0db8103a1e77926bd166a86" dependencies = [ "serde", "serde_json", @@ -16416,7 +16473,7 @@ dependencies = [ "thiserror 2.0.18", "tokio", "toml 0.8.23", - "tough", + "tough 0.22.0", "tufaceous-artifact 0.1.0", "tufaceous-brand-metadata 0.1.0", "url", @@ -16715,7 +16772,7 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tokio-util", - "tough", + "tough 0.22.0", "tufaceous 0.1.0", "tufaceous-artifact 0.1.0", "tufaceous-brand-metadata 0.1.0", @@ -17497,7 +17554,7 @@ dependencies = [ "tokio-stream", "tokio-util", "toml 0.8.23", - "tough", + "tough 0.22.0", "transceiver-controller 0.1.1 (git+https://github.com/oxidecomputer/transceiver-control)", "tufaceous 0.2.0", "tufaceous-artifact 0.2.0", diff --git a/wicketd/src/artifacts/store.rs b/wicketd/src/artifacts/store.rs index 1fda57e1d8e..cdbb9cb81b3 100644 --- a/wicketd/src/artifacts/store.rs +++ b/wicketd/src/artifacts/store.rs @@ -40,6 +40,7 @@ impl WicketdArtifactStore { slog::debug!(self.log, "setting repository"); let by_hash = repo .handles() + .chain(repo.installinator_v1_handles()) .map(|handle| (handle.artifact().hash, handle)) .collect(); self.replace(Inner { by_hash, repo }); diff --git a/wicketd/src/update_tracker.rs b/wicketd/src/update_tracker.rs index 975a5fb62d5..043a8fa9ad2 100644 --- a/wicketd/src/update_tracker.rs +++ b/wicketd/src/update_tracker.rs @@ -1453,13 +1453,26 @@ impl UpdateDriver { UpdateStepId::SettingInstallinatorImageId, "Setting installinator image ID", async move |_cx| { - let tags = KnownArtifactTags::InstallinatorDocument; - let artifact = - repo.artifacts().get_only(&tags).map_err(|error| { - UpdateTerminalError::MissingArtifact { tags, error } - })?; + let hash = + if let Some(hash) = repo.installinator_v1_document() { + // Set the installinator document hash to the + // original v1 document, if one was present. This + // allows older Installinators to function. + hash + } else { + let tags = KnownArtifactTags::InstallinatorDocument; + repo.artifacts() + .get_only(&tags) + .map_err(|error| { + UpdateTerminalError::MissingArtifact { + tags, + error, + } + })? + .hash + }; let installinator_image_id = InstallinatorImageId { - host_phase_2: artifact.hash.to_string(), + host_phase_2: hash.to_string(), control_plane: ArtifactHash([0; 32]).to_string(), update_id: update_cx.update_id, }; diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index a9423821520..74fd490114f 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -82,6 +82,7 @@ indexmap = { version = "2.14.0", features = ["serde"] } ipnet = { version = "2.11.0", features = ["serde"] } ipnetwork = { version = "0.21.1", features = ["schemars", "serde"] } itertools = { version = "0.13.0" } +jiff = { version = "0.2.34", features = ["serde"] } lalrpop-util = { version = "0.19.12" } lazy_static = { version = "1.5.0", default-features = false, features = ["spin_no_std"] } libc = { version = "0.2.185", features = ["extra_traits"] } @@ -230,6 +231,7 @@ indexmap = { version = "2.14.0", features = ["serde"] } ipnet = { version = "2.11.0", features = ["serde"] } ipnetwork = { version = "0.21.1", features = ["schemars", "serde"] } itertools = { version = "0.13.0" } +jiff = { version = "0.2.34", features = ["serde"] } lalrpop-util = { version = "0.19.12" } lazy_static = { version = "1.5.0", default-features = false, features = ["spin_no_std"] } libc = { version = "0.2.185", features = ["extra_traits"] } From 4c96befa0e11429195e79bdf742e078f84d75396 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 21 Jul 2026 12:01:35 -0700 Subject: [PATCH 06/24] tufaceous#51 is merged --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 29714246a4f..aa604b99ee8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16350,7 +16350,7 @@ dependencies = [ [[package]] name = "tufaceous" version = "0.2.0" -source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#d24aec34fcbdf5fda0db8103a1e77926bd166a86" +source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#38730b689596098c61c2c683fe4848712f2e40da" dependencies = [ "atomicwrites", "aws-lc-rs", @@ -16402,7 +16402,7 @@ dependencies = [ [[package]] name = "tufaceous-artifact" version = "0.2.0" -source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#d24aec34fcbdf5fda0db8103a1e77926bd166a86" +source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#38730b689596098c61c2c683fe4848712f2e40da" dependencies = [ "byte-wrapper", "daft", @@ -16430,7 +16430,7 @@ dependencies = [ [[package]] name = "tufaceous-brand-metadata" version = "0.2.0" -source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#d24aec34fcbdf5fda0db8103a1e77926bd166a86" +source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#38730b689596098c61c2c683fe4848712f2e40da" dependencies = [ "serde", "serde_json", From eef2c3dfc7c0a05b7f31e34e2720c1f22e39e2ab Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 27 Jul 2026 11:33:35 -0700 Subject: [PATCH 07/24] flip --- common/src/update/mupdate_override.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/src/update/mupdate_override.rs b/common/src/update/mupdate_override.rs index d867cadfebe..6610961f349 100644 --- a/common/src/update/mupdate_override.rs +++ b/common/src/update/mupdate_override.rs @@ -31,11 +31,11 @@ impl MupdateOverrideInfo { pub const FILE_NAME: &'static str = "mupdate-override.json"; } +/// An artifact hash written to the install dataset. This is used only for +/// debugging. #[derive( Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Deserialize, Serialize, )] -/// An artifact hash written to the install dataset. This is used only for -/// debugging. pub struct MupdateOverrideHashId { /// The kind of artifact this is. This is not intended to be mapped to an /// artifact in a repository. From b69d99b12b5b5a4ba9a7643eadea2c2a385b72e6 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 27 Jul 2026 11:37:04 -0700 Subject: [PATCH 08/24] remove mostly-unused field --- wicket/src/state/update.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/wicket/src/state/update.rs b/wicket/src/state/update.rs index 078ca8eddd8..067d099dbbf 100644 --- a/wicket/src/state/update.rs +++ b/wicket/src/state/update.rs @@ -26,7 +26,6 @@ use tufaceous_artifact_v2::{ArtifactVersion, KnownArtifactTags}; pub struct RackUpdateState { pub items: BTreeMap, pub system_version: Option, - pub artifacts: Vec, pub artifact_versions: BTreeMap>, // The update item currently selected is recorded in // state.rack_state.selected. @@ -76,14 +75,13 @@ impl RackUpdateState { ), }) .collect(), - artifacts: vec![], artifact_versions: BTreeMap::default(), status_view_displayed: false, } } pub fn item_state(&self, component: ComponentId) -> UpdateItemState<'_> { - if self.artifacts.is_empty() { + if self.artifact_versions.is_empty() { UpdateItemState::AwaitingRepository } else { match &self.items[&component].state { @@ -107,9 +105,8 @@ impl RackUpdateState { reports: EventReportMap, ) { self.system_version = system_version; - self.artifacts = artifacts; self.artifact_versions.clear(); - for a in &self.artifacts { + for a in artifacts { if let Ok(known) = KnownArtifactTags::from_tags(a.tags.clone()) { self.artifact_versions .entry(known) From 5424f49e70f45016d2ce55539cf55a58fe510ec2 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 27 Jul 2026 11:37:23 -0700 Subject: [PATCH 09/24] we know the host OS version we're updating to --- wicket/src/ui/panes/update.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/wicket/src/ui/panes/update.rs b/wicket/src/ui/panes/update.rs index 708ff52b622..45c5cad54da 100644 --- a/wicket/src/ui/panes/update.rs +++ b/wicket/src/ui/panes/update.rs @@ -33,8 +33,8 @@ use ratatui::widgets::{ }; use slog::{Logger, info, o}; use tufaceous_artifact_v2::{ - ArtifactVersion, KnownArtifactTags, RotBootloaderTags, RotKeyTableHash, - RotTags, + ArtifactVersion, KnownArtifactTags, OsPhase2Tags, OsVariant, + RotBootloaderTags, RotKeyTableHash, RotTags, }; use tui_tree_widget::{Tree, TreeItem, TreeState}; use wicket_common::inventory::RotSlot; @@ -2465,7 +2465,14 @@ fn artifact_version( .unwrap_or("NO MATCH") .to_string(), UpdateComponent::Host => match id { - ComponentId::Sled(_) => "UNKNOWN".to_string(), + ComponentId::Sled(_) => versions + .get(&KnownArtifactTags::OsPhase2(OsPhase2Tags { + os_variant: OsVariant::Host, + })) + .and_then(|versions| versions.first()) + .map(ArtifactVersion::as_str) + .unwrap_or("NO MATCH") + .to_string(), // Switches and PSCs do not have a host. ComponentId::Switch(_) | ComponentId::Psc(_) => "N/A".to_string(), }, From e2c7c786e2638880f7dd8c1466937ee3389f2372 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 27 Jul 2026 11:41:28 -0700 Subject: [PATCH 10/24] comment on the `kind` values for older wicket --- installinator/src/dispatch.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/installinator/src/dispatch.rs b/installinator/src/dispatch.rs index 8a4a4bc042c..06eccfe0dc6 100644 --- a/installinator/src/dispatch.rs +++ b/installinator/src/dispatch.rs @@ -231,6 +231,9 @@ impl InstallOpts { "Downloading installinator document", async move |cx| { let installinator_doc_id = ArtifactLookupId { + // v2-aware Wicket doesn't read this field, but this is + // necessary to get the Installinator document from an + // older Wicket. kind: "installinator_document", hash: lookup_id.document, }; @@ -512,12 +515,19 @@ async fn download_all_artifacts( cx.with_nested_engine(|engine| { for artifact in artifacts { let kind = match &artifact.kind { + // v2-aware Wicket doesn't read `kind` on ArtifactLookupId, but + // using the correct value is necessary to get the artifact from + // an older Wicket. InstallinatorArtifactKind::HostPhase2 => "host_phase_2", InstallinatorArtifactKind::MeasurementCorpus => { "measurement_corpus" } - InstallinatorArtifactKind::Zone { .. } => "zone", InstallinatorArtifactKind::ControlPlane => "control_plane", + + // This kind doesn't exist on an older Wicket, and it won't be + // present in an Installinator document if we fetched it from an + // older Wicket, so this value is arbitrary. + InstallinatorArtifactKind::Zone { .. } => "zone", }; let id = ArtifactLookupId { kind, hash: artifact.hash }; From 8a917b64349a575bff76019ddefd8450b785ea37 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 27 Jul 2026 11:43:44 -0700 Subject: [PATCH 11/24] ensure we extract some zones from the tarball --- installinator/src/dispatch.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/installinator/src/dispatch.rs b/installinator/src/dispatch.rs index 06eccfe0dc6..ab4eece64a7 100644 --- a/installinator/src/dispatch.rs +++ b/installinator/src/dispatch.rs @@ -712,6 +712,10 @@ async fn read_legacy_control_plane_tarball( data, }); } + ensure!( + !zones.is_empty(), + "no zones were extracted from control plane tarball" + ); Ok(zones) }) .await? From eca0063b7459392176c5f7c7f6b9047e1b9e3561 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 28 Jul 2026 08:59:36 -0700 Subject: [PATCH 12/24] remove unused deps from Cargo.toml --- Cargo.lock | 2 -- installinator/Cargo.toml | 1 - wicketd/Cargo.toml | 1 - 3 files changed, 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d556d7447e3..99002b922ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5897,7 +5897,6 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tokio-stream", - "tufaceous 0.2.0", "tufaceous-artifact 0.2.0", ] @@ -17597,7 +17596,6 @@ dependencies = [ "hickory-resolver 0.25.2", "http", "http-body-util", - "hubtools 0.4.7 (git+https://github.com/oxidecomputer/hubtools.git?rev=2b1ef9b38d75563ea800baa3b17327eec17b1b7a)", "hyper", "iddqd", "illumos-utils", diff --git a/installinator/Cargo.toml b/installinator/Cargo.toml index e4fff08d017..2bde74aaf3b 100644 --- a/installinator/Cargo.toml +++ b/installinator/Cargo.toml @@ -49,7 +49,6 @@ tar.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["full"] } tufaceous-artifact-v2.workspace = true -tufaceous-v2.workspace = true oxide-update-engine.workspace = true oxide-update-engine-types.workspace = true omicron-workspace-hack.workspace = true diff --git a/wicketd/Cargo.toml b/wicketd/Cargo.toml index 4545cefedd2..824202b0198 100644 --- a/wicketd/Cargo.toml +++ b/wicketd/Cargo.toml @@ -28,7 +28,6 @@ hex.workspace = true hickory-resolver.workspace = true http.workspace = true http-body-util.workspace = true -hubtools.workspace = true hyper.workspace = true iddqd.workspace = true illumos-utils.workspace = true From 1a0ecbe178f3b0f40b521c2ada62816dc8e0bcab Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 28 Jul 2026 09:53:32 -0700 Subject: [PATCH 13/24] comments comments comments --- common/src/update/mupdate_override.rs | 7 +++-- installinator-common/src/progress.rs | 7 +++-- installinator/src/artifact.rs | 2 +- installinator/src/dispatch.rs | 6 ++-- wicket-common/src/artifact.rs | 6 ++++ wicket/src/ui/panes/update.rs | 3 +- wicketd-api/src/lib.rs | 13 ++------- wicketd/src/update_tracker.rs | 34 ++++++++++++++++++++-- wicketd/tests/integration_tests/updates.rs | 6 ++-- 9 files changed, 56 insertions(+), 28 deletions(-) diff --git a/common/src/update/mupdate_override.rs b/common/src/update/mupdate_override.rs index 6610961f349..8f309acdbea 100644 --- a/common/src/update/mupdate_override.rs +++ b/common/src/update/mupdate_override.rs @@ -19,10 +19,11 @@ pub struct MupdateOverrideInfo { /// A UUID that identifies a MUPdate that occurred. pub mupdate_uuid: MupdateOverrideUuid, - /// Artifact hashes written out to the install dataset. + /// Artifact hashes written out to the install dataset. Not used by the + /// control plane itself. /// - /// Currently includes the host phase 2 and composite control plane - /// artifacts. + /// Currently includes the host phase 2 image, each zone, and each + /// measurement corpus. pub hash_ids: BTreeSet, } diff --git a/installinator-common/src/progress.rs b/installinator-common/src/progress.rs index d1fed496814..66d14e4a95e 100644 --- a/installinator-common/src/progress.rs +++ b/installinator-common/src/progress.rs @@ -61,8 +61,9 @@ pub enum InstallinatorComponent { /// The measurement corpus component. MeasurementCorpus, - /// A component that means "all components", used for downloads and writes - /// for now. It is possible that this component will go away in the future. + /// A component that means "all components", used for downloads, writes, and + /// hardware scans for now. It is possible that this component will go away + /// in the future. All, /// Future variants that might be unknown. @@ -85,7 +86,7 @@ pub enum InstallinatorStepId { #[serde(rename_all = "snake_case", tag = "reason")] pub enum InstallinatorStepMetadata { Write { - /// The destination being formatted or written to. + /// The destination being written to. /// /// Available with format and destination events. #[schemars(schema_with = "path_schema_opt")] diff --git a/installinator/src/artifact.rs b/installinator/src/artifact.rs index bb532d39131..4fe0a4d2067 100644 --- a/installinator/src/artifact.rs +++ b/installinator/src/artifact.rs @@ -188,7 +188,7 @@ impl ArtifactClient { } } -/// Wicket prior to Tufaceous v1 looks up artifacts by both kind and hash. The +/// Wicket prior to Tufaceous v2 looks up artifacts by both kind and hash. The /// kind must be correct in order for an artifact to be retrieved. When the /// Tufaceous v2 transition is complete, only the hash will be used to look up /// artifacts, and the `kind` value at the API layer can be replaced with any diff --git a/installinator/src/dispatch.rs b/installinator/src/dispatch.rs index ab4eece64a7..f5b112ccb8f 100644 --- a/installinator/src/dispatch.rs +++ b/installinator/src/dispatch.rs @@ -661,9 +661,9 @@ pub(crate) fn stderr_env_drain( builder.build() } -/// If the Wicket on the other end is older and using Tufaceous v1, control -/// plane zones won't be split into their own files. We have to read the -/// individual zone artifacts out of the tarball we've downloaded. +/// If the Wicket on the other end has a v1 repo, it will serve the original v1 +/// Installinator document, which lists the control plane tarball. We have to +/// read the individual zone artifacts out of the tarball we've downloaded. /// /// This function can be removed at the same time the /// `InstallinatorArtifactKind::ControlPlane` variant is removed at the end of diff --git a/wicket-common/src/artifact.rs b/wicket-common/src/artifact.rs index 5fb928f3a12..436e33c5215 100644 --- a/wicket-common/src/artifact.rs +++ b/wicket-common/src/artifact.rs @@ -10,6 +10,9 @@ use serde::Serialize; use tufaceous_artifact_v2::Artifact; use tufaceous_artifact_v2::ArtifactVersion; +/// The identifying components of an [`Artifact`], sent over the wire from +/// wicketd to the wicket client to show versions of artifacts we will be +/// updating to. #[derive( Debug, Clone, @@ -22,7 +25,10 @@ use tufaceous_artifact_v2::ArtifactVersion; JsonSchema, )] pub struct ArtifactId { + /// The Tufaceous tags used to select the artifact. pub tags: BTreeMap, + + /// The artifact's version string. pub version: ArtifactVersion, } diff --git a/wicket/src/ui/panes/update.rs b/wicket/src/ui/panes/update.rs index 45c5cad54da..819343b7bae 100644 --- a/wicket/src/ui/panes/update.rs +++ b/wicket/src/ui/panes/update.rs @@ -2411,7 +2411,8 @@ fn artifact_version( ) -> String { // NOTE: This function currently assumes that all artifacts in the repo of // the same general kind (SP, ROT, bootloader) are of the same version; we - // don't filter on board or slot. + // don't filter on board or slot. (Wicketd does not verify this assumption, + // either.) match update_component { UpdateComponent::RotBootloader => { let Some(sign) = inventory diff --git a/wicketd-api/src/lib.rs b/wicketd-api/src/lib.rs index 035fe000401..56bf60a8e76 100644 --- a/wicketd-api/src/lib.rs +++ b/wicketd-api/src/lib.rs @@ -464,17 +464,8 @@ pub enum GetInventoryResponse { pub struct GetArtifactsAndEventReportsResponse { pub system_version: Option, - /// Map of artifacts we ingested from the most-recently-uploaded TUF - /// repository to a list of artifacts we're serving over the bootstrap - /// network. In some cases the list of artifacts being served will have - /// length 1 (when we're serving the artifact directly); in other cases the - /// artifact in the TUF repo contains multiple nested artifacts inside it - /// (e.g., RoT artifacts contain both A and B images), and we serve the list - /// of extracted artifacts but not the original combination. - /// - /// Conceptually, this is a `BTreeMap>`, but - /// JSON requires string keys for maps, so we give back a vec of pairs - /// instead. + /// List of artifacts we ingested from the most-recently-uploaded TUF + /// repository. pub artifacts: Vec, pub event_reports: BTreeMap>, diff --git a/wicketd/src/update_tracker.rs b/wicketd/src/update_tracker.rs index 323f4051c61..6401ae6979a 100644 --- a/wicketd/src/update_tracker.rs +++ b/wicketd/src/update_tracker.rs @@ -378,6 +378,14 @@ impl UpdateTracker { } /// Updates the repository stored inside the update tracker. + /// + /// This verifies that the uploaded repository is consistent (all TUF + /// metadata has valid signatures and all targets have a correct hash) but + /// does not verify the TUF signatures against a valid trust root or verify + /// the expiration timestamps; see RFD 721. + /// + /// It also does not verify that the artifacts used by Wicket and + /// Installinator are present. pub(crate) async fn put_repository( &self, stream: impl Stream> + Send + 'static, @@ -831,7 +839,7 @@ impl UpdateTrackerData { )); } - // Set the new artifacts_with_plan. + // Set the new repo. self.artifact_store.set_repository(repository); // Reset all running data: a new repository means starting afresh. @@ -1487,11 +1495,31 @@ impl UpdateDriver { })? .hash }; + + // In the beginning, Installinator only had a host phase + // 2 image and a control plane zones tarball to fetch + // from Wicket. Wicket communicates these artifact hashes + // to Installinator by sending a JSON-encoded request + // to MGS, which encodes the image ID as CBOR and sets + // it as a key to be retrieved from IPCC. Because Wicket + // and Installinator are always updated before MGS, it is + // difficult to change the wire format of this chain. + // + // When we introduced the Installinator document artifact we + // overloaded the definition of the image ID to communicate + // that Installinator should instead fetch the document. let installinator_image_id = InstallinatorImageId { - host_phase_2: hash.to_string(), - control_plane: ArtifactHash([0; 32]).to_string(), update_id: update_cx.update_id, + + // A zeroed control plane image hash indicates that + // Installinator should fetch an Installinator document, + // with the hash stored as `host_phase_2`. + control_plane: ArtifactHash([0; 32]).to_string(), + // The Installinator document hash, since + // `control_plane` is a zero hash. + host_phase_2: hash.to_string(), }; + update_cx .mgs_client .sp_installinator_image_id_set( diff --git a/wicketd/tests/integration_tests/updates.rs b/wicketd/tests/integration_tests/updates.rs index 0d42c5ee5d4..5812d9baeee 100644 --- a/wicketd/tests/integration_tests/updates.rs +++ b/wicketd/tests/integration_tests/updates.rs @@ -53,7 +53,7 @@ use wicketd_commission_types_versions::latest::update::{ UpdateStepStatus, UpdateTargets, }; -/// The list of zone file names defined in fake-non-semver.toml. +/// The list of zone file names defined in Tufaceous's `FAKE_ZONES`. static FAKE_NON_SEMVER_ZONE_FILE_NAMES: &[&str] = &[ "clickhouse.tar.gz", "clickhouse_keeper.tar.gz", @@ -526,8 +526,8 @@ async fn installinator_fetch_impl( // Check that the host and control plane artifacts were downloaded // correctly. // - // The control plane zone names here are defined in `fake-non-semver.toml` - // which we load above. + // The control plane zone names here are defined in Tufaceous's `FAKE_ZONES` + // which we have a copy of above. for file_name in [HOST_PHASE_2_FILE_NAME.to_owned()].into_iter().chain( FAKE_NON_SEMVER_ZONE_FILE_NAMES.iter().map(|z| format!("install/{z}")), ) { From 725b46233eeb714ba04758479dfd367858d49c7e Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 28 Jul 2026 09:54:02 -0700 Subject: [PATCH 14/24] interrogate_rot_bootloader no longer returns Option --- wicketd/src/update_tracker.rs | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/wicketd/src/update_tracker.rs b/wicketd/src/update_tracker.rs index 6401ae6979a..86b13917127 100644 --- a/wicketd/src/update_tracker.rs +++ b/wicketd/src/update_tracker.rs @@ -1049,20 +1049,9 @@ impl UpdateDriver { } let rot_bootloader_interrogation = - match rot_bootloader_interrogation + rot_bootloader_interrogation .into_value(cx.token()) - .await - { - Some(v) => v, - None => { - return StepSkipped::new( - (), - "Skipping bootloader update, \ - check interrogation step", - ) - .into(); - } - }; + .await; let bootloader_has_this_version = rot_bootloader_interrogation @@ -1994,7 +1983,7 @@ impl UpdateContext { async fn interrogate_rot_bootloader( &self, repo: &Arc, - ) -> Result>, UpdateTerminalError> { + ) -> Result, UpdateTerminalError> { // Read the caboose of the currently running version (always 0) let caboose = self .mgs_client @@ -2020,13 +2009,13 @@ impl UpdateContext { })?; let make_result = |active_version| { - Some(RotInterrogation { + RotInterrogation { // We always update slot 1 slot_to_update: 1, artifact_to_apply: artifact_to_apply.clone(), sp: self.sp, active_version, - }) + } }; let message = format!( From 614aee0c7b401abce1be099488d6ac036b0c806d Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 28 Jul 2026 10:47:38 -0700 Subject: [PATCH 15/24] don't panic on duplicate file names (and don't recompute hashes we already know) --- installinator/src/dispatch.rs | 67 ++++++++++++--- installinator/src/write.rs | 149 +++++++++------------------------- 2 files changed, 97 insertions(+), 119 deletions(-) diff --git a/installinator/src/dispatch.rs b/installinator/src/dispatch.rs index f5b112ccb8f..cc9d37c1667 100644 --- a/installinator/src/dispatch.rs +++ b/installinator/src/dispatch.rs @@ -27,7 +27,7 @@ use tufaceous_artifact_v2::{ }; use crate::{ - ArtifactWriter, MeasurementToWrite, WriteDestination, ZoneToWrite, + ArtifactToWrite, ArtifactWriter, WriteDestination, artifact::{ArtifactIdOpts, ArtifactLookupId}, fetch::{FetchArtifactBackend, FetchedArtifact, HttpFetchBackend}, peers::{DiscoveryMechanism, LastKnownPeer}, @@ -274,6 +274,31 @@ impl InstallOpts { image but found {host_phase_2_count}" ); + // We expect that measurement corpus and zone file names + // are unique. + let mut measurements = BTreeSet::new(); + let mut zones = BTreeSet::new(); + for artifact in &json.artifacts { + match artifact.kind { + InstallinatorArtifactKind::MeasurementCorpus => { + ensure!( + measurements.insert(&artifact.file_name), + "measurement with file name {} seen twice", + artifact.file_name + ); + } + InstallinatorArtifactKind::Zone { .. } => { + ensure!( + zones.insert(&artifact.file_name), + "zone with file name {} seen twice", + artifact.file_name + ); + } + InstallinatorArtifactKind::HostPhase2 + | InstallinatorArtifactKind::ControlPlane => {} + } + } + StepSuccess::new(json.artifacts).into() }, ) @@ -327,6 +352,7 @@ impl InstallOpts { let mut host_phase_2 = None; let mut zones = Vec::new(); + let mut zone_file_names = BTreeSet::new(); let mut measurement_corpus = Vec::new(); for (artifact, step) in downloads.into_value(cx.token()).await @@ -338,10 +364,10 @@ impl InstallOpts { kind: "measurement_corpus".into(), hash: artifact.hash, }; - measurement_corpus.push(MeasurementToWrite { + measurement_corpus.push(ArtifactToWrite { id, - name: artifact.file_name, - artifact: data, + file_name: artifact.file_name, + data, }); } InstallinatorArtifactKind::HostPhase2 => { @@ -352,25 +378,46 @@ impl InstallOpts { host_phase_2 = Some((id, data)); } InstallinatorArtifactKind::Zone { zone_name } => { + // We already checked this against other zones + // in the same Installinator document, but we + // might have extracted a legacy control plane + // tarball. (We can remove this check once that + // branch is removed.) + ensure!( + zone_file_names + .insert(artifact.file_name.clone()), + "zone with file name {} seen twice", + artifact.file_name + ); let id = MupdateOverrideHashId { kind: format!("zone-{zone_name}"), hash: artifact.hash, }; - zones.push(ZoneToWrite { + zones.push(ArtifactToWrite { id, file_name: artifact.file_name, data, }); } + // When this branch is removed, remove the duplicate + // zone file name check in the `Zone` branch. InstallinatorArtifactKind::ControlPlane => { - zones.extend( + for zone in read_legacy_control_plane_tarball(data) .await .context( "failed to read control \ plane tarball", - )?, - ); + )? + { + ensure!( + zone_file_names + .insert(zone.file_name.clone()), + "zone with file name {} seen twice", + artifact.file_name + ); + zones.push(zone); + } } } } @@ -670,7 +717,7 @@ pub(crate) fn stderr_env_drain( /// the transition. async fn read_legacy_control_plane_tarball( data: BufList, -) -> Result> { +) -> Result> { // None of this is blocking I/O since it's copying and hashing memory, but // the tarball is large enough that we should probably do this on its own // worker thread. @@ -700,7 +747,7 @@ async fn read_legacy_control_plane_tarball( hasher.update(&buf[..n]); } - zones.push(ZoneToWrite { + zones.push(ArtifactToWrite { id: MupdateOverrideHashId { // We normally write the zone name as found in oxide.json // but this string is for debugging and reading the inner diff --git a/installinator/src/write.rs b/installinator/src/write.rs index 9484900b365..f75b7be6793 100644 --- a/installinator/src/write.rs +++ b/installinator/src/write.rs @@ -14,7 +14,6 @@ use async_trait::async_trait; use buf_list::BufList; use bytes::Buf; use camino::{Utf8Path, Utf8PathBuf}; -use iddqd::IdOrdMap; use illumos_utils::zpool::{Zpool, ZpoolName}; use installinator_common::{ ControlPlaneZonesSpec, ControlPlaneZonesStepId, RawDiskWriter, StepContext, @@ -39,7 +38,6 @@ use tokio::{ fs, fs::File, io::{AsyncWrite, AsyncWriteExt}, - task::JoinSet, }; use tufaceous_artifact_v2::ArtifactHash; @@ -229,8 +227,8 @@ impl<'a> ArtifactWriter<'a> { mupdate_id: MupdateUuid, host_phase_2_id: &'a MupdateOverrideHashId, host_phase_2_data: &'a BufList, - control_plane_zones: &'a [ZoneToWrite], - measurement_corpus: &'a [MeasurementToWrite], + control_plane_zones: &'a [ArtifactToWrite], + measurement_corpus: &'a [ArtifactToWrite], destination: WriteDestination, ) -> Self { let drives = destination @@ -564,10 +562,10 @@ impl SlotWriteContext<'_> { struct ArtifactsToWrite<'a> { host_phase_2_id: &'a MupdateOverrideHashId, host_phase_2_data: &'a BufList, - control_plane_zones: &'a [ZoneToWrite], + control_plane_zones: &'a [ArtifactToWrite], mupdate_id: MupdateUuid, mupdate_override_uuid: MupdateOverrideUuid, - measurement_corpus: &'a [MeasurementToWrite], + measurement_corpus: &'a [ArtifactToWrite], } impl ArtifactsToWrite<'_> { @@ -646,16 +644,21 @@ impl ArtifactsToWrite<'_> { } } -pub(crate) struct ZoneToWrite { +pub(crate) struct ArtifactToWrite { pub(crate) id: MupdateOverrideHashId, pub(crate) file_name: String, pub(crate) data: BufList, } -pub(crate) struct MeasurementToWrite { - pub(crate) id: MupdateOverrideHashId, - pub(crate) name: String, - pub(crate) artifact: BufList, +impl ArtifactToWrite { + fn to_install_metadata(&self) -> OmicronInstallMetadata { + let file_size: usize = self.data.num_bytes(); + OmicronInstallMetadata::new_v2( + self.file_name.clone(), + u64::try_from(file_size).expect("usize fits in u64"), + self.id.hash, + ) + } } struct ControlPlaneZoneWriteContext<'a> { @@ -663,9 +666,9 @@ struct ControlPlaneZoneWriteContext<'a> { clean_output_directory: bool, output_directory: &'a Utf8Path, measurement_directory: &'a Utf8Path, - zones: &'a [ZoneToWrite], + zones: &'a [ArtifactToWrite], host_phase_2_id: &'a MupdateOverrideHashId, - measurement_corpus: &'a [MeasurementToWrite], + measurement_corpus: &'a [ArtifactToWrite], mupdate_id: MupdateUuid, mupdate_override_uuid: MupdateOverrideUuid, } @@ -788,7 +791,7 @@ impl ControlPlaneZoneWriteContext<'_> { ) .register(); - for ZoneToWrite { file_name, data, .. } in self.zones { + for ArtifactToWrite { file_name, data, .. } in self.zones { let out_path = self.output_directory.join(file_name); transport = engine .new_step( @@ -866,7 +869,7 @@ impl ControlPlaneZoneWriteContext<'_> { .register(); for data in self.measurement_corpus { - let name = data.name.clone(); + let name = data.file_name.clone(); let out_path = self.measurement_directory.join(&name); transport = engine .new_step( @@ -878,7 +881,7 @@ impl ControlPlaneZoneWriteContext<'_> { write_artifact_impl( WriteComponent::MeasurementCorpus, slot, - data.artifact.clone(), + data.data.clone(), &out_path, transport, &cx, @@ -941,13 +944,18 @@ impl ControlPlaneZoneWriteContext<'_> { } async fn omicron_measurement_manifest_artifact(&self) -> BufList { - let files = compute_measurement_hashes(self.measurement_corpus).await; - let omicron_zone_manifest = OmicronInstallManifest { source: OmicronInstallManifestSource::Installinator { mupdate_id: self.mupdate_id, }, - files, + // This is an `IdOrdMap` keyed on the file name. We already + // confirmed that all measurement artifacts have a unique file name + // when we read the Installinator document. + files: self + .measurement_corpus + .iter() + .map(ArtifactToWrite::to_install_metadata) + .collect(), }; let json_bytes = serde_json::to_vec(&omicron_zone_manifest) .expect("this serialization is infallible"); @@ -955,13 +963,19 @@ impl ControlPlaneZoneWriteContext<'_> { } async fn omicron_zone_manifest_artifact(&self) -> BufList { - let files = compute_zone_hashes(&self.zones).await; - let omicron_zone_manifest = OmicronInstallManifest { source: OmicronInstallManifestSource::Installinator { mupdate_id: self.mupdate_id, }, - files, + // This is an `IdOrdMap` keyed on the file name. We already + // confirmed that all zones have a unique file name when we read the + // Installinator document (or when we extracted the legacy control + // plane tarball). + files: self + .zones + .iter() + .map(ArtifactToWrite::to_install_metadata) + .collect(), }; let json_bytes = serde_json::to_vec(&omicron_zone_manifest) .expect("this serialization is infallible"); @@ -969,89 +983,6 @@ impl ControlPlaneZoneWriteContext<'_> { } } -/// Computes the measurement hash IDs. -/// -/// Hash computation is done in parallel on blocking tasks. -/// -/// # Panics -/// -/// Panics if the runtime shuts down causing a task abort, or a task panics. -async fn compute_measurement_hashes( - images: &[MeasurementToWrite], -) -> IdOrdMap { - let mut tasks = JoinSet::new(); - for m in images { - let file_name = m.name.clone(); - let data: BufList = m.artifact.clone(); - // Compute hashes in parallel. - tasks.spawn_blocking(move || { - let mut hasher = Sha256::new(); - for d in &data { - hasher.update(&d); - } - let hash = hasher.finalize(); - OmicronInstallMetadata::new_v2( - file_name, - u64::try_from(data.num_bytes()).unwrap(), - ArtifactHash(hash.into()), - ) - }); - } - - let mut output = IdOrdMap::new(); - while let Some(res) = tasks.join_next().await { - // Propagate panics across tasks—this is the standard pattern we follow - // in installinator. - output - .insert_unique(res.expect("task panicked")) - .expect("filenames are unique"); - } - output -} - -/// Computes the zone hash IDs. -/// -/// Hash computation is done in parallel on blocking tasks. -/// -/// # Panics -/// -/// Panics if the runtime shuts down causing a task abort, or a task panics. -async fn compute_zone_hashes( - images: &[ZoneToWrite], -) -> IdOrdMap { - let mut tasks = JoinSet::new(); - for ZoneToWrite { file_name, data, .. } in images { - let file_name = file_name.clone(); - // data is a BufList so is relatively cheap to clone. - let data: BufList = data.clone(); - // Compute hashes in parallel. - tasks.spawn_blocking(move || { - let mut len = 0; - let mut hasher = Sha256::new(); - for buf in data { - len += buf.len(); - hasher.update(&buf); - } - let hash = hasher.finalize(); - OmicronInstallMetadata::new_v2( - file_name, - u64::try_from(len).unwrap(), - ArtifactHash(hash.into()), - ) - }); - } - - let mut output = IdOrdMap::new(); - while let Some(res) = tasks.join_next().await { - // Propagate panics across tasks—this is the standard pattern we follow - // in installinator. - output - .insert_unique(res.expect("task panicked")) - .expect("filenames are unique"); - } - output -} - fn remove_contents_of(path: &Utf8Path) -> io::Result<()> { use std::fs; @@ -1461,7 +1392,7 @@ mod tests { // Assemble our one control plane artifact into a 1-long list of zone // images. - let control_plane_zone_images = vec![ZoneToWrite { + let control_plane_zone_images = vec![ArtifactToWrite { id: MupdateOverrideHashId { kind: "zone".into(), hash: ArtifactHash([0; 32]), @@ -1473,13 +1404,13 @@ mod tests { data: artifact_control_plane.clone(), }]; - let all_corpus = vec![MeasurementToWrite { + let all_corpus = vec![ArtifactToWrite { id: MupdateOverrideHashId { kind: "measurement_corpus".into(), hash: ArtifactHash([0; 32]), }, - artifact: artifact_measurement_corpus, - name: "blah".to_string(), + file_name: "blah".to_string(), + data: artifact_measurement_corpus, }]; let mut writer = ArtifactWriter::new( MupdateUuid::new_v4(), From 1565c64dac3d0f37a3700a80304949cb63be4b82 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 28 Jul 2026 16:49:31 -0700 Subject: [PATCH 16/24] i forgot comments end up in this thing --- openapi/wicketd.json | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/openapi/wicketd.json b/openapi/wicketd.json index b8bf3e7e0aa..5b44af1fa96 100644 --- a/openapi/wicketd.json +++ b/openapi/wicketd.json @@ -921,16 +921,23 @@ ] }, "ArtifactId": { + "description": "The identifying components of an [`Artifact`], sent over the wire from wicketd to the wicket client to show versions of artifacts we will be updating to.", "type": "object", "properties": { "tags": { + "description": "The Tufaceous tags used to select the artifact.", "type": "object", "additionalProperties": { "type": "string" } }, "version": { - "$ref": "#/components/schemas/ArtifactVersion" + "description": "The artifact's version string.", + "allOf": [ + { + "$ref": "#/components/schemas/ArtifactVersion" + } + ] } }, "required": [ @@ -2000,7 +2007,7 @@ "type": "object", "properties": { "artifacts": { - "description": "Map of artifacts we ingested from the most-recently-uploaded TUF repository to a list of artifacts we're serving over the bootstrap network. In some cases the list of artifacts being served will have length 1 (when we're serving the artifact directly); in other cases the artifact in the TUF repo contains multiple nested artifacts inside it (e.g., RoT artifacts contain both A and B images), and we serve the list of extracted artifacts but not the original combination.\n\nConceptually, this is a `BTreeMap>`, but JSON requires string keys for maps, so we give back a vec of pairs instead.", + "description": "List of artifacts we ingested from the most-recently-uploaded TUF repository.", "type": "array", "items": { "$ref": "#/components/schemas/ArtifactId" From 4615e827bc59f2a532da3e3ab17143fdb37ca6b1 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 29 Jul 2026 10:00:30 -0700 Subject: [PATCH 17/24] remove `hash_ids` from MupdateOverrideInfo --- common/src/update/mupdate_override.rs | 24 ------- installinator/src/dispatch.rs | 33 ++------- installinator/src/write.rs | 70 +++++++------------ .../resolvable-files-examples/src/lib.rs | 9 +-- wicketd/tests/integration_tests/updates.rs | 27 +------ 5 files changed, 37 insertions(+), 126 deletions(-) diff --git a/common/src/update/mupdate_override.rs b/common/src/update/mupdate_override.rs index 8f309acdbea..d111fe24ed7 100644 --- a/common/src/update/mupdate_override.rs +++ b/common/src/update/mupdate_override.rs @@ -4,11 +4,8 @@ //! Type definitions for the MUPdate override (RFD 556). -use std::collections::BTreeSet; - use omicron_uuid_kinds::MupdateOverrideUuid; use serde::{Deserialize, Serialize}; -use tufaceous_artifact_v2::ArtifactHash; /// MUPdate override information, typically serialized as JSON (RFD 556). /// @@ -18,30 +15,9 @@ use tufaceous_artifact_v2::ArtifactHash; pub struct MupdateOverrideInfo { /// A UUID that identifies a MUPdate that occurred. pub mupdate_uuid: MupdateOverrideUuid, - - /// Artifact hashes written out to the install dataset. Not used by the - /// control plane itself. - /// - /// Currently includes the host phase 2 image, each zone, and each - /// measurement corpus. - pub hash_ids: BTreeSet, } impl MupdateOverrideInfo { /// The name of the file on the install dataset. pub const FILE_NAME: &'static str = "mupdate-override.json"; } - -/// An artifact hash written to the install dataset. This is used only for -/// debugging. -#[derive( - Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Deserialize, Serialize, -)] -pub struct MupdateOverrideHashId { - /// The kind of artifact this is. This is not intended to be mapped to an - /// artifact in a repository. - pub kind: String, - - /// The hash of the artifact. - pub hash: ArtifactHash, -} diff --git a/installinator/src/dispatch.rs b/installinator/src/dispatch.rs index cc9d37c1667..7d1bb86307b 100644 --- a/installinator/src/dispatch.rs +++ b/installinator/src/dispatch.rs @@ -16,7 +16,6 @@ use installinator_common::{ StepWarning, UpdateEngine, }; use omicron_common::FileKv; -use omicron_common::update::MupdateOverrideHashId; use oxide_update_engine::StepResult; use sha2::{Digest, Sha256}; use sled_hardware::{DataLinks, is_oxide_sled}; @@ -360,24 +359,16 @@ impl InstallOpts { let data = step.into_value(cx.token()).await; match &artifact.kind { InstallinatorArtifactKind::MeasurementCorpus => { - let id = MupdateOverrideHashId { - kind: "measurement_corpus".into(), - hash: artifact.hash, - }; measurement_corpus.push(ArtifactToWrite { - id, file_name: artifact.file_name, data, + hash: artifact.hash, }); } InstallinatorArtifactKind::HostPhase2 => { - let id = MupdateOverrideHashId { - kind: "host_phase2".into(), - hash: artifact.hash, - }; - host_phase_2 = Some((id, data)); + host_phase_2 = Some((artifact.hash, data)); } - InstallinatorArtifactKind::Zone { zone_name } => { + InstallinatorArtifactKind::Zone { .. } => { // We already checked this against other zones // in the same Installinator document, but we // might have extracted a legacy control plane @@ -389,14 +380,10 @@ impl InstallOpts { "zone with file name {} seen twice", artifact.file_name ); - let id = MupdateOverrideHashId { - kind: format!("zone-{zone_name}"), - hash: artifact.hash, - }; zones.push(ArtifactToWrite { - id, file_name: artifact.file_name, data, + hash: artifact.hash, }); } // When this branch is removed, remove the duplicate @@ -423,14 +410,14 @@ impl InstallOpts { } // We checked there is at least one host phase 2 artifact // when we read the installinator document. - let (host_phase_2_id, host_phase_2_data) = host_phase_2 + let (host_phase_2_hash, host_phase_2_data) = host_phase_2 .expect( "already checked presence of host phase 2 artifact", ); let mut writer = ArtifactWriter::new( lookup_id.update_id, - &host_phase_2_id, + host_phase_2_hash, &host_phase_2_data, &zones, &measurement_corpus, @@ -748,15 +735,9 @@ async fn read_legacy_control_plane_tarball( } zones.push(ArtifactToWrite { - id: MupdateOverrideHashId { - // We normally write the zone name as found in oxide.json - // but this string is for debugging and reading the inner - // tarball is annoying. - kind: format!("zone-{file_name}"), - hash: ArtifactHash(hasher.finalize().into()), - }, file_name: file_name.to_owned(), data, + hash: ArtifactHash(hasher.finalize().into()), }); } ensure!( diff --git a/installinator/src/write.rs b/installinator/src/write.rs index f75b7be6793..90e3d12abe0 100644 --- a/installinator/src/write.rs +++ b/installinator/src/write.rs @@ -23,7 +23,7 @@ use installinator_common::{ use omicron_common::{ disk::M2Slot, update::{ - MupdateOverrideHashId, MupdateOverrideInfo, OmicronInstallManifest, + MupdateOverrideInfo, OmicronInstallManifest, OmicronInstallManifestSource, OmicronInstallMetadata, }, }; @@ -225,7 +225,7 @@ pub(crate) struct ArtifactWriter<'a> { impl<'a> ArtifactWriter<'a> { pub(crate) fn new( mupdate_id: MupdateUuid, - host_phase_2_id: &'a MupdateOverrideHashId, + host_phase_2_hash: ArtifactHash, host_phase_2_data: &'a BufList, control_plane_zones: &'a [ArtifactToWrite], measurement_corpus: &'a [ArtifactToWrite], @@ -248,7 +248,7 @@ impl<'a> ArtifactWriter<'a> { is_host_phase_2_block_device: destination .is_host_phase_2_block_device, artifacts: ArtifactsToWrite { - host_phase_2_id, + host_phase_2_hash, host_phase_2_data, control_plane_zones, mupdate_id, @@ -516,7 +516,7 @@ impl SlotWriteContext<'_> { .unwrap() .map_err(WriteError::ChecksumValidationError)?; - if computed_hash == self.artifacts.host_phase_2_id.hash { + if computed_hash == self.artifacts.host_phase_2_hash { StepSuccess::new(()) .with_message(format!( "validated hash {computed_hash} \ @@ -527,7 +527,7 @@ impl SlotWriteContext<'_> { Err(WriteError::ChecksumValidationError(anyhow!( "expected {} but computed {computed_hash} \ for host phase 2 written to {slot:?}", - self.artifacts.host_phase_2_id.hash + self.artifacts.host_phase_2_hash ))) } } @@ -560,7 +560,7 @@ impl SlotWriteContext<'_> { #[derive(Copy, Clone)] struct ArtifactsToWrite<'a> { - host_phase_2_id: &'a MupdateOverrideHashId, + host_phase_2_hash: ArtifactHash, host_phase_2_data: &'a BufList, control_plane_zones: &'a [ArtifactToWrite], mupdate_id: MupdateUuid, @@ -591,7 +591,7 @@ impl ArtifactsToWrite<'_> { info!( log, "Failed to write host phase 2 image"; - "artifact_id" => ?self.host_phase_2_id, + "hash" => ?self.host_phase_2_hash, InlineErrorChain::new(&error), ); })?; @@ -616,7 +616,6 @@ impl ArtifactsToWrite<'_> { output_directory: &destinations.control_plane_dir, measurement_directory: &destinations.measurement_dir, zones: self.control_plane_zones, - host_phase_2_id: self.host_phase_2_id, measurement_corpus: self.measurement_corpus, mupdate_id: self.mupdate_id, mupdate_override_uuid: self.mupdate_override_uuid, @@ -645,9 +644,9 @@ impl ArtifactsToWrite<'_> { } pub(crate) struct ArtifactToWrite { - pub(crate) id: MupdateOverrideHashId, pub(crate) file_name: String, pub(crate) data: BufList, + pub(crate) hash: ArtifactHash, } impl ArtifactToWrite { @@ -656,7 +655,7 @@ impl ArtifactToWrite { OmicronInstallMetadata::new_v2( self.file_name.clone(), u64::try_from(file_size).expect("usize fits in u64"), - self.id.hash, + self.hash, ) } } @@ -667,7 +666,6 @@ struct ControlPlaneZoneWriteContext<'a> { output_directory: &'a Utf8Path, measurement_directory: &'a Utf8Path, zones: &'a [ArtifactToWrite], - host_phase_2_id: &'a MupdateOverrideHashId, measurement_corpus: &'a [ArtifactToWrite], mupdate_id: MupdateUuid, mupdate_override_uuid: MupdateOverrideUuid, @@ -929,15 +927,8 @@ impl ControlPlaneZoneWriteContext<'_> { &self, mupdate_override_uuid: MupdateOverrideUuid, ) -> BufList { - let mut hash_ids = BTreeSet::new(); - hash_ids.insert(self.host_phase_2_id.clone()); - hash_ids.extend(self.zones.iter().map(|z| z.id.clone())); - hash_ids.extend(self.measurement_corpus.iter().map(|z| z.id.clone())); - - let mupdate_override = MupdateOverrideInfo { - mupdate_uuid: mupdate_override_uuid, - hash_ids, - }; + let mupdate_override = + MupdateOverrideInfo { mupdate_uuid: mupdate_override_uuid }; let json_bytes = serde_json::to_vec(&mupdate_override) .expect("this serialization is infallible"); BufList::from(json_bytes) @@ -1332,21 +1323,18 @@ mod tests { let artifact_measurement_corpus: BufList = data3.into_iter().map(Bytes::from).collect(); - let host_id = MupdateOverrideHashId { - kind: "host_phase2".into(), - hash: { - // The `validate_written_host_phase_2_hash()` will fail unless - // we give the actual hash of the host phase 2 data, so compute - // it here. - // - // We currently don't have any equivalent check on the control - // plane, so it can use `dummy_artifact_hash_id` instead. - let mut hasher = Sha256::new(); - for chunk in artifact_host.iter() { - hasher.update(chunk); - } - ArtifactHash(hasher.finalize().into()) - }, + let host_hash = { + // The `validate_written_host_phase_2_hash()` will fail unless + // we give the actual hash of the host phase 2 data, so compute + // it here. + // + // We currently don't have any equivalent check on the control + // plane, so it can use `dummy_artifact_hash_id` instead. + let mut hasher = Sha256::new(); + for chunk in artifact_host.iter() { + hasher.update(chunk); + } + ArtifactHash(hasher.finalize().into()) }; // XXX: note we don't assert on the number of attempts it took to write @@ -1393,28 +1381,22 @@ mod tests { // Assemble our one control plane artifact into a 1-long list of zone // images. let control_plane_zone_images = vec![ArtifactToWrite { - id: MupdateOverrideHashId { - kind: "zone".into(), - hash: ArtifactHash([0; 32]), - }, file_name: destination_control_plane .file_name() .unwrap() .to_string(), data: artifact_control_plane.clone(), + hash: ArtifactHash([0; 32]), }]; let all_corpus = vec![ArtifactToWrite { - id: MupdateOverrideHashId { - kind: "measurement_corpus".into(), - hash: ArtifactHash([0; 32]), - }, file_name: "blah".to_string(), data: artifact_measurement_corpus, + hash: ArtifactHash([0; 32]), }]; let mut writer = ArtifactWriter::new( MupdateUuid::new_v4(), - &host_id, + host_hash, &artifact_host, &control_plane_zone_images, &all_corpus, diff --git a/sled-agent/resolvable-files-examples/src/lib.rs b/sled-agent/resolvable-files-examples/src/lib.rs index 54b8deb9d75..83e4bea8909 100644 --- a/sled-agent/resolvable-files-examples/src/lib.rs +++ b/sled-agent/resolvable-files-examples/src/lib.rs @@ -7,7 +7,7 @@ //! This crate provides helpers for generating example values for zone image //! resolver types. -use std::{collections::BTreeSet, fs, io, sync::LazyLock}; +use std::{fs, io, sync::LazyLock}; use camino::{Utf8Path, Utf8PathBuf}; use camino_tempfile_ext::{ @@ -182,12 +182,7 @@ impl WriteInstallDatasetContext { /// Returns the override information for the mupdate. pub fn override_info(&self) -> MupdateOverrideInfo { - MupdateOverrideInfo { - mupdate_uuid: self.mupdate_override_uuid, - // The hash IDs are not used for validation, so leave this - // empty. - hash_ids: BTreeSet::new(), - } + MupdateOverrideInfo { mupdate_uuid: self.mupdate_override_uuid } } pub fn zone_manifest(&self) -> OmicronInstallManifest { diff --git a/wicketd/tests/integration_tests/updates.rs b/wicketd/tests/integration_tests/updates.rs index 5812d9baeee..9fcbbee265e 100644 --- a/wicketd/tests/integration_tests/updates.rs +++ b/wicketd/tests/integration_tests/updates.rs @@ -20,7 +20,7 @@ use maplit::btreeset; use omicron_common::{ disk::DiskIdentity, update::{ - MupdateOverrideHashId, MupdateOverrideInfo, OmicronInstallManifest, + MupdateOverrideInfo, OmicronInstallManifest, OmicronInstallManifestSource, }, }; @@ -34,7 +34,7 @@ use sled_agent_resolvable_files::ZoneImageSourceResolver; use sled_agent_types::resolvable_files::MupdateOverrideNonBootResult; use sled_storage::config::MountConfig; use tokio::sync::oneshot; -use tufaceous_artifact_v2::{InstallinatorArtifactKind, KnownArtifactTags}; +use tufaceous_artifact_v2::KnownArtifactTags; use tufaceous_v2::{Repository, edit::RepositoryEditor}; use wicket::OutputKind; use wicket_common::{ @@ -423,27 +423,6 @@ async fn installinator_fetch_impl( let log = wicketd_testctx.log(); let temp_dir = Utf8TempDir::new().expect("temp dir created"); - let mut expected_ids = BTreeSet::new(); - for artifact in repo.artifacts() { - let Some(kind) = - artifact.known_tags().and_then(|tags| tags.to_installinator()) - else { - continue; - }; - let kind = match kind { - InstallinatorArtifactKind::Zone { zone_name } => { - format!("zone-{zone_name}") - } - InstallinatorArtifactKind::HostPhase2 => "host_phase2".into(), - InstallinatorArtifactKind::MeasurementCorpus => { - "measurement_corpus".into() - } - InstallinatorArtifactKind::ControlPlane => unreachable!(), - }; - expected_ids - .insert(MupdateOverrideHashId { kind, hash: artifact.hash }); - } - let installinator_doc_hash = repo .artifacts() .get_only(&KnownArtifactTags::InstallinatorDocument) @@ -553,8 +532,6 @@ async fn installinator_fetch_impl( serde_json::from_slice::(&a_override_bytes) .expect("mupdate override file successfully deserialized"); - assert_eq!(a_override_info.hash_ids, expected_ids); - // Ensure that the B path also had the same file written out. let b_override_path = b_path.join("install").join(MupdateOverrideInfo::FILE_NAME); From 8a3024dacf57934638b424f1987ad78ec1c027a5 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 29 Jul 2026 10:15:56 -0700 Subject: [PATCH 18/24] clean up get_rot_rkth a bit --- wicketd/src/update_tracker.rs | 41 +++++------------------------------ 1 file changed, 6 insertions(+), 35 deletions(-) diff --git a/wicketd/src/update_tracker.rs b/wicketd/src/update_tracker.rs index 86b13917127..0d59f7f677b 100644 --- a/wicketd/src/update_tracker.rs +++ b/wicketd/src/update_tracker.rs @@ -2115,48 +2115,19 @@ impl UpdateContext { /// Via `client`, ask the target RoT for its CMPA page, then extract /// and return the RKTH. async fn get_rot_rkth(&self) -> Result<[u8; 32], UpdateTerminalError> { - let cmpa = match self - .mgs_client + self.mgs_client .sp_rot_cmpa_get( &self.sp.typ, self.sp.slot, SpComponent::ROT.const_as_str(), ) .await - { - Ok(response) => { + .map_err(anyhow::Error::from) + .and_then(|response| { let data = response.into_inner().base64_data; - self.decode_rot_page(&data).map_err(|error| { - UpdateTerminalError::GetRotCmpaFailed { error } - })? - } - // TODO is there a better way to check the _specific_ error response - // we get here? We only have a couple of strings; we could check the - // error string contents for something like "WrongVersion", but - // that's pretty fragile. Instead we'll treat any error response - // here as a "fallback to previous behavior". - Err(err @ gateway_client::Error::ErrorResponse(_)) => { - error!( - self.log, - "Failed to get RoT CMPA; unable to choose from \ - available RoT artifacts"; - "err" => %err, - ); - return Err(UpdateTerminalError::GetRotCmpaFailed { - error: err.into(), - }); - } - // For any other error (e.g., comms failures), just fail as normal. - Err(err) => { - return Err(UpdateTerminalError::GetRotCmpaFailed { - error: err.into(), - }); - } - }; - let cmpa = CMPAPage::from_bytes(&cmpa).map_err(|err| { - UpdateTerminalError::GetRotCmpaFailed { error: err.into() } - })?; - Ok(cmpa.rotkh) + Ok(CMPAPage::from_bytes(&self.decode_rot_page(&data)?)?.rotkh) + }) + .map_err(|error| UpdateTerminalError::GetRotCmpaFailed { error }) } /// Decode a base64-encoded RoT page we received from MGS. From 1fed1d5c2037725fddef237997d4177a7e358456 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 29 Jul 2026 10:56:08 -0700 Subject: [PATCH 19/24] uphold the invariant --- wicketd/src/update_tracker.rs | 64 +++++++++++++++++------------------ 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/wicketd/src/update_tracker.rs b/wicketd/src/update_tracker.rs index 0d59f7f677b..26847862d1a 100644 --- a/wicketd/src/update_tracker.rs +++ b/wicketd/src/update_tracker.rs @@ -139,6 +139,7 @@ impl SpUpdateData { #[derive(Debug)] enum UploadTrampolinePhase2ToMgsStatus { + Starting, Running { hash: ArtifactHash }, Done { hash: ArtifactHash, uploaded_image_id: HostPhase2RecoveryImageId }, Failed(Arc), @@ -151,7 +152,8 @@ impl UploadTrampolinePhase2ToMgsStatus { | UploadTrampolinePhase2ToMgsStatus::Done { hash, .. } => { Some(*hash) } - UploadTrampolinePhase2ToMgsStatus::Failed(_) => None, + UploadTrampolinePhase2ToMgsStatus::Starting + | UploadTrampolinePhase2ToMgsStatus::Failed(_) => None, } } } @@ -347,34 +349,15 @@ impl UpdateTracker { &self, repo: &Arc, ) -> UploadTrampolinePhase2ToMgs { - match repo.get_handle(&TRAMPOLINE_PHASE_2_TAGS) { - Ok(handle) => { - let (status_tx, status_rx) = watch::channel( - UploadTrampolinePhase2ToMgsStatus::Running { - hash: handle.artifact().hash, - }, - ); - let task = tokio::spawn(upload_trampoline_phase_2_to_mgs( - self.mgs_client.clone(), - handle, - status_tx, - self.log.clone(), - )); - UploadTrampolinePhase2ToMgs { status: status_rx, task } - } - Err(error) => { - let error = UpdateTerminalError::MissingArtifact { - tags: TRAMPOLINE_PHASE_2_TAGS, - error, - }; - let (_, status) = - watch::channel(UploadTrampolinePhase2ToMgsStatus::Failed( - Arc::new(anyhow!(error)), - )); - let task = tokio::spawn(futures::future::ready(())); - UploadTrampolinePhase2ToMgs { status, task } - } - } + let (status_tx, status_rx) = + watch::channel(UploadTrampolinePhase2ToMgsStatus::Starting); + let task = tokio::spawn(upload_trampoline_phase_2_to_mgs( + self.mgs_client.clone(), + Arc::clone(repo), + status_tx, + self.log.clone(), + )); + UploadTrampolinePhase2ToMgs { status: status_rx, task } } /// Updates the repository stored inside the update tracker. @@ -1427,6 +1410,7 @@ impl UpdateDriver { loop { match &*upload_trampoline_phase_2_to_mgs.borrow_and_update() { + UploadTrampolinePhase2ToMgsStatus::Starting | UploadTrampolinePhase2ToMgsStatus::Running { .. } => { // fall through to `.changed()` below }, @@ -2526,7 +2510,7 @@ enum ComponentUpdateStage { async fn upload_trampoline_phase_2_to_mgs( mgs_client: gateway_client::Client, - artifact: ArtifactHandle, + repo: Arc, status: watch::Sender, log: Logger, ) { @@ -2537,7 +2521,23 @@ async fn upload_trampoline_phase_2_to_mgs( let mut attempt = 1; let final_status = loop { - let image_stream = match artifact.stream().await { + let artifact = match repo.artifacts().get_only(&TRAMPOLINE_PHASE_2_TAGS) + { + Ok(artifact) => artifact, + Err(error) => { + break UploadTrampolinePhase2ToMgsStatus::Failed(Arc::new( + anyhow::Error::new(UpdateTerminalError::MissingArtifact { + tags: TRAMPOLINE_PHASE_2_TAGS, + error, + }), + )); + } + }; + status.send_replace(UploadTrampolinePhase2ToMgsStatus::Running { + hash: artifact.hash, + }); + + let image_stream = match repo.read_artifact(&artifact).await { Ok(stream) => stream, Err(err) => { error!( @@ -2559,7 +2559,7 @@ async fn upload_trampoline_phase_2_to_mgs( { Ok(response) => { break UploadTrampolinePhase2ToMgsStatus::Done { - hash: artifact.artifact().hash, + hash: artifact.hash, uploaded_image_id: response.into_inner(), }; } From c91b5b1b24dec2660f3ba8f7840f7445d4b34474 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 29 Jul 2026 12:22:00 -0700 Subject: [PATCH 20/24] fix Cargo.lock mismerge --- Cargo.lock | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 74072143881..635f94eb2b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4507,11 +4507,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi", "wasip2", - "wasm-bindgen", ] [[package]] @@ -12053,15 +12051,16 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.13" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "aws-lc-rs", "bytes", - "getrandom 0.3.4", + "getrandom 0.4.1", "lru-slab", - "rand 0.9.2", + "rand 0.10.1", + "rand_pcg", "ring", "rustc-hash", "rustls", @@ -12215,6 +12214,15 @@ dependencies = [ "rand 0.9.2", ] +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.0", +] + [[package]] name = "rand_seeder" version = "0.4.0" From 8e6acd88d80d1b0051801d746913a0288f5530fe Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 29 Jul 2026 12:53:53 -0700 Subject: [PATCH 21/24] comment on v1_json_schema --- gateway-types/versions/src/initial/update.rs | 7 +++++++ installinator-api/src/lib.rs | 3 +++ 2 files changed, 10 insertions(+) diff --git a/gateway-types/versions/src/initial/update.rs b/gateway-types/versions/src/initial/update.rs index 98db8cee722..5f4802fc51d 100644 --- a/gateway-types/versions/src/initial/update.rs +++ b/gateway-types/versions/src/initial/update.rs @@ -97,6 +97,10 @@ pub struct UpdatePreparationProgress { #[serde(rename_all = "snake_case")] pub struct InstallinatorImageId { pub update_id: MupdateUuid, + + // Tufaceous v2 introduces a new JSON schema for `ArtifactHash` that is + // wire-compatible but perceived as different by drift. Continue using the + // old schema in this API version. #[schemars(schema_with = "ArtifactHash::v1_json_schema")] pub host_phase_2: ArtifactHash, #[schemars(schema_with = "ArtifactHash::v1_json_schema")] @@ -118,6 +122,9 @@ pub enum HostPhase2Progress { /// Identity of a host phase2 recovery image. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct HostPhase2RecoveryImageId { + // Tufaceous v2 introduces a new JSON schema for `ArtifactHash` that is + // wire-compatible but perceived as different by drift. Continue using the + // old schema in this API version. #[schemars(schema_with = "ArtifactHash::v1_json_schema")] pub sha256_hash: ArtifactHash, } diff --git a/installinator-api/src/lib.rs b/installinator-api/src/lib.rs index 1f38fab7412..be0bbb41b3a 100644 --- a/installinator-api/src/lib.rs +++ b/installinator-api/src/lib.rs @@ -71,6 +71,9 @@ pub struct GetArtifactPathParams { pub kind: String, /// The hash of the artifact. + // Tufaceous v2 introduces a new JSON schema for `ArtifactHash` that is + // wire-compatible but perceived as different by drift. Continue using the + // old schema in this API version. #[schemars(schema_with = "ArtifactHash::v1_json_schema")] pub hash: ArtifactHash, } From 3442de6296cefb7fe14156908f9fb0802f579696 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 29 Jul 2026 13:06:54 -0700 Subject: [PATCH 22/24] add context to fetch_artifact calls --- installinator/src/dispatch.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/installinator/src/dispatch.rs b/installinator/src/dispatch.rs index 7d1bb86307b..a954356401a 100644 --- a/installinator/src/dispatch.rs +++ b/installinator/src/dispatch.rs @@ -243,7 +243,8 @@ impl InstallOpts { &last_known_peer, log, ) - .await?; + .await + .context("failed to fetch installinator document")?; // Read the document as JSON. // @@ -577,11 +578,12 @@ async fn download_all_artifacts( InstallinatorComponent::ControlPlaneZone } }; + let file_name = artifact.file_name.clone(); let step = engine .new_step( component, InstallinatorStepId::Download, - format!("Downloading {}", artifact.file_name), + format!("Downloading {}", file_name), async move |cx2| { let fetched = fetch_artifact( &cx2, @@ -590,7 +592,10 @@ async fn download_all_artifacts( last_known_peer, log, ) - .await?; + .await + .with_context(|| { + format!("failed to fetch {file_name}") + })?; StepSuccess::new(fetched.artifact) .with_metadata( InstallinatorCompletionMetadata::Download { From a7c28c9ee2f5bf8583e2053ad7b5a1fa71ac7033 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 29 Jul 2026 13:12:03 -0700 Subject: [PATCH 23/24] get_rot_rkth returns RotKeyTableHash --- wicketd/src/update_tracker.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/wicketd/src/update_tracker.rs b/wicketd/src/update_tracker.rs index 26847862d1a..bd825846197 100644 --- a/wicketd/src/update_tracker.rs +++ b/wicketd/src/update_tracker.rs @@ -1983,10 +1983,9 @@ impl UpdateContext { UpdateTerminalError::GetRotBootloaderCabooseFailed { error } })?; - let rkth = self.get_rot_rkth().await?; let tags = KnownArtifactTags::RotBootloader(RotBootloaderTags { rot_board: caboose.board, - rot_rkth: Some(RotKeyTableHash::from_bytes(rkth)), + rot_rkth: Some(self.get_rot_rkth().await?), }); let artifact_to_apply = repo.get_handle(&tags).map_err(|error| { UpdateTerminalError::MissingArtifact { tags, error } @@ -2062,10 +2061,9 @@ impl UpdateContext { })? .into_inner(); - let rkth = self.get_rot_rkth().await?; let tags = KnownArtifactTags::Rot(RotTags { rot_board: caboose.board, - rot_rkth: Some(RotKeyTableHash::from_bytes(rkth)), + rot_rkth: Some(self.get_rot_rkth().await?), rot_slot: slot_tag, }); let artifact_to_apply = repo.get_handle(&tags).map_err(|error| { @@ -2098,7 +2096,9 @@ impl UpdateContext { /// Via `client`, ask the target RoT for its CMPA page, then extract /// and return the RKTH. - async fn get_rot_rkth(&self) -> Result<[u8; 32], UpdateTerminalError> { + async fn get_rot_rkth( + &self, + ) -> Result { self.mgs_client .sp_rot_cmpa_get( &self.sp.typ, @@ -2109,7 +2109,8 @@ impl UpdateContext { .map_err(anyhow::Error::from) .and_then(|response| { let data = response.into_inner().base64_data; - Ok(CMPAPage::from_bytes(&self.decode_rot_page(&data)?)?.rotkh) + let cmpa = CMPAPage::from_bytes(&self.decode_rot_page(&data)?)?; + Ok(RotKeyTableHash::from_bytes(cmpa.rotkh)) }) .map_err(|error| UpdateTerminalError::GetRotCmpaFailed { error }) } From 848ff9a15c0a2018da211ccc44d5cab5186b691d Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 29 Jul 2026 19:37:33 -0700 Subject: [PATCH 24/24] remove a long-wrong comment --- wicketd/src/update_tracker.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/wicketd/src/update_tracker.rs b/wicketd/src/update_tracker.rs index bd825846197..1fc2a6ad816 100644 --- a/wicketd/src/update_tracker.rs +++ b/wicketd/src/update_tracker.rs @@ -160,9 +160,6 @@ impl UploadTrampolinePhase2ToMgsStatus { #[derive(Debug)] struct UploadTrampolinePhase2ToMgs { - // The tuple is the ID of the Trampoline image and a boolean for whether or - // not it is complete. The upload task retries forever until it succeeds, so - // we don't need to keep a "tried but failed" variant here. status: watch::Receiver, task: JoinHandle<()>, }